8855 lines
1.4 MiB
8855 lines
1.4 MiB
[2026-04-23 08:56:21,777] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:964584] bf16 support detected, enabling for this configuration.
|
||
[2026-04-23 08:56:21,837] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:964584] baseline 0.000GB ()
|
||
[2026-04-23 08:56:21,837] [INFO] [axolotl.cli.config.load_cfg:341] [PID:964584] config:
|
||
{
|
||
"activation_offloading": true,
|
||
"adam_beta1": 0.9,
|
||
"adam_beta2": 0.95,
|
||
"axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_cf_v3_1000.yaml",
|
||
"base_model": "Qwen/Qwen3-8B",
|
||
"base_model_config": "Qwen/Qwen3-8B",
|
||
"batch_size": 32,
|
||
"bf16": true,
|
||
"capabilities": {
|
||
"bf16": true,
|
||
"compute_capability": "sm_90",
|
||
"fp8": true,
|
||
"n_gpu": 4,
|
||
"n_node": 1,
|
||
"tf32": true
|
||
},
|
||
"chat_template": "chatml",
|
||
"context_parallel_size": 1,
|
||
"dataloader_num_workers": 4,
|
||
"dataloader_pin_memory": true,
|
||
"dataloader_prefetch_factor": 256,
|
||
"dataset_num_proc": 288,
|
||
"dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v3-1000",
|
||
"datasets": [
|
||
{
|
||
"ds_type": "parquet",
|
||
"message_property_mappings": {
|
||
"content": "content",
|
||
"role": "role"
|
||
},
|
||
"path": "laion/CoderForge-Preview-v3-1000",
|
||
"trust_remote_code": false
|
||
}
|
||
],
|
||
"ddp": true,
|
||
"deepspeed": {
|
||
"bf16": {
|
||
"enabled": true
|
||
},
|
||
"gradient_accumulation_steps": "auto",
|
||
"gradient_clipping": "auto",
|
||
"train_batch_size": "auto",
|
||
"train_micro_batch_size_per_gpu": "auto",
|
||
"wall_clock_breakdown": false,
|
||
"zero_optimization": {
|
||
"contiguous_gradients": true,
|
||
"gather_16bit_weights_on_model_save": true,
|
||
"max_live_parameters": 0,
|
||
"max_reuse_distance": 0,
|
||
"overlap_comm": true,
|
||
"reduce_bucket_size": "auto",
|
||
"stage": 3,
|
||
"stage3_param_persistence_threshold": "auto",
|
||
"stage3_prefetch_bucket_size": "auto",
|
||
"sub_group_size": 0
|
||
}
|
||
},
|
||
"device": "cuda:0",
|
||
"device_map": {
|
||
"": 0
|
||
},
|
||
"dion_rank_fraction": 1.0,
|
||
"dion_rank_multiple_of": 1,
|
||
"eaft_alpha": 1.0,
|
||
"eaft_k": 20,
|
||
"env_capabilities": {
|
||
"torch_version": "2.9.1"
|
||
},
|
||
"eval_batch_size": 1,
|
||
"eval_causal_lm_metrics": [
|
||
"sacrebleu",
|
||
"comet",
|
||
"ter",
|
||
"chrf"
|
||
],
|
||
"eval_max_new_tokens": 128,
|
||
"eval_table_size": 0,
|
||
"evals_per_epoch": 0,
|
||
"experimental_skip_move_to_device": true,
|
||
"flash_attention": true,
|
||
"fp16": false,
|
||
"generate_samples": false,
|
||
"generation_do_sample": true,
|
||
"generation_max_new_tokens": 50,
|
||
"generation_prompt_ratio": 0.5,
|
||
"generation_temperature": 0.7,
|
||
"gradient_accumulation_steps": 8,
|
||
"gradient_checkpointing": true,
|
||
"gradient_checkpointing_kwargs": {
|
||
"use_reentrant": true
|
||
},
|
||
"include_tkps": true,
|
||
"layer_offloading": false,
|
||
"learning_rate": 1e-05,
|
||
"lisa_layers_attribute": "model.layers",
|
||
"load_best_model_at_end": false,
|
||
"load_in_4bit": false,
|
||
"load_in_8bit": false,
|
||
"local_rank": 0,
|
||
"logging_steps": 1,
|
||
"lora_dropout": 0.0,
|
||
"loraplus_lr_embedding": 1e-06,
|
||
"loss_watchdog_patience": 3,
|
||
"loss_watchdog_threshold": 5.0,
|
||
"lr_scheduler": "cosine",
|
||
"max_grad_norm": 1.0,
|
||
"mean_resizing_embeddings": false,
|
||
"merge_method": "memory_efficient",
|
||
"micro_batch_size": 1,
|
||
"model_config_type": "qwen3",
|
||
"num_epochs": 3.0,
|
||
"num_generation_samples": 3,
|
||
"optimizer": "adamw_torch",
|
||
"otel_metrics_host": "localhost",
|
||
"otel_metrics_port": 8000,
|
||
"output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B",
|
||
"pretrain_multipack_attn": true,
|
||
"profiler_steps_start": 0,
|
||
"qlora_sharded_model_loading": false,
|
||
"quantize_moe_experts": false,
|
||
"ray_num_workers": 1,
|
||
"resources_per_worker": {
|
||
"GPU": 1
|
||
},
|
||
"sample_packing_bin_size": 200,
|
||
"sample_packing_group_size": 100000,
|
||
"save_only_model": false,
|
||
"save_safetensors": true,
|
||
"save_strategy": "epoch",
|
||
"sequence_len": 32768,
|
||
"shuffle_before_merging_datasets": false,
|
||
"shuffle_merged_datasets": true,
|
||
"skip_prepare_dataset": false,
|
||
"streaming_multipack_buffer_size": 10000,
|
||
"strict": false,
|
||
"tensor_parallel_size": 1,
|
||
"tf32": false,
|
||
"tiled_mlp_use_original_mlp": true,
|
||
"tokenizer_config": "Qwen/Qwen3-8B",
|
||
"tokenizer_save_jinja_files": true,
|
||
"torch_dtype": "torch.bfloat16",
|
||
"train_on_inputs": false,
|
||
"trl": {
|
||
"async_prefetch": false,
|
||
"log_completions": false,
|
||
"mask_truncated_completions": false,
|
||
"ref_model_mixup_alpha": 0.9,
|
||
"ref_model_sync_steps": 64,
|
||
"replay_buffer_size": 0,
|
||
"replay_recompute_logps": true,
|
||
"reroll_max_groups": 1,
|
||
"reroll_start_fraction": 1.0,
|
||
"reward_num_workers": 1,
|
||
"scale_rewards": true,
|
||
"skip_zero_advantage_batches": true,
|
||
"sync_ref_model": false,
|
||
"use_data_producer": false,
|
||
"use_vllm": false,
|
||
"vllm_lora_sync": false,
|
||
"vllm_server_host": "0.0.0.0",
|
||
"vllm_server_port": 8000
|
||
},
|
||
"use_otel_metrics": false,
|
||
"use_ray": false,
|
||
"val_set_size": 0.0,
|
||
"vllm": {
|
||
"device": "auto",
|
||
"dtype": "auto",
|
||
"gpu_memory_utilization": 0.9,
|
||
"host": "0.0.0.0",
|
||
"port": 8000
|
||
},
|
||
"wandb_name": "cf-v3-1000-axolotl__Qwen3-8B",
|
||
"warmup_ratio": 0.1875,
|
||
"weight_decay": 0.01,
|
||
"world_size": 4
|
||
}
|
||
[2026-04-23 08:56:21,980] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:964584] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
|
||
[2026-04-23 08:56:22,579] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:964584] EOS: 151645 / <|im_end|>
|
||
[2026-04-23 08:56:22,579] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:964584] BOS: None / None
|
||
[2026-04-23 08:56:22,579] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:964584] PAD: 151643 / <|endoftext|>
|
||
[2026-04-23 08:56:22,579] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:964584] UNK: None / None
|
||
[2026-04-23 08:56:22,724] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:964584] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v3-1000/9ed68e2ecc181422c0bac4f9f1d233ae...
|
||
[2026-04-23 08:56:22,770] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:964584] total_num_tokens: 9_002_408
|
||
[2026-04-23 08:56:22,989] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:964584] `total_supervised_tokens: 2_055_840`
|
||
[2026-04-23 08:56:22,989] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:964584] total_num_steps: 31
|
||
[2026-04-23 08:56:22,989] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:964584] Maximum number of steps set at 31
|
||
[2026-04-23 08:56:23,028] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:964584] loading tokenizer... Qwen/Qwen3-8B
|
||
[2026-04-23 08:56:23,289] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:964584] EOS: 151645 / <|im_end|>
|
||
[2026-04-23 08:56:23,289] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:964584] BOS: None / None
|
||
[2026-04-23 08:56:23,289] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:964584] PAD: 151643 / <|endoftext|>
|
||
[2026-04-23 08:56:23,289] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:964584] UNK: None / None
|
||
[2026-04-23 08:56:23,289] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:964584] Loading model
|
||
[2026-04-23 08:56:23,300] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:964584] Patched Trainer.evaluation_loop with nanmean loss calculation
|
||
[2026-04-23 08:56:23,301] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:964584] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
|
||
[2026-04-23 08:56:23,667] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:964584] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4
|
||
[2026-04-23 08:56:23,668] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:964584] CheckpointWrapper patched to forward DeepSpeed attributes
|
||
[2026-04-23 08:56:54,302] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:964584] Converting modules to torch.bfloat16
|
||
[2026-04-23 08:56:54,305] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:964584] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved)
|
||
[2026-04-23 08:57:00,721] [INFO] [axolotl.train.save_initial_configs:421] [PID:964584] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B...
|
||
[2026-04-23 08:57:00,838] [INFO] [axolotl.train.save_initial_configs:426] [PID:964584] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B...
|
||
[2026-04-23 08:57:00,841] [INFO] [axolotl.train.execute_training:222] [PID:964584] Starting trainer...
|
||
[2026-04-23 08:57:01,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:01,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_id to wrapped module Qwen3DecoderLayer
|
||
0%| | 0/31 [00:00<?, ?it/s][2026-04-23 08:57:03,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:04,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:05,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:06,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:10,580] [WARNING] [trl.models.activation_offloading.verify_sufficient_virtual_memory:178] [PID:964584] [RANK 0] curr_pct=71.4% > self.virtual_memory_safe_pct=60% of virtual memory used
|
||
[2026-04-23 08:57:18,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:18,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:19,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:20,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:31,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:32,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:33,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:41,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:42,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:43,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:52,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:53,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:57:54,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:05,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:05,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:05,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:05,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:05,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:06,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:07,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:16,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:17,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:18,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:27,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:28,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:29,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
3%|▎ | 1/31 [01:36<48:16, 96.56s/it]
{'loss': '0.8091', 'grad_norm': '15.09', 'learning_rate': '0', 'ppl': '2.246', 'memory/max_active (GiB)': '74.97', 'memory/max_allocated (GiB)': '73.5', 'memory/device_reserved (GiB)': '86.85', 'tokens/train_per_sec_per_gpu': '59.02', 'tokens/total': 898176, 'tokens/trainable': 207069, 'epoch': '0.09756'}
|
||
3%|▎ | 1/31 [01:36<48:16, 96.56s/it][2026-04-23 08:58:39,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:39,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:40,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:41,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:48,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:49,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:50,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:58:59,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:00,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:01,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:01,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:01,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:10,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:11,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:12,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:12,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:12,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:12,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:20,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:21,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:22,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:34,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:34,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:34,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:34,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:34,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:35,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:36,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:37,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:47,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:47,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:47,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:48,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:49,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:58,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 08:59:59,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:00,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:07,686] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
6%|▋ | 2/31 [03:05<44:24, 91.87s/it]
{'loss': '0.8537', 'grad_norm': '16.35', 'learning_rate': '2e-06', 'ppl': '2.348', 'memory/max_active (GiB)': '91.59', 'memory/max_allocated (GiB)': '91.59', 'memory/device_reserved (GiB)': '92.88', 'tokens/train_per_sec_per_gpu': '65.14', 'tokens/total': 1759936, 'tokens/trainable': 388084, 'epoch': '0.1951'}
|
||
6%|▋ | 2/31 [03:05<44:24, 91.87s/it][2026-04-23 09:00:07,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:07,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:07,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:08,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:09,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:10,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:20,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:21,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:22,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:22,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:22,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:22,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:32,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:33,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:34,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:44,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:45,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:46,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:55,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:56,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:00:57,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:07,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:08,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:09,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:09,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:09,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:09,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:19,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:20,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:21,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:21,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:21,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:21,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:30,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:30,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:31,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:32,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:33,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,126] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
10%|▉ | 3/31 [04:41<43:50, 93.96s/it]
{'loss': '0.786', 'grad_norm': '13.72', 'learning_rate': '4e-06', 'ppl': '2.195', 'memory/max_active (GiB)': '90.64', 'memory/max_allocated (GiB)': '90.64', 'memory/device_reserved (GiB)': '92.61', 'tokens/train_per_sec_per_gpu': '78.27', 'tokens/total': 2670784, 'tokens/trainable': 589584, 'epoch': '0.2927'}
|
||
10%|▉ | 3/31 [04:41<43:50, 93.96s/it][2026-04-23 09:01:44,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:44,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:45,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:46,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:47,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:47,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:47,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:47,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:57,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:58,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:01:59,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:00,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:00,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:00,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:00,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:09,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:10,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:11,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:20,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:21,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:22,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:22,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:22,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:22,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:32,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:33,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:34,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:34,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:43,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:44,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:45,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:54,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:55,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:02:56,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:05,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:06,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:07,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:15,936] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
13%|█▎ | 4/31 [06:13<41:53, 93.11s/it]
{'loss': '0.6883', 'grad_norm': '8.815', 'learning_rate': '6e-06', 'ppl': '1.99', 'memory/max_active (GiB)': '90.29', 'memory/max_allocated (GiB)': '90.29', 'memory/device_reserved (GiB)': '92.67', 'tokens/train_per_sec_per_gpu': '78.25', 'tokens/total': 3553664, 'tokens/trainable': 801454, 'epoch': '0.3902'}
|
||
13%|█▎ | 4/31 [06:13<41:53, 93.11s/it][2026-04-23 09:03:15,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:16,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:17,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:18,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:27,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:27,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:27,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:28,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:29,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:39,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:39,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:40,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:41,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:42,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:42,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:42,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:51,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:52,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:03:53,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:02,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:03,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:04,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:13,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:14,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:15,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:25,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:26,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:27,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:38,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:39,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:40,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:40,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:40,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:40,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:40,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,534] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
16%|█▌ | 5/31 [07:48<40:34, 93.65s/it]
{'loss': '0.5448', 'grad_norm': '7.169', 'learning_rate': '8e-06', 'ppl': '1.724', 'memory/max_active (GiB)': '91.24', 'memory/max_allocated (GiB)': '91.24', 'memory/device_reserved (GiB)': '92.67', 'tokens/train_per_sec_per_gpu': '63.43', 'tokens/total': 4455808, 'tokens/trainable': 1000579, 'epoch': '0.4878'}
|
||
16%|█▌ | 5/31 [07:48<40:34, 93.65s/it][2026-04-23 09:04:50,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:50,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:51,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:04:52,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:02,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:02,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:02,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:02,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:03,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:04,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:05,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:05,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:13,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:13,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:13,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:14,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:15,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:16,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:16,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:16,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:26,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:27,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:28,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:28,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:36,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:37,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:44,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:45,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:46,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:57,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:58,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:05:59,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:08,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:09,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:10,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:10,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:10,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:10,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:10,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,347] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
19%|█▉ | 6/31 [09:17<38:28, 92.34s/it]
{'loss': '0.4627', 'grad_norm': '1.812', 'learning_rate': '1e-05', 'ppl': '1.588', 'memory/max_active (GiB)': '92.31', 'memory/max_allocated (GiB)': '92.31', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '80.16', 'tokens/total': 5316736, 'tokens/trainable': 1203944, 'epoch': '0.5854'}
|
||
19%|█▉ | 6/31 [09:17<38:28, 92.34s/it][2026-04-23 09:06:20,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:20,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:21,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:22,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:32,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:33,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:34,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:44,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:45,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:46,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:55,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:56,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:06:57,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:06,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:06,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:07,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:08,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:09,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:09,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:09,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:09,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:19,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:19,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:19,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:19,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:19,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:20,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:21,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:29,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:30,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:31,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:41,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:42,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:43,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:43,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,537] [WARNING] [stage3.py:2497:step] 13 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
23%|██▎ | 7/31 [10:52<37:10, 92.95s/it]
{'loss': '0.4532', 'grad_norm': '1.562', 'learning_rate': '9.964e-06', 'ppl': '1.573', 'memory/max_active (GiB)': '92.06', 'memory/max_allocated (GiB)': '92.06', 'memory/device_reserved (GiB)': '92.85', 'tokens/train_per_sec_per_gpu': '60.22', 'tokens/total': 6198848, 'tokens/trainable': 1406676, 'epoch': '0.6829'}
|
||
23%|██▎ | 7/31 [10:52<37:10, 92.95s/it][2026-04-23 09:07:54,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:54,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:55,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:07:56,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:05,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:05,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:05,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:06,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:07,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:08,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:08,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:08,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:08,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:18,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:19,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:20,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:21,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:21,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:21,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:30,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:31,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:32,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:43,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:44,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:45,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:45,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:45,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:45,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:55,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:55,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:55,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:55,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:56,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:08:57,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:07,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:07,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:08,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:09,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:10,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:10,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:19,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:20,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,052] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
26%|██▌ | 8/31 [12:26<35:49, 93.45s/it]
{'loss': '0.4041', 'grad_norm': '1.043', 'learning_rate': '9.855e-06', 'ppl': '1.498', 'memory/max_active (GiB)': '90.29', 'memory/max_allocated (GiB)': '90.29', 'memory/device_reserved (GiB)': '92.54', 'tokens/train_per_sec_per_gpu': '57.36', 'tokens/total': 7110464, 'tokens/trainable': 1618273, 'epoch': '0.7805'}
|
||
26%|██▌ | 8/31 [12:26<35:49, 93.45s/it][2026-04-23 09:09:29,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:29,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:30,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:31,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:31,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:31,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:31,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:40,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:41,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:42,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:52,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:53,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:09:54,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:06,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:07,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:16,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:17,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:26,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:27,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:28,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:39,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:40,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:50,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:51,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:10:52,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,631] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
29%|██▉ | 9/31 [13:58<34:03, 92.86s/it]
{'loss': '0.3758', 'grad_norm': '0.8805', 'learning_rate': '9.675e-06', 'ppl': '1.456', 'memory/max_active (GiB)': '92.19', 'memory/max_allocated (GiB)': '92.19', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '68.53', 'tokens/total': 7983104, 'tokens/trainable': 1820107, 'epoch': '0.878'}
|
||
29%|██▉ | 9/31 [13:58<34:03, 92.86s/it][2026-04-23 09:11:00,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:00,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:01,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:02,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:10,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:11,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:12,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:22,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:23,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:24,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:24,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:24,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:24,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:24,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:33,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:34,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:35,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:44,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:45,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:46,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:55,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:56,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:11:57,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:08,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:09,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:16,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:16,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:16,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:17,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:18,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,393] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
32%|███▏ | 10/31 [15:27<32:10, 91.91s/it]
{'loss': '0.3601', 'grad_norm': '0.7456', 'learning_rate': '9.427e-06', 'ppl': '1.433', 'memory/max_active (GiB)': '92.06', 'memory/max_allocated (GiB)': '92.06', 'memory/device_reserved (GiB)': '92.84', 'tokens/train_per_sec_per_gpu': '71.5', 'tokens/total': 8868288, 'tokens/trainable': 2023086, 'epoch': '0.9756'}
|
||
32%|███▏ | 10/31 [15:27<32:10, 91.91s/it][2026-04-23 09:12:30,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:30,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:31,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:32,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:43,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:44,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:12:54,564] [WARNING] [stage3.py:2497:step] 2 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
35%|███▌ | 11/31 [15:52<23:43, 71.18s/it]
{'loss': '0.3371', 'grad_norm': '0.7523', 'learning_rate': '9.115e-06', 'ppl': '1.401', 'memory/max_active (GiB)': '87.95', 'memory/max_allocated (GiB)': '87.95', 'memory/device_reserved (GiB)': '90.81', 'tokens/train_per_sec_per_gpu': '269.8', 'tokens/total': 9100416, 'tokens/trainable': 2076964, 'epoch': '1'}
|
||
35%|███▌ | 11/31 [15:52<23:43, 71.18s/it][2026-04-23 09:12:57,152] [INFO] [axolotl.core.trainers.base._save:741] [PID:964584] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B/checkpoint-11
|
||
|
||
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
|
||
Writing model shards: 100%|██████████| 1/1 [00:06<00:00, 6.87s/it][A
Writing model shards: 100%|██████████| 1/1 [00:06<00:00, 6.87s/it]
|
||
[2026-04-23 09:13:18,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:18,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:18,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:18,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:18,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:19,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:20,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:21,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:21,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:21,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:21,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:31,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:32,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:33,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:41,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:42,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:43,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:43,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:43,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:53,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:54,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:13:55,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:05,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:06,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:07,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:15,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:16,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:17,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:28,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:29,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:30,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:30,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:30,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:39,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:39,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:39,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:39,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:40,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:41,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:51,957] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
39%|███▊ | 12/31 [17:49<26:59, 85.23s/it]
{'loss': '0.3438', 'grad_norm': '0.6627', 'learning_rate': '8.743e-06', 'ppl': '1.41', 'memory/max_active (GiB)': '89.11', 'memory/max_allocated (GiB)': '89.11', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '74.99', 'tokens/total': 9988736, 'tokens/trainable': 2282599, 'epoch': '1.098'}
|
||
39%|███▊ | 12/31 [17:49<26:59, 85.23s/it][2026-04-23 09:14:51,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:52,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:53,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:54,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:54,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:14:54,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:03,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:04,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:05,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:16,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:17,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:18,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:28,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:29,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:30,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:30,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:30,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:38,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:39,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:40,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:41,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:41,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:51,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:52,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:53,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:53,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:15:53,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:02,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:02,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:02,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:03,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:04,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:14,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:15,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:16,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:16,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:16,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,304] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
42%|████▏ | 13/31 [19:23<26:23, 87.99s/it]
{'loss': '0.316', 'grad_norm': '0.6492', 'learning_rate': '8.316e-06', 'ppl': '1.372', 'memory/max_active (GiB)': '85.45', 'memory/max_allocated (GiB)': '85.45', 'memory/device_reserved (GiB)': '92.85', 'tokens/train_per_sec_per_gpu': '83.88', 'tokens/total': 10890496, 'tokens/trainable': 2476517, 'epoch': '1.195'}
|
||
42%|████▏ | 13/31 [19:23<26:23, 87.99s/it][2026-04-23 09:16:26,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:26,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:27,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:28,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:38,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:39,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:49,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:50,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:16:51,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:02,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:03,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:04,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:13,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:14,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:21,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:22,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:23,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:23,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:23,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:23,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:23,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:32,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:32,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:32,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:32,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:32,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:33,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:34,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:44,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:45,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:46,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:54,886] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
45%|████▌ | 14/31 [20:52<24:58, 88.17s/it]
{'loss': '0.3142', 'grad_norm': '0.552', 'learning_rate': '7.84e-06', 'ppl': '1.369', 'memory/max_active (GiB)': '89.46', 'memory/max_allocated (GiB)': '89.46', 'memory/device_reserved (GiB)': '91.25', 'tokens/train_per_sec_per_gpu': '68.61', 'tokens/total': 11754688, 'tokens/trainable': 2675574, 'epoch': '1.293'}
|
||
45%|████▌ | 14/31 [20:52<24:58, 88.17s/it][2026-04-23 09:17:54,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:54,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:55,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:56,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:57,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:17:57,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:07,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:08,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:09,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:18,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:19,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:20,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:21,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:30,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:30,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:30,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:30,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:31,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:32,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:33,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:33,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:33,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:33,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:44,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:45,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:46,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:56,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:56,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:56,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:56,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:57,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:18:58,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:07,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:07,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:08,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:09,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:19,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:20,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:21,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:31,819] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
48%|████▊ | 15/31 [22:29<24:13, 90.81s/it]
{'loss': '0.2985', 'grad_norm': '0.5145', 'learning_rate': '7.324e-06', 'ppl': '1.348', 'memory/max_active (GiB)': '89.46', 'memory/max_allocated (GiB)': '89.46', 'memory/device_reserved (GiB)': '92.67', 'tokens/train_per_sec_per_gpu': '65.62', 'tokens/total': 12684224, 'tokens/trainable': 2887437, 'epoch': '1.39'}
|
||
48%|████▊ | 15/31 [22:29<24:13, 90.81s/it][2026-04-23 09:19:31,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:31,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:31,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:32,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:33,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:34,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:34,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:44,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:45,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:46,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:56,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:56,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:56,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:56,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:56,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:57,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:19:58,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:08,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:09,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:10,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:21,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:22,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:23,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:23,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:23,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:23,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:34,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:35,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:36,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:47,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:48,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:57,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:58,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:20:59,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,458] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
52%|█████▏ | 16/31 [24:06<23:13, 92.87s/it]
{'loss': '0.2974', 'grad_norm': '0.4643', 'learning_rate': '6.773e-06', 'ppl': '1.346', 'memory/max_active (GiB)': '90.43', 'memory/max_allocated (GiB)': '90.43', 'memory/device_reserved (GiB)': '92.38', 'tokens/train_per_sec_per_gpu': '76.24', 'tokens/total': 13611520, 'tokens/trainable': 3099917, 'epoch': '1.488'}
|
||
52%|█████▏ | 16/31 [24:06<23:13, 92.87s/it][2026-04-23 09:21:09,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:09,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:10,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:11,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:17,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:17,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:17,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:17,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:18,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:19,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:26,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:27,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:28,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:39,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:40,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:41,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:41,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:41,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:41,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:49,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:49,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:49,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:49,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:49,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:50,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:21:51,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:01,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:02,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:03,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:12,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:13,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:14,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:23,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:24,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:25,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:34,793] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
55%|█████▍ | 17/31 [25:32<21:08, 90.60s/it]
{'loss': '0.2965', 'grad_norm': '0.4375', 'learning_rate': '6.197e-06', 'ppl': '1.345', 'memory/max_active (GiB)': '91.94', 'memory/max_allocated (GiB)': '91.94', 'memory/device_reserved (GiB)': '92.84', 'tokens/train_per_sec_per_gpu': '74.68', 'tokens/total': 14465856, 'tokens/trainable': 3300361, 'epoch': '1.585'}
|
||
55%|█████▍ | 17/31 [25:32<21:08, 90.60s/it][2026-04-23 09:22:34,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:34,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:34,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:35,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:36,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:37,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:37,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:47,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:48,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:49,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:22:59,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:00,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:01,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:01,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:01,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:01,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:01,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:10,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:11,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:12,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:12,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:12,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:12,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:20,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:20,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:21,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:22,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:31,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:32,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:41,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:42,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:43,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:53,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:54,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:55,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:55,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:55,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:55,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:23:55,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:04,851] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
58%|█████▊ | 18/31 [27:02<19:35, 90.44s/it]
{'loss': '0.3018', 'grad_norm': '0.4241', 'learning_rate': '5.603e-06', 'ppl': '1.352', 'memory/max_active (GiB)': '90.77', 'memory/max_allocated (GiB)': '90.77', 'memory/device_reserved (GiB)': '92.52', 'tokens/train_per_sec_per_gpu': '65.08', 'tokens/total': 15327808, 'tokens/trainable': 3497396, 'epoch': '1.683'}
|
||
58%|█████▊ | 18/31 [27:02<19:35, 90.44s/it][2026-04-23 09:24:04,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:04,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:04,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:05,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:06,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:13,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:14,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:15,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:15,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:15,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:25,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:26,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:27,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:36,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:37,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:38,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:47,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:47,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:47,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:48,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:49,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:50,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:50,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:24:50,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:00,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:01,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:02,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:02,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:02,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:02,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:02,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:12,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:13,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:14,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:26,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:27,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,585] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
61%|██████▏ | 19/31 [28:32<18:02, 90.23s/it]
{'loss': '0.3036', 'grad_norm': '0.4186', 'learning_rate': '5e-06', 'ppl': '1.355', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.81', 'tokens/train_per_sec_per_gpu': '67.39', 'tokens/total': 16206400, 'tokens/trainable': 3698782, 'epoch': '1.78'}
|
||
61%|██████▏ | 19/31 [28:32<18:02, 90.23s/it][2026-04-23 09:25:34,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:34,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:35,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:36,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:44,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:45,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:46,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:55,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:56,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:25:57,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:08,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:08,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:09,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:10,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:11,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:11,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:20,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:21,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:22,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:29,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:30,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:31,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:41,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:42,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:43,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:53,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:54,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:26:55,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:04,705] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
65%|██████▍ | 20/31 [30:02<16:32, 90.19s/it]
{'loss': '0.2789', 'grad_norm': '0.3937', 'learning_rate': '4.397e-06', 'ppl': '1.322', 'memory/max_active (GiB)': '92.31', 'memory/max_allocated (GiB)': '92.31', 'memory/device_reserved (GiB)': '92.83', 'tokens/train_per_sec_per_gpu': '66.97', 'tokens/total': 17077248, 'tokens/trainable': 3901105, 'epoch': '1.878'}
|
||
65%|██████▍ | 20/31 [30:02<16:32, 90.19s/it][2026-04-23 09:27:04,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:04,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:04,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:04,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:04,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:05,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:06,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:15,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:16,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:17,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:26,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:27,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:28,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:36,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:36,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:36,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:37,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:38,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:48,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:49,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:50,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:50,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:50,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:27:50,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:00,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:01,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:02,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:13,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:14,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:15,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:15,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:15,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:15,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:26,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:27,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:28,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:38,825] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
68%|██████▊ | 21/31 [31:36<15:13, 91.37s/it]
{'loss': '0.2808', 'grad_norm': '0.4225', 'learning_rate': '3.803e-06', 'ppl': '1.324', 'memory/max_active (GiB)': '91.23', 'memory/max_allocated (GiB)': '91.23', 'memory/device_reserved (GiB)': '92.89', 'tokens/train_per_sec_per_gpu': '76.26', 'tokens/total': 17974976, 'tokens/trainable': 4097983, 'epoch': '1.976'}
|
||
68%|██████▊ | 21/31 [31:36<15:13, 91.37s/it][2026-04-23 09:28:38,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:38,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:38,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:39,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:40,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:49,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:50,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,297] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:28:51,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:02,318] [WARNING] [stage3.py:2497:step] 3 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
71%|███████ | 22/31 [31:59<10:39, 71.00s/it]
{'loss': '0.2504', 'grad_norm': '0.5612', 'learning_rate': '3.227e-06', 'ppl': '1.285', 'memory/max_active (GiB)': '88.05', 'memory/max_allocated (GiB)': '88.05', 'memory/device_reserved (GiB)': '89.9', 'tokens/train_per_sec_per_gpu': '324.3', 'tokens/total': 18203648, 'tokens/trainable': 4154716, 'epoch': '2'}
|
||
71%|███████ | 22/31 [31:59<10:39, 71.00s/it][2026-04-23 09:29:04,989] [INFO] [axolotl.core.trainers.base._save:741] [PID:964584] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B/checkpoint-22
|
||
|
||
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
|
||
Writing model shards: 100%|██████████| 1/1 [00:06<00:00, 6.97s/it][A
Writing model shards: 100%|██████████| 1/1 [00:06<00:00, 6.97s/it]
|
||
[2026-04-23 09:29:29,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:29,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:29,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:30,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:31,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:32,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:32,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:41,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:42,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:43,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:53,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:54,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:29:55,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:03,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:03,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:03,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:03,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:04,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:05,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:14,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:15,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:16,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:26,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:27,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:28,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:29,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:29,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:29,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:38,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:39,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:40,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:49,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:50,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:30:51,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:00,811] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
74%|███████▍ | 23/31 [33:58<11:22, 85.25s/it]
{'loss': '0.272', 'grad_norm': '0.9863', 'learning_rate': '2.676e-06', 'ppl': '1.313', 'memory/max_active (GiB)': '89.59', 'memory/max_allocated (GiB)': '89.59', 'memory/device_reserved (GiB)': '92.71', 'tokens/train_per_sec_per_gpu': '68.45', 'tokens/total': 19071296, 'tokens/trainable': 4351781, 'epoch': '2.098'}
|
||
74%|███████▍ | 23/31 [33:58<11:22, 85.25s/it][2026-04-23 09:31:00,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:00,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:00,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:01,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:02,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:03,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:03,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:12,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:13,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:14,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:23,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:24,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:25,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:25,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:25,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:25,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:33,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:33,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:33,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:33,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:33,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:34,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:35,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:44,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:45,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:46,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:57,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:58,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:59,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:59,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:31:59,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:08,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:08,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:09,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:10,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:11,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:21,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:22,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:23,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,082] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
77%|███████▋ | 24/31 [35:31<10:13, 87.66s/it]
{'loss': '0.2675', 'grad_norm': '0.3964', 'learning_rate': '2.16e-06', 'ppl': '1.307', 'memory/max_active (GiB)': '89.94', 'memory/max_allocated (GiB)': '89.94', 'memory/device_reserved (GiB)': '92.69', 'tokens/train_per_sec_per_gpu': '69.55', 'tokens/total': 19945536, 'tokens/trainable': 4553328, 'epoch': '2.195'}
|
||
77%|███████▋ | 24/31 [35:31<10:13, 87.66s/it][2026-04-23 09:32:34,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:34,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:35,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:42,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:43,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:44,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:54,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:55,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:32:56,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:06,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:07,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:08,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:18,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:19,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:20,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:20,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:20,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:20,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:31,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:32,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:33,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:44,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:45,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:46,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:47,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:47,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:47,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:56,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:57,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:58,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:58,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:58,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:33:58,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:08,759] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
81%|████████ | 25/31 [37:06<08:58, 89.76s/it]
{'loss': '0.2445', 'grad_norm': '0.3436', 'learning_rate': '1.684e-06', 'ppl': '1.277', 'memory/max_active (GiB)': '91.94', 'memory/max_allocated (GiB)': '91.94', 'memory/device_reserved (GiB)': '92.82', 'tokens/train_per_sec_per_gpu': '68.79', 'tokens/total': 20844928, 'tokens/trainable': 4758146, 'epoch': '2.293'}
|
||
81%|████████ | 25/31 [37:06<08:58, 89.76s/it][2026-04-23 09:34:08,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:08,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:08,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:08,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:09,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:10,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:11,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:21,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:22,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:30,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:30,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:30,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:30,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:31,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:32,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:42,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:43,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:44,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:53,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:54,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:34:55,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:06,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:07,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:08,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:18,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:19,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:20,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:20,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:20,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:20,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:30,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:30,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:30,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:31,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:32,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:33,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,582] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
84%|████████▍ | 26/31 [38:42<07:37, 91.58s/it]
{'loss': '0.2485', 'grad_norm': '0.3626', 'learning_rate': '1.257e-06', 'ppl': '1.282', 'memory/max_active (GiB)': '92.06', 'memory/max_allocated (GiB)': '92.06', 'memory/device_reserved (GiB)': '92.83', 'tokens/train_per_sec_per_gpu': '66.34', 'tokens/total': 21751360, 'tokens/trainable': 4957920, 'epoch': '2.39'}
|
||
84%|████████▍ | 26/31 [38:42<07:37, 91.58s/it][2026-04-23 09:35:44,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:44,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:45,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:46,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:56,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:57,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:58,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:58,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:35:58,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:08,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:09,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:10,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:19,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:20,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:21,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:32,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:33,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:34,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:43,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:44,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:45,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:55,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:56,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:36:57,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:08,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:09,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:10,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:19,997] [WARNING] [stage3.py:2497:step] 13 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
87%|████████▋ | 27/31 [40:17<06:10, 92.73s/it]
{'loss': '0.258', 'grad_norm': '0.3698', 'learning_rate': '8.851e-07', 'ppl': '1.294', 'memory/max_active (GiB)': '91.35', 'memory/max_allocated (GiB)': '91.35', 'memory/device_reserved (GiB)': '92.83', 'tokens/train_per_sec_per_gpu': '66.55', 'tokens/total': 22644544, 'tokens/trainable': 5158254, 'epoch': '2.488'}
|
||
87%|████████▋ | 27/31 [40:17<06:10, 92.73s/it][2026-04-23 09:37:20,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:20,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:21,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:22,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:22,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:22,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:22,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:31,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:31,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:31,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:31,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:31,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:32,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:33,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:44,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:45,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:46,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:55,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:55,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:56,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:37:57,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:05,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:06,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:07,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:17,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:17,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:17,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:18,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:19,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:20,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:20,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:30,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:31,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:32,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:41,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:41,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:41,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:41,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:42,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:43,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:52,939] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
90%|█████████ | 28/31 [41:50<04:38, 92.80s/it]
{'loss': '0.2665', 'grad_norm': '0.3795', 'learning_rate': '5.727e-07', 'ppl': '1.305', 'memory/max_active (GiB)': '88.64', 'memory/max_allocated (GiB)': '88.64', 'memory/device_reserved (GiB)': '92.12', 'tokens/train_per_sec_per_gpu': '62.95', 'tokens/total': 23529472, 'tokens/trainable': 5358282, 'epoch': '2.585'}
|
||
90%|█████████ | 28/31 [41:50<04:38, 92.80s/it][2026-04-23 09:38:52,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:53,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:38:54,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:02,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:03,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:04,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:13,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:14,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:15,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:15,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:15,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:15,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:15,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:22,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:23,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:24,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:34,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:35,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:36,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:45,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:45,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:46,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:47,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:48,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:48,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:48,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:58,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:39:59,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:00,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:08,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:09,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:17,980] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
94%|█████████▎| 29/31 [43:15<03:00, 90.47s/it]
{'loss': '0.2648', 'grad_norm': '0.3643', 'learning_rate': '3.249e-07', 'ppl': '1.303', 'memory/max_active (GiB)': '90.53', 'memory/max_allocated (GiB)': '90.53', 'memory/device_reserved (GiB)': '92.77', 'tokens/train_per_sec_per_gpu': '67.28', 'tokens/total': 24384064, 'tokens/trainable': 5559608, 'epoch': '2.683'}
|
||
94%|█████████▎| 29/31 [43:15<03:00, 90.47s/it][2026-04-23 09:40:18,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:18,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:19,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:20,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:29,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:30,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:31,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:41,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:42,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:43,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:53,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:54,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:40:55,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:03,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:04,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:05,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:15,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:16,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:17,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:26,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:27,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:28,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:39,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:40,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:41,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:41,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:41,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:41,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:41,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:49,994] [WARNING] [stage3.py:2497:step] 13 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
97%|█████████▋| 30/31 [44:47<01:30, 90.93s/it]
{'loss': '0.2678', 'grad_norm': '0.337', 'learning_rate': '1.453e-07', 'ppl': '1.307', 'memory/max_active (GiB)': '90.64', 'memory/max_allocated (GiB)': '90.64', 'memory/device_reserved (GiB)': '92.4', 'tokens/train_per_sec_per_gpu': '71.15', 'tokens/total': 25272896, 'tokens/trainable': 5772058, 'epoch': '2.78'}
|
||
97%|█████████▋| 30/31 [44:47<01:30, 90.93s/it][2026-04-23 09:41:50,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:50,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:51,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:41:52,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:03,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:04,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:05,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:05,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:05,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:05,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:05,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:14,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:15,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:16,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:25,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:26,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:27,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:28,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:38,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:39,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:40,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:50,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:50,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:50,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:50,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:51,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:42:52,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:02,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:03,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:04,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:13,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:14,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:15,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:964584] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
|
||
[2026-04-23 09:43:26,510] [WARNING] [stage3.py:2497:step] 13 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
|
||
100%|██████████| 31/31 [46:23<00:00, 92.61s/it]
{'loss': '0.2794', 'grad_norm': '0.426', 'learning_rate': '3.646e-08', 'ppl': '1.322', 'memory/max_active (GiB)': '90.53', 'memory/max_allocated (GiB)': '90.53', 'memory/device_reserved (GiB)': '92.59', 'tokens/train_per_sec_per_gpu': '69.7', 'tokens/total': 26190464, 'tokens/trainable': 5980322, 'epoch': '2.878'}
|
||
100%|██████████| 31/31 [46:23<00:00, 92.61s/it][2026-04-23 09:43:31,456] [INFO] [axolotl.core.trainers.base._save:741] [PID:964584] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B/checkpoint-31
|
||
|
||
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
|
||
Writing model shards: 100%|██████████| 1/1 [00:07<00:00, 7.30s/it][A
Writing model shards: 100%|██████████| 1/1 [00:07<00:00, 7.30s/it]
|
||
{'train_runtime': '2822', 'train_samples_per_second': '0.352', 'train_steps_per_second': '0.011', 'train_loss': '0.3783', 'memory/max_active (GiB)': '31.77', 'memory/max_allocated (GiB)': '31.77', 'memory/device_reserved (GiB)': '92.24', 'epoch': '2.878', 'tokens/train_per_sec_per_gpu': '0'}
|
||
100%|██████████| 31/31 [47:02<00:00, 92.61s/it]
100%|██████████| 31/31 [47:02<00:00, 91.03s/it]
|
||
[2026-04-23 09:44:05,437] [INFO] [axolotl.train.save_trained_model:241] [PID:964584] Training completed! Saving trained model to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B.
|
||
[2026-04-23 09:44:06,989] [INFO] [axolotl.core.trainers.base._save:741] [PID:964584] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B
|
||
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:04<00:00, 4.68s/it]
Writing model shards: 100%|██████████| 1/1 [00:04<00:00, 4.68s/it]
|
||
[2026-04-23 09:44:11,791] [INFO] [axolotl.train.save_trained_model:355] [PID:964584] Model successfully saved to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-1000-axolotl__Qwen3-8B
|