diff --git a/qwen3_6_scripts/patch_xformers_sdpa_seq.py b/qwen3_6_scripts/patch_xformers_sdpa_seq.py index 496abc1..d4309a0 100644 --- a/qwen3_6_scripts/patch_xformers_sdpa_seq.py +++ b/qwen3_6_scripts/patch_xformers_sdpa_seq.py @@ -29,6 +29,10 @@ flash attention kernel（ixformer / cudnnFlashAttnForward）。 关闭选项），原意是防止 profiling OOM。但 _run_sdpa_fallback 已通过 Q-tiling 解决了该问题，chunked prefill 反而会把推理路径从 _run_sdpa_fallback 切换到 _forward_prefix_pytorch，属于不必要的行为变更，因此一并禁用该自动逻辑。 + 同时解除 EngineArgs.create_*_config 中强制 enforce_eager=True 和 + disable_custom_all_reduce=True 的硬编码，让 Iluvatar 环境可以实际验证 + CUDA Graph / custom all-reduce；需要回退时仍可通过命令行显式传 + --enforce-eager --disable-custom-all-reduce。 Deploy: python3 modified_scripts/patch_xformers_sdpa_seq.py @@ -91,6 +95,13 @@ _ARG_NEW_BLOCK = """\ # handles long-context memory without chunked prefill\ """ +_ARG_FORCE_EAGER_OLD = " enforce_eager=True," +_ARG_FORCE_EAGER_NEW = " enforce_eager=self.enforce_eager," +_ARG_FORCE_ALLREDUCE_OLD = " disable_custom_all_reduce=True," +_ARG_FORCE_ALLREDUCE_NEW = ( + " disable_custom_all_reduce=self.disable_custom_all_reduce," +) + FALLBACK_METHOD = ''' def _run_sdpa_fallback( self, @@ -275,6 +286,26 @@ def patch_arg_utils(path): else: print(" [warn] target block not found — check arg_utils.py version") + if _ARG_FORCE_EAGER_NEW in content: + print(" [skip] enforce_eager already respects CLI") + elif _ARG_FORCE_EAGER_OLD in content: + content = content.replace(_ARG_FORCE_EAGER_OLD, + _ARG_FORCE_EAGER_NEW, 1) + print(" [ok] enforce_eager now respects CLI") + changed = True + else: + print(" [warn] enforce_eager assignment not found") + + if _ARG_FORCE_ALLREDUCE_NEW in content: + print(" [skip] custom all-reduce already respects CLI") + elif _ARG_FORCE_ALLREDUCE_OLD in content: + content = content.replace(_ARG_FORCE_ALLREDUCE_OLD, + _ARG_FORCE_ALLREDUCE_NEW, 1) + print(" [ok] custom all-reduce now respects CLI") + changed = True + else: + print(" [warn] custom all-reduce assignment not found") + if changed: with open(path, "w") as f: f.write(content) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 4825f0f..0a5a288 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -851,7 +851,7 @@ class EngineArgs: max_model_len=self.max_model_len, quantization=self.quantization, quantization_param_path=self.quantization_param_path, - enforce_eager=True, + enforce_eager=self.enforce_eager, max_context_len_to_capture=self.max_context_len_to_capture, max_seq_len_to_capture=self.max_seq_len_to_capture, max_logprobs=self.max_logprobs, @@ -927,7 +927,7 @@ class EngineArgs: tensor_parallel_size=self.tensor_parallel_size, worker_use_ray=self.worker_use_ray, max_parallel_loading_workers=self.max_parallel_loading_workers, - disable_custom_all_reduce=True, + disable_custom_all_reduce=self.disable_custom_all_reduce, tokenizer_pool_config=TokenizerPoolConfig.create_config( self.tokenizer_pool_size, self.tokenizer_pool_type, diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md index 3a31ab4..a12288a 100644 --- a/worklogs/decode_analysis_report_2026-07-14.md +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -431,3 +431,107 @@ self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(...) - 直接在当前可用 xFormers 路径插桩。 - 重点记录每 token decode 中 attention、MoE、sampler、TP 同步的耗时。 - 当前平均 GPU 利用率低，profiling 比继续盲调参数更有价值。 + +## 2026-07-14：解除强制 eager / custom all-reduce 禁用验证 + +### 修改内容 + +本轮先处理 `vllm/engine/arg_utils.py` 中两个会掩盖真实性能路径的硬编码： + +```python +enforce_eager=True +disable_custom_all_reduce=True +``` + +改为尊重 CLI / dataclass 参数： + +```python +enforce_eager=self.enforce_eager +disable_custom_all_reduce=self.disable_custom_all_reduce +``` + +同时更新 `qwen3_6_scripts/patch_xformers_sdpa_seq.py`，让后续重新执行 patchops 时也会保留该行为。服务器实际运行路径已确认： + +```text +PYTHONPATH=/usr/local/corex/lib/python3/dist-packages:/usr/local/corex/lib64/python3/dist-packages +``` + +因此运行时同时 patch 了： + +- `/usr/local/corex/lib/python3/dist-packages/vllm/engine/arg_utils.py` +- `/usr/local/corex/lib64/python3/dist-packages/vllm/engine/arg_utils.py` + +远端运行时备份： + +- `arg_utils.py.bak_20260714_eager_allreduce` + +### 实验 1：CUDA Graph + custom all-reduce 同时开启 + +启动命令不再带 `--enforce-eager`，也不带 `--disable-custom-all-reduce`。 + +日志确认配置已生效： + +```text +disable_custom_all_reduce=False +enforce_eager=False +use_async_output_proc=True +``` + +结果：服务未能完成启动，长时间卡在 CUDA Graph capture 阶段。 + +关键日志： + +```text +Capturing the model for CUDA graphs. +[W CUDAGraph.cpp:145] Warning: Waiting for pending NCCL work to finish before starting graph capture. +``` + +判断：当前 Iluvatar BI-V100 + CoreX + xFormers + TP=4 路径下，CUDA Graph capture 不可直接启用。它没有快速报错，而是卡在 graph capture / NCCL pending work 阶段，风险比普通参数不兼容更高。短期不建议继续沿 CUDA Graph 方向盲试。 + +本地归档： + +- `worklogs/remote_results/2026-07-14-eager-allreduce/server_exp_graph_allreduce_seq2_b8192.log` + +### 实验 2：仅开启 custom all-reduce，继续 eager + +启动命令保留 `--enforce-eager`，但不再带 `--disable-custom-all-reduce`。 + +日志确认： + +```text +enforce_eager=True +disable_custom_all_reduce=False +``` + +服务可以正常启动并通过 `/health`。 + +decode microbench 结果： + +| 配置 | 成功率 | TTFT P90 | Output TPS P10/req | Aggregate Output TPS | 对比旧结果 | +| --- | ---: | ---: | ---: | ---: | --- | +| short c1, 256 tok, custom AR on | 100% | 3.47s | 8.47 | 7.94 | 旧 full-parser c1 为 P10 8.74 / aggregate 8.34，略降 | +| short c2, 128 tok, custom AR on | 100% | 1.46s | 4.22 | 8.07 | 旧 parser-off c2 aggregate 7.23，略升但口径不完全相同 | + +本地归档： + +- `worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c1_t256_r3.json` +- `worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c2_t128_r4.json` +- `worklogs/remote_results/2026-07-14-eager-allreduce/server_exp_eager_custom_ar_seq2_b8192_retry.log` + +### 结论 + +1. 之前的硬编码确实屏蔽了真实配置，本轮已经解除，并确认修改落在实际运行的 CoreX site-packages 路径里。 +2. CUDA Graph 当前不兼容或存在严重启动卡死问题，不适合作为短期主优化方向。 +3. custom all-reduce 可以启动和推理，但收益有限：c2 聚合吞吐有小幅提升，c1 无提升。 +4. 当前 decode 吞吐仍在 8 tok/s 左右，距离 Output TPS P10 >= 20 仍有明显差距，瓶颈不只是 all-reduce 开关。 + +### 下一步 profiling 方向 + +优先进入代码级 profiling，而不是继续调 CLI 开关： + +1. 在 `ModelRunner.execute_model` 统计模型 forward、logits、sample 的阶段耗时。 +2. 在 Qwen MoE 层统计 attention、MoE expert、MoE gate、TP all-reduce 的耗时占比。 +3. 在 fused MoE 路径统计 topk、expert kernel、activation、sum 的耗时。 +4. 用 `ENGINEX_PROFILE_DECODE=1` 这类环境变量控制插桩，只在短压测时开启，避免污染正式结果。 + +初步判断：custom all-reduce 不是第一大瓶颈；更可能的主战场是 xFormers decode attention、MoE 小 batch kernel、以及 TP 下大量小 kernel / 同步造成的低 GPU 利用率。 diff --git a/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c1_t256_r3.json b/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c1_t256_r3.json new file mode 100644 index 0000000..56b7545 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c1_t256_r3.json @@ -0,0 +1,63 @@ +{ + "created_at": "2026-07-14T11:00:49", + "label": "eager_custom_ar_short_c1_t256_r3", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 3, + "max_tokens": 256, + "wall_sec": 96.7806523796171, + "success_rate": 1.0, + "ttft_p50_sec": 1.0768930949270725, + "ttft_p90_sec": 3.4729231126606463, + "output_tps_p10_per_request": 8.472860425455258, + "output_tps_p50_per_request": 8.483117808158335, + "aggregate_output_tps": 7.935470376739762, + "prompt_tokens": 117, + "cached_tokens": 64, + "completion_tokens": 768, + "reasoning_tokens": 768, + "chars": 2635, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 34.2951952572912, + "ttft_sec": 4.07193061709404, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 256, + "output_tps": 8.470296079779487, + "chars": 916, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.25290015526116, + "ttft_sec": 1.0768930949270725, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 256, + "output_tps": 8.483561111586171, + "chars": 856, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.230418637394905, + "ttft_sec": 1.05283466540277, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 256, + "output_tps": 8.483117808158335, + "chars": 863, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c2_t128_r4.json b/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c2_t128_r4.json new file mode 100644 index 0000000..3abf83a --- /dev/null +++ b/worklogs/remote_results/2026-07-14-eager-allreduce/decode_eager_custom_ar_short_c2_t128_r4.json @@ -0,0 +1,75 @@ +{ + "created_at": "2026-07-14T11:01:53", + "label": "eager_custom_ar_short_c2_t128_r4", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 2, + "requests": 4, + "max_tokens": 128, + "wall_sec": 63.41567398421466, + "success_rate": 1.0, + "ttft_p50_sec": 1.449904115870595, + "ttft_p90_sec": 1.4555151607841255, + "output_tps_p10_per_request": 4.218399789762928, + "output_tps_p50_per_request": 4.2304733122443885, + "aggregate_output_tps": 8.073713765581775, + "prompt_tokens": 156, + "cached_tokens": 128, + "completion_tokens": 512, + "reasoning_tokens": 512, + "chars": 1816, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 31.78747241385281, + "ttft_sec": 1.4443142116069794, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 4.21841388911607, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.787043346092105, + "ttft_sec": 1.4437402617186308, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 4.2183937471830095, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.626181228086352, + "ttft_sec": 1.4555242210626602, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 4.242532735372708, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.625997802242637, + "ttft_sec": 1.4554940201342106, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 4.24255428163934, + "chars": 454, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-eager-allreduce/server_exp_eager_custom_ar_seq2_b8192_retry.log b/worklogs/remote_results/2026-07-14-eager-allreduce/server_exp_eager_custom_ar_seq2_b8192_retry.log new file mode 100644 index 0000000..13d4616 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-eager-allreduce/server_exp_eager_custom_ar_seq2_b8192_retry.log @@ -0,0 +1,398 @@ +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 10:55:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-14 10:55:03.598198: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-14 10:55:03.649443: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-14 10:55:09 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-14 10:55:09 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=100000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.95, num_gpu_blocks_override=None, max_num_batched_tokens=8192, max_num_seqs=2, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=32768, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-14 10:55:09 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-14 10:55:20 config.py:887] Defaulting to use mp for distributed inference +INFO 07-14 10:55:20 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=8192. +WARNING 07-14 10:55:20 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-14 10:55:20 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=100000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-14 10:55:20 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-14 10:55:20 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-14 10:55:20 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 10:55:20 selector.py:115] Using XFormers backend. +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 10:55:22 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 10:55:22 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 10:55:22 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:29 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:29 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:29 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:29 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:29 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:29 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-14 10:55:30 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=46879, remote_subscribe_port=None) +INFO 07-14 10:55:30 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:30 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:30 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 10:55:30 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 10:55:30 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:30 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9684) INFO 07-14 10:55:30 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:30 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9683) INFO 07-14 10:55:30 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=9685) INFO 07-14 10:55:30 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=53133, remote_subscribe_port=None) +INFO 07-14 10:46:28 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=8115) INFO 07-14 10:46:28 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=8114) INFO 07-14 10:46:28 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=8116) INFO 07-14 10:46:28 model_runner.py:1065] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 10:46:28 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 10:46:28 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=8116) INFO 07-14 10:46:28 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=8114) INFO 07-14 10:46:28 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=8115) INFO 07-14 10:46:28 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=8116) INFO 07-14 10:46:28 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=8114) INFO 07-14 10:46:28 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=8115) INFO 07-14 10:46:28 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00