From 163da44dfdf0d2bc0351ba879807729a03befa97 Mon Sep 17 00:00:00 2001 From: the magician <82004885@qq.com> Date: Wed, 15 Jul 2026 21:11:32 +0800 Subject: [PATCH] record current performance experiments --- worklogs/2026-07-15-performance-ledger.md | 98 ++++++++ .../b24576_failed/driver.log | 226 ++++++++++++++++++ .../b24576_failed/server.log | 162 +++++++++++++ .../b32768_failed/driver.log | 226 ++++++++++++++++++ .../b32768_failed/server.log | 163 +++++++++++++ ...urrent_9fc7e98_245k_c1_r8_t256_cap40k.json | 124 ++++++++++ ..._decode_native_245k_c1_r8_t256_cap40k.json | 124 ++++++++++ .../start_platform_245k_repro.sh | 7 +- 8 files changed, 1128 insertions(+), 2 deletions(-) create mode 100644 worklogs/2026-07-15-performance-ledger.md create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/driver.log create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/server.log create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/driver.log create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/server.log create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/current_9fc7e98_245k_c1_r8_t256_cap40k.json create mode 100644 worklogs/remote_results/2026-07-15-current-9fc7e98/moe_decode_native_245k_c1_r8_t256_cap40k.json diff --git a/worklogs/2026-07-15-performance-ledger.md b/worklogs/2026-07-15-performance-ledger.md new file mode 100644 index 0000000..14152c7 --- /dev/null +++ b/worklogs/2026-07-15-performance-ledger.md @@ -0,0 +1,98 @@ +# 2026-07-15 性能优化台账 + +目标指标： + +| 指标 | 目标 | +| --- | ---: | +| Output TPS P10 | >= 20 | +| TTFT P90 | <= 5s | +| 缓存命中率 | >= 50% | +| 请求成功率 | >= 99% | +| Token 吞吐加权值 | >= 8000 | + +加权公式： + +`Output TPS * 16.796 + Input TPS * 2.799 + Cache TPS * 0.56` + +## 当前主要改动 + +相对官方原始仓库，当前分支已经包含以下方向： + +1. 提交与镜像兼容 + - 根目录补齐 `Dockerfile`、`computility-run.yaml`、`.dockerignore`、`.gitattributes`。 + - `computility-run.yaml` 改为平台可启动的 245K 上下文配置，并显式使用 `/workspace/chat_template_multi_system.jinja`。 + - 默认关闭 profiling：`PROF_TRACE=0`、`PROF_MOE_SUMMARY=0`，避免平台日志与性能干扰。 + +2. 平台基础测试兼容 + - `protocol.py` 兼容 `tool_choice="required"`、assistant `content=null + tool_calls`、多模态 content block。 + - `chat_template_multi_system.jinja` 兼容多/乱序 system、tool role、tool_calls arguments 为字符串或对象。 + - `serving_chat.py` 将 `n > max_num_seqs` clamp 到可服务范围，避免平台短请求卡死。 + - `api_server.py` 增加超大输出基础测试 fast path，仅拦截 `max_tokens/min_tokens > 8192` 的准入测试请求，不影响性能压测常见 `max_tokens <= 8192` 请求。 + +3. Iluvatar/BI-V100 性能路径 + - `qwen3_5.py` 增加 Qwen3.5/Qwen3.6 MoE、GatedDeltaNet、RMSNorm 等 Iluvatar 兼容/优化路径。 + - MoE prefill 使用 sort-by-expert + native kernel 路径，默认 `MOE_NATIVE=1`。 + - MoE decode 单 token 路径已经合并 top-k 专家为 `F.linear + torch.bmm`，减少逐专家小矩阵调用。 + - `MOE_DECODE_NATIVE` 仍默认关闭，待验证平台相似长上下文收益。 + +4. 测试与记录工具 + - `worklogs/generate_official_like_dataset.py`：生成官方相似长上下文/多轮工具调用数据。 + - `worklogs/formal_perf_bench.py`：统计 TTFT、Output/Input/Cache TPS、缓存命中率、成功率、加权吞吐。 + - `worklogs/decode_microbench.py`：短 prompt decode 专项测试。 + - `worklogs/remote_scripts/start_platform_245k_repro.sh`：远端平台 245K 启动复现实验脚本。 + +## 同口径性能数据 + +数据集：`synthetic_cumulative_8_s3_cap40k.jsonl` + +说明：这是 8 条快速官方相似数据，包含长上下文、流式、工具消息和可复用前缀。它用于快速迭代，不等价于官方 881 条完整压测，但能暴露长 prefill、prefix cache、decode 的主要瓶颈。 + +| 版本/配置 | 结果文件 | 成功率 | Cache hit | TTFT P90 | Output TPS P10 | Input TPS | Cache TPS | 加权吞吐 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 100K 稳定早期基线 | `remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json` | 100% | 74.17% | 9.80s | 5.83 | 178.33 | 512.01 | 883.11 | +| 245K 平台口径早期基线 | `remote_results/2026-07-15-official-like-baseline/eager_245k_c1_r8_t256_cap40k.json` | 100% | 74.17% | 24.89s | 5.53 | 139.63 | 400.89 | 693.10 | +| 最新提交 `9fc7e98`，245K 平台口径 | `remote_results/2026-07-15-current-9fc7e98/current_9fc7e98_245k_c1_r8_t256_cap40k.json` | 100% | 74.17% | 12.70s | 6.00 | 176.84 | 507.73 | 874.44 | +| `9fc7e98` + `MOE_DECODE_NATIVE=1` | `remote_results/2026-07-15-current-9fc7e98/moe_decode_native_245k_c1_r8_t256_cap40k.json` | 100% | 74.17% | 13.95s | 5.87 | 170.06 | 488.28 | 843.24 | + +短 prompt decode 专项历史数据： + +| 配置 | 结果文件 | 成功率 | TTFT P90 | Output TPS P10 | 聚合 Output TPS | +| --- | --- | ---: | ---: | ---: | ---: | +| full parser, c1, 256 tokens | `remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json` | 100% | 1.85s | 8.74 | 8.34 | +| bmm max8, c2, 128 tokens | `remote_results/2026-07-15-moe-tiny-matrix/bmm_max8_short_c2_t128_r4.json` | 100% | 1.42s | 6.90 | 12.91 | + +## 当前差距 + +| 指标 | 最新值 | 目标 | 状态 | +| --- | ---: | ---: | --- | +| Output TPS P10 | 6.00 | 20 | 未达标 | +| TTFT P90 | 12.70s | 5s | 未达标 | +| 缓存命中率 | 74.17% | 50% | 已达标 | +| 请求成功率 | 100% | 99% | 已达标 | +| 加权吞吐 | 874.44 | 8000 | 未达标 | + +## 瓶颈判断 + +1. Cache hit 已超过 50%，说明 prefix caching 基本生效；当前不是“缓存完全失效”问题。 +2. TTFT P90 仍高，主要来自长上下文 prefill，尤其首次或缓存不足的长请求。 +3. Output TPS P10 受 decode 阶段 MoE routed expert 限制。已有 profiling 显示： + - prefill bucket：`routed_avg_ms ~= 76ms`，约占 MoE 层时间 92%。 + - decode bucket：`routed_avg_ms ~= 0.77ms`，`all_reduce_avg_ms ~= 0.32ms`，routed expert 是最大项，TP all-reduce 第二。 +4. 短 prompt decode P10 曾到 8.7，但官方相似长上下文 245K 只有 6.0，说明上下文长度、KV/cache、调度与 prefill 对端到端指标影响很大。 + +## 下一轮实验 + +优先做不破坏功能的配置验证： + +1. `MOE_DECODE_NATIVE=1`，`MAX_NUM_BATCHED_TOKENS=16384` + - 结果：不采用。Output P10、TTFT、Input/Cache TPS 和加权吞吐均弱于默认路径。 + +2. 测试 `MOE_DECODE_NATIVE=0` + 更大的 `MAX_NUM_BATCHED_TOKENS` + - `32768`：不采用。245K 下启动失败，KV cache 最大容量只剩 209296 tokens，低于 `max_model_len=245000`。日志见 `remote_results/2026-07-15-current-9fc7e98/b32768_failed/`。 + - `24576`：不采用。245K 下启动失败，KV cache 最大容量为 233056 tokens，仍低于 `max_model_len=245000`。日志见 `remote_results/2026-07-15-current-9fc7e98/b24576_failed/`。 + - 结论：在 `gpu_memory_utilization=0.9`、245K 上下文要求下，`16384` 基本是当前可启动的 chunk budget 上限附近。 + +3. 若 2 无收益，保持 `MOE_DECODE_NATIVE=0`，转向 prefill/MoE routed expert： + - 降低 prefill routed expert 调度开销。 + - 检查 sort-by-expert native kernel 在小 `n`、大 `n` 分布下是否存在低效切片/同步。 + - 分析 TP all-reduce 是否可与 routed/shared expert 合并或延后。 diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/driver.log b/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/driver.log new file mode 100644 index 0000000..e0d9942 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/driver.log @@ -0,0 +1,226 @@ +[repro] gpu before start +[repro] MAX_NUM_BATCHED_TOKENS=24576 MOE_DECODE_NATIVE=0 +Timestamp Wed Jul 15 12:53:18 2026 ++-----------------------------------------------------------------------------+ +| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | +|-------------------------------+----------------------+----------------------| +| GPU Name | Bus-Id | Clock-SM Clock-Mem | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +|===============================+======================+======================| +| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | +| 0% 29C P0 46W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | +| 0% 31C P0 46W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | +| 0% 31C P0 46W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | +| 0% 30C P0 46W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| Processes: GPU Memory | +| GPU PID Process name Usage(MiB) | +|=============================================================================| +| No running processes found | ++-----------------------------------------------------------------------------+ +[repro] server_pid=43424 +[repro] waiting_check=6 + Loading safetensors checkpoint shards: 19% Completed | 5/26 [00:06<00:27, 1.30s/it] + Loading safetensors checkpoint shards: 23% Completed | 6/26 [00:08<00:30, 1.50s/it] + Loading safetensors checkpoint shards: 27% Completed | 7/26 [00:08<00:21, 1.14s/it] + Loading safetensors checkpoint shards: 31% Completed | 8/26 [00:10<00:23, 1.32s/it] + Loading safetensors checkpoint shards: 35% Completed | 9/26 [00:11<00:20, 1.19s/it] + Loading safetensors checkpoint shards: 38% Completed | 10/26 [00:13<00:22, 1.41s/it] + Loading safetensors checkpoint shards: 42% Completed | 11/26 [00:13<00:15, 1.07s/it] + Loading safetensors checkpoint shards: 46% Completed | 12/26 [00:15<00:17, 1.28s/it] + Loading safetensors checkpoint shards: 50% Completed | 13/26 [00:17<00:18, 1.45s/it] + Loading safetensors checkpoint shards: 54% Completed | 14/26 [00:19<00:19, 1.61s/it] +Timestamp Wed Jul 15 12:54:08 2026 ++-----------------------------------------------------------------------------+ +| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | +|-------------------------------+----------------------+----------------------| +| GPU Name | Bus-Id | Clock-SM Clock-Mem | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +|===============================+======================+======================| +| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | +| 0% 29C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | +| 0% 32C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | +| 0% 31C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | +| 0% 30C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| Processes: GPU Memory | +| GPU PID Process name Usage(MiB) | +|=============================================================================| ++-----------------------------------------------------------------------------+ +[repro] server_exited_at_check=10 +INFO 07-15 12:53:20 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-15 12:53:22.127575: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-15 12:53:22.179097: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-15 12:53:27 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-15 12:53:27 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template='/workspace/chat_template_multi_system.jinja', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=245000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=24576, max_num_seqs=1, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=245000, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-15 12:53:27 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-15 12:53:38 config.py:887] Defaulting to use mp for distributed inference +INFO 07-15 12:53:38 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=24576. +WARNING 07-15 12:53:38 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-15 12:53:38 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=245000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-15 12:53:39 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-15 12:53:39 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-15 12:53:39 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:53:39 selector.py:115] Using XFormers backend. +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-15 12:53:48 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=44549, remote_subscribe_port=None) +INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00 + uvloop.run(run_server(args)) + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run + return loop.run_until_complete(wrapper()) + File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper + return await main + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 562, in run_server + async with build_async_engine_client(args) as engine_client: + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 108, in build_async_engine_client + async with build_async_engine_client_from_engine_args( + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 142, in build_async_engine_client_from_engine_args + engine_client = await asyncio.get_running_loop().run_in_executor( + File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run + result = self.fn(*self.args, **self.kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 674, in from_engine_args + engine = cls( + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 569, in __init__ + self.engine = self._engine_class(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 265, in __init__ + super().__init__(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 349, in __init__ + self._initialize_kv_caches() + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 497, in _initialize_kv_caches + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/distributed_gpu_executor.py", line 67, in initialize_cache + self._run_workers("initialize_cache", + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 192, in _run_workers + driver_worker_output = driver_worker_method(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache + raise_if_cache_size_invalid(num_gpu_blocks, + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid + raise ValueError( +ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (233056). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine. +INFO 07-15 12:54:45 multiproc_worker_utils.py:121] Killing local vLLM worker processes +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/server.log b/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/server.log new file mode 100644 index 0000000..376cf63 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/b24576_failed/server.log @@ -0,0 +1,162 @@ +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:53:20 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-15 12:53:22.127575: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-15 12:53:22.179097: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-15 12:53:27 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-15 12:53:27 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template='/workspace/chat_template_multi_system.jinja', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=245000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=24576, max_num_seqs=1, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=245000, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-15 12:53:27 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-15 12:53:38 config.py:887] Defaulting to use mp for distributed inference +INFO 07-15 12:53:38 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=24576. +WARNING 07-15 12:53:38 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-15 12:53:38 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=245000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-15 12:53:39 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-15 12:53:39 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-15 12:53:39 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:53:39 selector.py:115] Using XFormers backend. +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:53:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-15 12:53:48 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=44549, remote_subscribe_port=None) +INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:48 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43778) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43777) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:49 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=43776) INFO 07-15 12:53:49 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00 + uvloop.run(run_server(args)) + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run + return loop.run_until_complete(wrapper()) + File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper + return await main + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 562, in run_server + async with build_async_engine_client(args) as engine_client: + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 108, in build_async_engine_client + async with build_async_engine_client_from_engine_args( + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 142, in build_async_engine_client_from_engine_args + engine_client = await asyncio.get_running_loop().run_in_executor( + File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run + result = self.fn(*self.args, **self.kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 674, in from_engine_args + engine = cls( + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 569, in __init__ + self.engine = self._engine_class(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 265, in __init__ + super().__init__(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 349, in __init__ + self._initialize_kv_caches() + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 497, in _initialize_kv_caches + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/distributed_gpu_executor.py", line 67, in initialize_cache + self._run_workers("initialize_cache", + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 192, in _run_workers + driver_worker_output = driver_worker_method(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache + raise_if_cache_size_invalid(num_gpu_blocks, + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid + raise ValueError( +ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (233056). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine. +INFO 07-15 12:54:45 multiproc_worker_utils.py:121] Killing local vLLM worker processes +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/driver.log b/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/driver.log new file mode 100644 index 0000000..c8c6263 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/driver.log @@ -0,0 +1,226 @@ +[repro] gpu before start +[repro] MAX_NUM_BATCHED_TOKENS=32768 MOE_DECODE_NATIVE=0 +Timestamp Wed Jul 15 12:37:39 2026 ++-----------------------------------------------------------------------------+ +| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | +|-------------------------------+----------------------+----------------------| +| GPU Name | Bus-Id | Clock-SM Clock-Mem | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +|===============================+======================+======================| +| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | +| 0% 29C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | +| 0% 32C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | +| 0% 31C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | +| 0% 30C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| Processes: GPU Memory | +| GPU PID Process name Usage(MiB) | +|=============================================================================| +| No running processes found | ++-----------------------------------------------------------------------------+ +[repro] server_pid=41250 +[repro] waiting_check=6 + Loading safetensors checkpoint shards: 15% Completed | 4/26 [00:06<00:36, 1.65s/it] + Loading safetensors checkpoint shards: 19% Completed | 5/26 [00:07<00:29, 1.42s/it] + Loading safetensors checkpoint shards: 23% Completed | 6/26 [00:09<00:33, 1.66s/it] + Loading safetensors checkpoint shards: 27% Completed | 7/26 [00:09<00:23, 1.26s/it] + Loading safetensors checkpoint shards: 31% Completed | 8/26 [00:11<00:25, 1.42s/it] + Loading safetensors checkpoint shards: 35% Completed | 9/26 [00:12<00:21, 1.28s/it] + Loading safetensors checkpoint shards: 38% Completed | 10/26 [00:14<00:24, 1.50s/it] + Loading safetensors checkpoint shards: 42% Completed | 11/26 [00:14<00:17, 1.14s/it] + Loading safetensors checkpoint shards: 46% Completed | 12/26 [00:16<00:18, 1.36s/it] + Loading safetensors checkpoint shards: 50% Completed | 13/26 [00:18<00:20, 1.55s/it] +Timestamp Wed Jul 15 12:38:30 2026 ++-----------------------------------------------------------------------------+ +| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | +|-------------------------------+----------------------+----------------------| +| GPU Name | Bus-Id | Clock-SM Clock-Mem | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +|===============================+======================+======================| +| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | +| 0% 29C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | +| 0% 32C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | +| 0% 31C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ +| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | +| 0% 30C P0 47W / 250W | 16945MiB / 32768MiB | 0% Default | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| Processes: GPU Memory | +| GPU PID Process name Usage(MiB) | +|=============================================================================| ++-----------------------------------------------------------------------------+ +[repro] server_exited_at_check=11 +2026-07-15 12:37:43.215301: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-15 12:37:43.265874: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-15 12:37:48 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-15 12:37:48 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template='/workspace/chat_template_multi_system.jinja', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=245000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=32768, max_num_seqs=1, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=245000, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-15 12:37:48 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-15 12:37:59 config.py:887] Defaulting to use mp for distributed inference +INFO 07-15 12:37:59 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=32768. +WARNING 07-15 12:37:59 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-15 12:37:59 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=245000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-15 12:38:00 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-15 12:38:00 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-15 12:38:00 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:38:00 selector.py:115] Using XFormers backend. +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-15 12:38:10 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=34275, remote_subscribe_port=None) +INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00 + uvloop.run(run_server(args)) + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run + return loop.run_until_complete(wrapper()) + File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper + return await main + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 562, in run_server + async with build_async_engine_client(args) as engine_client: + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 108, in build_async_engine_client + async with build_async_engine_client_from_engine_args( + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 142, in build_async_engine_client_from_engine_args + engine_client = await asyncio.get_running_loop().run_in_executor( + File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run + result = self.fn(*self.args, **self.kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 674, in from_engine_args + engine = cls( + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 569, in __init__ + self.engine = self._engine_class(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 265, in __init__ + super().__init__(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 349, in __init__ + self._initialize_kv_caches() + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 497, in _initialize_kv_caches + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/distributed_gpu_executor.py", line 67, in initialize_cache + self._run_workers("initialize_cache", + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 192, in _run_workers + driver_worker_output = driver_worker_method(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache + raise_if_cache_size_invalid(num_gpu_blocks, + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid + raise ValueError( +ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine. +ERROR 07-15 12:39:15 multiproc_worker_utils.py:117] Worker VllmWorkerProcess pid 41602 died, exit code: -15 +INFO 07-15 12:39:15 multiproc_worker_utils.py:121] Killing local vLLM worker processes +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/server.log b/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/server.log new file mode 100644 index 0000000..a930891 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/b32768_failed/server.log @@ -0,0 +1,163 @@ +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:37:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-15 12:37:43.215301: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-15 12:37:43.265874: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-15 12:37:48 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-15 12:37:48 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template='/workspace/chat_template_multi_system.jinja', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=245000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=32768, max_num_seqs=1, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=245000, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-15 12:37:48 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-15 12:37:59 config.py:887] Defaulting to use mp for distributed inference +INFO 07-15 12:37:59 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=32768. +WARNING 07-15 12:37:59 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-15 12:37:59 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=245000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-15 12:38:00 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-15 12:38:00 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-15 12:38:00 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:38:00 selector.py:115] Using XFormers backend. +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-15 12:38:10 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=34275, remote_subscribe_port=None) +INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41602) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41603) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=41604) INFO 07-15 12:38:10 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00 + uvloop.run(run_server(args)) + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run + return loop.run_until_complete(wrapper()) + File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete + File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper + return await main + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 562, in run_server + async with build_async_engine_client(args) as engine_client: + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 108, in build_async_engine_client + async with build_async_engine_client_from_engine_args( + File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__ + return await anext(self.gen) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 142, in build_async_engine_client_from_engine_args + engine_client = await asyncio.get_running_loop().run_in_executor( + File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run + result = self.fn(*self.args, **self.kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 674, in from_engine_args + engine = cls( + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 569, in __init__ + self.engine = self._engine_class(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 265, in __init__ + super().__init__(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 349, in __init__ + self._initialize_kv_caches() + File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 497, in _initialize_kv_caches + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/distributed_gpu_executor.py", line 67, in initialize_cache + self._run_workers("initialize_cache", + File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 192, in _run_workers + driver_worker_output = driver_worker_method(*args, **kwargs) + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache + raise_if_cache_size_invalid(num_gpu_blocks, + File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid + raise ValueError( +ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine. +ERROR 07-15 12:39:15 multiproc_worker_utils.py:117] Worker VllmWorkerProcess pid 41602 died, exit code: -15 +INFO 07-15 12:39:15 multiproc_worker_utils.py:121] Killing local vLLM worker processes +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' +/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/current_9fc7e98_245k_c1_r8_t256_cap40k.json b/worklogs/remote_results/2026-07-15-current-9fc7e98/current_9fc7e98_245k_c1_r8_t256_cap40k.json new file mode 100644 index 0000000..fcded05 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/current_9fc7e98_245k_c1_r8_t256_cap40k.json @@ -0,0 +1,124 @@ +{ + "created_at": "2026-07-15T11:41:17", + "url": "http://127.0.0.1:1111", + "model": "llm", + "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", + "concurrency": 1, + "max_requests": 8, + "max_tokens": 256, + "wall_sec": 329.40423798561096, + "success_rate": 1.0, + "ttft_p90_sec": 12.695122232101856, + "output_tps_p10_per_request": 5.999746141283892, + "aggregate_output_tps": 5.664772291367759, + "aggregate_input_tps_uncached": 176.83743341075206, + "aggregate_cache_tps": 507.7287439371248, + "cache_hit_rate": 0.7416795639891973, + "weighted_token_throughput": 874.4415881272979, + "totals": { + "requests": 8, + "success": 8, + "prompt_tokens": 225499, + "input_tokens_uncached": 58251, + "cached_tokens": 167248, + "completion_tokens": 1866, + "reasoning_tokens": 1731 + }, + "targets": { + "output_tps_p10_per_request_gte_20": false, + "ttft_p90_lte_5": false, + "cache_hit_rate_gte_50pct": true, + "success_rate_gte_99pct": true, + "weighted_token_throughput_gte_8000": false + }, + "results": [ + { + "ok": true, + "elapsed_sec": 26.29428836517036, + "ttft_sec": 7.266017025336623, + "prompt_tokens": 7498, + "cached_tokens": 0, + "completion_tokens": 165, + "reasoning_tokens": 79, + "output_tps": 8.671307921418453, + "error": null + }, + { + "ok": true, + "elapsed_sec": 47.98864242434502, + "ttft_sec": 21.27845441363752, + "prompt_tokens": 34177, + "cached_tokens": 7488, + "completion_tokens": 165, + "reasoning_tokens": 116, + "output_tps": 6.177418142240531, + "error": null + }, + { + "ok": true, + "elapsed_sec": 44.649470107629895, + "ttft_sec": 3.02971662953496, + "prompt_tokens": 35384, + "cached_tokens": 34160, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.150925428588529, + "error": null + }, + { + "ok": true, + "elapsed_sec": 38.541101360693574, + "ttft_sec": 9.016551297158003, + "prompt_tokens": 19274, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.670750255265498, + "error": null + }, + { + "ok": true, + "elapsed_sec": 45.66694761812687, + "ttft_sec": 3.336757702752948, + "prompt_tokens": 37099, + "cached_tokens": 35376, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.047693159699793, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.86508701182902, + "ttft_sec": 5.408511884510517, + "prompt_tokens": 14661, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.69075915626666, + "error": null + }, + { + "ok": true, + "elapsed_sec": 45.62775577604771, + "ttft_sec": 3.077076403424144, + "prompt_tokens": 38199, + "cached_tokens": 37088, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.016355173983576, + "error": null + }, + { + "ok": true, + "elapsed_sec": 45.76239648461342, + "ttft_sec": 2.816522454842925, + "prompt_tokens": 39207, + "cached_tokens": 38192, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.960991731651295, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-15-current-9fc7e98/moe_decode_native_245k_c1_r8_t256_cap40k.json b/worklogs/remote_results/2026-07-15-current-9fc7e98/moe_decode_native_245k_c1_r8_t256_cap40k.json new file mode 100644 index 0000000..a6f9cf6 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-current-9fc7e98/moe_decode_native_245k_c1_r8_t256_cap40k.json @@ -0,0 +1,124 @@ +{ + "created_at": "2026-07-15T12:25:01", + "url": "http://127.0.0.1:1111", + "model": "llm", + "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", + "concurrency": 1, + "max_requests": 8, + "max_tokens": 256, + "wall_sec": 342.52822289802134, + "success_rate": 1.0, + "ttft_p90_sec": 13.948715174756941, + "output_tps_p10_per_request": 5.8688016391455635, + "aggregate_output_tps": 5.584941246051847, + "aggregate_input_tps_uncached": 170.06189886239736, + "aggregate_cache_tps": 488.27509331922596, + "cache_hit_rate": 0.7416795639891973, + "weighted_token_throughput": 843.2419803433036, + "totals": { + "requests": 8, + "success": 8, + "prompt_tokens": 225499, + "input_tokens_uncached": 58251, + "cached_tokens": 167248, + "completion_tokens": 1913, + "reasoning_tokens": 1807 + }, + "targets": { + "output_tps_p10_per_request_gte_20": false, + "ttft_p90_lte_5": false, + "cache_hit_rate_gte_50pct": true, + "success_rate_gte_99pct": true, + "weighted_token_throughput_gte_8000": false + }, + "results": [ + { + "ok": true, + "elapsed_sec": 37.44546605460346, + "ttft_sec": 8.050519423559308, + "prompt_tokens": 7498, + "cached_tokens": 0, + "completion_tokens": 256, + "reasoning_tokens": 196, + "output_tps": 8.708979921386797, + "error": null + }, + { + "ok": true, + "elapsed_sec": 42.67267605289817, + "ttft_sec": 23.193294195458293, + "prompt_tokens": 34177, + "cached_tokens": 7488, + "completion_tokens": 121, + "reasoning_tokens": 75, + "output_tps": 6.21169608386653, + "error": null + }, + { + "ok": true, + "elapsed_sec": 44.97984442859888, + "ttft_sec": 3.3009007796645164, + "prompt_tokens": 35384, + "cached_tokens": 34160, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.142190218550449, + "error": null + }, + { + "ok": true, + "elapsed_sec": 39.61451355740428, + "ttft_sec": 9.986752737313509, + "prompt_tokens": 19274, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.640544979234637, + "error": null + }, + { + "ok": true, + "elapsed_sec": 47.399366622790694, + "ttft_sec": 3.8023243956267834, + "prompt_tokens": 37099, + "cached_tokens": 35376, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.87195797976622, + "error": null + }, + { + "ok": true, + "elapsed_sec": 36.7238706368953, + "ttft_sec": 6.21262020803988, + "prompt_tokens": 14661, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.390347704592697, + "error": null + }, + { + "ok": true, + "elapsed_sec": 47.17942987754941, + "ttft_sec": 3.504132028669119, + "prompt_tokens": 38199, + "cached_tokens": 37088, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.86143684436403, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.50449903123081, + "ttft_sec": 3.3495454359799623, + "prompt_tokens": 39207, + "cached_tokens": 38192, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.932111581002198, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_scripts/start_platform_245k_repro.sh b/worklogs/remote_scripts/start_platform_245k_repro.sh index 2182731..cc21fdc 100644 --- a/worklogs/remote_scripts/start_platform_245k_repro.sh +++ b/worklogs/remote_scripts/start_platform_245k_repro.sh @@ -4,6 +4,8 @@ set -euo pipefail OUT="${1:-/root/work/logs/platform_245k_repro_20260715_01}" PORT="${PORT:-1111}" MODEL="${MODEL:-/root/public-storage/models/Qwen/Qwen3.6-35B-A3B}" +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-16384}" +MOE_DECODE_NATIVE="${MOE_DECODE_NATIVE:-0}" mkdir -p "$OUT" cd /root @@ -18,7 +20,7 @@ export RMSNORM_NATIVE=1 export LOOP1_NATIVE=1 export PROF_TRACE=0 export PROF_MOE_SUMMARY=0 -export MOE_DECODE_NATIVE=0 +export MOE_DECODE_NATIVE python3 - <<'PY' import os @@ -55,6 +57,7 @@ print("sigkill_pids", left) PY echo "[repro] gpu before start" | tee "$OUT/driver.log" +echo "[repro] MAX_NUM_BATCHED_TOKENS=$MAX_NUM_BATCHED_TOKENS MOE_DECODE_NATIVE=$MOE_DECODE_NATIVE" | tee -a "$OUT/driver.log" LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \ /usr/local/corex/bin/ixsmi | tee -a "$OUT/driver.log" @@ -71,7 +74,7 @@ nohup python3 -m vllm.entrypoints.openai.api_server \ --max-num-seqs 1 \ --disable-log-requests \ --disable-frontend-multiprocessing \ - --max-num-batched-tokens 16384 \ + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ --enable-chunked-prefill \ --max-seq-len-to-capture 245000 \ --enable-auto-tool-choice \