164 lines
24 KiB
Plaintext
164 lines
24 KiB
Plaintext
/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
||
import pynvml # type: ignore[import]
|
||
INFO 07-15 12:37:41 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
|
||
2026-07-15 12:37:43.215301: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
|
||
2026-07-15 12:37:43.265874: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
|
||
To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
||
WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.
|
||
INFO 07-15 12:37:48 api_server.py:530] vLLM API server version 0.6.3
|
||
INFO 07-15 12:37:48 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template='/workspace/chat_template_multi_system.jinja', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=245000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=32768, max_num_seqs=1, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=245000, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False)
|
||
INFO 07-15 12:37:48 config.py:1670] Downcasting torch.float32 to torch.float16.
|
||
INFO 07-15 12:37:59 config.py:887] Defaulting to use mp for distributed inference
|
||
INFO 07-15 12:37:59 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=32768.
|
||
WARNING 07-15 12:37:59 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used
|
||
INFO 07-15 12:37:59 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=245000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None)
|
||
WARNING 07-15 12:38:00 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||
INFO 07-15 12:38:00 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager
|
||
INFO 07-15 12:38:00 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
INFO 07-15 12:38:00 selector.py:115] Using XFormers backend.
|
||
/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
||
import pynvml # type: ignore[import]
|
||
/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
||
import pynvml # type: ignore[import]
|
||
/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
|
||
import pynvml # type: ignore[import]
|
||
INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
|
||
INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
|
||
INFO 07-15 12:38:02 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
|
||
WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.
|
||
WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.
|
||
WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:09 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:09 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:09 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:09 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:09 multiproc_worker_utils.py:216] Worker ready; awaiting tasks
|
||
INFO 07-15 12:38:10 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=<vllm.distributed.device_communicators.shm_broadcast.ShmRingBuffer object at 0x7f011af40f10>, local_subscribe_port=34275, remote_subscribe_port=None)
|
||
INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B...
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B...
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B...
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:10 model_runner.py:1117] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B...
|
||
INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
INFO 07-15 12:38:10 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:10 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:10 selector.py:115] Using XFormers backend.
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:10 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default).
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:10 selector.py:115] Using XFormers backend.
|
||
|
||
Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00<?, ?it/s]
|
||
|
||
Loading safetensors checkpoint shards: 4% Completed | 1/26 [00:01<00:47, 1.89s/it]
|
||
|
||
Loading safetensors checkpoint shards: 8% Completed | 2/26 [00:02<00:25, 1.08s/it]
|
||
|
||
Loading safetensors checkpoint shards: 12% Completed | 3/26 [00:04<00:33, 1.45s/it]
|
||
|
||
Loading safetensors checkpoint shards: 15% Completed | 4/26 [00:06<00:36, 1.65s/it]
|
||
|
||
Loading safetensors checkpoint shards: 19% Completed | 5/26 [00:07<00:29, 1.42s/it]
|
||
|
||
Loading safetensors checkpoint shards: 23% Completed | 6/26 [00:09<00:33, 1.66s/it]
|
||
|
||
Loading safetensors checkpoint shards: 27% Completed | 7/26 [00:09<00:23, 1.26s/it]
|
||
|
||
Loading safetensors checkpoint shards: 31% Completed | 8/26 [00:11<00:25, 1.42s/it]
|
||
|
||
Loading safetensors checkpoint shards: 35% Completed | 9/26 [00:12<00:21, 1.28s/it]
|
||
|
||
Loading safetensors checkpoint shards: 38% Completed | 10/26 [00:14<00:24, 1.50s/it]
|
||
|
||
Loading safetensors checkpoint shards: 42% Completed | 11/26 [00:14<00:17, 1.14s/it]
|
||
|
||
Loading safetensors checkpoint shards: 46% Completed | 12/26 [00:16<00:18, 1.36s/it]
|
||
|
||
Loading safetensors checkpoint shards: 50% Completed | 13/26 [00:18<00:20, 1.55s/it]
|
||
|
||
Loading safetensors checkpoint shards: 54% Completed | 14/26 [00:20<00:21, 1.76s/it]
|
||
|
||
Loading safetensors checkpoint shards: 58% Completed | 15/26 [00:21<00:16, 1.48s/it]
|
||
|
||
Loading safetensors checkpoint shards: 62% Completed | 16/26 [00:24<00:18, 1.81s/it]
|
||
|
||
Loading safetensors checkpoint shards: 65% Completed | 17/26 [00:24<00:12, 1.41s/it]
|
||
|
||
Loading safetensors checkpoint shards: 69% Completed | 18/26 [00:26<00:12, 1.56s/it]
|
||
|
||
Loading safetensors checkpoint shards: 73% Completed | 19/26 [00:28<00:11, 1.62s/it]
|
||
|
||
Loading safetensors checkpoint shards: 77% Completed | 20/26 [00:31<00:11, 1.91s/it]
|
||
|
||
Loading safetensors checkpoint shards: 81% Completed | 21/26 [00:32<00:08, 1.67s/it]
|
||
|
||
Loading safetensors checkpoint shards: 85% Completed | 22/26 [00:33<00:05, 1.41s/it]
|
||
|
||
Loading safetensors checkpoint shards: 88% Completed | 23/26 [00:35<00:05, 1.75s/it]
|
||
|
||
Loading safetensors checkpoint shards: 92% Completed | 24/26 [00:37<00:03, 1.83s/it]
|
||
|
||
Loading safetensors checkpoint shards: 96% Completed | 25/26 [00:39<00:01, 1.90s/it]
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m INFO 07-15 12:38:50 model_runner.py:1128] Loading model weights took 16.2303 GB
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m INFO 07-15 12:38:50 model_runner.py:1128] Loading model weights took 16.2303 GB
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m INFO 07-15 12:38:50 model_runner.py:1128] Loading model weights took 16.2303 GB
|
||
|
||
Loading safetensors checkpoint shards: 100% Completed | 26/26 [00:40<00:00, 1.49s/it]
|
||
|
||
Loading safetensors checkpoint shards: 100% Completed | 26/26 [00:40<00:00, 1.55s/it]
|
||
|
||
INFO 07-15 12:38:51 model_runner.py:1128] Loading model weights took 16.2303 GB
|
||
INFO 07-15 12:39:15 distributed_gpu_executor.py:57] # GPU blocks: 13081, # CPU blocks: 6553
|
||
INFO 07-15 12:39:15 distributed_gpu_executor.py:61] Maximum concurrency for 245000 tokens per request: 0.85x
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] Exception in worker VllmWorkerProcess while processing method initialize_cache: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine., Traceback (most recent call last):
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_worker_utils.py", line 224, in _run_worker_process
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] output = executor(*args, **kwargs)
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise_if_cache_size_invalid(num_gpu_blocks,
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise ValueError(
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
|
||
[1;36m(VllmWorkerProcess pid=41602)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231]
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] Exception in worker VllmWorkerProcess while processing method initialize_cache: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine., Traceback (most recent call last):
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_worker_utils.py", line 224, in _run_worker_process
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] output = executor(*args, **kwargs)
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise_if_cache_size_invalid(num_gpu_blocks,
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise ValueError(
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
|
||
[1;36m(VllmWorkerProcess pid=41603)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231]
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] Exception in worker VllmWorkerProcess while processing method initialize_cache: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine., Traceback (most recent call last):
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_worker_utils.py", line 224, in _run_worker_process
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] output = executor(*args, **kwargs)
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise_if_cache_size_invalid(num_gpu_blocks,
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] raise ValueError(
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231] ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
|
||
[1;36m(VllmWorkerProcess pid=41604)[0;0m ERROR 07-15 12:39:15 multiproc_worker_utils.py:231]
|
||
Traceback (most recent call last):
|
||
File "/usr/local/lib/python3.10/runpy.py", line 196, in _run_module_as_main
|
||
return _run_code(code, main_globals, None,
|
||
File "/usr/local/lib/python3.10/runpy.py", line 86, in _run_code
|
||
exec(code, run_globals)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 595, in <module>
|
||
uvloop.run(run_server(args))
|
||
File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run
|
||
return loop.run_until_complete(wrapper())
|
||
File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||
File "/usr/local/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper
|
||
return await main
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 562, in run_server
|
||
async with build_async_engine_client(args) as engine_client:
|
||
File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__
|
||
return await anext(self.gen)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 108, in build_async_engine_client
|
||
async with build_async_engine_client_from_engine_args(
|
||
File "/usr/local/lib/python3.10/contextlib.py", line 199, in __aenter__
|
||
return await anext(self.gen)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/entrypoints/openai/api_server.py", line 142, in build_async_engine_client_from_engine_args
|
||
engine_client = await asyncio.get_running_loop().run_in_executor(
|
||
File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run
|
||
result = self.fn(*self.args, **self.kwargs)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 674, in from_engine_args
|
||
engine = cls(
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 569, in __init__
|
||
self.engine = self._engine_class(*args, **kwargs)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/async_llm_engine.py", line 265, in __init__
|
||
super().__init__(*args, **kwargs)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 349, in __init__
|
||
self._initialize_kv_caches()
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/engine/llm_engine.py", line 497, in _initialize_kv_caches
|
||
self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/distributed_gpu_executor.py", line 67, in initialize_cache
|
||
self._run_workers("initialize_cache",
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 192, in _run_workers
|
||
driver_worker_output = driver_worker_method(*args, **kwargs)
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 262, in initialize_cache
|
||
raise_if_cache_size_invalid(num_gpu_blocks,
|
||
File "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/worker.py", line 492, in raise_if_cache_size_invalid
|
||
raise ValueError(
|
||
ValueError: The model's max seq len (245000) is larger than the maximum number of tokens that can be stored in KV cache (209296). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
|
||
ERROR 07-15 12:39:15 multiproc_worker_utils.py:117] Worker VllmWorkerProcess pid 41602 died, exit code: -15
|
||
INFO 07-15 12:39:15 multiproc_worker_utils.py:121] Killing local vLLM worker processes
|
||
/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown
|
||
warnings.warn('resource_tracker: There appear to be %d '
|
||
/usr/local/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||
warnings.warn('resource_tracker: There appear to be %d '
|