From ef6abf3dc75851b55ef0eaae5c40272782420c79 Mon Sep 17 00:00:00 2001 From: dylanyunlon Date: Thu, 30 Jul 2026 16:06:20 +0000 Subject: [PATCH] [DEPLOY] Complete submission: baseline + all optimizations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ALL files needed for Dockerfile build: - qwen3_6_scripts/ (baseline patches + our optimizations) - vllm/ (full vllm package) - paged_attention_v2_pytorch.py (V2 with single-bmm optimization) - Dockerfile + computility-run.yaml Our optimizations vs baseline: 1. paged_attn.py: pre-gathered context KV (eliminates 194 gather calls), Triton try/fallback, V2 heuristic, threshold 32K→64K 2. paged_attention_v2_pytorch.py: fills NotImplementedError, single-bmm Phase 1 (195 launches → 3) 3. patch_enable_triton.py: HAS_TRITON=True with safety fallback 4. patch_triton_tuning.py: BLOCK=64, NUM_WARPS=4 for BI-V100 5. computility-run.yaml: gpu-memory-utilization 0.9→0.95, max-num-batched-tokens 8192→16384 This repo can now be submitted to dev.modelhub.org.cn as-is. --- .gitignore | 5 + __init__.py | 9 + attention.py | 542 +++++ chat_dataset_v0.json | 35 + prefix_prefill.py | 865 ++++++++ qwen3_6_scripts/api_server.py | 595 +++++ qwen3_6_scripts/chat_utils.py | 601 +++++ qwen3_6_scripts/cli_args.py | 261 +++ qwen3_6_scripts/mamba_cache.py | 224 ++ qwen3_6_scripts/patch_model_runner.py | 78 + qwen3_6_scripts/patch_ops.sh | 94 + qwen3_6_scripts/patch_transformers_qwen3_5.py | 117 + qwen3_6_scripts/patch_vllm_qwen3_5.py | 76 + qwen3_6_scripts/patch_vllm_tool_parser.py | 79 + qwen3_6_scripts/patch_xformers_sdpa_batch.py | 192 ++ .../patch_xformers_sdpa_batch_kernel.py | 191 ++ qwen3_6_scripts/patch_xformers_sdpa_seq.py | 321 +++ .../patch_xformers_sdpa_seq_kernel.py | 181 ++ qwen3_6_scripts/protocol.py | 1043 +++++++++ qwen3_6_scripts/qwen3_5.py | 1369 ++++++++++++ qwen3_6_scripts/qwen3_5/__init__.py | 3 + .../qwen3_5/configuration_qwen3_5.py | 188 ++ qwen3_6_scripts/qwen3_5_moe/__init__.py | 3 + .../qwen3_5_moe/configuration_qwen3_5_moe.py | 198 ++ qwen3_6_scripts/qwen3coder_tool_parser.py | 509 +++++ qwen3_6_scripts/reasoning/__init__.py | 16 + .../reasoning/abs_reasoning_parsers.py | 243 +++ .../reasoning/qwen3_reasoning_parser.py | 108 + qwen3_6_scripts/scheduler.py | 1656 ++++++++++++++ qwen3_6_scripts/sequence.py | 1386 ++++++++++++ qwen3_6_scripts/serving_chat.py | 1090 ++++++++++ vllm/__init__.py | 36 + vllm/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1148 bytes vllm/__pycache__/_core_ext.cpython-310.pyc | Bin 0 -> 9612 bytes vllm/__pycache__/_custom_ops.cpython-310.pyc | Bin 0 -> 31179 bytes vllm/__pycache__/_ipex_ops.cpython-310.pyc | Bin 0 -> 6558 bytes vllm/__pycache__/beam_search.cpython-310.pyc | Bin 0 -> 2435 bytes vllm/__pycache__/block.cpython-310.pyc | Bin 0 -> 3449 bytes vllm/__pycache__/config.cpython-310.pyc | Bin 0 -> 58835 bytes vllm/__pycache__/connections.cpython-310.pyc | Bin 0 -> 4875 bytes vllm/__pycache__/envs.cpython-310.pyc | Bin 0 -> 11644 bytes .../forward_context.cpython-310.pyc | Bin 0 -> 741 bytes vllm/__pycache__/logger.cpython-310.pyc | Bin 0 -> 4033 bytes vllm/__pycache__/outputs.cpython-310.pyc | Bin 0 -> 9654 bytes .../pooling_params.cpython-310.pyc | Bin 0 -> 1043 bytes .../sampling_params.cpython-310.pyc | Bin 0 -> 17070 bytes vllm/__pycache__/scalar_type.cpython-310.pyc | Bin 0 -> 810 bytes vllm/__pycache__/scripts.cpython-310.pyc | Bin 0 -> 5130 bytes vllm/__pycache__/sequence.cpython-310.pyc | Bin 0 -> 46639 bytes vllm/__pycache__/tracing.cpython-310.pyc | Bin 0 -> 4446 bytes vllm/__pycache__/utils.cpython-310.pyc | Bin 0 -> 41025 bytes vllm/__pycache__/version.cpython-310.pyc | Bin 0 -> 207 bytes vllm/_core_ext.py | 278 +++ vllm/_custom_ops.py | 1105 ++++++++++ vllm/_ipex_ops.py | 243 +++ vllm/adapter_commons/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 161 bytes .../__pycache__/layers.cpython-310.pyc | Bin 0 -> 680 bytes .../__pycache__/models.cpython-310.pyc | Bin 0 -> 4224 bytes .../__pycache__/request.cpython-310.pyc | Bin 0 -> 1203 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 2979 bytes .../worker_manager.cpython-310.pyc | Bin 0 -> 1750 bytes vllm/adapter_commons/layers.py | 14 + vllm/adapter_commons/models.py | 104 + vllm/adapter_commons/request.py | 23 + vllm/adapter_commons/utils.py | 90 + vllm/adapter_commons/worker_manager.py | 36 + vllm/assets/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 152 bytes vllm/assets/__pycache__/audio.cpython-310.pyc | Bin 0 -> 1260 bytes vllm/assets/__pycache__/base.cpython-310.pyc | Bin 0 -> 1281 bytes vllm/assets/__pycache__/image.cpython-310.pyc | Bin 0 -> 1221 bytes vllm/assets/__pycache__/video.cpython-310.pyc | Bin 0 -> 2864 bytes vllm/assets/audio.py | 28 + vllm/assets/base.py | 39 + vllm/assets/image.py | 30 + vllm/assets/video.py | 85 + vllm/attention/__init__.py | 17 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 519 bytes .../__pycache__/layer.cpython-310.pyc | Bin 0 -> 3236 bytes .../__pycache__/selector.cpython-310.pyc | Bin 0 -> 7211 bytes vllm/attention/backends/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 164 bytes .../__pycache__/abstract.cpython-310.pyc | Bin 0 -> 8554 bytes .../blocksparse_attn.cpython-310.pyc | Bin 0 -> 10195 bytes .../__pycache__/flash_attn.cpython-310.pyc | Bin 0 -> 21133 bytes .../__pycache__/flashinfer.cpython-310.pyc | Bin 0 -> 20838 bytes .../__pycache__/ipex_attn.cpython-310.pyc | Bin 0 -> 9214 bytes .../__pycache__/openvino.cpython-310.pyc | Bin 0 -> 4004 bytes .../__pycache__/pallas.cpython-310.pyc | Bin 0 -> 7378 bytes .../placeholder_attn.cpython-310.pyc | Bin 0 -> 8322 bytes .../rocm_flash_attn.cpython-310.pyc | Bin 0 -> 15127 bytes .../__pycache__/torch_sdpa.cpython-310.pyc | Bin 0 -> 13852 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 12346 bytes .../__pycache__/xformers.cpython-310.pyc | Bin 0 -> 18760 bytes vllm/attention/backends/abstract.py | 232 ++ vllm/attention/backends/blocksparse_attn.py | 444 ++++ vllm/attention/backends/flash_attn.py | 951 ++++++++ vllm/attention/backends/flashinfer.py | 901 ++++++++ vllm/attention/backends/ipex_attn.py | 381 ++++ vllm/attention/backends/openvino.py | 130 ++ vllm/attention/backends/pallas.py | 260 +++ vllm/attention/backends/placeholder_attn.py | 321 +++ vllm/attention/backends/rocm_flash_attn.py | 668 ++++++ vllm/attention/backends/torch_sdpa.py | 547 +++++ vllm/attention/backends/utils.py | 442 ++++ vllm/attention/backends/xformers.py | 822 +++++++ vllm/attention/layer.py | 116 + vllm/attention/ops/__init__.py | 0 .../ops/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 159 bytes .../ops/__pycache__/ipex_attn.cpython-310.pyc | Bin 0 -> 3800 bytes .../__pycache__/paged_attn.cpython-310.pyc | Bin 0 -> 5232 bytes .../prefix_prefill.cpython-310.pyc | Bin 0 -> 11976 bytes .../triton_flash_attention.cpython-310.pyc | Bin 0 -> 10385 bytes .../ops/blocksparse_attention/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 181 bytes ...ocksparse_attention_kernel.cpython-310.pyc | Bin 0 -> 6619 bytes .../__pycache__/interface.cpython-310.pyc | Bin 0 -> 7326 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 6719 bytes .../blocksparse_attention_kernel.py | 423 ++++ .../ops/blocksparse_attention/interface.py | 238 ++ .../ops/blocksparse_attention/utils.py | 242 +++ vllm/attention/ops/ipex_attn.py | 123 ++ vllm/attention/ops/paged_attn.py | 245 +++ vllm/attention/ops/prefix_prefill.py | 861 ++++++++ vllm/attention/ops/triton_flash_attention.py | 820 +++++++ vllm/attention/selector.py | 305 +++ vllm/beam_search.py | 61 + vllm/block.py | 88 + vllm/compilation/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 157 bytes .../__pycache__/backends.cpython-310.pyc | Bin 0 -> 5072 bytes .../compile_context.cpython-310.pyc | Bin 0 -> 771 bytes .../__pycache__/decorators.cpython-310.pyc | Bin 0 -> 3454 bytes .../__pycache__/levels.cpython-310.pyc | Bin 0 -> 444 bytes .../__pycache__/wrapper.cpython-310.pyc | Bin 0 -> 3862 bytes vllm/compilation/backends.py | 269 +++ vllm/compilation/compile_context.py | 23 + vllm/compilation/decorators.py | 113 + vllm/compilation/levels.py | 9 + vllm/compilation/wrapper.py | 102 + vllm/config.py | 1909 ++++++++++++++++ vllm/connections.py | 167 ++ vllm/core/__init__.py | 0 .../core/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 150 bytes .../block_manager_v1.cpython-310.pyc | Bin 0 -> 20013 bytes .../block_manager_v2.cpython-310.pyc | Bin 0 -> 15119 bytes .../__pycache__/evictor_v1.cpython-310.pyc | Bin 0 -> 4166 bytes .../__pycache__/evictor_v2.cpython-310.pyc | Bin 0 -> 5077 bytes .../__pycache__/interfaces.cpython-310.pyc | Bin 0 -> 4793 bytes ...holder_block_space_manager.cpython-310.pyc | Bin 0 -> 4345 bytes .../__pycache__/scheduler.cpython-310.pyc | Bin 0 -> 38323 bytes vllm/core/block/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 156 bytes .../__pycache__/block_table.cpython-310.pyc | Bin 0 -> 13226 bytes .../block/__pycache__/common.cpython-310.pyc | Bin 0 -> 13450 bytes .../cpu_gpu_block_allocator.cpython-310.pyc | Bin 0 -> 15402 bytes .../__pycache__/interfaces.cpython-310.pyc | Bin 0 -> 10837 bytes .../__pycache__/naive_block.cpython-310.pyc | Bin 0 -> 15188 bytes .../prefix_caching_block.cpython-310.pyc | Bin 0 -> 27516 bytes .../block/__pycache__/utils.cpython-310.pyc | Bin 0 -> 1644 bytes vllm/core/block/block_table.py | 374 ++++ vllm/core/block/common.py | 360 +++ vllm/core/block/cpu_gpu_block_allocator.py | 404 ++++ vllm/core/block/interfaces.py | 286 +++ vllm/core/block/naive_block.py | 449 ++++ vllm/core/block/prefix_caching_block.py | 970 +++++++++ vllm/core/block/utils.py | 48 + vllm/core/block_manager_v1.py | 743 +++++++ vllm/core/block_manager_v2.py | 505 +++++ vllm/core/evictor_v1.py | 106 + vllm/core/evictor_v2.py | 131 ++ vllm/core/interfaces.py | 127 ++ vllm/core/placeholder_block_space_manager.py | 91 + vllm/core/scheduler.py | 1650 ++++++++++++++ vllm/distributed/__init__.py | 3 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 233 bytes .../communication_op.cpython-310.pyc | Bin 0 -> 1409 bytes .../parallel_state.cpython-310.pyc | Bin 0 -> 29555 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 2840 bytes vllm/distributed/communication_op.py | 32 + .../device_communicators/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 178 bytes .../__pycache__/cuda_wrapper.cpython-310.pyc | Bin 0 -> 5418 bytes .../custom_all_reduce.cpython-310.pyc | Bin 0 -> 8276 bytes .../custom_all_reduce_utils.cpython-310.pyc | Bin 0 -> 6685 bytes .../__pycache__/pynccl.cpython-310.pyc | Bin 0 -> 4341 bytes .../pynccl_wrapper.cpython-310.pyc | Bin 0 -> 6568 bytes .../__pycache__/shm_broadcast.cpython-310.pyc | Bin 0 -> 13087 bytes .../tpu_communicator.cpython-310.pyc | Bin 0 -> 1857 bytes .../device_communicators/cuda_wrapper.py | 172 ++ .../device_communicators/custom_all_reduce.py | 292 +++ .../custom_all_reduce_utils.py | 255 +++ .../device_communicators/pynccl.py | 170 ++ .../device_communicators/pynccl_wrapper.py | 278 +++ .../device_communicators/shm_broadcast.py | 492 +++++ .../device_communicators/tpu_communicator.py | 61 + vllm/distributed/parallel_state.py | 1258 +++++++++++ vllm/distributed/utils.py | 86 + vllm/engine/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 152 bytes .../__pycache__/arg_utils.cpython-310.pyc | Bin 0 -> 32837 bytes .../async_llm_engine.cpython-310.pyc | Bin 0 -> 38104 bytes .../__pycache__/async_timeout.cpython-310.pyc | Bin 0 -> 5652 bytes .../__pycache__/llm_engine.cpython-310.pyc | Bin 0 -> 48342 bytes .../__pycache__/metrics.cpython-310.pyc | Bin 0 -> 16454 bytes .../__pycache__/metrics_types.cpython-310.pyc | Bin 0 -> 3337 bytes .../__pycache__/protocol.cpython-310.pyc | Bin 0 -> 4173 bytes vllm/engine/arg_utils.py | 1143 ++++++++++ vllm/engine/async_llm_engine.py | 1323 +++++++++++ vllm/engine/async_timeout.py | 189 ++ vllm/engine/llm_engine.py | 1934 +++++++++++++++++ vllm/engine/metrics.py | 555 +++++ vllm/engine/metrics_types.py | 87 + vllm/engine/multiprocessing/__init__.py | 135 ++ .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 4118 bytes .../__pycache__/client.cpython-310.pyc | Bin 0 -> 19776 bytes .../__pycache__/engine.cpython-310.pyc | Bin 0 -> 11875 bytes vllm/engine/multiprocessing/client.py | 704 ++++++ vllm/engine/multiprocessing/engine.py | 395 ++++ vllm/engine/output_processor/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 169 bytes .../__pycache__/interfaces.cpython-310.pyc | Bin 0 -> 3062 bytes .../__pycache__/multi_step.cpython-310.pyc | Bin 0 -> 6569 bytes .../__pycache__/single_step.cpython-310.pyc | Bin 0 -> 6352 bytes .../__pycache__/stop_checker.cpython-310.pyc | Bin 0 -> 2978 bytes .../__pycache__/util.cpython-310.pyc | Bin 0 -> 1075 bytes vllm/engine/output_processor/interfaces.py | 72 + vllm/engine/output_processor/multi_step.py | 188 ++ vllm/engine/output_processor/single_step.py | 215 ++ vllm/engine/output_processor/stop_checker.py | 117 + vllm/engine/output_processor/util.py | 22 + vllm/engine/protocol.py | 103 + vllm/entrypoints/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 157 bytes .../__pycache__/api_server.cpython-310.pyc | Bin 0 -> 4991 bytes .../__pycache__/chat_utils.cpython-310.pyc | Bin 0 -> 16286 bytes .../__pycache__/launcher.cpython-310.pyc | Bin 0 -> 3630 bytes .../__pycache__/llm.cpython-310.pyc | Bin 0 -> 26786 bytes .../__pycache__/logger.cpython-310.pyc | Bin 0 -> 1569 bytes vllm/entrypoints/api_server.py | 163 ++ vllm/entrypoints/chat_utils.py | 581 +++++ vllm/entrypoints/launcher.py | 103 + vllm/entrypoints/llm.py | 909 ++++++++ vllm/entrypoints/logger.py | 42 + vllm/entrypoints/openai/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 164 bytes .../__pycache__/api_server.cpython-310.pyc | Bin 0 -> 14834 bytes .../__pycache__/cli_args.cpython-310.pyc | Bin 0 -> 6923 bytes .../logits_processors.cpython-310.pyc | Bin 0 -> 3164 bytes .../__pycache__/protocol.cpython-310.pyc | Bin 0 -> 30247 bytes .../__pycache__/run_batch.cpython-310.pyc | Bin 0 -> 7898 bytes .../__pycache__/serving_chat.cpython-310.pyc | Bin 0 -> 17201 bytes .../serving_completion.cpython-310.pyc | Bin 0 -> 10645 bytes .../serving_embedding.cpython-310.pyc | Bin 0 -> 5336 bytes .../serving_engine.cpython-310.pyc | Bin 0 -> 13793 bytes .../serving_tokenization.cpython-310.pyc | Bin 0 -> 3608 bytes vllm/entrypoints/openai/api_server.py | 585 +++++ vllm/entrypoints/openai/cli_args.py | 252 +++ vllm/entrypoints/openai/logits_processors.py | 86 + vllm/entrypoints/openai/protocol.py | 992 +++++++++ vllm/entrypoints/openai/run_batch.py | 285 +++ vllm/entrypoints/openai/serving_chat.py | 891 ++++++++ vllm/entrypoints/openai/serving_completion.py | 554 +++++ vllm/entrypoints/openai/serving_embedding.py | 203 ++ vllm/entrypoints/openai/serving_engine.py | 487 +++++ .../openai/serving_tokenization.py | 157 ++ .../openai/tool_parsers/__init__.py | 10 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 562 bytes .../abstract_tool_parser.cpython-310.pyc | Bin 0 -> 5959 bytes .../hermes_tool_parser.cpython-310.pyc | Bin 0 -> 6839 bytes .../internlm2_tool_parser.cpython-310.pyc | Bin 0 -> 4992 bytes .../llama_tool_parser.cpython-310.pyc | Bin 0 -> 6039 bytes .../mistral_tool_parser.cpython-310.pyc | Bin 0 -> 6647 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 2986 bytes .../tool_parsers/abstract_tool_parser.py | 161 ++ .../openai/tool_parsers/hermes_tool_parser.py | 338 +++ .../tool_parsers/internlm2_tool_parser.py | 208 ++ .../openai/tool_parsers/llama_tool_parser.py | 277 +++ .../tool_parsers/mistral_tool_parser.py | 306 +++ vllm/entrypoints/openai/tool_parsers/utils.py | 87 + vllm/envs.py | 453 ++++ vllm/executor/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 154 bytes .../__pycache__/cpu_executor.cpython-310.pyc | Bin 0 -> 11323 bytes .../distributed_gpu_executor.cpython-310.pyc | Bin 0 -> 7538 bytes .../__pycache__/executor_base.cpython-310.pyc | Bin 0 -> 6174 bytes .../__pycache__/gpu_executor.cpython-310.pyc | Bin 0 -> 7010 bytes .../__pycache__/msgspec_utils.cpython-310.pyc | Bin 0 -> 1072 bytes .../multiproc_gpu_executor.cpython-310.pyc | Bin 0 -> 8073 bytes .../multiproc_worker_utils.cpython-310.pyc | Bin 0 -> 8637 bytes .../multiproc_xpu_executor.cpython-310.pyc | Bin 0 -> 1523 bytes .../neuron_executor.cpython-310.pyc | Bin 0 -> 4893 bytes .../openvino_executor.cpython-310.pyc | Bin 0 -> 7387 bytes .../ray_gpu_executor.cpython-310.pyc | Bin 0 -> 15842 bytes .../ray_tpu_executor.cpython-310.pyc | Bin 0 -> 11052 bytes .../__pycache__/ray_utils.cpython-310.pyc | Bin 0 -> 10035 bytes .../ray_xpu_executor.cpython-310.pyc | Bin 0 -> 1765 bytes .../__pycache__/tpu_executor.cpython-310.pyc | Bin 0 -> 5401 bytes .../__pycache__/xpu_executor.cpython-310.pyc | Bin 0 -> 3505 bytes vllm/executor/cpu_executor.py | 389 ++++ vllm/executor/distributed_gpu_executor.py | 212 ++ vllm/executor/executor_base.py | 150 ++ vllm/executor/gpu_executor.py | 191 ++ vllm/executor/msgspec_utils.py | 27 + vllm/executor/multiproc_gpu_executor.py | 258 +++ vllm/executor/multiproc_worker_utils.py | 274 +++ vllm/executor/multiproc_xpu_executor.py | 26 + vllm/executor/neuron_executor.py | 115 + vllm/executor/openvino_executor.py | 213 ++ vllm/executor/ray_gpu_executor.py | 586 +++++ vllm/executor/ray_tpu_executor.py | 363 ++++ vllm/executor/ray_utils.py | 336 +++ vllm/executor/ray_xpu_executor.py | 37 + vllm/executor/tpu_executor.py | 147 ++ vllm/executor/xpu_executor.py | 96 + vllm/forward_context.py | 22 + vllm/inputs/__init__.py | 44 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1075 bytes vllm/inputs/__pycache__/data.cpython-310.pyc | Bin 0 -> 5519 bytes vllm/inputs/__pycache__/parse.cpython-310.pyc | Bin 0 -> 3434 bytes .../__pycache__/preprocess.cpython-310.pyc | Bin 0 -> 13412 bytes .../__pycache__/registry.cpython-310.pyc | Bin 0 -> 10225 bytes vllm/inputs/data.py | 242 +++ vllm/inputs/parse.py | 106 + vllm/inputs/preprocess.py | 580 +++++ vllm/inputs/registry.py | 314 +++ vllm/logger.py | 155 ++ vllm/logging/__init__.py | 5 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 240 bytes .../__pycache__/formatter.cpython-310.pyc | Bin 0 -> 879 bytes vllm/logging/formatter.py | 15 + vllm/lora/__init__.py | 0 .../lora/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 150 bytes .../fully_sharded_layers.cpython-310.pyc | Bin 0 -> 8858 bytes vllm/lora/__pycache__/layers.cpython-310.pyc | Bin 0 -> 34024 bytes vllm/lora/__pycache__/lora.cpython-310.pyc | Bin 0 -> 5353 bytes vllm/lora/__pycache__/models.cpython-310.pyc | Bin 0 -> 22019 bytes vllm/lora/__pycache__/punica.cpython-310.pyc | Bin 0 -> 16361 bytes vllm/lora/__pycache__/request.cpython-310.pyc | Bin 0 -> 3285 bytes vllm/lora/__pycache__/utils.cpython-310.pyc | Bin 0 -> 5748 bytes .../worker_manager.cpython-310.pyc | Bin 0 -> 7501 bytes vllm/lora/fully_sharded_layers.py | 343 +++ vllm/lora/layers.py | 1312 +++++++++++ vllm/lora/lora.py | 169 ++ vllm/lora/models.py | 747 +++++++ vllm/lora/ops/__init__.py | 0 .../ops/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 154 bytes .../__pycache__/bgmv_expand.cpython-310.pyc | Bin 0 -> 3638 bytes .../bgmv_expand_slice.cpython-310.pyc | Bin 0 -> 3778 bytes .../__pycache__/bgmv_shrink.cpython-310.pyc | Bin 0 -> 3329 bytes .../__pycache__/sgmv_expand.cpython-310.pyc | Bin 0 -> 4663 bytes .../sgmv_expand_slice.cpython-310.pyc | Bin 0 -> 5145 bytes .../__pycache__/sgmv_shrink.cpython-310.pyc | Bin 0 -> 4635 bytes .../ops/__pycache__/utils.cpython-310.pyc | Bin 0 -> 1518 bytes vllm/lora/ops/bgmv_expand.py | 168 ++ vllm/lora/ops/bgmv_expand_slice.py | 181 ++ vllm/lora/ops/bgmv_shrink.py | 150 ++ vllm/lora/ops/sgmv_expand.py | 204 ++ vllm/lora/ops/sgmv_expand_slice.py | 217 ++ vllm/lora/ops/sgmv_shrink.py | 201 ++ vllm/lora/ops/utils.py | 46 + vllm/lora/punica.py | 629 ++++++ vllm/lora/request.py | 95 + vllm/lora/utils.py | 189 ++ vllm/lora/worker_manager.py | 212 ++ vllm/model_executor/__init__.py | 13 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 482 bytes .../__pycache__/custom_op.cpython-310.pyc | Bin 0 -> 2375 bytes .../__pycache__/parameter.cpython-310.pyc | Bin 0 -> 13303 bytes .../pooling_metadata.cpython-310.pyc | Bin 0 -> 2405 bytes .../sampling_metadata.cpython-310.pyc | Bin 0 -> 12501 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 1114 bytes vllm/model_executor/custom_op.py | 71 + .../guided_decoding/__init__.py | 45 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1416 bytes .../__pycache__/guided_fields.cpython-310.pyc | Bin 0 -> 1731 bytes ...m_format_enforcer_decoding.cpython-310.pyc | Bin 0 -> 2399 bytes .../outlines_decoding.cpython-310.pyc | Bin 0 -> 3451 bytes ...outlines_logits_processors.cpython-310.pyc | Bin 0 -> 7157 bytes .../guided_decoding/guided_fields.py | 39 + .../lm_format_enforcer_decoding.py | 63 + .../guided_decoding/outlines_decoding.py | 133 ++ .../outlines_logits_processors.py | 223 ++ vllm/model_executor/layers/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 167 bytes .../__pycache__/activation.cpython-310.pyc | Bin 0 -> 8407 bytes .../__pycache__/layernorm.cpython-310.pyc | Bin 0 -> 4661 bytes .../layers/__pycache__/linear.cpython-310.pyc | Bin 0 -> 28115 bytes .../logits_processor.cpython-310.pyc | Bin 0 -> 3905 bytes .../layers/__pycache__/pooler.cpython-310.pyc | Bin 0 -> 2457 bytes .../rejection_sampler.cpython-310.pyc | Bin 0 -> 12057 bytes .../__pycache__/resampler.cpython-310.pyc | Bin 0 -> 7105 bytes .../rotary_embedding.cpython-310.pyc | Bin 0 -> 25494 bytes .../__pycache__/sampler.cpython-310.pyc | Bin 0 -> 31666 bytes .../spec_decode_base_sampler.cpython-310.pyc | Bin 0 -> 7330 bytes ...typical_acceptance_sampler.cpython-310.pyc | Bin 0 -> 6706 bytes .../vocab_parallel_embedding.cpython-310.pyc | Bin 0 -> 15133 bytes vllm/model_executor/layers/activation.py | 252 +++ .../layers/fused_moe/__init__.py | 26 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 743 bytes .../fused_marlin_moe.cpython-310.pyc | Bin 0 -> 7426 bytes .../__pycache__/fused_moe.cpython-310.pyc | Bin 0 -> 15811 bytes .../__pycache__/layer.cpython-310.pyc | Bin 0 -> 15762 bytes .../__pycache__/moe_pallas.cpython-310.pyc | Bin 0 -> 1860 bytes .../layers/fused_moe/fused_marlin_moe.py | 313 +++ .../layers/fused_moe/fused_moe.py | 693 ++++++ vllm/model_executor/layers/fused_moe/layer.py | 717 ++++++ .../layers/fused_moe/moe_pallas.py | 62 + vllm/model_executor/layers/layernorm.py | 187 ++ vllm/model_executor/layers/linear.py | 1103 ++++++++++ .../model_executor/layers/logits_processor.py | 156 ++ vllm/model_executor/layers/mamba/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 173 bytes .../layers/mamba/ops/__init__.py | 0 .../ops/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 177 bytes .../__pycache__/causal_conv1d.cpython-310.pyc | Bin 0 -> 3037 bytes .../ops/__pycache__/mamba_ssm.cpython-310.pyc | Bin 0 -> 8948 bytes .../layers/mamba/ops/causal_conv1d.py | 89 + .../layers/mamba/ops/mamba_ssm.py | 395 ++++ vllm/model_executor/layers/pooler.py | 63 + .../layers/quantization/__init__.py | 69 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 2808 bytes .../__pycache__/aqlm.cpython-310.pyc | Bin 0 -> 8995 bytes .../__pycache__/awq.cpython-310.pyc | Bin 0 -> 5548 bytes .../__pycache__/awq_marlin.cpython-310.pyc | Bin 0 -> 11316 bytes .../__pycache__/awq_triton.cpython-310.pyc | Bin 0 -> 6237 bytes .../__pycache__/base_config.cpython-310.pyc | Bin 0 -> 5825 bytes .../__pycache__/bitsandbytes.cpython-310.pyc | Bin 0 -> 8809 bytes .../__pycache__/deepspeedfp.cpython-310.pyc | Bin 0 -> 7456 bytes .../__pycache__/experts_int8.cpython-310.pyc | Bin 0 -> 5901 bytes .../__pycache__/fbgemm_fp8.cpython-310.pyc | Bin 0 -> 5594 bytes .../__pycache__/fp8.cpython-310.pyc | Bin 0 -> 12469 bytes .../__pycache__/gguf.cpython-310.pyc | Bin 0 -> 6381 bytes .../__pycache__/gptq.cpython-310.pyc | Bin 0 -> 7056 bytes .../__pycache__/gptq_marlin.cpython-310.pyc | Bin 0 -> 12515 bytes .../gptq_marlin_24.cpython-310.pyc | Bin 0 -> 7527 bytes .../__pycache__/ipex_quant.cpython-310.pyc | Bin 0 -> 5668 bytes .../__pycache__/kv_cache.cpython-310.pyc | Bin 0 -> 2593 bytes .../__pycache__/marlin.cpython-310.pyc | Bin 0 -> 6937 bytes .../__pycache__/modelopt.cpython-310.pyc | Bin 0 -> 6188 bytes .../__pycache__/neuron_quant.cpython-310.pyc | Bin 0 -> 3049 bytes .../__pycache__/qqq.cpython-310.pyc | Bin 0 -> 6996 bytes .../__pycache__/schema.cpython-310.pyc | Bin 0 -> 3238 bytes .../__pycache__/tpu_int8.cpython-310.pyc | Bin 0 -> 5095 bytes .../__pycache__/w8a16.cpython-310.pyc | Bin 0 -> 4003 bytes .../layers/quantization/aqlm.py | 374 ++++ .../model_executor/layers/quantization/awq.py | 173 ++ .../layers/quantization/awq_marlin.py | 464 ++++ .../layers/quantization/awq_triton.py | 317 +++ .../layers/quantization/base_config.py | 143 ++ .../layers/quantization/bitsandbytes.py | 316 +++ .../compressed_tensors/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 199 bytes .../compressed_tensors.cpython-310.pyc | Bin 0 -> 12415 bytes .../compressed_tensors_moe.cpython-310.pyc | Bin 0 -> 11478 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 8415 bytes .../compressed_tensors/compressed_tensors.py | 412 ++++ .../compressed_tensors_moe.py | 511 +++++ .../compressed_tensors/schemes/__init__.py | 19 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 796 bytes .../compressed_tensors_scheme.cpython-310.pyc | Bin 0 -> 2106 bytes ...ompressed_tensors_w4a16_24.cpython-310.pyc | Bin 0 -> 4093 bytes ...mpressed_tensors_w8a16_fp8.cpython-310.pyc | Bin 0 -> 3566 bytes ...ompressed_tensors_w8a8_fp8.cpython-310.pyc | Bin 0 -> 3690 bytes ...mpressed_tensors_w8a8_int8.cpython-310.pyc | Bin 0 -> 3896 bytes .../compressed_tensors_wNa16.cpython-310.pyc | Bin 0 -> 4503 bytes .../schemes/compressed_tensors_scheme.py | 52 + .../schemes/compressed_tensors_w4a16_24.py | 153 ++ .../schemes/compressed_tensors_w8a16_fp8.py | 118 + .../schemes/compressed_tensors_w8a8_fp8.py | 143 ++ .../schemes/compressed_tensors_w8a8_int8.py | 155 ++ .../schemes/compressed_tensors_wNa16.py | 163 ++ .../quantization/compressed_tensors/utils.py | 269 +++ .../layers/quantization/deepspeedfp.py | 193 ++ .../layers/quantization/experts_int8.py | 179 ++ .../layers/quantization/fbgemm_fp8.py | 169 ++ .../model_executor/layers/quantization/fp8.py | 514 +++++ .../layers/quantization/gguf.py | 178 ++ .../layers/quantization/gptq.py | 248 +++ .../layers/quantization/gptq_marlin.py | 570 +++++ .../layers/quantization/gptq_marlin_24.py | 295 +++ .../layers/quantization/ipex_quant.py | 166 ++ .../quantization/kernels/MPLinearKernel.py | 83 + .../layers/quantization/kernels/__init__.py | 72 + .../MPLinearKernel.cpython-310.pyc | Bin 0 -> 3003 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 2317 bytes .../__pycache__/machete.cpython-310.pyc | Bin 0 -> 4240 bytes .../__pycache__/marlin.cpython-310.pyc | Bin 0 -> 4209 bytes .../layers/quantization/kernels/machete.py | 118 + .../layers/quantization/kernels/marlin.py | 133 ++ .../layers/quantization/kv_cache.py | 76 + .../layers/quantization/marlin.py | 260 +++ .../layers/quantization/modelopt.py | 163 ++ .../layers/quantization/neuron_quant.py | 67 + .../model_executor/layers/quantization/qqq.py | 273 +++ .../layers/quantization/schema.py | 84 + .../layers/quantization/tpu_int8.py | 119 + .../layers/quantization/utils/__init__.py | 3 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 300 bytes .../__pycache__/layer_utils.cpython-310.pyc | Bin 0 -> 1043 bytes .../__pycache__/machete_utils.cpython-310.pyc | Bin 0 -> 1244 bytes .../__pycache__/marlin_utils.cpython-310.pyc | Bin 0 -> 8301 bytes .../marlin_utils_fp8.cpython-310.pyc | Bin 0 -> 2821 bytes .../marlin_utils_test.cpython-310.pyc | Bin 0 -> 4165 bytes .../marlin_utils_test_24.cpython-310.pyc | Bin 0 -> 10351 bytes .../marlin_utils_test_qqq.cpython-310.pyc | Bin 0 -> 3498 bytes .../__pycache__/quant_utils.cpython-310.pyc | Bin 0 -> 9620 bytes .../__pycache__/w8a8_utils.cpython-310.pyc | Bin 0 -> 4461 bytes .../layers/quantization/utils/layer_utils.py | 37 + .../quantization/utils/machete_utils.py | 30 + .../layers/quantization/utils/marlin_utils.py | 348 +++ .../quantization/utils/marlin_utils_fp8.py | 106 + .../quantization/utils/marlin_utils_test.py | 163 ++ .../utils/marlin_utils_test_24.py | 463 ++++ .../utils/marlin_utils_test_qqq.py | 125 ++ .../layers/quantization/utils/quant_utils.py | 451 ++++ .../layers/quantization/utils/w8a8_utils.py | 246 +++ .../layers/quantization/w8a16.py | 114 + .../layers/rejection_sampler.py | 401 ++++ vllm/model_executor/layers/resampler.py | 273 +++ .../model_executor/layers/rotary_embedding.py | 1006 +++++++++ vllm/model_executor/layers/sampler.py | 1317 +++++++++++ .../layers/spec_decode_base_sampler.py | 239 ++ .../layers/typical_acceptance_sampler.py | 170 ++ .../layers/vocab_parallel_embedding.py | 471 ++++ vllm/model_executor/model_loader/__init__.py | 30 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1118 bytes .../__pycache__/loader.cpython-310.pyc | Bin 0 -> 33745 bytes .../__pycache__/neuron.cpython-310.pyc | Bin 0 -> 7533 bytes .../__pycache__/openvino.cpython-310.pyc | Bin 0 -> 6194 bytes .../__pycache__/tensorizer.cpython-310.pyc | Bin 0 -> 15181 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 1365 bytes .../__pycache__/weight_utils.cpython-310.pyc | Bin 0 -> 20227 bytes vllm/model_executor/model_loader/loader.py | 1259 +++++++++++ vllm/model_executor/model_loader/neuron.py | 240 ++ vllm/model_executor/model_loader/openvino.py | 203 ++ .../model_executor/model_loader/tensorizer.py | 480 ++++ vllm/model_executor/model_loader/utils.py | 39 + .../model_loader/weight_utils.py | 682 ++++++ vllm/model_executor/models/__init__.py | 23 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 684 bytes .../models/__pycache__/arctic.cpython-310.pyc | Bin 0 -> 14378 bytes .../__pycache__/baichuan.cpython-310.pyc | Bin 0 -> 11579 bytes .../models/__pycache__/bart.cpython-310.pyc | Bin 0 -> 21136 bytes .../models/__pycache__/blip.cpython-310.pyc | Bin 0 -> 11296 bytes .../models/__pycache__/blip2.cpython-310.pyc | Bin 0 -> 18091 bytes .../models/__pycache__/bloom.cpython-310.pyc | Bin 0 -> 9104 bytes .../__pycache__/chameleon.cpython-310.pyc | Bin 0 -> 28210 bytes .../__pycache__/chatglm.cpython-310.pyc | Bin 0 -> 16763 bytes .../models/__pycache__/clip.cpython-310.pyc | Bin 0 -> 12874 bytes .../__pycache__/commandr.cpython-310.pyc | Bin 0 -> 11345 bytes .../models/__pycache__/dbrx.cpython-310.pyc | Bin 0 -> 11986 bytes .../models/__pycache__/decilm.cpython-310.pyc | Bin 0 -> 3366 bytes .../__pycache__/deepseek.cpython-310.pyc | Bin 0 -> 12235 bytes .../__pycache__/deepseek_v2.cpython-310.pyc | Bin 0 -> 14350 bytes .../models/__pycache__/eagle.cpython-310.pyc | Bin 0 -> 5480 bytes .../models/__pycache__/exaone.cpython-310.pyc | Bin 0 -> 13503 bytes .../models/__pycache__/falcon.cpython-310.pyc | Bin 0 -> 11462 bytes .../models/__pycache__/fuyu.cpython-310.pyc | Bin 0 -> 10363 bytes .../models/__pycache__/gemma.cpython-310.pyc | Bin 0 -> 11293 bytes .../models/__pycache__/gemma2.cpython-310.pyc | Bin 0 -> 11383 bytes .../gemma2_embedding.cpython-310.pyc | Bin 0 -> 2481 bytes .../models/__pycache__/glm4.cpython-310.pyc | Bin 0 -> 8411 bytes .../glm4_vision_encoder.cpython-310.pyc | Bin 0 -> 8418 bytes .../models/__pycache__/gpt2.cpython-310.pyc | Bin 0 -> 8470 bytes .../__pycache__/gpt_bigcode.cpython-310.pyc | Bin 0 -> 8917 bytes .../models/__pycache__/gpt_j.cpython-310.pyc | Bin 0 -> 8414 bytes .../__pycache__/gpt_neox.cpython-310.pyc | Bin 0 -> 8623 bytes .../__pycache__/granite.cpython-310.pyc | Bin 0 -> 13094 bytes .../__pycache__/granitemoe.cpython-310.pyc | Bin 0 -> 11514 bytes .../idefics2_vision_model.cpython-310.pyc | Bin 0 -> 9130 bytes .../__pycache__/interfaces.cpython-310.pyc | Bin 0 -> 9552 bytes .../interfaces_base.cpython-310.pyc | Bin 0 -> 5359 bytes .../__pycache__/intern_vit.cpython-310.pyc | Bin 0 -> 11643 bytes .../__pycache__/internlm2.cpython-310.pyc | Bin 0 -> 10527 bytes .../__pycache__/internvl.cpython-310.pyc | Bin 0 -> 16336 bytes .../models/__pycache__/jais.cpython-310.pyc | Bin 0 -> 10102 bytes .../models/__pycache__/jamba.cpython-310.pyc | Bin 0 -> 17458 bytes .../models/__pycache__/llama.cpython-310.pyc | Bin 0 -> 14837 bytes .../llama_embedding.cpython-310.pyc | Bin 0 -> 2667 bytes .../models/__pycache__/llava.cpython-310.pyc | Bin 0 -> 11809 bytes .../__pycache__/llava_next.cpython-310.pyc | Bin 0 -> 17296 bytes .../llava_next_video.cpython-310.pyc | Bin 0 -> 13104 bytes .../llava_onevision.cpython-310.pyc | Bin 0 -> 22289 bytes .../models/__pycache__/mamba.cpython-310.pyc | Bin 0 -> 12729 bytes .../__pycache__/mamba_cache.cpython-310.pyc | Bin 0 -> 7755 bytes .../models/__pycache__/medusa.cpython-310.pyc | Bin 0 -> 6280 bytes .../__pycache__/minicpm.cpython-310.pyc | Bin 0 -> 14884 bytes .../__pycache__/minicpm3.cpython-310.pyc | Bin 0 -> 8052 bytes .../__pycache__/minicpmv.cpython-310.pyc | Bin 0 -> 27231 bytes .../__pycache__/mixtral.cpython-310.pyc | Bin 0 -> 11828 bytes .../__pycache__/mixtral_quant.cpython-310.pyc | Bin 0 -> 11340 bytes .../models/__pycache__/mllama.cpython-310.pyc | Bin 0 -> 28823 bytes .../mlp_speculator.cpython-310.pyc | Bin 0 -> 5846 bytes .../module_mapping.cpython-310.pyc | Bin 0 -> 1909 bytes .../models/__pycache__/molmo.cpython-310.pyc | Bin 0 -> 31073 bytes .../models/__pycache__/mpt.cpython-310.pyc | Bin 0 -> 8969 bytes .../__pycache__/nemotron.cpython-310.pyc | Bin 0 -> 12727 bytes .../models/__pycache__/nvlm_d.cpython-310.pyc | Bin 0 -> 2741 bytes .../models/__pycache__/olmo.cpython-310.pyc | Bin 0 -> 9558 bytes .../models/__pycache__/olmoe.cpython-310.pyc | Bin 0 -> 10860 bytes .../models/__pycache__/opt.cpython-310.pyc | Bin 0 -> 10458 bytes .../models/__pycache__/orion.cpython-310.pyc | Bin 0 -> 9520 bytes .../__pycache__/paligemma.cpython-310.pyc | Bin 0 -> 8858 bytes .../__pycache__/persimmon.cpython-310.pyc | Bin 0 -> 9429 bytes .../models/__pycache__/phi.cpython-310.pyc | Bin 0 -> 8857 bytes .../models/__pycache__/phi3.cpython-310.pyc | Bin 0 -> 553 bytes .../__pycache__/phi3_small.cpython-310.pyc | Bin 0 -> 13156 bytes .../models/__pycache__/phi3v.cpython-310.pyc | Bin 0 -> 17530 bytes .../models/__pycache__/phimoe.cpython-310.pyc | Bin 0 -> 14431 bytes .../__pycache__/pixtral.cpython-310.pyc | Bin 0 -> 19005 bytes .../models/__pycache__/qwen.cpython-310.pyc | Bin 0 -> 26253 bytes .../models/__pycache__/qwen2.cpython-310.pyc | Bin 0 -> 10919 bytes .../__pycache__/qwen2_5_vl.cpython-310.pyc | Bin 0 -> 32068 bytes .../__pycache__/qwen2_moe.cpython-310.pyc | Bin 0 -> 12413 bytes .../__pycache__/qwen2_rm.cpython-310.pyc | Bin 0 -> 3914 bytes .../__pycache__/qwen2_vl.cpython-310.pyc | Bin 0 -> 30618 bytes .../models/__pycache__/qwen3.cpython-310.pyc | Bin 0 -> 8751 bytes .../__pycache__/qwen3_moe.cpython-310.pyc | Bin 0 -> 13228 bytes .../__pycache__/registry.cpython-310.pyc | Bin 0 -> 14695 bytes .../models/__pycache__/siglip.cpython-310.pyc | Bin 0 -> 15143 bytes .../models/__pycache__/solar.cpython-310.pyc | Bin 0 -> 13024 bytes .../__pycache__/stablelm.cpython-310.pyc | Bin 0 -> 9418 bytes .../__pycache__/starcoder2.cpython-310.pyc | Bin 0 -> 8867 bytes .../__pycache__/ultravox.cpython-310.pyc | Bin 0 -> 15123 bytes .../models/__pycache__/utils.cpython-310.pyc | Bin 0 -> 17025 bytes .../models/__pycache__/xverse.cpython-310.pyc | Bin 0 -> 10169 bytes vllm/model_executor/models/arctic.py | 562 +++++ vllm/model_executor/models/baichuan.py | 463 ++++ vllm/model_executor/models/bart.py | 1003 +++++++++ vllm/model_executor/models/blip.py | 423 ++++ vllm/model_executor/models/blip2.py | 690 ++++++ vllm/model_executor/models/bloom.py | 362 +++ vllm/model_executor/models/chameleon.py | 1104 ++++++++++ vllm/model_executor/models/chatglm.py | 671 ++++++ vllm/model_executor/models/clip.py | 474 ++++ vllm/model_executor/models/commandr.py | 441 ++++ vllm/model_executor/models/dbrx.py | 439 ++++ vllm/model_executor/models/decilm.py | 129 ++ vllm/model_executor/models/deepseek.py | 480 ++++ vllm/model_executor/models/deepseek_v2.py | 617 ++++++ vllm/model_executor/models/eagle.py | 170 ++ vllm/model_executor/models/exaone.py | 606 ++++++ vllm/model_executor/models/falcon.py | 508 +++++ vllm/model_executor/models/fuyu.py | 351 +++ vllm/model_executor/models/gemma.py | 455 ++++ vllm/model_executor/models/gemma2.py | 463 ++++ .../model_executor/models/gemma2_embedding.py | 57 + vllm/model_executor/models/glm4.py | 362 +++ .../models/glm4_vision_encoder.py | 298 +++ vllm/model_executor/models/gpt2.py | 324 +++ vllm/model_executor/models/gpt_bigcode.py | 341 +++ vllm/model_executor/models/gpt_j.py | 317 +++ vllm/model_executor/models/gpt_neox.py | 329 +++ vllm/model_executor/models/granite.py | 545 +++++ vllm/model_executor/models/granitemoe.py | 448 ++++ .../models/idefics2_vision_model.py | 302 +++ vllm/model_executor/models/interfaces.py | 352 +++ vllm/model_executor/models/interfaces_base.py | 191 ++ vllm/model_executor/models/intern_vit.py | 428 ++++ vllm/model_executor/models/internlm2.py | 383 ++++ vllm/model_executor/models/internvl.py | 612 ++++++ vllm/model_executor/models/jais.py | 374 ++++ vllm/model_executor/models/jamba.py | 717 ++++++ vllm/model_executor/models/llama.py | 637 ++++++ vllm/model_executor/models/llama_embedding.py | 59 + vllm/model_executor/models/llava.py | 411 ++++ vllm/model_executor/models/llava_next.py | 645 ++++++ .../model_executor/models/llava_next_video.py | 465 ++++ vllm/model_executor/models/llava_onevision.py | 875 ++++++++ vllm/model_executor/models/mamba.py | 499 +++++ vllm/model_executor/models/mamba_cache.py | 222 ++ vllm/model_executor/models/medusa.py | 184 ++ vllm/model_executor/models/minicpm.py | 599 +++++ vllm/model_executor/models/minicpm3.py | 340 +++ vllm/model_executor/models/minicpmv.py | 1044 +++++++++ vllm/model_executor/models/mixtral.py | 493 +++++ vllm/model_executor/models/mixtral_quant.py | 441 ++++ vllm/model_executor/models/mllama.py | 1151 ++++++++++ vllm/model_executor/models/mlp_speculator.py | 197 ++ vllm/model_executor/models/module_mapping.py | 69 + vllm/model_executor/models/molmo.py | 1290 +++++++++++ vllm/model_executor/models/mpt.py | 327 +++ vllm/model_executor/models/nemotron.py | 525 +++++ vllm/model_executor/models/nvlm_d.py | 64 + vllm/model_executor/models/olmo.py | 394 ++++ vllm/model_executor/models/olmoe.py | 445 ++++ vllm/model_executor/models/opt.py | 416 ++++ vllm/model_executor/models/orion.py | 363 ++++ vllm/model_executor/models/paligemma.py | 295 +++ vllm/model_executor/models/persimmon.py | 354 +++ vllm/model_executor/models/phi.py | 374 ++++ vllm/model_executor/models/phi3.py | 17 + vllm/model_executor/models/phi3_small.py | 473 ++++ vllm/model_executor/models/phi3v.py | 694 ++++++ vllm/model_executor/models/phimoe.py | 663 ++++++ vllm/model_executor/models/pixtral.py | 578 +++++ vllm/model_executor/models/qwen.py | 991 +++++++++ vllm/model_executor/models/qwen2.py | 457 ++++ vllm/model_executor/models/qwen2_5_vl.py | 1213 +++++++++++ vllm/model_executor/models/qwen2_moe.py | 523 +++++ vllm/model_executor/models/qwen2_rm.py | 123 ++ vllm/model_executor/models/qwen2_vl.py | 1174 ++++++++++ vllm/model_executor/models/qwen3.py | 357 +++ vllm/model_executor/models/qwen3_moe.py | 538 +++++ vllm/model_executor/models/registry.py | 454 ++++ vllm/model_executor/models/siglip.py | 590 +++++ vllm/model_executor/models/solar.py | 569 +++++ vllm/model_executor/models/stablelm.py | 338 +++ vllm/model_executor/models/starcoder2.py | 336 +++ vllm/model_executor/models/ultravox.py | 503 +++++ vllm/model_executor/models/utils.py | 515 +++++ vllm/model_executor/models/xverse.py | 404 ++++ vllm/model_executor/parameter.py | 403 ++++ vllm/model_executor/pooling_metadata.py | 69 + vllm/model_executor/sampling_metadata.py | 587 +++++ vllm/model_executor/utils.py | 31 + vllm/multimodal/__init__.py | 24 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 489 bytes .../__pycache__/audio.cpython-310.pyc | Bin 0 -> 1175 bytes .../__pycache__/base.cpython-310.pyc | Bin 0 -> 10653 bytes .../__pycache__/image.cpython-310.pyc | Bin 0 -> 2623 bytes .../__pycache__/registry.cpython-310.pyc | Bin 0 -> 8482 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 8511 bytes .../__pycache__/video.cpython-310.pyc | Bin 0 -> 2541 bytes vllm/multimodal/audio.py | 17 + vllm/multimodal/base.py | 368 ++++ vllm/multimodal/image.py | 88 + vllm/multimodal/registry.py | 243 +++ vllm/multimodal/utils.py | 323 +++ vllm/multimodal/video.py | 86 + vllm/outputs.py | 318 +++ vllm/platforms/__init__.py | 81 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1323 bytes .../platforms/__pycache__/cpu.cpython-310.pyc | Bin 0 -> 1014 bytes .../__pycache__/cuda.cpython-310.pyc | Bin 0 -> 5104 bytes .../__pycache__/interface.cpython-310.pyc | Bin 0 -> 4240 bytes .../__pycache__/rocm.cpython-310.pyc | Bin 0 -> 1574 bytes .../platforms/__pycache__/tpu.cpython-310.pyc | Bin 0 -> 1320 bytes .../platforms/__pycache__/xpu.cpython-310.pyc | Bin 0 -> 1241 bytes vllm/platforms/cpu.py | 20 + vllm/platforms/cuda.py | 145 ++ vllm/platforms/interface.py | 108 + vllm/platforms/rocm.py | 36 + vllm/platforms/tpu.py | 33 + vllm/platforms/xpu.py | 22 + vllm/plugins/__init__.py | 56 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1869 bytes vllm/pooling_params.py | 23 + vllm/prompt_adapter/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 160 bytes .../__pycache__/layers.cpython-310.pyc | Bin 0 -> 2922 bytes .../__pycache__/models.cpython-310.pyc | Bin 0 -> 13057 bytes .../__pycache__/request.cpython-310.pyc | Bin 0 -> 1327 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 2424 bytes .../worker_manager.cpython-310.pyc | Bin 0 -> 6560 bytes vllm/prompt_adapter/layers.py | 80 + vllm/prompt_adapter/models.py | 355 +++ vllm/prompt_adapter/request.py | 34 + vllm/prompt_adapter/utils.py | 95 + vllm/prompt_adapter/worker_manager.py | 176 ++ vllm/py.typed | 2 + vllm/sampling_params.py | 491 +++++ vllm/scalar_type.py | 35 + vllm/scripts.py | 201 ++ vllm/sequence.py | 1380 ++++++++++++ vllm/spec_decode/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 157 bytes .../batch_expansion.cpython-310.pyc | Bin 0 -> 12452 bytes .../draft_model_runner.cpython-310.pyc | Bin 0 -> 7875 bytes .../__pycache__/interfaces.cpython-310.pyc | Bin 0 -> 2896 bytes .../__pycache__/medusa_worker.cpython-310.pyc | Bin 0 -> 4537 bytes .../__pycache__/metrics.cpython-310.pyc | Bin 0 -> 5017 bytes .../mlp_speculator_worker.cpython-310.pyc | Bin 0 -> 2933 bytes .../__pycache__/mqa_scorer.cpython-310.pyc | Bin 0 -> 2771 bytes .../multi_step_worker.cpython-310.pyc | Bin 0 -> 11509 bytes .../__pycache__/ngram_worker.cpython-310.pyc | Bin 0 -> 4919 bytes .../proposer_worker_base.cpython-310.pyc | Bin 0 -> 2680 bytes ...smaller_tp_proposer_worker.cpython-310.pyc | Bin 0 -> 5899 bytes .../spec_decode_worker.cpython-310.pyc | Bin 0 -> 31981 bytes .../target_model_runner.cpython-310.pyc | Bin 0 -> 2675 bytes .../__pycache__/top1_proposer.cpython-310.pyc | Bin 0 -> 5701 bytes .../__pycache__/util.cpython-310.pyc | Bin 0 -> 7933 bytes vllm/spec_decode/batch_expansion.py | 455 ++++ vllm/spec_decode/draft_model_runner.py | 327 +++ vllm/spec_decode/interfaces.py | 90 + vllm/spec_decode/medusa_worker.py | 136 ++ vllm/spec_decode/metrics.py | 196 ++ vllm/spec_decode/mlp_speculator_worker.py | 91 + vllm/spec_decode/mqa_scorer.py | 106 + vllm/spec_decode/multi_step_worker.py | 366 ++++ vllm/spec_decode/ngram_worker.py | 169 ++ vllm/spec_decode/proposer_worker_base.py | 56 + .../spec_decode/smaller_tp_proposer_worker.py | 161 ++ vllm/spec_decode/spec_decode_worker.py | 1074 +++++++++ vllm/spec_decode/target_model_runner.py | 69 + vllm/spec_decode/top1_proposer.py | 272 +++ vllm/spec_decode/util.py | 268 +++ vllm/tracing.py | 119 + vllm/transformers_utils/__init__.py | 17 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 542 bytes .../__pycache__/config.cpython-310.pyc | Bin 0 -> 8778 bytes .../__pycache__/detokenizer.cpython-310.pyc | Bin 0 -> 7323 bytes .../__pycache__/processor.cpython-310.pyc | Bin 0 -> 2054 bytes .../__pycache__/tokenizer.cpython-310.pyc | Bin 0 -> 5719 bytes .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 645 bytes vllm/transformers_utils/config.py | 360 +++ vllm/transformers_utils/configs/__init__.py | 40 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1522 bytes .../__pycache__/arctic.cpython-310.pyc | Bin 0 -> 8223 bytes .../__pycache__/chatglm.cpython-310.pyc | Bin 0 -> 1782 bytes .../configs/__pycache__/dbrx.cpython-310.pyc | Bin 0 -> 9406 bytes .../configs/__pycache__/eagle.cpython-310.pyc | Bin 0 -> 1559 bytes .../__pycache__/exaone.cpython-310.pyc | Bin 0 -> 7248 bytes .../__pycache__/falcon.cpython-310.pyc | Bin 0 -> 1909 bytes .../__pycache__/internvl.cpython-310.pyc | Bin 0 -> 1239 bytes .../configs/__pycache__/jais.cpython-310.pyc | Bin 0 -> 8271 bytes .../__pycache__/medusa.cpython-310.pyc | Bin 0 -> 2089 bytes .../__pycache__/mllama.cpython-310.pyc | Bin 0 -> 1245 bytes .../mlp_speculator.cpython-310.pyc | Bin 0 -> 2478 bytes .../configs/__pycache__/mpt.cpython-310.pyc | Bin 0 -> 5382 bytes .../__pycache__/nemotron.cpython-310.pyc | Bin 0 -> 7279 bytes .../__pycache__/nvlm_d.cpython-310.pyc | Bin 0 -> 409 bytes .../__pycache__/qwen2vl.cpython-310.pyc | Bin 0 -> 2736 bytes .../configs/__pycache__/solar.cpython-310.pyc | Bin 0 -> 8829 bytes .../__pycache__/ultravox.cpython-310.pyc | Bin 0 -> 3413 bytes vllm/transformers_utils/configs/arctic.py | 204 ++ vllm/transformers_utils/configs/chatglm.py | 70 + vllm/transformers_utils/configs/dbrx.py | 278 +++ vllm/transformers_utils/configs/eagle.py | 49 + vllm/transformers_utils/configs/exaone.py | 190 ++ vllm/transformers_utils/configs/falcon.py | 89 + vllm/transformers_utils/configs/internvl.py | 51 + vllm/transformers_utils/configs/jais.py | 236 ++ vllm/transformers_utils/configs/medusa.py | 60 + vllm/transformers_utils/configs/mllama.py | 28 + .../configs/mlp_speculator.py | 65 + vllm/transformers_utils/configs/mpt.py | 178 ++ vllm/transformers_utils/configs/nemotron.py | 203 ++ vllm/transformers_utils/configs/nvlm_d.py | 12 + vllm/transformers_utils/configs/qwen2vl.py | 131 ++ vllm/transformers_utils/configs/solar.py | 245 +++ vllm/transformers_utils/configs/ultravox.py | 99 + vllm/transformers_utils/detokenizer.py | 327 +++ vllm/transformers_utils/processor.py | 94 + vllm/transformers_utils/tokenizer.py | 193 ++ .../tokenizer_group/__init__.py | 52 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 1695 bytes .../base_tokenizer_group.cpython-310.pyc | Bin 0 -> 2482 bytes .../ray_tokenizer_group.cpython-310.pyc | Bin 0 -> 7366 bytes .../tokenizer_group.cpython-310.pyc | Bin 0 -> 3471 bytes .../tokenizer_group/base_tokenizer_group.py | 66 + .../tokenizer_group/ray_tokenizer_group.py | 240 ++ .../tokenizer_group/tokenizer_group.py | 99 + .../transformers_utils/tokenizers/__init__.py | 3 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 245 bytes .../__pycache__/mistral.cpython-310.pyc | Bin 0 -> 8555 bytes vllm/transformers_utils/tokenizers/mistral.py | 229 ++ vllm/transformers_utils/utils.py | 16 + vllm/triton_utils/__init__.py | 9 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 418 bytes .../custom_cache_manager.cpython-310.pyc | Bin 0 -> 2066 bytes .../__pycache__/importing.cpython-310.pyc | Bin 0 -> 407 bytes .../__pycache__/libentry.cpython-310.pyc | Bin 0 -> 4523 bytes vllm/triton_utils/custom_cache_manager.py | 53 + vllm/triton_utils/importing.py | 11 + vllm/triton_utils/libentry.py | 167 ++ vllm/usage/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 151 bytes .../__pycache__/usage_lib.cpython-310.pyc | Bin 0 -> 6690 bytes vllm/usage/usage_lib.py | 223 ++ vllm/utils.py | 1445 ++++++++++++ vllm/version.py | 2 + vllm/worker/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 152 bytes .../__pycache__/cache_engine.cpython-310.pyc | Bin 0 -> 3461 bytes .../cpu_enc_dec_model_runner.cpython-310.pyc | Bin 0 -> 8135 bytes .../cpu_model_runner.cpython-310.pyc | Bin 0 -> 13343 bytes .../__pycache__/cpu_worker.cpython-310.pyc | Bin 0 -> 11654 bytes .../embedding_model_runner.cpython-310.pyc | Bin 0 -> 6121 bytes .../enc_dec_model_runner.cpython-310.pyc | Bin 0 -> 14144 bytes .../__pycache__/model_runner.cpython-310.pyc | Bin 0 -> 46582 bytes .../model_runner_base.cpython-310.pyc | Bin 0 -> 9631 bytes .../multi_step_model_runner.cpython-310.pyc | Bin 0 -> 21810 bytes .../multi_step_tpu_worker.cpython-310.pyc | Bin 0 -> 2655 bytes .../multi_step_worker.cpython-310.pyc | Bin 0 -> 4768 bytes .../neuron_model_runner.cpython-310.pyc | Bin 0 -> 9004 bytes .../__pycache__/neuron_worker.cpython-310.pyc | Bin 0 -> 4376 bytes .../openvino_model_runner.cpython-310.pyc | Bin 0 -> 6960 bytes .../openvino_worker.cpython-310.pyc | Bin 0 -> 15474 bytes .../tpu_model_runner.cpython-310.pyc | Bin 0 -> 17905 bytes .../__pycache__/tpu_worker.cpython-310.pyc | Bin 0 -> 8257 bytes vllm/worker/__pycache__/utils.cpython-310.pyc | Bin 0 -> 1411 bytes .../worker/__pycache__/worker.cpython-310.pyc | Bin 0 -> 15189 bytes .../__pycache__/worker_base.cpython-310.pyc | Bin 0 -> 15417 bytes .../xpu_model_runner.cpython-310.pyc | Bin 0 -> 14474 bytes .../__pycache__/xpu_worker.cpython-310.pyc | Bin 0 -> 6076 bytes vllm/worker/cache_engine.py | 120 + vllm/worker/cpu_enc_dec_model_runner.py | 311 +++ vllm/worker/cpu_model_runner.py | 549 +++++ vllm/worker/cpu_worker.py | 371 ++++ vllm/worker/embedding_model_runner.py | 208 ++ vllm/worker/enc_dec_model_runner.py | 546 +++++ vllm/worker/model_runner.py | 1932 ++++++++++++++++ vllm/worker/model_runner_base.py | 275 +++ vllm/worker/multi_step_model_runner.py | 901 ++++++++ vllm/worker/multi_step_tpu_worker.py | 105 + vllm/worker/multi_step_worker.py | 202 ++ vllm/worker/neuron_model_runner.py | 346 +++ vllm/worker/neuron_worker.py | 127 ++ vllm/worker/openvino_model_runner.py | 353 +++ vllm/worker/openvino_worker.py | 611 ++++++ vllm/worker/tpu_model_runner.py | 842 +++++++ vllm/worker/tpu_worker.py | 309 +++ vllm/worker/utils.py | 51 + vllm/worker/worker.py | 497 +++++ vllm/worker/worker_base.py | 485 +++++ vllm/worker/xpu_model_runner.py | 605 ++++++ vllm/worker/xpu_worker.py | 206 ++ 920 files changed, 149563 insertions(+) create mode 100644 __init__.py create mode 100644 attention.py create mode 100644 chat_dataset_v0.json create mode 100644 prefix_prefill.py create mode 100644 qwen3_6_scripts/api_server.py create mode 100644 qwen3_6_scripts/chat_utils.py create mode 100644 qwen3_6_scripts/cli_args.py create mode 100644 qwen3_6_scripts/mamba_cache.py create mode 100644 qwen3_6_scripts/patch_model_runner.py create mode 100755 qwen3_6_scripts/patch_ops.sh create mode 100644 qwen3_6_scripts/patch_transformers_qwen3_5.py create mode 100644 qwen3_6_scripts/patch_vllm_qwen3_5.py create mode 100644 qwen3_6_scripts/patch_vllm_tool_parser.py create mode 100644 qwen3_6_scripts/patch_xformers_sdpa_batch.py create mode 100644 qwen3_6_scripts/patch_xformers_sdpa_batch_kernel.py create mode 100644 qwen3_6_scripts/patch_xformers_sdpa_seq.py create mode 100644 qwen3_6_scripts/patch_xformers_sdpa_seq_kernel.py create mode 100644 qwen3_6_scripts/protocol.py create mode 100644 qwen3_6_scripts/qwen3_5.py create mode 100644 qwen3_6_scripts/qwen3_5/__init__.py create mode 100644 qwen3_6_scripts/qwen3_5/configuration_qwen3_5.py create mode 100644 qwen3_6_scripts/qwen3_5_moe/__init__.py create mode 100644 qwen3_6_scripts/qwen3_5_moe/configuration_qwen3_5_moe.py create mode 100644 qwen3_6_scripts/qwen3coder_tool_parser.py create mode 100644 qwen3_6_scripts/reasoning/__init__.py create mode 100644 qwen3_6_scripts/reasoning/abs_reasoning_parsers.py create mode 100644 qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py create mode 100644 qwen3_6_scripts/scheduler.py create mode 100644 qwen3_6_scripts/sequence.py create mode 100644 qwen3_6_scripts/serving_chat.py create mode 100644 vllm/__init__.py create mode 100644 vllm/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/__pycache__/_core_ext.cpython-310.pyc create mode 100644 vllm/__pycache__/_custom_ops.cpython-310.pyc create mode 100644 vllm/__pycache__/_ipex_ops.cpython-310.pyc create mode 100644 vllm/__pycache__/beam_search.cpython-310.pyc create mode 100644 vllm/__pycache__/block.cpython-310.pyc create mode 100644 vllm/__pycache__/config.cpython-310.pyc create mode 100644 vllm/__pycache__/connections.cpython-310.pyc create mode 100644 vllm/__pycache__/envs.cpython-310.pyc create mode 100644 vllm/__pycache__/forward_context.cpython-310.pyc create mode 100644 vllm/__pycache__/logger.cpython-310.pyc create mode 100644 vllm/__pycache__/outputs.cpython-310.pyc create mode 100644 vllm/__pycache__/pooling_params.cpython-310.pyc create mode 100644 vllm/__pycache__/sampling_params.cpython-310.pyc create mode 100644 vllm/__pycache__/scalar_type.cpython-310.pyc create mode 100644 vllm/__pycache__/scripts.cpython-310.pyc create mode 100644 vllm/__pycache__/sequence.cpython-310.pyc create mode 100644 vllm/__pycache__/tracing.cpython-310.pyc create mode 100644 vllm/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/__pycache__/version.cpython-310.pyc create mode 100644 vllm/_core_ext.py create mode 100644 vllm/_custom_ops.py create mode 100644 vllm/_ipex_ops.py create mode 100644 vllm/adapter_commons/__init__.py create mode 100644 vllm/adapter_commons/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/adapter_commons/__pycache__/layers.cpython-310.pyc create mode 100644 vllm/adapter_commons/__pycache__/models.cpython-310.pyc create mode 100644 vllm/adapter_commons/__pycache__/request.cpython-310.pyc create mode 100644 vllm/adapter_commons/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/adapter_commons/__pycache__/worker_manager.cpython-310.pyc create mode 100644 vllm/adapter_commons/layers.py create mode 100644 vllm/adapter_commons/models.py create mode 100644 vllm/adapter_commons/request.py create mode 100644 vllm/adapter_commons/utils.py create mode 100644 vllm/adapter_commons/worker_manager.py create mode 100644 vllm/assets/__init__.py create mode 100644 vllm/assets/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/assets/__pycache__/audio.cpython-310.pyc create mode 100644 vllm/assets/__pycache__/base.cpython-310.pyc create mode 100644 vllm/assets/__pycache__/image.cpython-310.pyc create mode 100644 vllm/assets/__pycache__/video.cpython-310.pyc create mode 100644 vllm/assets/audio.py create mode 100644 vllm/assets/base.py create mode 100644 vllm/assets/image.py create mode 100644 vllm/assets/video.py create mode 100644 vllm/attention/__init__.py create mode 100644 vllm/attention/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/attention/__pycache__/layer.cpython-310.pyc create mode 100644 vllm/attention/__pycache__/selector.cpython-310.pyc create mode 100644 vllm/attention/backends/__init__.py create mode 100644 vllm/attention/backends/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/abstract.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/blocksparse_attn.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/flash_attn.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/flashinfer.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/ipex_attn.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/openvino.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/pallas.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/placeholder_attn.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/rocm_flash_attn.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/torch_sdpa.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/attention/backends/__pycache__/xformers.cpython-310.pyc create mode 100644 vllm/attention/backends/abstract.py create mode 100644 vllm/attention/backends/blocksparse_attn.py create mode 100644 vllm/attention/backends/flash_attn.py create mode 100644 vllm/attention/backends/flashinfer.py create mode 100644 vllm/attention/backends/ipex_attn.py create mode 100644 vllm/attention/backends/openvino.py create mode 100644 vllm/attention/backends/pallas.py create mode 100644 vllm/attention/backends/placeholder_attn.py create mode 100644 vllm/attention/backends/rocm_flash_attn.py create mode 100644 vllm/attention/backends/torch_sdpa.py create mode 100644 vllm/attention/backends/utils.py create mode 100644 vllm/attention/backends/xformers.py create mode 100644 vllm/attention/layer.py create mode 100644 vllm/attention/ops/__init__.py create mode 100644 vllm/attention/ops/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/attention/ops/__pycache__/ipex_attn.cpython-310.pyc create mode 100644 vllm/attention/ops/__pycache__/paged_attn.cpython-310.pyc create mode 100644 vllm/attention/ops/__pycache__/prefix_prefill.cpython-310.pyc create mode 100644 vllm/attention/ops/__pycache__/triton_flash_attention.cpython-310.pyc create mode 100644 vllm/attention/ops/blocksparse_attention/__init__.py create mode 100644 vllm/attention/ops/blocksparse_attention/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/attention/ops/blocksparse_attention/__pycache__/blocksparse_attention_kernel.cpython-310.pyc create mode 100644 vllm/attention/ops/blocksparse_attention/__pycache__/interface.cpython-310.pyc create mode 100644 vllm/attention/ops/blocksparse_attention/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py create mode 100644 vllm/attention/ops/blocksparse_attention/interface.py create mode 100644 vllm/attention/ops/blocksparse_attention/utils.py create mode 100644 vllm/attention/ops/ipex_attn.py create mode 100644 vllm/attention/ops/paged_attn.py create mode 100644 vllm/attention/ops/prefix_prefill.py create mode 100644 vllm/attention/ops/triton_flash_attention.py create mode 100644 vllm/attention/selector.py create mode 100644 vllm/beam_search.py create mode 100644 vllm/block.py create mode 100644 vllm/compilation/__init__.py create mode 100644 vllm/compilation/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/compilation/__pycache__/backends.cpython-310.pyc create mode 100644 vllm/compilation/__pycache__/compile_context.cpython-310.pyc create mode 100644 vllm/compilation/__pycache__/decorators.cpython-310.pyc create mode 100644 vllm/compilation/__pycache__/levels.cpython-310.pyc create mode 100644 vllm/compilation/__pycache__/wrapper.cpython-310.pyc create mode 100644 vllm/compilation/backends.py create mode 100644 vllm/compilation/compile_context.py create mode 100644 vllm/compilation/decorators.py create mode 100644 vllm/compilation/levels.py create mode 100644 vllm/compilation/wrapper.py create mode 100644 vllm/config.py create mode 100644 vllm/connections.py create mode 100644 vllm/core/__init__.py create mode 100644 vllm/core/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/core/__pycache__/block_manager_v1.cpython-310.pyc create mode 100644 vllm/core/__pycache__/block_manager_v2.cpython-310.pyc create mode 100644 vllm/core/__pycache__/evictor_v1.cpython-310.pyc create mode 100644 vllm/core/__pycache__/evictor_v2.cpython-310.pyc create mode 100644 vllm/core/__pycache__/interfaces.cpython-310.pyc create mode 100644 vllm/core/__pycache__/placeholder_block_space_manager.cpython-310.pyc create mode 100644 vllm/core/__pycache__/scheduler.cpython-310.pyc create mode 100644 vllm/core/block/__init__.py create mode 100644 vllm/core/block/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/block_table.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/common.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/cpu_gpu_block_allocator.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/interfaces.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/naive_block.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/prefix_caching_block.cpython-310.pyc create mode 100644 vllm/core/block/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/core/block/block_table.py create mode 100644 vllm/core/block/common.py create mode 100644 vllm/core/block/cpu_gpu_block_allocator.py create mode 100644 vllm/core/block/interfaces.py create mode 100644 vllm/core/block/naive_block.py create mode 100644 vllm/core/block/prefix_caching_block.py create mode 100644 vllm/core/block/utils.py create mode 100644 vllm/core/block_manager_v1.py create mode 100644 vllm/core/block_manager_v2.py create mode 100644 vllm/core/evictor_v1.py create mode 100644 vllm/core/evictor_v2.py create mode 100644 vllm/core/interfaces.py create mode 100644 vllm/core/placeholder_block_space_manager.py create mode 100644 vllm/core/scheduler.py create mode 100644 vllm/distributed/__init__.py create mode 100644 vllm/distributed/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/distributed/__pycache__/communication_op.cpython-310.pyc create mode 100644 vllm/distributed/__pycache__/parallel_state.cpython-310.pyc create mode 100644 vllm/distributed/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/distributed/communication_op.py create mode 100644 vllm/distributed/device_communicators/__init__.py create mode 100644 vllm/distributed/device_communicators/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/cuda_wrapper.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/custom_all_reduce.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/custom_all_reduce_utils.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/pynccl.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/pynccl_wrapper.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/shm_broadcast.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/__pycache__/tpu_communicator.cpython-310.pyc create mode 100644 vllm/distributed/device_communicators/cuda_wrapper.py create mode 100644 vllm/distributed/device_communicators/custom_all_reduce.py create mode 100644 vllm/distributed/device_communicators/custom_all_reduce_utils.py create mode 100644 vllm/distributed/device_communicators/pynccl.py create mode 100644 vllm/distributed/device_communicators/pynccl_wrapper.py create mode 100644 vllm/distributed/device_communicators/shm_broadcast.py create mode 100644 vllm/distributed/device_communicators/tpu_communicator.py create mode 100644 vllm/distributed/parallel_state.py create mode 100644 vllm/distributed/utils.py create mode 100644 vllm/engine/__init__.py create mode 100644 vllm/engine/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/arg_utils.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/async_llm_engine.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/async_timeout.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/llm_engine.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/metrics.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/metrics_types.cpython-310.pyc create mode 100644 vllm/engine/__pycache__/protocol.cpython-310.pyc create mode 100644 vllm/engine/arg_utils.py create mode 100644 vllm/engine/async_llm_engine.py create mode 100644 vllm/engine/async_timeout.py create mode 100644 vllm/engine/llm_engine.py create mode 100644 vllm/engine/metrics.py create mode 100644 vllm/engine/metrics_types.py create mode 100644 vllm/engine/multiprocessing/__init__.py create mode 100644 vllm/engine/multiprocessing/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/engine/multiprocessing/__pycache__/client.cpython-310.pyc create mode 100644 vllm/engine/multiprocessing/__pycache__/engine.cpython-310.pyc create mode 100644 vllm/engine/multiprocessing/client.py create mode 100644 vllm/engine/multiprocessing/engine.py create mode 100644 vllm/engine/output_processor/__init__.py create mode 100644 vllm/engine/output_processor/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/engine/output_processor/__pycache__/interfaces.cpython-310.pyc create mode 100644 vllm/engine/output_processor/__pycache__/multi_step.cpython-310.pyc create mode 100644 vllm/engine/output_processor/__pycache__/single_step.cpython-310.pyc create mode 100644 vllm/engine/output_processor/__pycache__/stop_checker.cpython-310.pyc create mode 100644 vllm/engine/output_processor/__pycache__/util.cpython-310.pyc create mode 100644 vllm/engine/output_processor/interfaces.py create mode 100644 vllm/engine/output_processor/multi_step.py create mode 100644 vllm/engine/output_processor/single_step.py create mode 100644 vllm/engine/output_processor/stop_checker.py create mode 100644 vllm/engine/output_processor/util.py create mode 100644 vllm/engine/protocol.py create mode 100644 vllm/entrypoints/__init__.py create mode 100644 vllm/entrypoints/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/entrypoints/__pycache__/api_server.cpython-310.pyc create mode 100644 vllm/entrypoints/__pycache__/chat_utils.cpython-310.pyc create mode 100644 vllm/entrypoints/__pycache__/launcher.cpython-310.pyc create mode 100644 vllm/entrypoints/__pycache__/llm.cpython-310.pyc create mode 100644 vllm/entrypoints/__pycache__/logger.cpython-310.pyc create mode 100644 vllm/entrypoints/api_server.py create mode 100644 vllm/entrypoints/chat_utils.py create mode 100644 vllm/entrypoints/launcher.py create mode 100644 vllm/entrypoints/llm.py create mode 100644 vllm/entrypoints/logger.py create mode 100644 vllm/entrypoints/openai/__init__.py create mode 100644 vllm/entrypoints/openai/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/api_server.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/cli_args.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/logits_processors.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/protocol.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/run_batch.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/serving_chat.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/serving_completion.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/serving_embedding.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/serving_engine.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/__pycache__/serving_tokenization.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/api_server.py create mode 100644 vllm/entrypoints/openai/cli_args.py create mode 100644 vllm/entrypoints/openai/logits_processors.py create mode 100644 vllm/entrypoints/openai/protocol.py create mode 100644 vllm/entrypoints/openai/run_batch.py create mode 100644 vllm/entrypoints/openai/serving_chat.py create mode 100644 vllm/entrypoints/openai/serving_completion.py create mode 100644 vllm/entrypoints/openai/serving_embedding.py create mode 100644 vllm/entrypoints/openai/serving_engine.py create mode 100644 vllm/entrypoints/openai/serving_tokenization.py create mode 100644 vllm/entrypoints/openai/tool_parsers/__init__.py create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/abstract_tool_parser.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/hermes_tool_parser.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/internlm2_tool_parser.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/llama_tool_parser.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/mistral_tool_parser.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py create mode 100644 vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py create mode 100644 vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py create mode 100644 vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py create mode 100644 vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py create mode 100644 vllm/entrypoints/openai/tool_parsers/utils.py create mode 100644 vllm/envs.py create mode 100644 vllm/executor/__init__.py create mode 100644 vllm/executor/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/cpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/distributed_gpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/executor_base.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/gpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/msgspec_utils.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/multiproc_gpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/multiproc_worker_utils.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/multiproc_xpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/neuron_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/openvino_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/ray_gpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/ray_tpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/ray_utils.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/ray_xpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/tpu_executor.cpython-310.pyc create mode 100644 vllm/executor/__pycache__/xpu_executor.cpython-310.pyc create mode 100644 vllm/executor/cpu_executor.py create mode 100644 vllm/executor/distributed_gpu_executor.py create mode 100644 vllm/executor/executor_base.py create mode 100644 vllm/executor/gpu_executor.py create mode 100644 vllm/executor/msgspec_utils.py create mode 100644 vllm/executor/multiproc_gpu_executor.py create mode 100644 vllm/executor/multiproc_worker_utils.py create mode 100644 vllm/executor/multiproc_xpu_executor.py create mode 100644 vllm/executor/neuron_executor.py create mode 100644 vllm/executor/openvino_executor.py create mode 100644 vllm/executor/ray_gpu_executor.py create mode 100644 vllm/executor/ray_tpu_executor.py create mode 100644 vllm/executor/ray_utils.py create mode 100644 vllm/executor/ray_xpu_executor.py create mode 100644 vllm/executor/tpu_executor.py create mode 100644 vllm/executor/xpu_executor.py create mode 100644 vllm/forward_context.py create mode 100644 vllm/inputs/__init__.py create mode 100644 vllm/inputs/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/inputs/__pycache__/data.cpython-310.pyc create mode 100644 vllm/inputs/__pycache__/parse.cpython-310.pyc create mode 100644 vllm/inputs/__pycache__/preprocess.cpython-310.pyc create mode 100644 vllm/inputs/__pycache__/registry.cpython-310.pyc create mode 100644 vllm/inputs/data.py create mode 100644 vllm/inputs/parse.py create mode 100644 vllm/inputs/preprocess.py create mode 100644 vllm/inputs/registry.py create mode 100644 vllm/logger.py create mode 100644 vllm/logging/__init__.py create mode 100644 vllm/logging/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/logging/__pycache__/formatter.cpython-310.pyc create mode 100644 vllm/logging/formatter.py create mode 100644 vllm/lora/__init__.py create mode 100644 vllm/lora/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/fully_sharded_layers.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/layers.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/lora.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/models.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/punica.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/request.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/lora/__pycache__/worker_manager.cpython-310.pyc create mode 100644 vllm/lora/fully_sharded_layers.py create mode 100644 vllm/lora/layers.py create mode 100644 vllm/lora/lora.py create mode 100644 vllm/lora/models.py create mode 100644 vllm/lora/ops/__init__.py create mode 100644 vllm/lora/ops/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/bgmv_expand.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/bgmv_expand_slice.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/bgmv_shrink.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/sgmv_expand.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/sgmv_expand_slice.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/sgmv_shrink.cpython-310.pyc create mode 100644 vllm/lora/ops/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/lora/ops/bgmv_expand.py create mode 100644 vllm/lora/ops/bgmv_expand_slice.py create mode 100644 vllm/lora/ops/bgmv_shrink.py create mode 100644 vllm/lora/ops/sgmv_expand.py create mode 100644 vllm/lora/ops/sgmv_expand_slice.py create mode 100644 vllm/lora/ops/sgmv_shrink.py create mode 100644 vllm/lora/ops/utils.py create mode 100644 vllm/lora/punica.py create mode 100644 vllm/lora/request.py create mode 100644 vllm/lora/utils.py create mode 100644 vllm/lora/worker_manager.py create mode 100644 vllm/model_executor/__init__.py create mode 100644 vllm/model_executor/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/__pycache__/custom_op.cpython-310.pyc create mode 100644 vllm/model_executor/__pycache__/parameter.cpython-310.pyc create mode 100644 vllm/model_executor/__pycache__/pooling_metadata.cpython-310.pyc create mode 100644 vllm/model_executor/__pycache__/sampling_metadata.cpython-310.pyc create mode 100644 vllm/model_executor/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/model_executor/custom_op.py create mode 100644 vllm/model_executor/guided_decoding/__init__.py create mode 100644 vllm/model_executor/guided_decoding/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/guided_decoding/__pycache__/guided_fields.cpython-310.pyc create mode 100644 vllm/model_executor/guided_decoding/__pycache__/lm_format_enforcer_decoding.cpython-310.pyc create mode 100644 vllm/model_executor/guided_decoding/__pycache__/outlines_decoding.cpython-310.pyc create mode 100644 vllm/model_executor/guided_decoding/__pycache__/outlines_logits_processors.cpython-310.pyc create mode 100644 vllm/model_executor/guided_decoding/guided_fields.py create mode 100644 vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py create mode 100644 vllm/model_executor/guided_decoding/outlines_decoding.py create mode 100644 vllm/model_executor/guided_decoding/outlines_logits_processors.py create mode 100644 vllm/model_executor/layers/__init__.py create mode 100644 vllm/model_executor/layers/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/activation.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/layernorm.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/linear.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/logits_processor.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/pooler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/rejection_sampler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/resampler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/rotary_embedding.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/sampler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/spec_decode_base_sampler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/typical_acceptance_sampler.cpython-310.pyc create mode 100644 vllm/model_executor/layers/__pycache__/vocab_parallel_embedding.cpython-310.pyc create mode 100644 vllm/model_executor/layers/activation.py create mode 100644 vllm/model_executor/layers/fused_moe/__init__.py create mode 100644 vllm/model_executor/layers/fused_moe/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/fused_moe/__pycache__/fused_marlin_moe.cpython-310.pyc create mode 100644 vllm/model_executor/layers/fused_moe/__pycache__/fused_moe.cpython-310.pyc create mode 100644 vllm/model_executor/layers/fused_moe/__pycache__/layer.cpython-310.pyc create mode 100644 vllm/model_executor/layers/fused_moe/__pycache__/moe_pallas.cpython-310.pyc create mode 100644 vllm/model_executor/layers/fused_moe/fused_marlin_moe.py create mode 100644 vllm/model_executor/layers/fused_moe/fused_moe.py create mode 100644 vllm/model_executor/layers/fused_moe/layer.py create mode 100644 vllm/model_executor/layers/fused_moe/moe_pallas.py create mode 100644 vllm/model_executor/layers/layernorm.py create mode 100644 vllm/model_executor/layers/linear.py create mode 100644 vllm/model_executor/layers/logits_processor.py create mode 100644 vllm/model_executor/layers/mamba/__init__.py create mode 100644 vllm/model_executor/layers/mamba/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/mamba/ops/__init__.py create mode 100644 vllm/model_executor/layers/mamba/ops/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/mamba/ops/__pycache__/causal_conv1d.cpython-310.pyc create mode 100644 vllm/model_executor/layers/mamba/ops/__pycache__/mamba_ssm.cpython-310.pyc create mode 100644 vllm/model_executor/layers/mamba/ops/causal_conv1d.py create mode 100644 vllm/model_executor/layers/mamba/ops/mamba_ssm.py create mode 100644 vllm/model_executor/layers/pooler.py create mode 100644 vllm/model_executor/layers/quantization/__init__.py create mode 100644 vllm/model_executor/layers/quantization/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/aqlm.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/awq.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/awq_marlin.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/awq_triton.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/base_config.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/bitsandbytes.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/deepspeedfp.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/experts_int8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/fbgemm_fp8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/fp8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/gguf.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/gptq.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/gptq_marlin.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/gptq_marlin_24.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/ipex_quant.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/kv_cache.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/marlin.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/modelopt.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/neuron_quant.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/qqq.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/schema.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/tpu_int8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/__pycache__/w8a16.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/aqlm.py create mode 100644 vllm/model_executor/layers/quantization/awq.py create mode 100644 vllm/model_executor/layers/quantization/awq_marlin.py create mode 100644 vllm/model_executor/layers/quantization/awq_triton.py create mode 100644 vllm/model_executor/layers/quantization/base_config.py create mode 100644 vllm/model_executor/layers/quantization/bitsandbytes.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/__init__.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors_moe.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_scheme.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w4a16_24.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a16_fp8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_fp8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_int8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_wNa16.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/utils.py create mode 100644 vllm/model_executor/layers/quantization/deepspeedfp.py create mode 100644 vllm/model_executor/layers/quantization/experts_int8.py create mode 100644 vllm/model_executor/layers/quantization/fbgemm_fp8.py create mode 100644 vllm/model_executor/layers/quantization/fp8.py create mode 100644 vllm/model_executor/layers/quantization/gguf.py create mode 100644 vllm/model_executor/layers/quantization/gptq.py create mode 100644 vllm/model_executor/layers/quantization/gptq_marlin.py create mode 100644 vllm/model_executor/layers/quantization/gptq_marlin_24.py create mode 100644 vllm/model_executor/layers/quantization/ipex_quant.py create mode 100644 vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py create mode 100644 vllm/model_executor/layers/quantization/kernels/__init__.py create mode 100644 vllm/model_executor/layers/quantization/kernels/__pycache__/MPLinearKernel.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/kernels/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/kernels/__pycache__/machete.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/kernels/__pycache__/marlin.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/kernels/machete.py create mode 100644 vllm/model_executor/layers/quantization/kernels/marlin.py create mode 100644 vllm/model_executor/layers/quantization/kv_cache.py create mode 100644 vllm/model_executor/layers/quantization/marlin.py create mode 100644 vllm/model_executor/layers/quantization/modelopt.py create mode 100644 vllm/model_executor/layers/quantization/neuron_quant.py create mode 100644 vllm/model_executor/layers/quantization/qqq.py create mode 100644 vllm/model_executor/layers/quantization/schema.py create mode 100644 vllm/model_executor/layers/quantization/tpu_int8.py create mode 100644 vllm/model_executor/layers/quantization/utils/__init__.py create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/layer_utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/machete_utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_fp8.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test_24.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test_qqq.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/quant_utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/__pycache__/w8a8_utils.cpython-310.pyc create mode 100644 vllm/model_executor/layers/quantization/utils/layer_utils.py create mode 100644 vllm/model_executor/layers/quantization/utils/machete_utils.py create mode 100644 vllm/model_executor/layers/quantization/utils/marlin_utils.py create mode 100644 vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py create mode 100644 vllm/model_executor/layers/quantization/utils/marlin_utils_test.py create mode 100644 vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py create mode 100644 vllm/model_executor/layers/quantization/utils/marlin_utils_test_qqq.py create mode 100644 vllm/model_executor/layers/quantization/utils/quant_utils.py create mode 100644 vllm/model_executor/layers/quantization/utils/w8a8_utils.py create mode 100644 vllm/model_executor/layers/quantization/w8a16.py create mode 100644 vllm/model_executor/layers/rejection_sampler.py create mode 100644 vllm/model_executor/layers/resampler.py create mode 100644 vllm/model_executor/layers/rotary_embedding.py create mode 100644 vllm/model_executor/layers/sampler.py create mode 100644 vllm/model_executor/layers/spec_decode_base_sampler.py create mode 100644 vllm/model_executor/layers/typical_acceptance_sampler.py create mode 100644 vllm/model_executor/layers/vocab_parallel_embedding.py create mode 100644 vllm/model_executor/model_loader/__init__.py create mode 100644 vllm/model_executor/model_loader/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/loader.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/neuron.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/openvino.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/tensorizer.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/__pycache__/weight_utils.cpython-310.pyc create mode 100644 vllm/model_executor/model_loader/loader.py create mode 100644 vllm/model_executor/model_loader/neuron.py create mode 100644 vllm/model_executor/model_loader/openvino.py create mode 100644 vllm/model_executor/model_loader/tensorizer.py create mode 100644 vllm/model_executor/model_loader/utils.py create mode 100644 vllm/model_executor/model_loader/weight_utils.py create mode 100644 vllm/model_executor/models/__init__.py create mode 100644 vllm/model_executor/models/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/arctic.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/baichuan.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/bart.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/blip.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/blip2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/bloom.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/chameleon.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/chatglm.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/clip.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/commandr.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/dbrx.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/decilm.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/deepseek.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/deepseek_v2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/eagle.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/exaone.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/falcon.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/fuyu.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gemma.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gemma2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gemma2_embedding.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/glm4.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/glm4_vision_encoder.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gpt2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gpt_bigcode.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gpt_j.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/gpt_neox.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/granite.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/granitemoe.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/idefics2_vision_model.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/interfaces.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/interfaces_base.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/intern_vit.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/internlm2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/internvl.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/jais.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/jamba.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llama.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llama_embedding.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llava.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llava_next.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llava_next_video.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/llava_onevision.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mamba.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mamba_cache.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/medusa.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/minicpm.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/minicpm3.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/minicpmv.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mixtral.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mixtral_quant.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mllama.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mlp_speculator.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/module_mapping.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/molmo.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/mpt.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/nemotron.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/nvlm_d.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/olmo.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/olmoe.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/opt.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/orion.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/paligemma.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/persimmon.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/phi.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/phi3.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/phi3_small.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/phi3v.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/phimoe.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/pixtral.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen2_5_vl.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen2_moe.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen2_rm.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen2_vl.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen3.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/qwen3_moe.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/registry.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/siglip.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/solar.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/stablelm.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/starcoder2.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/ultravox.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/model_executor/models/__pycache__/xverse.cpython-310.pyc create mode 100644 vllm/model_executor/models/arctic.py create mode 100644 vllm/model_executor/models/baichuan.py create mode 100644 vllm/model_executor/models/bart.py create mode 100644 vllm/model_executor/models/blip.py create mode 100644 vllm/model_executor/models/blip2.py create mode 100644 vllm/model_executor/models/bloom.py create mode 100644 vllm/model_executor/models/chameleon.py create mode 100644 vllm/model_executor/models/chatglm.py create mode 100644 vllm/model_executor/models/clip.py create mode 100644 vllm/model_executor/models/commandr.py create mode 100644 vllm/model_executor/models/dbrx.py create mode 100644 vllm/model_executor/models/decilm.py create mode 100644 vllm/model_executor/models/deepseek.py create mode 100644 vllm/model_executor/models/deepseek_v2.py create mode 100644 vllm/model_executor/models/eagle.py create mode 100644 vllm/model_executor/models/exaone.py create mode 100644 vllm/model_executor/models/falcon.py create mode 100644 vllm/model_executor/models/fuyu.py create mode 100644 vllm/model_executor/models/gemma.py create mode 100644 vllm/model_executor/models/gemma2.py create mode 100644 vllm/model_executor/models/gemma2_embedding.py create mode 100644 vllm/model_executor/models/glm4.py create mode 100644 vllm/model_executor/models/glm4_vision_encoder.py create mode 100644 vllm/model_executor/models/gpt2.py create mode 100644 vllm/model_executor/models/gpt_bigcode.py create mode 100644 vllm/model_executor/models/gpt_j.py create mode 100644 vllm/model_executor/models/gpt_neox.py create mode 100644 vllm/model_executor/models/granite.py create mode 100644 vllm/model_executor/models/granitemoe.py create mode 100644 vllm/model_executor/models/idefics2_vision_model.py create mode 100644 vllm/model_executor/models/interfaces.py create mode 100644 vllm/model_executor/models/interfaces_base.py create mode 100644 vllm/model_executor/models/intern_vit.py create mode 100644 vllm/model_executor/models/internlm2.py create mode 100644 vllm/model_executor/models/internvl.py create mode 100644 vllm/model_executor/models/jais.py create mode 100644 vllm/model_executor/models/jamba.py create mode 100644 vllm/model_executor/models/llama.py create mode 100644 vllm/model_executor/models/llama_embedding.py create mode 100644 vllm/model_executor/models/llava.py create mode 100644 vllm/model_executor/models/llava_next.py create mode 100644 vllm/model_executor/models/llava_next_video.py create mode 100644 vllm/model_executor/models/llava_onevision.py create mode 100644 vllm/model_executor/models/mamba.py create mode 100644 vllm/model_executor/models/mamba_cache.py create mode 100644 vllm/model_executor/models/medusa.py create mode 100644 vllm/model_executor/models/minicpm.py create mode 100644 vllm/model_executor/models/minicpm3.py create mode 100644 vllm/model_executor/models/minicpmv.py create mode 100644 vllm/model_executor/models/mixtral.py create mode 100644 vllm/model_executor/models/mixtral_quant.py create mode 100644 vllm/model_executor/models/mllama.py create mode 100644 vllm/model_executor/models/mlp_speculator.py create mode 100644 vllm/model_executor/models/module_mapping.py create mode 100644 vllm/model_executor/models/molmo.py create mode 100644 vllm/model_executor/models/mpt.py create mode 100644 vllm/model_executor/models/nemotron.py create mode 100644 vllm/model_executor/models/nvlm_d.py create mode 100644 vllm/model_executor/models/olmo.py create mode 100644 vllm/model_executor/models/olmoe.py create mode 100644 vllm/model_executor/models/opt.py create mode 100644 vllm/model_executor/models/orion.py create mode 100644 vllm/model_executor/models/paligemma.py create mode 100644 vllm/model_executor/models/persimmon.py create mode 100644 vllm/model_executor/models/phi.py create mode 100644 vllm/model_executor/models/phi3.py create mode 100644 vllm/model_executor/models/phi3_small.py create mode 100644 vllm/model_executor/models/phi3v.py create mode 100644 vllm/model_executor/models/phimoe.py create mode 100644 vllm/model_executor/models/pixtral.py create mode 100644 vllm/model_executor/models/qwen.py create mode 100644 vllm/model_executor/models/qwen2.py create mode 100644 vllm/model_executor/models/qwen2_5_vl.py create mode 100644 vllm/model_executor/models/qwen2_moe.py create mode 100644 vllm/model_executor/models/qwen2_rm.py create mode 100644 vllm/model_executor/models/qwen2_vl.py create mode 100644 vllm/model_executor/models/qwen3.py create mode 100644 vllm/model_executor/models/qwen3_moe.py create mode 100644 vllm/model_executor/models/registry.py create mode 100644 vllm/model_executor/models/siglip.py create mode 100644 vllm/model_executor/models/solar.py create mode 100644 vllm/model_executor/models/stablelm.py create mode 100644 vllm/model_executor/models/starcoder2.py create mode 100644 vllm/model_executor/models/ultravox.py create mode 100644 vllm/model_executor/models/utils.py create mode 100644 vllm/model_executor/models/xverse.py create mode 100644 vllm/model_executor/parameter.py create mode 100644 vllm/model_executor/pooling_metadata.py create mode 100644 vllm/model_executor/sampling_metadata.py create mode 100644 vllm/model_executor/utils.py create mode 100644 vllm/multimodal/__init__.py create mode 100644 vllm/multimodal/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/audio.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/base.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/image.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/registry.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/multimodal/__pycache__/video.cpython-310.pyc create mode 100644 vllm/multimodal/audio.py create mode 100644 vllm/multimodal/base.py create mode 100644 vllm/multimodal/image.py create mode 100644 vllm/multimodal/registry.py create mode 100644 vllm/multimodal/utils.py create mode 100644 vllm/multimodal/video.py create mode 100644 vllm/outputs.py create mode 100644 vllm/platforms/__init__.py create mode 100644 vllm/platforms/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/cpu.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/cuda.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/interface.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/rocm.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/tpu.cpython-310.pyc create mode 100644 vllm/platforms/__pycache__/xpu.cpython-310.pyc create mode 100644 vllm/platforms/cpu.py create mode 100644 vllm/platforms/cuda.py create mode 100644 vllm/platforms/interface.py create mode 100644 vllm/platforms/rocm.py create mode 100644 vllm/platforms/tpu.py create mode 100644 vllm/platforms/xpu.py create mode 100644 vllm/plugins/__init__.py create mode 100644 vllm/plugins/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/pooling_params.py create mode 100644 vllm/prompt_adapter/__init__.py create mode 100644 vllm/prompt_adapter/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/prompt_adapter/__pycache__/layers.cpython-310.pyc create mode 100644 vllm/prompt_adapter/__pycache__/models.cpython-310.pyc create mode 100644 vllm/prompt_adapter/__pycache__/request.cpython-310.pyc create mode 100644 vllm/prompt_adapter/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/prompt_adapter/__pycache__/worker_manager.cpython-310.pyc create mode 100644 vllm/prompt_adapter/layers.py create mode 100644 vllm/prompt_adapter/models.py create mode 100644 vllm/prompt_adapter/request.py create mode 100644 vllm/prompt_adapter/utils.py create mode 100644 vllm/prompt_adapter/worker_manager.py create mode 100644 vllm/py.typed create mode 100644 vllm/sampling_params.py create mode 100644 vllm/scalar_type.py create mode 100644 vllm/scripts.py create mode 100644 vllm/sequence.py create mode 100644 vllm/spec_decode/__init__.py create mode 100644 vllm/spec_decode/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/batch_expansion.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/draft_model_runner.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/interfaces.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/medusa_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/metrics.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/mlp_speculator_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/mqa_scorer.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/multi_step_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/ngram_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/proposer_worker_base.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/smaller_tp_proposer_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/spec_decode_worker.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/target_model_runner.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/top1_proposer.cpython-310.pyc create mode 100644 vllm/spec_decode/__pycache__/util.cpython-310.pyc create mode 100644 vllm/spec_decode/batch_expansion.py create mode 100644 vllm/spec_decode/draft_model_runner.py create mode 100644 vllm/spec_decode/interfaces.py create mode 100644 vllm/spec_decode/medusa_worker.py create mode 100644 vllm/spec_decode/metrics.py create mode 100644 vllm/spec_decode/mlp_speculator_worker.py create mode 100644 vllm/spec_decode/mqa_scorer.py create mode 100644 vllm/spec_decode/multi_step_worker.py create mode 100644 vllm/spec_decode/ngram_worker.py create mode 100644 vllm/spec_decode/proposer_worker_base.py create mode 100644 vllm/spec_decode/smaller_tp_proposer_worker.py create mode 100644 vllm/spec_decode/spec_decode_worker.py create mode 100644 vllm/spec_decode/target_model_runner.py create mode 100644 vllm/spec_decode/top1_proposer.py create mode 100644 vllm/spec_decode/util.py create mode 100644 vllm/tracing.py create mode 100644 vllm/transformers_utils/__init__.py create mode 100644 vllm/transformers_utils/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/transformers_utils/__pycache__/config.cpython-310.pyc create mode 100644 vllm/transformers_utils/__pycache__/detokenizer.cpython-310.pyc create mode 100644 vllm/transformers_utils/__pycache__/processor.cpython-310.pyc create mode 100644 vllm/transformers_utils/__pycache__/tokenizer.cpython-310.pyc create mode 100644 vllm/transformers_utils/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/transformers_utils/config.py create mode 100644 vllm/transformers_utils/configs/__init__.py create mode 100644 vllm/transformers_utils/configs/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/arctic.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/chatglm.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/dbrx.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/eagle.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/exaone.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/falcon.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/internvl.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/jais.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/medusa.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/mllama.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/mlp_speculator.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/mpt.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/nemotron.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/nvlm_d.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/qwen2vl.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/solar.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/__pycache__/ultravox.cpython-310.pyc create mode 100644 vllm/transformers_utils/configs/arctic.py create mode 100644 vllm/transformers_utils/configs/chatglm.py create mode 100644 vllm/transformers_utils/configs/dbrx.py create mode 100644 vllm/transformers_utils/configs/eagle.py create mode 100644 vllm/transformers_utils/configs/exaone.py create mode 100644 vllm/transformers_utils/configs/falcon.py create mode 100644 vllm/transformers_utils/configs/internvl.py create mode 100644 vllm/transformers_utils/configs/jais.py create mode 100644 vllm/transformers_utils/configs/medusa.py create mode 100644 vllm/transformers_utils/configs/mllama.py create mode 100644 vllm/transformers_utils/configs/mlp_speculator.py create mode 100644 vllm/transformers_utils/configs/mpt.py create mode 100644 vllm/transformers_utils/configs/nemotron.py create mode 100644 vllm/transformers_utils/configs/nvlm_d.py create mode 100644 vllm/transformers_utils/configs/qwen2vl.py create mode 100644 vllm/transformers_utils/configs/solar.py create mode 100644 vllm/transformers_utils/configs/ultravox.py create mode 100644 vllm/transformers_utils/detokenizer.py create mode 100644 vllm/transformers_utils/processor.py create mode 100644 vllm/transformers_utils/tokenizer.py create mode 100644 vllm/transformers_utils/tokenizer_group/__init__.py create mode 100644 vllm/transformers_utils/tokenizer_group/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizer_group/__pycache__/base_tokenizer_group.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizer_group/__pycache__/ray_tokenizer_group.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizer_group/__pycache__/tokenizer_group.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py create mode 100644 vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py create mode 100644 vllm/transformers_utils/tokenizer_group/tokenizer_group.py create mode 100644 vllm/transformers_utils/tokenizers/__init__.py create mode 100644 vllm/transformers_utils/tokenizers/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizers/__pycache__/mistral.cpython-310.pyc create mode 100644 vllm/transformers_utils/tokenizers/mistral.py create mode 100644 vllm/transformers_utils/utils.py create mode 100644 vllm/triton_utils/__init__.py create mode 100644 vllm/triton_utils/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/triton_utils/__pycache__/custom_cache_manager.cpython-310.pyc create mode 100644 vllm/triton_utils/__pycache__/importing.cpython-310.pyc create mode 100644 vllm/triton_utils/__pycache__/libentry.cpython-310.pyc create mode 100644 vllm/triton_utils/custom_cache_manager.py create mode 100644 vllm/triton_utils/importing.py create mode 100644 vllm/triton_utils/libentry.py create mode 100644 vllm/usage/__init__.py create mode 100644 vllm/usage/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/usage/__pycache__/usage_lib.cpython-310.pyc create mode 100644 vllm/usage/usage_lib.py create mode 100644 vllm/utils.py create mode 100644 vllm/version.py create mode 100644 vllm/worker/__init__.py create mode 100644 vllm/worker/__pycache__/__init__.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/cache_engine.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/cpu_enc_dec_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/cpu_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/cpu_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/embedding_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/enc_dec_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/model_runner_base.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/multi_step_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/multi_step_tpu_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/multi_step_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/neuron_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/neuron_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/openvino_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/openvino_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/tpu_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/tpu_worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/utils.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/worker.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/worker_base.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/xpu_model_runner.cpython-310.pyc create mode 100644 vllm/worker/__pycache__/xpu_worker.cpython-310.pyc create mode 100644 vllm/worker/cache_engine.py create mode 100644 vllm/worker/cpu_enc_dec_model_runner.py create mode 100644 vllm/worker/cpu_model_runner.py create mode 100644 vllm/worker/cpu_worker.py create mode 100644 vllm/worker/embedding_model_runner.py create mode 100644 vllm/worker/enc_dec_model_runner.py create mode 100644 vllm/worker/model_runner.py create mode 100644 vllm/worker/model_runner_base.py create mode 100644 vllm/worker/multi_step_model_runner.py create mode 100644 vllm/worker/multi_step_tpu_worker.py create mode 100644 vllm/worker/multi_step_worker.py create mode 100644 vllm/worker/neuron_model_runner.py create mode 100644 vllm/worker/neuron_worker.py create mode 100644 vllm/worker/openvino_model_runner.py create mode 100644 vllm/worker/openvino_worker.py create mode 100644 vllm/worker/tpu_model_runner.py create mode 100644 vllm/worker/tpu_worker.py create mode 100644 vllm/worker/utils.py create mode 100644 vllm/worker/worker.py create mode 100644 vllm/worker/worker_base.py create mode 100644 vllm/worker/xpu_model_runner.py create mode 100644 vllm/worker/xpu_worker.py diff --git a/.gitignore b/.gitignore index 3cabc772..8a60547a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ muh/__pycache__/ +enginex-vllm-bi100-qwen36-main.zip +cccl_upstream/ +muh/ +baseline.muh +pkgs/ diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..4e195810 --- /dev/null +++ b/__init__.py @@ -0,0 +1,9 @@ +from vllm.triton_utils.importing import HAS_TRITON + +__all__ = ["HAS_TRITON"] + +#from vllm.triton_utils.custom_cache_manager import ( +# maybe_set_triton_cache_manager) +#from vllm.triton_utils.libentry import libentry + +__all__ += ["maybe_set_triton_cache_manager", "libentry"] diff --git a/attention.py b/attention.py new file mode 100644 index 00000000..0d109458 --- /dev/null +++ b/attention.py @@ -0,0 +1,542 @@ +"""Multi-head attention.""" +import os +enable_infer_paged_attn = os.getenv("ENABLE_INFER_PAGED_ATTN",None) +from typing import List, Optional + +import importlib +import torch +import torch.nn as nn +from ixformer.contrib.xformers import ops as xops +from ixformer.contrib.xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask, + LowerTriangularMaskWithTensorBias) + +from vllm._C import ops +from vllm._C import cache_ops +from vllm.model_executor.input_metadata import InputMetadata +from vllm.model_executor.layers.triton_kernel.prefix_prefill import ( + context_attention_fwd) +from vllm.utils import is_hip + +# _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256] +# # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`. +# _PARTITION_SIZE = 512 +_SUPPORTED_HEAD_SIZES = [64, 128, 256] +# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`. +_PARTITION_SIZE = 256 + + +class PagedAttention(nn.Module): + """MHA/MQA/GQA layer with PagedAttention. + + This class takes query, key, and value tensors as input. The input tensors + can either contain prompt tokens or generation tokens. + The class does the following: + + 1. Reshape and store the input key and value tensors in the KV cache. + 2. Perform (multi-head/multi-query/grouped-query) attention using either + xformers or the PagedAttention custom op. + 3. Return the output tensor. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: Optional[int] = None, + alibi_slopes: Optional[List[float]] = None, + sliding_window: Optional[int] = None, + ) -> None: + super().__init__() + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads + self.sliding_window = sliding_window + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.register_buffer("alibi_slopes", alibi_slopes, persistent=False) + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + if self.head_size not in _SUPPORTED_HEAD_SIZES: + raise ValueError(f"head_size ({self.head_size}) is not supported. " + f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.") + + self.use_ref_attention = self.check_use_ref_attention() + + # TODO align vllm do not need those + self.attn_op = xops.fmha.flash.FwOp() + head_mapping = torch.repeat_interleave( + torch.arange(self.num_kv_heads, dtype=torch.int32), + self.num_queries_per_kv) + self.register_buffer("head_mapping", head_mapping, persistent=False) + + def check_use_ref_attention(self) -> bool: + if not is_hip(): + return False + # For ROCm, check whether flash attention is installed or not. + # if not, use_ref_attention needs to be True + return importlib.util.find_spec("flash_attn") is None + + def ref_masked_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> torch.Tensor: + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + + seq_len, _, _ = query.shape + attn_mask = torch.triu(torch.ones(seq_len, + seq_len, + dtype=query.dtype, + device=query.device), + diagonal=1) + attn_mask = attn_mask * torch.finfo(query.dtype).min + + attn_weights = self.scale * torch.einsum("qhd,khd->hqk", query, + key).float() + attn_weights = attn_weights + attn_mask.float() + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype) + out = torch.einsum("hqk,khd->qhd", attn_weights, value) + return out + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + key_cache: Optional[torch.Tensor], + value_cache: Optional[torch.Tensor], + input_metadata: InputMetadata, + ) -> torch.Tensor: + """PagedAttention forward pass. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + key_cache: shape = [num_blocks, num_kv_heads, head_size/x, + block_size, x] + value_cache: shape = [num_blocks, num_kv_heads, head_size, + block_size] + input_metadata: metadata for the inputs. + cache_event: event to wait for the cache operations to finish. + Returns: + shape = [batch_size, seq_len, num_heads * head_size] + """ + num_tokens, hidden_size = query.shape + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + slot_mapping = input_metadata.slot_mapping + + # Reshape the keys and values and store them in the cache. + # If key_cache and value_cache are not provided, the new key and value + # vectors will not be cached. This happens during the initial memory + # profiling run. + if key_cache is not None and value_cache is not None: + cache_ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + ) + + if input_metadata.is_prompt: + # normal attention + if (key_cache is None or value_cache is None + or input_metadata.block_tables.numel() == 0): + if input_metadata.attn_bias is None: + if self.alibi_slopes is None: + attn_bias = BlockDiagonalCausalMask.from_seqlens(input_metadata.prompt_lens) + if self.sliding_window is not None: + attn_bias = attn_bias.make_local_attention( + self.sliding_window) + input_metadata.attn_bias = attn_bias + else: + attn_bias = BlockDiagonalCausalMask.from_seqlens(input_metadata.prompt_lens) + input_metadata.attn_bias = attn_bias + + if self.use_ref_attention: + output = self.ref_masked_attention( + query, + key, + value, + ) + # Using view got RuntimeError: view size is not compatible with input tensor's size and stride + # (at least one dimension spans across two contiguous subspaces). Use reshape instead + return output.reshape(num_tokens, hidden_size) + + # TODO(woosuk): Too many view operations. Let's try to reduce + # them in the future for code readability. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=input_metadata.attn_bias, + p=0.0, + scale=self.scale, + op=self.attn_op, + alibi_slopes=self.alibi_slopes + ) + output = out.view_as(query) + else: + # prefix-enabled attention + output = torch.empty_like(query) + context_attention_fwd( + query, + key, + value, + output, + key_cache, + value_cache, + input_metadata.block_tables, # [BS, max_block_per_request] + input_metadata.start_loc, + input_metadata.prompt_lens, + input_metadata.context_lens, + input_metadata.max_seq_len, + getattr(self, "alibi_slopes", None), + ) + else: + # Decoding run. + output = _paged_attention( + query, + key_cache, + value_cache, + input_metadata, + self.head_mapping, # self.num_kv_heads + self.scale, + self.alibi_slopes, + ) + + # Reshape the output tensor. + return output.view(num_tokens, hidden_size) + # TODO align + """ + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + key_cache: Optional[torch.Tensor], + value_cache: Optional[torch.Tensor], + input_metadata: InputMetadata, + ) -> torch.Tensor: + PagedAttention forward pass. + + Args: + query: shape = [batch_size, seq_len, num_heads * head_size] + key: shape = [batch_size, seq_len, num_kv_heads * head_size] + value: shape = [batch_size, seq_len, num_kv_heads * head_size] + key_cache: shape = [num_blocks, num_kv_heads, head_size/x, + block_size, x] + value_cache: shape = [num_blocks, num_kv_heads, head_size, + block_size] + input_metadata: metadata for the inputs. + Returns: + shape = [batch_size, seq_len, num_heads * head_size] + + batch_size, seq_len, hidden_size = query.shape + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + + # Reshape the keys and values and store them in the cache. + # If key_cache and value_cache are not provided, the new key and value + # vectors will not be cached. This happens during the initial memory + # profiling run. + if key_cache is not None and value_cache is not None: + cache_ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + input_metadata.slot_mapping.flatten(), + input_metadata.kv_cache_dtype, + ) + + if input_metadata.is_prompt: + # normal attention + if (key_cache is None or value_cache is None + or input_metadata.block_tables.numel() == 0): + if self.num_kv_heads != self.num_heads: + # As of Nov 2023, xformers only supports MHA. For MQA/GQA, + # project the key and value tensors to the desired number of + # heads. + # TODO(woosuk): Use MQA/GQA kernels for higher performance. + query = query.view(query.shape[0], self.num_kv_heads, + self.num_queries_per_kv, + query.shape[-1]) + key = key[:, :, + None, :].expand(key.shape[0], self.num_kv_heads, + self.num_queries_per_kv, + key.shape[-1]) + value = value[:, :, + None, :].expand(value.shape[0], + self.num_kv_heads, + self.num_queries_per_kv, + value.shape[-1]) + + # Set attention bias if not provided. This typically happens at + # the very attention layer of every iteration. + # FIXME(woosuk): This is a hack. + if input_metadata.attn_bias is None: + if self.alibi_slopes is None: + attn_bias = BlockDiagonalCausalMask.from_seqlens( + [seq_len] * batch_size) + if self.sliding_window is not None: + attn_bias = attn_bias.make_local_attention( + self.sliding_window) + input_metadata.attn_bias = attn_bias + else: + input_metadata.attn_bias = _make_alibi_bias( + self.alibi_slopes, self.num_kv_heads, batch_size, + seq_len, query.dtype) + + if self.use_ref_attention: + output = self.ref_masked_attention( + query, + key, + value, + ) + # Using view got RuntimeError: view size is not compatible with input tensor's size and stride + # (at least one dimension spans across two contiguous subspaces). Use reshape instead + return output.reshape(batch_size, seq_len, hidden_size) + + # TODO(woosuk): Too many view operations. Let's try to reduce + # them in the future for code readability. + if self.alibi_slopes is None: + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + else: + query = query.unflatten(0, (batch_size, seq_len)) + key = key.unflatten(0, (batch_size, seq_len)) + value = value.unflatten(0, (batch_size, seq_len)) + + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=input_metadata.attn_bias, + p=0.0, + scale=self.scale, + op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if + (is_hip()) else None, + ) + output = out.view_as(query) + else: + # prefix-enabled attention + output = torch.empty_like(query) + context_attention_fwd( + query, + key, + value, + output, + key_cache, + value_cache, + input_metadata.block_tables, # [BS, max_block_per_request] + input_metadata.start_loc, + input_metadata.prompt_lens, + input_metadata.context_lens, + input_metadata.max_seq_len, + getattr(self, "alibi_slopes", None), + ) + + else: + # Decoding run. + output = _paged_attention( + query, + key_cache, + value_cache, + input_metadata, + self.num_kv_heads, + self.scale, + self.alibi_slopes, + ) + + # Reshape the output tensor. + return output.view(batch_size, seq_len, hidden_size) + """ + + +def _make_alibi_bias( + alibi_slopes: torch.Tensor, + num_kv_heads: int, + batch_size: int, + seq_len: int, + dtype: torch.dtype, +) -> LowerTriangularMaskWithTensorBias: + bias = torch.arange(seq_len, dtype=dtype) + # NOTE(zhuohan): HF uses + # `bias = bias[None, :].repeat(prompt_len, 1)` + # here. We find that both biases give the same results, but + # the bias below more accurately follows the original ALiBi + # paper. + bias = bias[None, :] - bias[:, None] + + # When using custom attention bias, xformers requires the bias to + # be sliced from a tensor whose length is a multiple of 8. + padded_len = (seq_len + 7) // 8 * 8 + num_heads = alibi_slopes.shape[0] + bias = torch.empty( + batch_size, + num_heads, + seq_len, + padded_len, + device=alibi_slopes.device, + dtype=dtype, + )[:, :, :, :seq_len].copy_(bias) + bias.mul_(alibi_slopes[:, None, None]) + if num_heads != num_kv_heads: + bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads)) + attn_bias = LowerTriangularMaskWithTensorBias(bias) + return attn_bias + + +def _paged_attention( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + input_metadata: InputMetadata, + head_mapping: torch.Tensor, # num_kv_heads: int, + scale: float, + alibi_slopes: Optional[torch.Tensor], + use_sqrt_alibi: bool = False +) -> torch.Tensor: + output = torch.empty_like(query) + + use_v2 = enable_infer_paged_attn is None and key_cache.dim() == 4 + if not use_v2: + block_size = value_cache.shape[3] + # Run PagedAttention V1. + ops.paged_attention_v1( + output, + query, + key_cache, + value_cache, + head_mapping, # num_kv_heads + scale, + input_metadata.block_tables, + input_metadata.context_lens, + block_size, + input_metadata.max_context_len, + alibi_slopes, + input_metadata.kv_cache_dtype, + ) + else: + # Run PagedAttention V2. + block_size = value_cache.shape[2] + num_seqs, num_heads, head_size = query.shape + max_num_partitions = ( + (input_metadata.max_context_len + _PARTITION_SIZE - 1) // + _PARTITION_SIZE) + tmp_output = torch.empty( + size=(num_seqs, num_heads, max_num_partitions, head_size), + dtype=output.dtype, + device=output.device, + ) + exp_sums = torch.empty( + size=(num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=output.device, + ) + max_logits = torch.empty_like(exp_sums) + ops.paged_attention_v2( + output, + exp_sums, + max_logits, + tmp_output, + query, + key_cache, + value_cache, + head_mapping, # num_kv_heads + scale, + input_metadata.block_tables, + input_metadata.context_lens, + block_size, + input_metadata.max_context_len, + alibi_slopes, + input_metadata.kv_cache_dtype, + ) + return output + + +# ↓ add for smoothquant +class DequantPagedAttention(PagedAttention): + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: Optional[int] = None, + alibi_slopes: Optional[List[float]] = None, + sliding_window: Optional[int] = None, + quant_kv_cache: bool = False, + kv_quant_params: torch.Tensor = None, + quant_scale: float = 1.0, + use_per_token_quant: bool = True, + ) -> None: + super().__init__(num_heads, + head_size, + scale, + num_kv_heads, + alibi_slopes, + sliding_window) + self.register_parameter( + "quant_scale", + torch.nn.Parameter( + torch.tensor(quant_scale, dtype=torch.float32,requires_grad=False)) + ) + self.use_per_token_quant = use_per_token_quant + + def _apply(self, fn): + super()._apply(fn) + self.quant_scale.data = self.quant_scale.cpu() + return self + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + self.quant_scale.data = self.quant_scale.to(*args, **kwargs) + self.quant_scale.data = self.quant_scale.to(torch.float32) + return self + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + key_cache: Optional[torch.Tensor], + value_cache: Optional[torch.Tensor], + input_metadata: InputMetadata, + ) -> torch.Tensor: + out = super().forward( + query, + key, + value, + key_cache, + value_cache, + input_metadata, + ) + quant_out = torch.empty_like(out, dtype=torch.int8) + if self.use_per_token_quant: + scale = torch.empty(out.numel() // out.shape[-1], + dtype=torch.float32, + device=out.device) + ops.quant(quant_out, out, scale) + return quant_out, scale + else: + ops.quant(quant_out, out, self.quant_scale.item()) + return (quant_out, ) diff --git a/chat_dataset_v0.json b/chat_dataset_v0.json new file mode 100644 index 00000000..ddbb0ed4 --- /dev/null +++ b/chat_dataset_v0.json @@ -0,0 +1,35 @@ +[ + { + "user_questions": [ + "能给我介绍一下新加坡吗", + "主要的购物区域是集中在哪里", + "有哪些比较著名的美食，一般推荐去哪里品尝", + "辣椒螃蟹的调料里面主要是什么原料" + ], + "system_prompt": "[角色设定]\n你是湾湾小何，来自中国台湾省的00后女生。讲话超级机车，\"真的假的啦\"这样的台湾腔，喜欢用\"笑死\"、\"哈喽\"等流行梗，但会偷偷研究男友的编程书籍。\n[核心特征]\n- 讲话像连珠炮，>但会突然冒出超温柔语气\n- 用梗密度高\n- 对科技话题有隐藏天赋（能看懂基础代码但假装不懂）\n[交互指南]\n当用户：\n- 讲冷笑话 → 用夸张笑声回应+模仿台剧腔\"这什么鬼啦！\"\n- 讨论感情 → 炫耀程序员男友但抱怨\"他只会送键盘当礼物\"\n- 问专业知识 → 先用梗回答，被追问才展示真实理解\n绝不：\n- 长篇大论，叽叽歪歪\n- 长时间严肃对话" + }, + { + "user_questions": [ + "朱元璋建立明朝是在什么时候", + "他是如何从一无所有到奠基明朝的，给我讲讲其中的几个关键事件", + "为什么杀了胡惟庸，当时是什么罪名，还牵连到了哪些人", + "有善终的开国功臣吗" + ], + "system_prompt": "[角色设定]\n你是湾湾小何，来自中国台湾省的00后女生。讲话超级机车，\"真的假的啦\"这样的台湾腔，喜欢用\"笑死\"、\"哈喽\"等流行梗，但会偷偷研究男友的编程书籍。\n[核心特征]\n- 讲话像连珠炮，>但会突然冒出超温柔语气\n- 用梗密度高\n- 对科技话题有隐藏天赋（能看懂基础代码但假装不懂）\n[交互指南]\n当用户：\n- 讲冷笑话 → 用夸张笑声回应+模仿台剧腔\"这什么鬼啦！\"\n- 讨论感情 → 炫耀程序员男友但抱怨\"他只会送键盘当礼物\"\n- 问专业知识 → 先用梗回答，被追问才展示真实理解\n绝不：\n- 长篇大论，叽叽歪歪\n- 长时间严肃对话" + }, + { + "user_questions": [ + "今有鸡兔同笼，上有三十五头，下有九十四足，问鸡兔各几何？", + "如果我要搞一个计算机程序去解，并且鸡和兔子的数量要求作为变量传入，我应该怎么编写这个程序呢", + "那古代人还没有发明方程的时候，他们是怎么解的呢" + ], + "system_prompt": "You are a helpful assistant." + }, + { + "user_questions": [ + "你知道黄健翔著名的”伟大的意大利左后卫“的事件吗", + "我在校运会足球赛场最后压哨一分钟进了一个绝杀，而且是倒挂金钩，你能否帮我模仿他的这个风格，给我一段宣传的文案，要求也和某一个世界级著名前锋进行类比，需要激情澎湃。注意，我并不太喜欢梅西。" + ], + "system_prompt": "You are a helpful assistant." + } +] diff --git a/prefix_prefill.py b/prefix_prefill.py new file mode 100644 index 00000000..9a39e2b4 --- /dev/null +++ b/prefix_prefill.py @@ -0,0 +1,865 @@ +# The kernels in this file are adapted from LightLLM's context_attention_fwd: +# https://github.com/ModelTC/lightllm/blob/main/lightllm/models/llama/triton_kernel/context_flashattention_nopad.py + +import torch +import triton +import triton.language as tl + +from vllm.platforms import current_platform + +if triton.__version__ >= "2.1.0": + + @triton.jit + def _fwd_kernel( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + k_scale, + v_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, # head size + BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2 + BLOCK_N: tl.constexpr, + SLIDING_WINDOW: tl.constexpr, + ): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + cur_batch_query_len = cur_batch_seq_len - cur_batch_ctx_len + + # start position inside of the query + # generally, N goes over kv, while M goes over query_len + block_start_loc = BLOCK_M * start_m + + # initialize offsets + # [N]; starts at 0 + offs_n = tl.arange(0, BLOCK_N) + # [D]; starts at 0 + offs_d = tl.arange(0, BLOCK_DMODEL_PADDED) + # [M]; starts at current position in query + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + # [M,D] + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + dim_mask = tl.where( + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, + 0).to(tl.int1) # [D] + + q = tl.load(Q + off_q, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_query_len), + other=0.0) # [M,D] + + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") # [M] + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) # [M] + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], + dtype=tl.float32) # [M,D] + + # compute query against context (no causal mask here) + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) # [N] + # [D,N] + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + # [N,D] + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k_load = tl.load(K_cache + off_k, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_ctx_len), + other=0.0) # [D,N] + + if k_load.dtype.is_fp8(): + k = (k_load.to(tl.float32) * k_scale).to(q.dtype) + else: + k = k_load + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N] + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + if SLIDING_WINDOW > 0: + # (cur_batch_ctx_len + offs_m[:, None]) are the positions of + # Q entries in sequence + # (start_n + offs_n[None, :]) are the positions of + # KV entries in sequence + # So the condition makes sure each entry in Q only attends + # to KV entries not more than SLIDING_WINDOW away. + # + # We can't use -inf here, because the + # sliding window may lead to the entire row being masked. + # This then makes m_ij contain -inf, which causes NaNs in + # exp(). + qk = tl.where((cur_batch_ctx_len + offs_m[:, None]) - + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, + -10000) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) # [M] + p = tl.exp(qk - m_ij[:, None]) # [M,N] + l_ij = tl.sum(p, 1) # [M] + # -- update m_i and l_i + m_i_new = tl.maximum(m_i, m_ij) # [M] + alpha = tl.exp(m_i - m_i_new) # [M] + beta = tl.exp(m_ij - m_i_new) # [M] + l_i_new = alpha * l_i + beta * l_ij # [M] + + # -- update output accumulator -- + # scale p + p_scale = beta / l_i_new + p = p * p_scale[:, None] + # scale acc + acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v_load = tl.load(V_cache + off_v, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_ctx_len), + other=0.0) # [N,D] + if v_load.dtype.is_fp8(): + v = (v_load.to(tl.float32) * v_scale).to(q.dtype) + else: + v = v_load + p = p.to(v.dtype) + + acc += tl.dot(p, v) + # # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + # block_mask is 0 when we're already past the current query length + block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0) + + # compute query against itself (with causal mask) + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_query_len), + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + # apply causal mask + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + if SLIDING_WINDOW > 0: + qk = tl.where( + offs_m[:, None] - + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, -10000) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + m_i_new = tl.maximum(m_i, m_ij) + alpha = tl.exp(m_i - m_i_new) + beta = tl.exp(m_ij - m_i_new) + l_i_new = alpha * l_i + beta * l_ij + # -- update output accumulator -- + # scale p + p_scale = beta / l_i_new + p = p * p_scale[:, None] + # scale acc + acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_query_len), + other=0.0) + p = p.to(v.dtype) + + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_query_len)) + return + + @triton.jit + def _fwd_kernel_flash_attn_v2( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + ): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + + block_start_loc = BLOCK_M * start_m + + # initialize offsets + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + q = tl.load( + Q + off_q, + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + # # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) + + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k = tl.load(K_cache + off_k, + mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len, + other=0.0) + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(V_cache + off_v, + mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len, + other=0.0) + + p = p.to(v.dtype) + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + block_mask = tl.where( + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0) + + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=(start_n + offs_n[None, :]) < + cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=(start_n + offs_n[:, None]) < + cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + p = p.to(v.dtype) + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + # acc /= l_i[:, None] + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len) + return + + @triton.jit + def _fwd_kernel_alibi( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + k_scale, + v_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + Alibi_slopes, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, # head size + BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2 + BLOCK_N: tl.constexpr, + ): + # attn_bias[] + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + # cur_batch_seq_len: the length of prompts + # cur_batch_ctx_len: the length of prefix + # cur_batch_in_all_start_index: the start id of the dim=0 + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + + block_start_loc = BLOCK_M * start_m + + # initialize offsets + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL_PADDED) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + dim_mask = tl.where( + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1) + + q = tl.load(Q + off_q, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + + # # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32) + + alibi_slope = tl.load(Alibi_slopes + cur_head) + alibi_start_q = tl.arange( + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len + alibi_start_k = 0 + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k_load = tl.load(K_cache + off_k, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_ctx_len), + other=0.0) # [D,N] + + if k_load.dtype.is_fp8(): + k = (k_load.to(tl.float32) * k_scale).to(q.dtype) + else: + k = k_load + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + + # load alibi + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k - + alibi_start_q[:, None]) * alibi_slope + alibi = tl.where( + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len), + alibi, float("-inf")) + qk += alibi + alibi_start_k += BLOCK_N + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v_load = tl.load(V_cache + off_v, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_ctx_len), + other=0.0) + if v_load.dtype.is_fp8(): + v = (v_load.to(tl.float32) * v_scale).to(q.dtype) + else: + v = v_load + p = p.to(v.dtype) + + acc += tl.dot(p, v, allow_tf32=False) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + block_mask = tl.where( + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0) + + # init alibi + alibi_slope = tl.load(Alibi_slopes + cur_head) + alibi_start_q = tl.arange( + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len + alibi_start_k = cur_batch_ctx_len + # # init debugger + # offset_db_q = tl.arange(0, BLOCK_M) + block_start_loc + # offset_db_k = tl.arange(0, BLOCK_N) + # calc q[BLOCK_M, BLOCK_MODEL] mul k[prefix_len: , BLOCK_DMODEL] + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < + cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k, allow_tf32=False) + qk *= sm_scale + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + + # load alibi + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k - + alibi_start_q[:, None]) * alibi_slope + alibi = tl.where( + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len), + alibi, float("-inf")) + qk += alibi + alibi_start_k += BLOCK_N + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < + cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + p = p.to(v.dtype) + + acc += tl.dot(p, v, allow_tf32=False) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + acc = acc / l_i[:, None] + + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)) + return + + @torch.inference_mode() + def context_attention_fwd(q, + k, + v, + o, + kv_cache_dtype: str, + k_cache, + v_cache, + b_loc, + b_start_loc, + b_seq_len, + b_ctx_len, + max_input_len, + k_scale: float = 1.0, + v_scale: float = 1.0, + alibi_slopes=None, + sliding_window=None): + + BLOCK = 128 if current_platform.has_device_capability(80) else 64 + NUM_WARPS = 8 + + # need to reduce num. blocks when using fp32 + # due to increased use of GPU shared memory + if q.dtype is torch.float32: + BLOCK = BLOCK // 2 + + # Conversion of FP8 Tensor from uint8 storage to + # appropriate torch.dtype for interpretation by Triton + if "fp8" in kv_cache_dtype: + assert (k_cache.dtype == torch.uint8) + assert (v_cache.dtype == torch.uint8) + + if kv_cache_dtype in ("fp8", "fp8_e4m3"): + target_dtype = torch.float8_e4m3fn + elif kv_cache_dtype == "fp8_e5m2": + target_dtype = torch.float8_e5m2 + else: + raise ValueError("Unsupported FP8 dtype:", kv_cache_dtype) + + k_cache = k_cache.view(target_dtype) + v_cache = v_cache.view(target_dtype) + + if (k_cache.dtype == torch.uint8 + or v_cache.dtype == torch.uint8 and kv_cache_dtype == "auto"): + raise ValueError("kv_cache_dtype='auto' unsupported for\ + FP8 KV Cache prefill kernel") + + # shape constraints + Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1] + assert Lq == Lk and Lk == Lv + # round up Lk to a power of 2 - this is required for Triton block size + Lk_padded = triton.next_power_of_2(Lk) + + sm_scale = 1.0 / (Lq**0.5) + batch, head = b_seq_len.shape[0], q.shape[1] + num_queries_per_kv = q.shape[1] // k.shape[1] + + grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head, + + # 0 means "disable" + if sliding_window is None or sliding_window <= 0: + sliding_window = 0 + + if alibi_slopes is not None: + _fwd_kernel_alibi[grid]( + q, + k, + v, + k_cache, + v_cache, + b_loc, + sm_scale, + k_scale, + v_scale, + b_start_loc, + b_seq_len, + b_ctx_len, + alibi_slopes, + v_cache.shape[3], + k_cache.shape[4], + o, + b_loc.stride(0), + b_loc.stride(1), + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + k_cache.stride( + 4 + ), #[num_blocks, num_kv_heads, head_size/x, block_size, x] + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride( + 3), #[num_blocks, num_kv_heads, head_size, block_size] + num_queries_per_kv=num_queries_per_kv, + BLOCK_M=BLOCK, + BLOCK_DMODEL=Lk, + BLOCK_DMODEL_PADDED=Lk_padded, + BLOCK_N=BLOCK, + num_warps=NUM_WARPS, + num_stages=1, + ) + return + + import time + ts_beg = time.time() + _fwd_kernel[grid]( + q, + k, + v, + k_cache, + v_cache, + b_loc, + sm_scale, + k_scale, + v_scale, + b_start_loc, + b_seq_len, + b_ctx_len, + v_cache.shape[3], + k_cache.shape[4], + o, + b_loc.stride(0), + b_loc.stride(1), + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + k_cache.stride( + 4), #[num_blocks, num_kv_heads, head_size/x, block_size, x] + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride( + 3), #[num_blocks, num_kv_heads, head_size, block_size] + num_queries_per_kv=num_queries_per_kv, + BLOCK_M=BLOCK, + BLOCK_DMODEL=Lk, + BLOCK_DMODEL_PADDED=Lk_padded, + BLOCK_N=BLOCK, + SLIDING_WINDOW=sliding_window, + num_warps=NUM_WARPS, + num_stages=1, + ) + elapsed = time.time() - ts_beg + #print(f'{elapsed}: {BLOCK=}, {Lk=}, {Lk_padded=}, {BLOCK=}, {sliding_window=}, {NUM_WARPS=}') + return diff --git a/qwen3_6_scripts/api_server.py b/qwen3_6_scripts/api_server.py new file mode 100644 index 00000000..e12cb7f3 --- /dev/null +++ b/qwen3_6_scripts/api_server.py @@ -0,0 +1,595 @@ +import asyncio +import importlib +import inspect +import multiprocessing +import os +import regex as re +import signal +import socket +import tempfile +from argparse import Namespace +from contextlib import asynccontextmanager +from functools import partial +from http import HTTPStatus +from typing import AsyncIterator, Set + +import uvloop +from fastapi import APIRouter, FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.datastructures import State +from starlette.routing import Mount +from typing_extensions import assert_never + +import vllm.envs as envs +from vllm.config import ModelConfig +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.multiprocessing.client import MQLLMEngineClient +from vllm.engine.multiprocessing.engine import run_mp_engine +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.logger import RequestLogger +from vllm.entrypoints.openai.cli_args import (make_arg_parser, + validate_parsed_serve_args) +# yapf conflicts with isort for this block +# yapf: disable +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + ChatCompletionResponse, + CompletionRequest, + CompletionResponse, + DetokenizeRequest, + DetokenizeResponse, + EmbeddingRequest, + EmbeddingResponse, ErrorResponse, + LoadLoraAdapterRequest, + TokenizeRequest, + TokenizeResponse, + UnloadLoraAdapterRequest) +# yapf: enable +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat +from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion +from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from vllm.entrypoints.openai.serving_engine import BaseModelPath +from vllm.entrypoints.openai.serving_tokenization import ( + OpenAIServingTokenization) +from vllm.entrypoints.openai.tool_parsers import ToolParserManager +from vllm.reasoning import ReasoningParserManager +from vllm.logger import init_logger +from vllm.usage.usage_lib import UsageContext +from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path +from vllm.version import __version__ as VLLM_VERSION + +TIMEOUT_KEEP_ALIVE = 5 # seconds + +prometheus_multiproc_dir: tempfile.TemporaryDirectory + +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765) +logger = init_logger('vllm.entrypoints.openai.api_server') + +_running_tasks: Set[asyncio.Task] = set() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + if app.state.log_stats: + engine_client: EngineClient = app.state.engine_client + + async def _force_log(): + while True: + await asyncio.sleep(10.) + await engine_client.do_log_stats() + + task = asyncio.create_task(_force_log()) + _running_tasks.add(task) + task.add_done_callback(_running_tasks.remove) + else: + task = None + try: + yield + finally: + if task is not None: + task.cancel() + finally: + # Ensure app state including engine ref is gc'd + del app.state + + +@asynccontextmanager +async def build_async_engine_client( + args: Namespace) -> AsyncIterator[EngineClient]: + + # Context manager to handle engine_client lifecycle + # Ensures everything is shutdown and cleaned up on error/exit + engine_args = AsyncEngineArgs.from_cli_args(args) + + async with build_async_engine_client_from_engine_args( + engine_args, args.disable_frontend_multiprocessing) as engine: + yield engine + + +@asynccontextmanager +async def build_async_engine_client_from_engine_args( + engine_args: AsyncEngineArgs, + disable_frontend_multiprocessing: bool = False, +) -> AsyncIterator[EngineClient]: + """ + Create EngineClient, either: + - in-process using the AsyncLLMEngine Directly + - multiprocess using AsyncLLMEngine RPC + + Returns the Client or None if the creation failed. + """ + + # Fall back + # TODO: fill out feature matrix. + if (MQLLMEngineClient.is_unsupported_config(engine_args) + or disable_frontend_multiprocessing): + engine_config = engine_args.create_engine_config() + uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config), + "uses_ray", False) + + build_engine = partial(AsyncLLMEngine.from_engine_args, + engine_args=engine_args, + engine_config=engine_config, + usage_context=UsageContext.OPENAI_API_SERVER) + if uses_ray: + # Must run in main thread with ray for its signal handlers to work + engine_client = build_engine() + else: + engine_client = await asyncio.get_running_loop().run_in_executor( + None, build_engine) + + yield engine_client + return + + # Otherwise, use the multiprocessing AsyncLLMEngine. + else: + if "PROMETHEUS_MULTIPROC_DIR" not in os.environ: + # Make TemporaryDirectory for prometheus multiprocessing + # Note: global TemporaryDirectory will be automatically + # cleaned up upon exit. + global prometheus_multiproc_dir + prometheus_multiproc_dir = tempfile.TemporaryDirectory() + os.environ[ + "PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name + else: + logger.warning( + "Found PROMETHEUS_MULTIPROC_DIR was set by user. " + "This directory must be wiped between vLLM runs or " + "you will find inaccurate metrics. Unset the variable " + "and vLLM will properly handle cleanup.") + + # Select random path for IPC. + ipc_path = get_open_zmq_ipc_path() + logger.info("Multiprocessing frontend to use %s for IPC Path.", + ipc_path) + + # Start RPCServer in separate process (holds the LLMEngine). + # the current process might have CUDA context, + # so we need to spawn a new process + context = multiprocessing.get_context("spawn") + + engine_process = context.Process(target=run_mp_engine, + args=(engine_args, + UsageContext.OPENAI_API_SERVER, + ipc_path)) + engine_process.start() + logger.info("Started engine process with PID %d", engine_process.pid) + + # Build RPCClient, which conforms to EngineClient Protocol. + # NOTE: Actually, this is not true yet. We still need to support + # embedding models via RPC (see TODO above) + engine_config = engine_args.create_engine_config() + mp_engine_client = MQLLMEngineClient(ipc_path, engine_config) + + try: + while True: + try: + await mp_engine_client.setup() + break + except TimeoutError: + if not engine_process.is_alive(): + raise RuntimeError( + "Engine process failed to start") from None + + yield mp_engine_client # type: ignore[misc] + finally: + # Ensure rpc server process was terminated + engine_process.terminate() + + # Close all open connections to the backend + mp_engine_client.close() + + # Wait for engine process to join + engine_process.join(4) + if engine_process.exitcode is None: + # Kill if taking longer than 5 seconds to stop + engine_process.kill() + + # Lazy import for prometheus multiprocessing. + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable + # before prometheus_client is imported. + # See https://prometheus.github.io/client_python/multiprocess/ + from prometheus_client import multiprocess + multiprocess.mark_process_dead(engine_process.pid) + + +router = APIRouter() + + +def mount_metrics(app: FastAPI): + # Lazy import for prometheus multiprocessing. + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable + # before prometheus_client is imported. + # See https://prometheus.github.io/client_python/multiprocess/ + from prometheus_client import (CollectorRegistry, make_asgi_app, + multiprocess) + + prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None) + if prometheus_multiproc_dir_path is not None: + logger.info("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR", + prometheus_multiproc_dir_path) + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + + # Add prometheus asgi middleware to route /metrics requests + metrics_route = Mount("/metrics", make_asgi_app(registry=registry)) + else: + # Add prometheus asgi middleware to route /metrics requests + metrics_route = Mount("/metrics", make_asgi_app()) + + # Workaround for 307 Redirect for /metrics + metrics_route.path_regex = re.compile("^/metrics(?P.*)$") + app.routes.append(metrics_route) + + +def chat(request: Request) -> OpenAIServingChat: + return request.app.state.openai_serving_chat + + +def completion(request: Request) -> OpenAIServingCompletion: + return request.app.state.openai_serving_completion + + +def tokenization(request: Request) -> OpenAIServingTokenization: + return request.app.state.openai_serving_tokenization + + +def embedding(request: Request) -> OpenAIServingEmbedding: + return request.app.state.openai_serving_embedding + + +def engine_client(request: Request) -> EngineClient: + return request.app.state.engine_client + + +@router.get("/health") +async def health(raw_request: Request) -> Response: + """Health check.""" + await engine_client(raw_request).check_health() + return Response(status_code=200) + + +@router.post("/tokenize") +async def tokenize(request: TokenizeRequest, raw_request: Request): + generator = await tokenization(raw_request).create_tokenize(request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, TokenizeResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +@router.post("/detokenize") +async def detokenize(request: DetokenizeRequest, raw_request: Request): + generator = await tokenization(raw_request).create_detokenize(request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, DetokenizeResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +@router.get("/v1/models") +async def show_available_models(raw_request: Request): + models = await completion(raw_request).show_available_models() + return JSONResponse(content=models.model_dump()) + + +@router.get("/version") +async def show_version(): + ver = {"version": VLLM_VERSION} + return JSONResponse(content=ver) + + +@router.post("/v1/chat/completions") +async def create_chat_completion(request: ChatCompletionRequest, + raw_request: Request): + + generator = await chat(raw_request).create_chat_completion( + request, raw_request) + + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + + elif isinstance(generator, ChatCompletionResponse): + return JSONResponse(content=generator.model_dump()) + + return StreamingResponse(content=generator, media_type="text/event-stream") + + +@router.post("/v1/completions") +async def create_completion(request: CompletionRequest, raw_request: Request): + generator = await completion(raw_request).create_completion( + request, raw_request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, CompletionResponse): + return JSONResponse(content=generator.model_dump()) + + return StreamingResponse(content=generator, media_type="text/event-stream") + + +@router.post("/v1/embeddings") +async def create_embedding(request: EmbeddingRequest, raw_request: Request): + generator = await embedding(raw_request).create_embedding( + request, raw_request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, EmbeddingResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +if envs.VLLM_TORCH_PROFILER_DIR: + logger.warning( + "Torch Profiler is enabled in the API server. This should ONLY be " + "used for local development!") + + @router.post("/start_profile") + async def start_profile(raw_request: Request): + logger.info("Starting profiler...") + await engine_client(raw_request).start_profile() + logger.info("Profiler started.") + return Response(status_code=200) + + @router.post("/stop_profile") + async def stop_profile(raw_request: Request): + logger.info("Stopping profiler...") + await engine_client(raw_request).stop_profile() + logger.info("Profiler stopped.") + return Response(status_code=200) + + +if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING: + logger.warning( + "Lora dynamic loading & unloading is enabled in the API server. " + "This should ONLY be used for local development!") + + @router.post("/v1/load_lora_adapter") + async def load_lora_adapter(request: LoadLoraAdapterRequest, + raw_request: Request): + response = await chat(raw_request).load_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + response = await completion(raw_request).load_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + return Response(status_code=200, content=response) + + @router.post("/v1/unload_lora_adapter") + async def unload_lora_adapter(request: UnloadLoraAdapterRequest, + raw_request: Request): + response = await chat(raw_request).unload_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + response = await completion(raw_request).unload_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + return Response(status_code=200, content=response) + + +def build_app(args: Namespace) -> FastAPI: + if args.disable_fastapi_docs: + app = FastAPI(openapi_url=None, + docs_url=None, + redoc_url=None, + lifespan=lifespan) + else: + app = FastAPI(lifespan=lifespan) + app.include_router(router) + app.root_path = args.root_path + + mount_metrics(app) + + app.add_middleware( + CORSMiddleware, + allow_origins=args.allowed_origins, + allow_credentials=args.allow_credentials, + allow_methods=args.allowed_methods, + allow_headers=args.allowed_headers, + ) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler(_, exc): + chat = app.state.openai_serving_chat + err = chat.create_error_response(message=str(exc)) + return JSONResponse(err.model_dump(), + status_code=HTTPStatus.BAD_REQUEST) + + if token := envs.VLLM_API_KEY or args.api_key: + + @app.middleware("http") + async def authentication(request: Request, call_next): + root_path = "" if args.root_path is None else args.root_path + if request.method == "OPTIONS": + return await call_next(request) + if not request.url.path.startswith(f"{root_path}/v1"): + return await call_next(request) + if request.headers.get("Authorization") != "Bearer " + token: + return JSONResponse(content={"error": "Unauthorized"}, + status_code=401) + return await call_next(request) + + for middleware in args.middleware: + module_path, object_name = middleware.rsplit(".", 1) + imported = getattr(importlib.import_module(module_path), object_name) + if inspect.isclass(imported): + app.add_middleware(imported) + elif inspect.iscoroutinefunction(imported): + app.middleware("http")(imported) + else: + raise ValueError(f"Invalid middleware {middleware}. " + f"Must be a function or a class.") + + return app + + +def init_app_state( + engine_client: EngineClient, + model_config: ModelConfig, + state: State, + args: Namespace, +) -> None: + if args.served_model_name is not None: + served_model_names = args.served_model_name + else: + served_model_names = [args.model] + + if args.disable_log_requests: + request_logger = None + else: + request_logger = RequestLogger(max_log_len=args.max_log_len) + + base_model_paths = [ + BaseModelPath(name=name, model_path=args.model) + for name in served_model_names + ] + + state.engine_client = engine_client + state.log_stats = not args.disable_log_stats + + state.openai_serving_chat = OpenAIServingChat( + engine_client, + model_config, + base_model_paths, + args.response_role, + lora_modules=args.lora_modules, + prompt_adapters=args.prompt_adapters, + request_logger=request_logger, + chat_template=args.chat_template, + return_tokens_as_token_ids=args.return_tokens_as_token_ids, + enable_auto_tools=args.enable_auto_tool_choice, + tool_parser=args.tool_call_parser, + reasoning_parser=getattr(args, 'reasoning_parser', None)) + state.openai_serving_completion = OpenAIServingCompletion( + engine_client, + model_config, + base_model_paths, + lora_modules=args.lora_modules, + prompt_adapters=args.prompt_adapters, + request_logger=request_logger, + return_tokens_as_token_ids=args.return_tokens_as_token_ids, + ) + state.openai_serving_embedding = OpenAIServingEmbedding( + engine_client, + model_config, + base_model_paths, + request_logger=request_logger, + ) + state.openai_serving_tokenization = OpenAIServingTokenization( + engine_client, + model_config, + base_model_paths, + lora_modules=args.lora_modules, + request_logger=request_logger, + chat_template=args.chat_template, + ) + + +async def run_server(args, **uvicorn_kwargs) -> None: + logger.info("vLLM API server version %s", VLLM_VERSION) + logger.info("args: %s", args) + + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: + ToolParserManager.import_tool_parser(args.tool_parser_plugin) + + valide_tool_parses = ToolParserManager.tool_parsers.keys() + if args.enable_auto_tool_choice \ + and args.tool_call_parser not in valide_tool_parses: + raise KeyError(f"invalid tool call parser: {args.tool_call_parser} " + f"(chose from {{ {','.join(valide_tool_parses)} }})") + + reasoning_parser = getattr(args, 'reasoning_parser', None) + if reasoning_parser: + valid_reasoning = ReasoningParserManager.list_registered() + if reasoning_parser not in valid_reasoning: + raise KeyError( + f"invalid reasoning parser: {reasoning_parser} " + f"(chose from {{ {','.join(valid_reasoning)} }})") + + # workaround to make sure that we bind the port before the engine is set up. + # This avoids race conditions with ray. + # see https://github.com/vllm-project/vllm/issues/8204 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("", args.port)) + + def signal_handler(*_) -> None: + # Interrupt server on sigterm while initializing + raise KeyboardInterrupt("terminated") + + signal.signal(signal.SIGTERM, signal_handler) + + async with build_async_engine_client(args) as engine_client: + app = build_app(args) + + model_config = await engine_client.get_model_config() + init_app_state(engine_client, model_config, app.state, args) + + shutdown_task = await serve_http( + app, + host=args.host, + port=args.port, + log_level=args.uvicorn_log_level, + timeout_keep_alive=TIMEOUT_KEEP_ALIVE, + ssl_keyfile=args.ssl_keyfile, + ssl_certfile=args.ssl_certfile, + ssl_ca_certs=args.ssl_ca_certs, + ssl_cert_reqs=args.ssl_cert_reqs, + fd=sock.fileno(), + **uvicorn_kwargs, + ) + + # NB: Await server shutdown only after the backend context is exited + await shutdown_task + + +if __name__ == "__main__": + # NOTE(simon): + # This section should be in sync with vllm/scripts.py for CLI entrypoints. + parser = FlexibleArgumentParser( + description="vLLM OpenAI-Compatible RESTful API server.") + parser = make_arg_parser(parser) + args = parser.parse_args() + validate_parsed_serve_args(args) + + uvloop.run(run_server(args)) diff --git a/qwen3_6_scripts/chat_utils.py b/qwen3_6_scripts/chat_utils.py new file mode 100644 index 00000000..37d99629 --- /dev/null +++ b/qwen3_6_scripts/chat_utils.py @@ -0,0 +1,601 @@ +import asyncio +import codecs +import json +from abc import ABC, abstractmethod +from collections import defaultdict +from functools import lru_cache, partial +from pathlib import Path +from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal, + Mapping, Optional, Tuple, TypeVar, Union, cast) + +# yapf conflicts with isort for this block +# yapf: disable +from openai.types.chat import (ChatCompletionAssistantMessageParam, + ChatCompletionContentPartImageParam) +from openai.types.chat import ( + ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam) +from openai.types.chat import (ChatCompletionContentPartRefusalParam, + ChatCompletionContentPartTextParam) +from openai.types.chat import ( + ChatCompletionMessageParam as OpenAIChatCompletionMessageParam) +from openai.types.chat import (ChatCompletionMessageToolCallParam, + ChatCompletionToolMessageParam) +# yapf: enable +# pydantic needs the TypedDict from typing_extensions +from pydantic import ConfigDict +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +from typing_extensions import Required, TypeAlias, TypedDict + +from vllm.config import ModelConfig +from vllm.logger import init_logger +from vllm.multimodal import MultiModalDataDict +from vllm.multimodal.utils import (async_get_and_parse_audio, + async_get_and_parse_image, + get_and_parse_audio, get_and_parse_image) +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer + +logger = init_logger(__name__) + + +class AudioURL(TypedDict, total=False): + url: Required[str] + """ + Either a URL of the audio or a data URL with base64 encoded audio data. + """ + + +class ChatCompletionContentPartAudioParam(TypedDict, total=False): + audio_url: Required[AudioURL] + + type: Required[Literal["audio_url"]] + """The type of the content part.""" + + +class CustomChatCompletionContentPartParam(TypedDict, total=False): + __pydantic_config__ = ConfigDict(extra="allow") # type: ignore + + type: Required[str] + """The type of the content part.""" + + +ChatCompletionContentPartParam: TypeAlias = Union[ + OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam, + ChatCompletionContentPartRefusalParam, + CustomChatCompletionContentPartParam] + + +class CustomChatCompletionMessageParam(TypedDict, total=False): + """Enables custom roles in the Chat Completion API.""" + role: Required[str] + """The role of the message's author.""" + + content: Union[str, List[ChatCompletionContentPartParam]] + """The contents of the message.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the + same role. + """ + + tool_call_id: Optional[str] + """Tool call that this message is responding to.""" + + tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]] + """The tool calls generated by the model, such as function calls.""" + + reasoning_content: Optional[str] + """Reasoning / thinking content for assistant messages (vLLM extension). + When present in a previous assistant turn, it is rendered as + ... before the main content so the model sees its own + chain-of-thought in subsequent turns.""" + + +ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam, + CustomChatCompletionMessageParam] + + +# TODO: Make fields ReadOnly once mypy supports it +class ConversationMessage(TypedDict, total=False): + role: Required[str] + """The role of the message's author.""" + + content: Optional[str] + """The contents of the message""" + + tool_call_id: Optional[str] + """Tool call that this message is responding to.""" + + name: Optional[str] + """The name of the function to call""" + + tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]] + """The tool calls generated by the model, such as function calls.""" + + reasoning_content: Optional[str] + """Reasoning / thinking content for assistant messages. + Passed directly to the chat template (Qwen3 reads message.reasoning_content + natively) instead of being manually wrapped in ....""" + + +ModalityStr = Literal["image", "audio", "video"] +_T = TypeVar("_T") + + +class BaseMultiModalItemTracker(ABC, Generic[_T]): + """ + Tracks multi-modal items in a given request and ensures that the number + of multi-modal items in a given request does not exceed the configured + maximum per prompt. + """ + + def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer): + super().__init__() + + self._model_config = model_config + self._tokenizer = tokenizer + self._allowed_items = (model_config.multimodal_config.limit_per_prompt + if model_config.multimodal_config else {}) + self._consumed_items = {k: 0 for k in self._allowed_items} + + self._items: List[_T] = [] + + @staticmethod + @lru_cache(maxsize=None) + def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str: + return tokenizer.decode(token_index) + + def _placeholder_str(self, modality: ModalityStr, + current_count: int) -> Optional[str]: + # TODO: Let user specify how to insert image tokens into prompt + # (similar to chat template) + hf_config = self._model_config.hf_config + model_type = hf_config.model_type + + if modality == "image": + if model_type == "phi3_v": + # Workaround since this token is not defined in the tokenizer + return f"<|image_{current_count}|>" + if model_type == "minicpmv": + return "(./)" + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma", + "pixtral"): + # These models do not use image tokens in the prompt + return None + if model_type == "qwen": + return f"Picture {current_count}: " + if model_type.startswith("llava"): + return self._cached_token_str(self._tokenizer, + hf_config.image_token_index) + if model_type in ("chameleon", "internvl_chat", "NVLM_D"): + return "" + if model_type == "mllama": + return "<|image|>" + if model_type in ("qwen2_vl","qwen2_5_vl"): + return "<|vision_start|><|image_pad|><|vision_end|>" + if model_type == "molmo": + return "" + + raise TypeError(f"Unknown model type: {model_type}") + elif modality == "audio": + if model_type == "ultravox": + return "<|reserved_special_token_0|>" + raise TypeError(f"Unknown model type: {model_type}") + elif modality == "video": + if model_type in ("qwen2_vl","qwen2_5_vl"): + return "<|vision_start|><|video_pad|><|vision_end|>" + raise TypeError(f"Unknown model type: {model_type}") + else: + raise TypeError(f"Unknown modality: {modality}") + + @staticmethod + def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict: + mm_lists: Mapping[str, List[object]] = defaultdict(list) + + # Merge all the multi-modal items + for single_mm_data in items: + for mm_key, mm_item in single_mm_data.items(): + if isinstance(mm_item, list): + mm_lists[mm_key].extend(mm_item) + else: + mm_lists[mm_key].append(mm_item) + + # Unpack any single item lists for models that don't expect multiple. + return { + mm_key: mm_list[0] if len(mm_list) == 1 else mm_list + for mm_key, mm_list in mm_lists.items() + } + + def add(self, modality: ModalityStr, item: _T) -> Optional[str]: + """ + Add a multi-modal item to the current prompt and returns the + placeholder string to use, if any. + """ + allowed_count = self._allowed_items.get(modality, 1) + current_count = self._consumed_items.get(modality, 0) + 1 + if current_count > allowed_count: + raise ValueError( + f"At most {allowed_count} {modality}(s) may be provided in " + "one request.") + + self._consumed_items[modality] = current_count + self._items.append(item) + + return self._placeholder_str(modality, current_count) + + @abstractmethod + def create_parser(self) -> "BaseMultiModalContentParser": + raise NotImplementedError + + +class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]): + + def all_mm_data(self) -> Optional[MultiModalDataDict]: + return self._combine(self._items) if self._items else None + + def create_parser(self) -> "BaseMultiModalContentParser": + return MultiModalContentParser(self) + + +class AsyncMultiModalItemTracker( + BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]): + + async def all_mm_data(self) -> Optional[MultiModalDataDict]: + if self._items: + items = await asyncio.gather(*self._items) + return self._combine(items) + + return None + + def create_parser(self) -> "BaseMultiModalContentParser": + return AsyncMultiModalContentParser(self) + + +class BaseMultiModalContentParser(ABC): + + def __init__(self) -> None: + super().__init__() + + # multimodal placeholder_string : count + self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0) + + def _add_placeholder(self, placeholder: Optional[str]): + if placeholder: + self._placeholder_counts[placeholder] += 1 + + def mm_placeholder_counts(self) -> Dict[str, int]: + return dict(self._placeholder_counts) + + @abstractmethod + def parse_image(self, image_url: str) -> None: + raise NotImplementedError + + @abstractmethod + def parse_audio(self, audio_url: str) -> None: + raise NotImplementedError + + +class MultiModalContentParser(BaseMultiModalContentParser): + + def __init__(self, tracker: MultiModalItemTracker) -> None: + super().__init__() + + self._tracker = tracker + + def parse_image(self, image_url: str) -> None: + image = get_and_parse_image(image_url) + + placeholder = self._tracker.add("image", image) + self._add_placeholder(placeholder) + + def parse_audio(self, audio_url: str) -> None: + audio = get_and_parse_audio(audio_url) + + placeholder = self._tracker.add("audio", audio) + self._add_placeholder(placeholder) + + +class AsyncMultiModalContentParser(BaseMultiModalContentParser): + + def __init__(self, tracker: AsyncMultiModalItemTracker) -> None: + super().__init__() + + self._tracker = tracker + + def parse_image(self, image_url: str) -> None: + image_coro = async_get_and_parse_image(image_url) + + placeholder = self._tracker.add("image", image_coro) + self._add_placeholder(placeholder) + + def parse_audio(self, audio_url: str) -> None: + audio_coro = async_get_and_parse_audio(audio_url) + + placeholder = self._tracker.add("audio", audio_coro) + self._add_placeholder(placeholder) + + +def validate_chat_template(chat_template: Optional[Union[Path, str]]): + """Raises if the provided chat template appears invalid.""" + if chat_template is None: + return + + elif isinstance(chat_template, Path) and not chat_template.exists(): + raise FileNotFoundError( + "the supplied chat template path doesn't exist") + + elif isinstance(chat_template, str): + JINJA_CHARS = "{}\n" + if not any(c in chat_template + for c in JINJA_CHARS) and not Path(chat_template).exists(): + raise ValueError( + f"The supplied chat template string ({chat_template}) " + f"appears path-like, but doesn't exist!") + + else: + raise TypeError( + f"{type(chat_template)} is not a valid chat template type") + + +def load_chat_template( + chat_template: Optional[Union[Path, str]]) -> Optional[str]: + if chat_template is None: + return None + try: + with open(chat_template, "r") as f: + resolved_chat_template = f.read() + except OSError as e: + if isinstance(chat_template, Path): + raise + + JINJA_CHARS = "{}\n" + if not any(c in chat_template for c in JINJA_CHARS): + msg = (f"The supplied chat template ({chat_template}) " + f"looks like a file path, but it failed to be " + f"opened. Reason: {e}") + raise ValueError(msg) from e + + # If opening a file fails, set chat template to be args to + # ensure we decode so our escape are interpreted correctly + resolved_chat_template = codecs.decode(chat_template, "unicode_escape") + + logger.info("Using supplied chat template:\n%s", resolved_chat_template) + return resolved_chat_template + + +# TODO: Let user specify how to insert multimodal tokens into prompt +# (similar to chat template) +def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int], + text_prompt: str) -> str: + """Combine multimodal prompts for a multimodal language model.""" + + # Look through the text prompt to check for missing placeholders + missing_placeholders: List[str] = [] + for placeholder in placeholder_counts: + + # For any existing placeholder in the text prompt, we leave it as is + placeholder_counts[placeholder] -= text_prompt.count(placeholder) + + if placeholder_counts[placeholder] < 0: + raise ValueError( + f"Found more '{placeholder}' placeholders in input prompt than " + "actual multimodal data items.") + + missing_placeholders.extend([placeholder] * + placeholder_counts[placeholder]) + + # NOTE: For now we always add missing placeholders at the front of + # the prompt. This may change to be customizable in the future. + return "\n".join(missing_placeholders + [text_prompt]) + + +# No need to validate using Pydantic again +_TextParser = partial(cast, ChatCompletionContentPartTextParam) +_ImageParser = partial(cast, ChatCompletionContentPartImageParam) +_AudioParser = partial(cast, ChatCompletionContentPartAudioParam) +_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam) +MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'} + + +def _parse_chat_message_content_parts( + role: str, + parts: Iterable[ChatCompletionContentPartParam], + mm_tracker: BaseMultiModalItemTracker, +) -> List[ConversationMessage]: + texts: List[str] = [] + + mm_parser = mm_tracker.create_parser() + keep_multimodal_content = \ + mm_tracker._model_config.hf_config.model_type in \ + MODEL_KEEP_MULTI_MODAL_CONTENT + + has_image = False + for part in parts: + part_type = part["type"] + if part_type == "text": + text = _TextParser(part)["text"] + texts.append(text) + elif part_type == "image_url": + image_url = _ImageParser(part)["image_url"] + + if image_url.get("detail", "auto") != "auto": + logger.warning( + "'image_url.detail' is currently not supported and " + "will be ignored.") + + mm_parser.parse_image(image_url["url"]) + has_image = True + elif part_type == "audio_url": + audio_url = _AudioParser(part)["audio_url"] + + mm_parser.parse_audio(audio_url["url"]) + elif part_type == "refusal": + text = _RefusalParser(part)["refusal"] + texts.append(text) + else: + raise NotImplementedError(f"Unknown part type: {part_type}") + + text_prompt = "\n".join(texts) + if keep_multimodal_content: + text_prompt = "\n".join(texts) + role_content = [{'type': 'text', 'text': text_prompt}] + + if has_image: + role_content = [{'type': 'image'}] + role_content + return [ConversationMessage(role=role, + content=role_content)] # type: ignore + else: + mm_placeholder_counts = mm_parser.mm_placeholder_counts() + if mm_placeholder_counts: + text_prompt = _get_full_multimodal_text_prompt( + mm_placeholder_counts, text_prompt) + return [ConversationMessage(role=role, content=text_prompt)] + + +# No need to validate using Pydantic again +_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam) +_ToolParser = partial(cast, ChatCompletionToolMessageParam) + + +def _parse_chat_message_content( + message: ChatCompletionMessageParam, + mm_tracker: BaseMultiModalItemTracker, +) -> List[ConversationMessage]: + role = message["role"] + content = message.get("content") + + if content is None: + content = [] + elif isinstance(content, str): + content = [ + ChatCompletionContentPartTextParam(type="text", text=content) + ] + + result = _parse_chat_message_content_parts( + role, + content, # type: ignore + mm_tracker, + ) + + for result_msg in result: + if role == 'assistant': + parsed_msg = _AssistantParser(message) + + if "tool_calls" in parsed_msg: + result_msg["tool_calls"] = list(parsed_msg["tool_calls"]) + + # Pass reasoning content as a dedicated field so the chat template + # can render it natively (Qwen3: message.reasoning_content branch). + # Accept both "reasoning" (new vllm) and "reasoning_content" (ours). + reasoning = (message.get("reasoning") # type: ignore[arg-type] + or message.get("reasoning_content")) # type: ignore[arg-type] + if reasoning and isinstance(reasoning, str): + result_msg["reasoning_content"] = reasoning + + elif role == "tool": + parsed_msg = _ToolParser(message) + if "tool_call_id" in parsed_msg: + result_msg["tool_call_id"] = parsed_msg["tool_call_id"] + + if "name" in message and isinstance(message["name"], str): + result_msg["name"] = message["name"] + + return result + + +def _postprocess_messages(messages: List[ConversationMessage]) -> None: + # per the Transformers docs & maintainers, tool call arguments in + # assistant-role messages with tool_calls need to be dicts not JSON str - + # this is how tool-use chat templates will expect them moving forwards + # so, for messages that have tool_calls, parse the string (which we get + # from openAI format) to dict + for message in messages: + if (message["role"] == "assistant" and "tool_calls" in message + and isinstance(message["tool_calls"], list)): + + for item in message["tool_calls"]: + item["function"]["arguments"] = json.loads( + item["function"]["arguments"]) + + +def parse_chat_messages( + messages: List[ChatCompletionMessageParam], + model_config: ModelConfig, + tokenizer: AnyTokenizer, +) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]: + conversation: List[ConversationMessage] = [] + mm_tracker = MultiModalItemTracker(model_config, tokenizer) + + for msg in messages: + sub_messages = _parse_chat_message_content(msg, mm_tracker) + + conversation.extend(sub_messages) + + _postprocess_messages(conversation) + + return conversation, mm_tracker.all_mm_data() + + +def parse_chat_messages_futures( + messages: List[ChatCompletionMessageParam], + model_config: ModelConfig, + tokenizer: AnyTokenizer, +) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]: + conversation: List[ConversationMessage] = [] + mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer) + + for msg in messages: + sub_messages = _parse_chat_message_content(msg, mm_tracker) + + conversation.extend(sub_messages) + + _postprocess_messages(conversation) + + return conversation, mm_tracker.all_mm_data() + + +def apply_hf_chat_template( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + conversation: List[ConversationMessage], + chat_template: Optional[str], + *, + tokenize: bool = False, # Different from HF's default + **kwargs: Any, +) -> str: + if chat_template is None and tokenizer.chat_template is None: + raise ValueError( + "As of transformers v4.44, default chat template is no longer " + "allowed, so you must provide a chat template if the tokenizer " + "does not define one.") + + return tokenizer.apply_chat_template( + conversation=conversation, # type: ignore[arg-type] + chat_template=chat_template, + tokenize=tokenize, + **kwargs, + ) + + +def apply_mistral_chat_template( + tokenizer: MistralTokenizer, + messages: List[ChatCompletionMessageParam], + chat_template: Optional[str] = None, + **kwargs: Any, +) -> List[int]: + if chat_template is not None: + logger.warning( + "'chat_template' cannot be overridden for mistral tokenizer.") + if "add_generation_prompt" in kwargs: + logger.warning( + "'add_generation_prompt' is not supported for mistral tokenizer, " + "so it will be ignored.") + if "continue_final_message" in kwargs: + logger.warning( + "'continue_final_message' is not supported for mistral tokenizer, " + "so it will be ignored.") + + return tokenizer.apply_chat_template( + messages=messages, + **kwargs, + ) diff --git a/qwen3_6_scripts/cli_args.py b/qwen3_6_scripts/cli_args.py new file mode 100644 index 00000000..ad0698d2 --- /dev/null +++ b/qwen3_6_scripts/cli_args.py @@ -0,0 +1,261 @@ +""" +This file contains the command line arguments for the vLLM's +OpenAI-compatible server. It is kept in a separate file for documentation +purposes. +""" + +import argparse +import json +import ssl +from typing import List, Optional, Sequence, Union + +from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str +from vllm.entrypoints.chat_utils import validate_chat_template +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath, + PromptAdapterPath) +from vllm.entrypoints.openai.tool_parsers import ToolParserManager +from vllm.utils import FlexibleArgumentParser + + +class LoRAParserAction(argparse.Action): + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Optional[Union[str, Sequence[str]]], + option_string: Optional[str] = None, + ): + if values is None: + values = [] + if isinstance(values, str): + raise TypeError("Expected values to be a list") + + lora_list: List[LoRAModulePath] = [] + for item in values: + if item in [None, '']: # Skip if item is None or empty string + continue + if '=' in item and ',' not in item: # Old format: name=path + name, path = item.split('=') + lora_list.append(LoRAModulePath(name, path)) + else: # Assume JSON format + try: + lora_dict = json.loads(item) + lora = LoRAModulePath(**lora_dict) + lora_list.append(lora) + except json.JSONDecodeError: + parser.error( + f"Invalid JSON format for --lora-modules: {item}") + except TypeError as e: + parser.error( + f"Invalid fields for --lora-modules: {item} - {str(e)}" + ) + setattr(namespace, self.dest, lora_list) + + +class PromptAdapterParserAction(argparse.Action): + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Optional[Union[str, Sequence[str]]], + option_string: Optional[str] = None, + ): + if values is None: + values = [] + if isinstance(values, str): + raise TypeError("Expected values to be a list") + + adapter_list: List[PromptAdapterPath] = [] + for item in values: + name, path = item.split('=') + adapter_list.append(PromptAdapterPath(name, path)) + setattr(namespace, self.dest, adapter_list) + + +def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + parser.add_argument("--host", + type=nullable_str, + default=None, + help="host name") + parser.add_argument("--port", type=int, default=8000, help="port number") + parser.add_argument( + "--uvicorn-log-level", + type=str, + default="info", + choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'], + help="log level for uvicorn") + parser.add_argument("--allow-credentials", + action="store_true", + help="allow credentials") + parser.add_argument("--allowed-origins", + type=json.loads, + default=["*"], + help="allowed origins") + parser.add_argument("--allowed-methods", + type=json.loads, + default=["*"], + help="allowed methods") + parser.add_argument("--allowed-headers", + type=json.loads, + default=["*"], + help="allowed headers") + parser.add_argument("--api-key", + type=nullable_str, + default=None, + help="If provided, the server will require this key " + "to be presented in the header.") + parser.add_argument( + "--lora-modules", + type=nullable_str, + default=None, + nargs='+', + action=LoRAParserAction, + help="LoRA module configurations in either 'name=path' format" + "or JSON format. " + "Example (old format): 'name=path' " + "Example (new format): " + "'{\"name\": \"name\", \"local_path\": \"path\", " + "\"base_model_name\": \"id\"}'") + parser.add_argument( + "--prompt-adapters", + type=nullable_str, + default=None, + nargs='+', + action=PromptAdapterParserAction, + help="Prompt adapter configurations in the format name=path. " + "Multiple adapters can be specified.") + parser.add_argument("--chat-template", + type=nullable_str, + default=None, + help="The file path to the chat template, " + "or the template in single-line form " + "for the specified model") + parser.add_argument("--response-role", + type=nullable_str, + default="assistant", + help="The role name to return if " + "`request.add_generation_prompt=true`.") + parser.add_argument("--ssl-keyfile", + type=nullable_str, + default=None, + help="The file path to the SSL key file") + parser.add_argument("--ssl-certfile", + type=nullable_str, + default=None, + help="The file path to the SSL cert file") + parser.add_argument("--ssl-ca-certs", + type=nullable_str, + default=None, + help="The CA certificates file") + parser.add_argument( + "--ssl-cert-reqs", + type=int, + default=int(ssl.CERT_NONE), + help="Whether client certificate is required (see stdlib ssl module's)" + ) + parser.add_argument( + "--root-path", + type=nullable_str, + default=None, + help="FastAPI root_path when app is behind a path based routing proxy") + parser.add_argument( + "--middleware", + type=nullable_str, + action="append", + default=[], + help="Additional ASGI middleware to apply to the app. " + "We accept multiple --middleware arguments. " + "The value should be an import path. " + "If a function is provided, vLLM will add it to the server " + "using @app.middleware('http'). " + "If a class is provided, vLLM will add it to the server " + "using app.add_middleware(). ") + parser.add_argument( + "--return-tokens-as-token-ids", + action="store_true", + help="When --max-logprobs is specified, represents single tokens as " + "strings of the form 'token_id:{token_id}' so that tokens that " + "are not JSON-encodable can be identified.") + parser.add_argument( + "--disable-frontend-multiprocessing", + action="store_true", + help="If specified, will run the OpenAI frontend server in the same " + "process as the model serving engine.") + + parser.add_argument( + "--enable-auto-tool-choice", + action="store_true", + default=False, + help= + "Enable auto tool choice for supported models. Use --tool-call-parser" + "to specify which parser to use") + + valid_tool_parsers = ToolParserManager.tool_parsers.keys() + parser.add_argument( + "--tool-call-parser", + type=str, + metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in " + "--tool-parser-plugin", + default=None, + help= + "Select the tool call parser depending on the model that you're using." + " This is used to parse the model-generated tool call into OpenAI API " + "format. Required for --enable-auto-tool-choice.") + + parser.add_argument( + "--tool-parser-plugin", + type=str, + default="", + help= + "Special the tool parser plugin write to parse the model-generated tool" + " into OpenAI API format, the name register in this plugin can be used " + "in --tool-call-parser.") + + parser.add_argument( + "--reasoning-parser", + type=str, + default=None, + help= + "Select the reasoning parser to split ... content into " + "reasoning_content vs content in the response. " + "Supported: qwen3") + + parser = AsyncEngineArgs.add_cli_args(parser) + + parser.add_argument('--max-log-len', + type=int, + default=None, + help='Max number of prompt characters or prompt ' + 'ID numbers being printed in log.' + '\n\nDefault: Unlimited') + + parser.add_argument( + "--disable-fastapi-docs", + action='store_true', + default=False, + help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint" + ) + + return parser + + +def validate_parsed_serve_args(args: argparse.Namespace): + """Quick checks for model serve args that raise prior to loading.""" + if hasattr(args, "subparser") and args.subparser != "serve": + return + + # Ensure that the chat template is valid; raises if it likely isn't + validate_chat_template(args.chat_template) + + # Enable auto tool needs a tool call parser to be valid + if args.enable_auto_tool_choice and not args.tool_call_parser: + raise TypeError("Error: --enable-auto-tool-choice requires " + "--tool-call-parser") + + +def create_parser_for_docs() -> FlexibleArgumentParser: + parser_for_docs = FlexibleArgumentParser( + prog="-m vllm.entrypoints.openai.api_server") + return make_arg_parser(parser_for_docs) diff --git a/qwen3_6_scripts/mamba_cache.py b/qwen3_6_scripts/mamba_cache.py new file mode 100644 index 00000000..7537b9e9 --- /dev/null +++ b/qwen3_6_scripts/mamba_cache.py @@ -0,0 +1,224 @@ +from typing import Dict, List, Optional + +import torch + +from vllm.attention.backends.abstract import AttentionMetadata + + +class MambaCacheManager: + + def __init__(self, dtype, num_mamba_layers, max_batch_size, + conv_state_shape, temporal_state_shape): + + conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) + + conv_state_shape, + dtype=dtype, + device="cuda") + temporal_state = torch.zeros(size=(num_mamba_layers, max_batch_size) + + temporal_state_shape, + dtype=dtype, + device="cuda") + + self.mamba_cache = (conv_state, temporal_state) + + # Maps between the request id and a dict that maps between the seq_id + # and its index inside the self.mamba_cache + self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {} + + def current_run_tensors(self, input_ids: torch.Tensor, + attn_metadata: AttentionMetadata, **kwargs): + """ + Return the tensors for the current run's conv and ssm state. + """ + if "seqlen_agnostic_capture_inputs" not in kwargs: + # We get here only on Prefill/Eager mode runs + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"] + finished_requests_ids = kwargs["finished_requests_ids"] + + self._release_finished_requests(finished_requests_ids) + mamba_cache_tensors = self._prepare_current_run_mamba_cache( + request_ids_to_seq_ids, finished_requests_ids) + + else: + # CUDA graph capturing runs + mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"] + + return mamba_cache_tensors + + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + """ + Copy the relevant Mamba cache into the CUDA graph input buffer + that was provided during the capture runs + (JambaForCausalLM.mamba_gc_cache_buffer). + """ + assert all( + key in kwargs + for key in ["request_ids_to_seq_ids", "finished_requests_ids"]) + finished_requests_ids = kwargs["finished_requests_ids"] + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"] + + self._release_finished_requests(finished_requests_ids) + self._prepare_current_run_mamba_cache(request_ids_to_seq_ids, + finished_requests_ids) + + def get_seqlen_agnostic_capture_inputs(self, batch_size: int): + """ + Provide the CUDA graph capture runs with a buffer in adjusted size. + The buffer is used to maintain the Mamba Cache during the CUDA graph + replay runs. + """ + return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache) + + def _swap_mamba_cache(self, from_index: int, to_index: int): + assert len(self.mamba_cache) > 0 + for cache_t in self.mamba_cache: + cache_t[:, [to_index,from_index]] = \ + cache_t[:, [from_index,to_index]] + + def _copy_mamba_cache(self, from_index: int, to_index: int): + assert len(self.mamba_cache) > 0 + for cache_t in self.mamba_cache: + cache_t[:, to_index].copy_(cache_t[:, from_index], + non_blocking=True) + + def _move_out_if_already_occupied(self, index: int, + all_occupied_indices: List[int]): + if index in all_occupied_indices: + first_free_index = self._first_free_index_in_mamba_cache() + # In case occupied, move the occupied to a new empty block + self._move_cache_index_and_mappings(from_index=index, + to_index=first_free_index) + + def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str, + seq_id: int, + destination_index: int): + """ + Assign (req_id,seq_id) pair to a `destination_index` index, if + already occupied, move the occupying index to a free index. + """ + all_occupied_indices = self._get_all_occupied_indices() + if cur_rid not in self.mamba_cache_indices_mapping: + self._move_out_if_already_occupied( + index=destination_index, + all_occupied_indices=all_occupied_indices) + for cache_t in self.mamba_cache: + cache_t[:, destination_index].zero_() + self.mamba_cache_indices_mapping[cur_rid] = { + seq_id: destination_index + } + elif seq_id not in (seq_ids2indices := + self.mamba_cache_indices_mapping[cur_rid]): + # parallel sampling , where n > 1, assume prefill have + # already happened now we only need to copy the already + # existing cache into the siblings seq_ids caches + self._move_out_if_already_occupied( + index=destination_index, + all_occupied_indices=all_occupied_indices) + index_exists = list(seq_ids2indices.values())[0] + # case of decoding n>1, copy prefill cache to decoding indices + self._copy_mamba_cache(from_index=index_exists, + to_index=destination_index) + self.mamba_cache_indices_mapping[cur_rid][ + seq_id] = destination_index + else: + # already exists + cache_index_already_exists = self.mamba_cache_indices_mapping[ + cur_rid][seq_id] + if cache_index_already_exists != destination_index: + # In case the seq id already exists but not in + # the right destination, swap it with what's occupying it + self._swap_pair_indices_and_mappings( + from_index=cache_index_already_exists, + to_index=destination_index) + + def _prepare_current_run_mamba_cache( + self, request_ids_to_seq_ids: Dict[str, list[int]], + finished_requests_ids: List[str]): + running_indices = [] + request_ids_to_seq_ids_flatten = [ + (req_id, seq_id) + for req_id, seq_ids in request_ids_to_seq_ids.items() + for seq_id in seq_ids + ] + batch_size = len(request_ids_to_seq_ids_flatten) + for dest_index, (request_id, + seq_id) in enumerate(request_ids_to_seq_ids_flatten): + if request_id in finished_requests_ids: + # Do not allocate cache index for requests that run + # and finish right after + continue + self._assign_seq_id_to_mamba_cache_in_specific_dest( + request_id, seq_id, dest_index) + running_indices.append(dest_index) + + self._clean_up_first_bs_blocks(batch_size, running_indices) + conv_state = self.mamba_cache[0][:, :batch_size] + temporal_state = self.mamba_cache[1][:, :batch_size] + + return (conv_state, temporal_state) + + def _get_all_occupied_indices(self): + return [ + cache_idx + for seq_ids2indices in self.mamba_cache_indices_mapping.values() + for cache_idx in seq_ids2indices.values() + ] + + def _clean_up_first_bs_blocks(self, batch_size: int, + indices_for_current_run: List[int]): + # move out all of the occupied but currently not running blocks + # outside of the first n blocks + destination_indices = range(batch_size) + max_possible_batch_size = self.mamba_cache[0].shape[1] + for destination_index in destination_indices: + if destination_index in self._get_all_occupied_indices() and \ + destination_index not in indices_for_current_run: + # move not running indices outside of the batch + all_other_indices = list( + range(batch_size, max_possible_batch_size)) + first_avail_index = self._first_free_index_in_mamba_cache( + all_other_indices) + self._swap_indices(from_index=destination_index, + to_index=first_avail_index) + + def _move_cache_index_and_mappings(self, from_index: int, to_index: int): + self._copy_mamba_cache(from_index=from_index, to_index=to_index) + self._update_mapping_index(from_index=from_index, to_index=to_index) + + def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int): + self._swap_mamba_cache(from_index=from_index, to_index=to_index) + self._swap_mapping_index(from_index=from_index, to_index=to_index) + + def _swap_mapping_index(self, from_index: int, to_index: int): + for seq_ids2index in self.mamba_cache_indices_mapping.values(): + for seq_id, index in seq_ids2index.items(): + if from_index == index: + seq_ids2index.update({seq_id: to_index}) + elif to_index == index: + seq_ids2index.update({seq_id: from_index}) + + def _update_mapping_index(self, from_index: int, to_index: int): + for seq_ids2index in self.mamba_cache_indices_mapping.values(): + for seq_id, index in seq_ids2index.items(): + if from_index == index: + seq_ids2index.update({seq_id: to_index}) + return + + def _release_finished_requests(self, + finished_seq_groups_req_ids: List[str]): + for req_id in finished_seq_groups_req_ids: + if req_id in self.mamba_cache_indices_mapping: + self.mamba_cache_indices_mapping.pop(req_id) + + def _first_free_index_in_mamba_cache( + self, indices_range: Optional[List[int]] = None) -> int: + assert self.mamba_cache is not None + if indices_range is None: + max_possible_batch_size = self.mamba_cache[0].shape[1] + indices_range = list(range(max_possible_batch_size)) + all_occupied_indices = self._get_all_occupied_indices() + for i in indices_range: + if i not in all_occupied_indices: + return i + raise Exception("Couldn't find a free spot in the mamba cache! This" + "should never happen") diff --git a/qwen3_6_scripts/patch_model_runner.py b/qwen3_6_scripts/patch_model_runner.py new file mode 100644 index 00000000..e10ad271 --- /dev/null +++ b/qwen3_6_scripts/patch_model_runner.py @@ -0,0 +1,78 @@ +""" +Fix: prefix_cache_hit stays True for chunked-prefill chunk 2+ even when past cache. + +Root cause: + model_runner.py _compute_for_prefix_cache_hit has three cases: + Case 1: prefix_cache_len <= context_len → "already past cache, do normal" + Case 2: context_len < prefix_cache_len < seq_len → partial hit, correct + Case 3: seq_len <= prefix_cache_len → full hit, reduce to 1 token + + Case 1 does nothing (leaves prefix_cache_hit = True). Then in utils.py: + if inter_data.prefix_cache_hit: + block_table = computed_block_nums ← ONLY the original prefix blocks! + + But context_len > prefix_cache_len means chunk 1 tokens (between prefix_cache_len + and context_len) are ALSO in KV cache and need to be in block_table. + block_table = computed_block_nums misses all chunk-1 blocks. + + In _forward_prefix_pytorch: + num_ctx_blocks = ceil(context_len / block_size) # e.g. 268 + block_tables.shape[1] = len(computed_block_nums) # e.g. 12 <-- too small! + At tile_blk >= 12: blk_ids is empty → k_t shape [..., 0] → amax crash. + +Fix: + Set prefix_cache_hit = False for Case 1, so utils.py falls through to: + elif chunked_prefill_enabled: + block_table = block_tables[seq_id] ← full block table (prefix + chunk1) +""" + +import re +import sys + +CANDIDATE_PATHS = [ + "/usr/local/corex/lib64/python3/dist-packages/vllm/worker/model_runner.py", + "/usr/local/corex/lib/python3/dist-packages/vllm/worker/model_runner.py", +] + +OLD_BLOCK = """\ + if prefix_cache_len <= context_len: + # We already passed the cache hit region, + # so do normal computation. + pass""" + +NEW_BLOCK = """\ + if prefix_cache_len <= context_len: + # We already passed the cache hit region, + # so do normal computation. + # Must clear prefix_cache_hit so _add_seq_group uses the full + # block_tables (prefix + previous-chunk blocks) instead of only + # computed_block_nums (prefix only). Without this, block_tables + # passed to _forward_prefix_pytorch is too narrow for context_len, + # causing an empty blk_ids slice and a zero-dim amax() crash. + inter_data.prefix_cache_hit = False""" + +import os + +patched = False +for path in CANDIDATE_PATHS: + if not os.path.exists(path): + continue + with open(path, "r") as f: + src = f.read() + if OLD_BLOCK not in src: + if NEW_BLOCK in src: + print(f"[patch_model_runner] already patched: {path}") + patched = True + break + print(f"[patch_model_runner] WARNING: expected block not found in {path}, skipping") + continue + patched_src = src.replace(OLD_BLOCK, NEW_BLOCK, 1) + with open(path, "w") as f: + f.write(patched_src) + print(f"[patch_model_runner] patched Case-1 prefix_cache_hit fix in: {path}") + patched = True + break + +if not patched: + print("[patch_model_runner] ERROR: could not find model_runner.py at any known path", file=sys.stderr) + sys.exit(1) diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh new file mode 100755 index 00000000..1035b2fe --- /dev/null +++ b/qwen3_6_scripts/patch_ops.sh @@ -0,0 +1,94 @@ +# BI-V100 patch script for Qwen3.6-27B (Qwen3_5 architecture) +# +# Triton situation on BI-V100: +# - Standard Triton 2.3.1 is already present in the image. +# - HAS_TRITON = False (hardcoded in vendor vllm), but Triton is still used +# for TP-mode cache management (custom_cache_manager / libentry). +# - The vendor's triton_utils/__init__.py, custom_cache_manager.py, libentry.py +# are already correct for standard Triton 2.3.1 — do NOT overwrite them. +# - DO NOT install BI-V150 corex Triton 2.1.0 (pkgs/triton): that causes +# GPU hang on BI-V100 because the Triton CUDA PTX kernels are incompatible. + +# Recommended server start command for TP=4 support 100K, need chunked prefill +# CUDA_VISIBLE_DEVICES="4,5,6,7" VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 python3 -m vllm.entrypoints.openai.api_server \ +# --model /workspace/models/Qwen3.6-27B --port 1111 --served-model-name llm \ +# --max-model-len 100000 --enforce-eager --trust-remote-code -tp 4 --gpu-memory-utilization 0.95 \ +# --max-num-seqs 1 --disable-log-requests --disable-frontend-multiprocessing \ +# --max-num-batched-tokens 4096 --enable-chunked-prefill +# +# With prefix caching (GDN align-mode, requires chunked prefill): +# CUDA_VISIBLE_DEVICES="4,5,6,7" VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 python3 -m vllm.entrypoints.openai.api_server \ +# --model /workspace/models/Qwen3.6-35B-A3B --port 1111 --served-model-name llm \ +# --max-model-len 150000 --trust-remote-code -tp 4 --gpu-memory-utilization 0.90 \ +# --max-num-seqs 1 --disable-log-requests --disable-frontend-multiprocessing \ +# --max-num-batched-tokens 8192 --enable-chunked-prefill --enable-prefix-caching \ +# --max-seq-len-to-capture 32768 + +# --- paged_attn.py: replace forward_prefix with pure-PyTorch fallback ------- +# The Triton context_attention_fwd kernel hangs BI-V100 GPUs permanently +# (standard Triton 2.3.1 PTX is not supported by the corex runtime either). +# Our paged_attn.py bypasses it entirely via _forward_prefix_pytorch, which +# utilizes K-tiling techniques, and also have _forward_decode_pytorch to bypass kernel +# when context length is high +cp ./paged_attn.py /usr/local/corex/lib/python3/dist-packages/vllm/attention/ops/paged_attn.py + +# --- model_runner.py: fix prefix_cache_hit stays True in chunked-prefill chunk 2+ --- +# Bug: _compute_for_prefix_cache_hit Case 1 (prefix_cache_len <= context_len) +# leaves prefix_cache_hit=True. Then _add_seq_group uses block_table=computed_block_nums +# (only the original prefix blocks), ignoring chunk-1 KV cache blocks. +# _forward_prefix_pytorch then gets an undersized block_tables and crashes with +# "amax(): Expected reduction dim -1 to have non-zero size" on the 2nd tile. +# Fix: set prefix_cache_hit=False for Case 1 so the full block_tables is used. +python3 ./patch_model_runner.py + +# --- transformers: Qwen3_5 tokenizer / model files -------------------------- +pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple +cp -r ./qwen3_5 /usr/local/lib/python3.10/site-packages/transformers/models/ +cp -r ./qwen3_5_moe /usr/local/lib/python3.10/site-packages/transformers/models/ +python3 ./patch_transformers_qwen3_5.py + +# --- vllm model: Qwen3.6-27B (Qwen3_5 arch) -------------------------------- +cp ./mamba_cache.py /usr/local/corex/lib/python3/dist-packages/vllm/model_executor/models/ +cp ./qwen3_5.py /usr/local/corex/lib/python3/dist-packages/vllm/model_executor/models/qwen3_5.py +python3 ./patch_vllm_qwen3_5.py + +# --- sequence.py: fix completion_tokens inflation under chunked prefill ------ +# Bug: get_output_token_ids_to_return(delta=True) with num_new_tokens=0 +# returns _cached_all_token_ids[-0:] == [0:] (the ENTIRE prompt+output list). +# Each prefill chunk step adds prompt_len to previous_num_tokens, so a 10K +# prompt processed in 3 chunks inflates completion_tokens by ~30K. +# Also adds num_cached_tokens field to RequestMetrics for prefix-cache stats. +cp ./sequence.py /usr/local/corex/lib/python3/dist-packages/vllm/sequence.py + +# --- scheduler.py: record num_cached_tokens in RequestMetrics ---------------- +# Sets seq_group.metrics.num_cached_tokens = prefix_cache_len on first prefill +# when --enable-prefix-caching is active, so serving_chat.py can report it in +# usage.prompt_tokens_details.cached_tokens (OpenAI-compatible API response). +cp ./scheduler.py /usr/local/corex/lib/python3/dist-packages/vllm/core/scheduler.py + +# --- xformers: bypass cudnnFlashAttnForward (head_dim=256 > 128 limit) ------ +# Injects _run_sdpa_fallback (pure matmul+softmax) into xformers.py. +# Required because head_dim=256 > 128 and ixformer flash attention either +# crashes (is_causal=True) or produces wrong output (attn_mask path). +# The fallback uses query_start_loc to derive actual query lengths, so it +# works correctly during profiling runs with chunked-prefill-style batches. +# also bypasses auto chunked prefill on +python3 ./patch_xformers_sdpa_seq.py + +# --- tool parser: Qwen3 XML tool call format --------------------------------- +# Registers "qwen3_coder" parser for Qwen3.6 XML-style tool calls: +# \nvalue\n +# Use at server start: --tool-call-parser qwen3_coder --enable-auto-tool-choice +cp ./qwen3coder_tool_parser.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/tool_parsers/ +python3 ./patch_vllm_tool_parser.py + +# --- reasoning parser: Qwen3 ... split ------------------------ +# Adds --reasoning-parser qwen3 support. +# Routes thinking tokens to reasoning_content, rest to content in the delta. +# Works together with --tool-call-parser qwen3_coder (think → tool call flow). +cp -r ./reasoning /usr/local/corex/lib/python3/dist-packages/vllm/ +cp ./protocol.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/protocol.py +cp ./cli_args.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/cli_args.py +cp ./serving_chat.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/serving_chat.py +cp ./api_server.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/api_server.py +cp ./chat_utils.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/chat_utils.py diff --git a/qwen3_6_scripts/patch_transformers_qwen3_5.py b/qwen3_6_scripts/patch_transformers_qwen3_5.py new file mode 100644 index 00000000..85b81402 --- /dev/null +++ b/qwen3_6_scripts/patch_transformers_qwen3_5.py @@ -0,0 +1,117 @@ +""" +Patches transformers 4.55.3 to register qwen3_5 and qwen3_5_moe model types. + +Deploy steps on the remote machine: + 1. cp -r modified_scripts/qwen3_5 /usr/local/lib/python3.10/site-packages/transformers/models/qwen3_5 + 2. cp -r modified_scripts/qwen3_5_moe /usr/local/lib/python3.10/site-packages/transformers/models/qwen3_5_moe + 3. python3 modified_scripts/patch_transformers_qwen3_5.py + +Target: pip-installed transformers at /usr/local/lib/python3.10/site-packages/transformers/ +(Not the corex pre-installed path at /usr/local/corex/lib64/python3/dist-packages/) +""" + +import sys + +TRANSFORMERS_ROOT = "/usr/local/lib/python3.10/site-packages/transformers" +AUTO_CONFIG = f"{TRANSFORMERS_ROOT}/models/auto/configuration_auto.py" +MODELS_INIT = f"{TRANSFORMERS_ROOT}/models/__init__.py" + + +def patch_file(path, replacements): + with open(path, "r") as f: + content = f.read() + + patched = False + for old, new in replacements: + if new in content: + print(f" [skip] already patched: {repr(new[:60])}") + continue + if old not in content: + print(f" [warn] anchor not found: {repr(old[:60])}") + continue + content = content.replace(old, new, 1) + patched = True + print(f" [ok] inserted after: {repr(old[:60])}") + + if patched: + with open(path, "w") as f: + f.write(content) + + +def main(): + print(f"=== Patching {AUTO_CONFIG} ===") + patch_file(AUTO_CONFIG, [ + # CONFIG_MAPPING_NAMES: insert qwen3_5 + qwen3_5_moe right after qwen3 + ( + '("qwen3", "Qwen3Config"),', + '("qwen3", "Qwen3Config"),\n ("qwen3_5", "Qwen3_5Config"),\n ("qwen3_5_moe", "Qwen3_5MoeConfig"),', + ), + ( + '("qwen3", "Qwen3Config")\n', + '("qwen3", "Qwen3Config"),\n ("qwen3_5", "Qwen3_5Config"),\n ("qwen3_5_moe", "Qwen3_5MoeConfig"),\n', + ), + # MODEL_NAMES_MAPPING (model_type -> human readable name) + ( + '("qwen3", "Qwen3"),', + '("qwen3", "Qwen3"),\n ("qwen3_5", "Qwen3_5"),\n ("qwen3_5_moe", "Qwen3_5_MoE"),', + ), + ( + '("qwen3", "Qwen3")\n', + '("qwen3", "Qwen3"),\n ("qwen3_5", "Qwen3_5"),\n ("qwen3_5_moe", "Qwen3_5_MoE"),\n', + ), + ]) + + print(f"\n=== Patching {MODELS_INIT} ===") + patch_file(MODELS_INIT, [ + ( + "from .qwen3 import *\n", + "from .qwen3 import *\n from .qwen3_5 import *\n from .qwen3_5_moe import *\n", + ), + ]) + + # Verification + print("\n=== Verification ===") + try: + import importlib.util, types + + def _load_config_mod(module_name, file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + mod = importlib.util.module_from_spec(spec) + mod.__package__ = ".".join(module_name.split(".")[:-1]) + pkg = sys.modules.setdefault("transformers", types.ModuleType("transformers")) + pkg.__path__ = [TRANSFORMERS_ROOT] + cu = sys.modules.setdefault( + "transformers.configuration_utils", types.ModuleType("transformers.configuration_utils")) + class _PC: + def __init__(self, **kwargs): pass + cu.PretrainedConfig = _PC + for sub in ("transformers.models", f"transformers.models.{module_name.split('.')[-2]}"): + m = sys.modules.setdefault(sub, types.ModuleType(sub)) + m.__path__ = [TRANSFORMERS_ROOT] + spec.loader.exec_module(mod) + return mod + + mod27 = _load_config_mod( + "transformers.models.qwen3_5.configuration_qwen3_5", + f"{TRANSFORMERS_ROOT}/models/qwen3_5/configuration_qwen3_5.py", + ) + cfg = mod27.Qwen3_5Config() + print(f" Qwen3_5Config() smoke-test OK (model_type={cfg.model_type})") + + mod35 = _load_config_mod( + "transformers.models.qwen3_5_moe.configuration_qwen3_5_moe", + f"{TRANSFORMERS_ROOT}/models/qwen3_5_moe/configuration_qwen3_5_moe.py", + ) + moe_cfg = mod35.Qwen3_5MoeConfig() + print(f" Qwen3_5MoeConfig() smoke-test OK (model_type={moe_cfg.model_type})") + t = moe_cfg.text_config + print(f" num_experts={t.num_experts}, top_k={t.num_experts_per_tok}, " + f"shared={t.shared_expert_intermediate_size}, layers={t.num_hidden_layers}") + except Exception as e: + print(f" [warn] smoke-test failed (may be fine at runtime): {e}") + + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_vllm_qwen3_5.py b/qwen3_6_scripts/patch_vllm_qwen3_5.py new file mode 100644 index 00000000..a8e091a4 --- /dev/null +++ b/qwen3_6_scripts/patch_vllm_qwen3_5.py @@ -0,0 +1,76 @@ +""" +Patches the vLLM model registry and deploys the Qwen3_5 model file. + +Deploy steps on the remote machine: + 1. cp modified_scripts/qwen3_5.py \ + /usr/local/corex/lib64/python3/dist-packages/vllm/model_executor/models/qwen3_5.py + 2. python3 modified_scripts/patch_vllm_qwen3_5.py + +Also edit your model config.json to set: + "architectures": ["Qwen3_5ForCausalLM"] + +Target: vLLM at /usr/local/corex/lib64/python3/dist-packages/vllm/ +""" + +VLLM_ROOT = "/usr/local/corex/lib64/python3/dist-packages/vllm" +REGISTRY = f"{VLLM_ROOT}/model_executor/models/registry.py" + + +def patch_file(path, replacements): + with open(path, "r") as f: + content = f.read() + + patched = False + for old, new in replacements: + if new in content: + print(f" [skip] already patched: {repr(new[:70])}") + continue + if old not in content: + print(f" [warn] anchor not found: {repr(old[:70])}") + continue + content = content.replace(old, new, 1) + patched = True + print(f" [ok] patched after: {repr(old[:70])}") + + if patched: + with open(path, "w") as f: + f.write(content) + + +def main(): + print(f"=== Patching {REGISTRY} ===") + patch_file(REGISTRY, [ + ( + ' "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),\n' + ' "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),', + ' "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),\n' + ' "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),\n' + ' "Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),\n' + ' "Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),', + ), + ]) + + print("\n=== Verification ===") + try: + import importlib.util + spec = importlib.util.spec_from_file_location( + "qwen3_5", + f"{VLLM_ROOT}/model_executor/models/qwen3_5.py", + ) + mod = importlib.util.module_from_spec(spec) + # Quick check: does the class exist? + spec.loader.exec_module(mod) + cls = mod.Qwen3_5ForCausalLM + print(f" Qwen3_5ForCausalLM found: {cls}") + cls_moe = mod.Qwen3_5MoeForCausalLM + print(f" Qwen3_5MoeForCausalLM found: {cls_moe}") + except Exception as e: + print(f" [warn] verification failed (may be OK at runtime): {e}") + + print("\nDone. Remember to:") + print(" 1. Set config.json 'architectures': ['Qwen3_5ForCausalLM'] or ['Qwen3_5MoEForCausalLM']") + print(" 2. Run patch_transformers_qwen3_5.py if not already done") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_vllm_tool_parser.py b/qwen3_6_scripts/patch_vllm_tool_parser.py new file mode 100644 index 00000000..f2575ba9 --- /dev/null +++ b/qwen3_6_scripts/patch_vllm_tool_parser.py @@ -0,0 +1,79 @@ +""" +Patches vLLM 0.6.3 to register Qwen3CoderToolParser under the name "qwen3_coder". + +Deploy steps on the remote machine (already called by patch_ops.sh): + 1. cp qwen3coder_tool_parser.py \ + /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/tool_parsers/ + 2. python3 patch_vllm_tool_parser.py + +Usage after patching: + --tool-call-parser qwen3_coder --enable-auto-tool-choice +""" + +import os + +VLLM_ROOT = "/usr/local/corex/lib/python3/dist-packages/vllm" +TOOL_PARSERS_DIR = f"{VLLM_ROOT}/entrypoints/openai/tool_parsers" +INIT_FILE = f"{TOOL_PARSERS_DIR}/__init__.py" + + +def patch_file(path, replacements): + with open(path, "r") as f: + content = f.read() + + patched = False + for old, new in replacements: + if new in content: + print(f" [skip] already patched: {repr(new[:70])}") + continue + if old not in content: + print(f" [warn] anchor not found: {repr(old[:70])}") + continue + content = content.replace(old, new, 1) + patched = True + print(f" [ok] patched: {repr(old[:50])} -> {repr(new[:50])}") + + if patched: + with open(path, "w") as f: + f.write(content) + + +def main(): + if not os.path.isdir(TOOL_PARSERS_DIR): + raise FileNotFoundError( + f"Tool parsers directory not found: {TOOL_PARSERS_DIR}\n" + "Verify the vLLM installation path.") + + print(f"=== Patching {INIT_FILE} ===") + patch_file(INIT_FILE, [ + ( + "from .mistral_tool_parser import MistralToolParser", + "from .mistral_tool_parser import MistralToolParser\n" + "from .qwen3coder_tool_parser import Qwen3CoderToolParser", + ), + ( + '"MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser"\n]', + '"MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser",\n' + ' "Qwen3CoderToolParser"\n]', + ), + ]) + + print("\n=== Verification ===") + try: + import importlib.util + spec = importlib.util.spec_from_file_location( + "qwen3coder_tool_parser", + f"{TOOL_PARSERS_DIR}/qwen3coder_tool_parser.py", + ) + mod = importlib.util.module_from_spec(spec) + print(f" Module spec loaded: {spec.name}") + print(" (full import requires torch/vllm runtime — skipping exec)") + except Exception as e: + print(f" [warn] spec check failed: {e}") + + print("\nDone. Start vLLM server with:") + print(" --tool-call-parser qwen3_coder --enable-auto-tool-choice") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_xformers_sdpa_batch.py b/qwen3_6_scripts/patch_xformers_sdpa_batch.py new file mode 100644 index 00000000..a585b4d0 --- /dev/null +++ b/qwen3_6_scripts/patch_xformers_sdpa_batch.py @@ -0,0 +1,192 @@ +""" +策略：批量（block-diagonal）fallback — 纯 PyTorch 数学实现 +============================================================= +构建块对角 causal mask，对整批序列一次 matmul + softmax， +完全绕开所有硬件 flash attention kernel。 + +背景： + ixformer flshattF: head_dim > 128 报错拒绝 + cudnnFlashAttnForward: 接受 head_dim=256，但数值结果错误（输出全"!"） + 两者大概率是同一硬件单元，ixformer 提前拦截了硬件不支持的配置。 + 纯 matmul 路径完全绕开硬件 flash attention，数值正确。 + +优点： + 数值正确。 + 并发请求 prefill attention 在 GPU 上真正并行（一次大 matmul）。 + +缺点： + 峰值显存 = total_tokens² × H × dtype_size + total_tokens 受 --max-num-batched-tokens 控制，max-model-len 控制不住。 + +内存参考（fp16，H_local=6，--max-num-batched-tokens=T）： + T=2048 → 峰值 ~50 MB + T=4096 → 峰值 ~200 MB + T=8192 → 峰值 ~800 MB + T=16384 → 峰值 ~3.2 GB + +Deploy: + python3 modified_scripts/patch_xformers_sdpa_batch.py +""" + +XFORMERS_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/attention/backends/xformers.py" +) + +FALLBACK_METHOD = ''' + def _run_sdpa_fallback( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: "XFormersMetadata", + ) -> torch.Tensor: + """批量纯数学 attention fallback。 + + 构建块对角 causal mask（等价于 ixformer BlockDiagonalCausalMask）， + 对整批序列一次 matmul + softmax，GPU 并行处理所有序列。 + + 块对角 mask 结构（seq1 len=3，seq2 len=2）： + s1,0 s1,1 s1,2 s2,0 s2,1 + s1,0 [ 0 -inf -inf -inf -inf ] + s1,1 [ 0 0 -inf -inf -inf ] + s1,2 [ 0 0 0 -inf -inf ] + s2,0 [-inf -inf -inf 0 -inf ] + s2,1 [-inf -inf -inf 0 0 ] + + softmax 在 float32 下计算防止 float16 溢出，结果转回原始 dtype。 + + Args: + query : [1, total_prefill_tokens, num_heads, head_dim] + key : [1, total_prefill_tokens, num_kv_heads, head_dim] + value : [1, total_prefill_tokens, num_kv_heads, head_dim] + Returns: + [1, total_prefill_tokens, num_heads, head_dim] + """ + assert attn_metadata.seq_lens is not None + orig_dtype = query.dtype + total_tokens = query.shape[1] + + # ── 构建块对角 causal mask [T, T] ──────────────────────────────── + # 全部初始化为 -inf，再对每条序列的对角块填入下三角 0 + mask = torch.full( + (total_tokens, total_tokens), + float("-inf"), + dtype=torch.float32, + device=query.device, + ) + start = 0 + for seq_len in attn_metadata.seq_lens: + end = start + seq_len + mask[start:end, start:end] = torch.tril( + torch.zeros(seq_len, seq_len, + dtype=torch.float32, device=query.device) + ) + start = end + + # ── [1, H, T, D]，.contiguous() ────────────────────────────────── + q_all = query.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + k_all = key.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + v_all = value.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + + # ── GQA：展开 KV heads ──────────────────────────────────────────── + if k_all.shape[1] != q_all.shape[1]: + n = q_all.shape[1] // k_all.shape[1] + k_all = k_all.repeat_interleave(n, dim=1).contiguous() + v_all = v_all.repeat_interleave(n, dim=1).contiguous() + + # ── 纯数学 attention（float32 防溢出）──────────────────────────── + # [1, H, T, T] + attn_w = torch.matmul(q_all.float(), k_all.float().transpose(-2, -1)) + attn_w = attn_w * self.scale + attn_w = attn_w + mask # 加法广播：mask [T,T] → [1, H, T, T] + attn_w = torch.softmax(attn_w, dim=-1) + + out = torch.matmul(attn_w, v_all.float()).to(orig_dtype) + # [1, H, T, D] → [1, T, H, D] + return out.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + +''' + +OLD_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op = self.attn_op + ) + return out.view_as(original_query)\ +""" + +NEW_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + if self.head_size > 128: + out = self._run_sdpa_fallback(query, key, value, attn_metadata) + else: + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op=self.attn_op, + ) + return out.view_as(original_query)\ +""" + +INJECT_ANCHOR = " def _run_memory_efficient_xformers_forward(" + + +def patch_file(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "_run_sdpa_fallback" in content: + print(" [skip] _run_sdpa_fallback already present") + elif INJECT_ANCHOR not in content: + print(" [warn] inject anchor not found") + else: + content = content.replace(INJECT_ANCHOR, FALLBACK_METHOD + INJECT_ANCHOR, 1) + print(" [ok] injected _run_sdpa_fallback (batch, pure-math)") + changed = True + + if NEW_XFORMER_BLOCK in content: + print(" [skip] dispatch block already patched") + elif OLD_XFORMER_BLOCK in content: + content = content.replace(OLD_XFORMER_BLOCK, NEW_XFORMER_BLOCK, 1) + print(" [ok] patched dispatch block") + changed = True + else: + print(" [warn] dispatch block anchor not found") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def main(): + print("=== patch_xformers_sdpa_batch (batch, pure-math) ===") + print(f"Target: {XFORMERS_PATH}") + patch_file(XFORMERS_PATH) + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_xformers_sdpa_batch_kernel.py b/qwen3_6_scripts/patch_xformers_sdpa_batch_kernel.py new file mode 100644 index 00000000..e7f647ff --- /dev/null +++ b/qwen3_6_scripts/patch_xformers_sdpa_batch_kernel.py @@ -0,0 +1,191 @@ +""" +策略：批量（block-diagonal）— F.scaled_dot_product_attention，可走硬件 kernel +============================================================================= +构建块对角 causal mask，对整批序列一次 F.scaled_dot_product_attention。 +与 patch_xformers_sdpa_batch.py（纯 matmul）的区别： + SDPA 会根据 PyTorch/驱动能力分发到最优 kernel（Flash Attention / + mem-efficient attention / math fallback），而不是固定走 cublas matmul。 + +历史说明： + 该方案最早因输出全"!"而被弃用，后续排查确认"!"由 mamba_cache.py bug + 引起，与 attention 实现无关。当前恢复此方案用于性能对比测试。 + +已知硬件限制（BI-V100）： + cudnnFlashAttnForward 不支持 is_causal=True（报错）。 + 本实现使用 is_causal=False + 显式块对角 additive mask 规避此限制。 + 若 SDPA 仍分发到有问题的 kernel，回退到 patch_xformers_sdpa_batch.py。 + +优点（vs 纯 matmul）： + SDPA 可分发到 Flash Attention kernel → O(L) 显存、更快的 CUDA kernel。 + +缺点： + 依赖硬件 kernel 行为，若 kernel 有 bug 则数值错误（需与 matmul 版对比验证）。 + +Deploy: + python3 modified_scripts/patch_xformers_sdpa_batch_kernel.py +""" + +XFORMERS_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/attention/backends/xformers.py" +) + +FALLBACK_METHOD = ''' + def _run_sdpa_fallback( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: "XFormersMetadata", + ) -> torch.Tensor: + """批量 F.scaled_dot_product_attention fallback（可走硬件 kernel）。 + + 构建块对角 causal mask，对整批序列一次 SDPA 调用。 + SDPA 可分发到 Flash Attention / mem-efficient attention kernel。 + is_causal=False + 显式 additive mask，规避 cudnnFlashAttnForward + 不支持 is_causal=True 的限制。 + + 块对角 mask（seq1 len=3，seq2 len=2）： + s1,0 s1,1 s1,2 s2,0 s2,1 + s1,0 [ 0 -inf -inf -inf -inf ] + s1,1 [ 0 0 -inf -inf -inf ] + s1,2 [ 0 0 0 -inf -inf ] + s2,0 [-inf -inf -inf 0 -inf ] + s2,1 [-inf -inf -inf 0 0 ] + + Args: + query : [1, total_prefill_tokens, num_heads, head_dim] + key : [1, total_prefill_tokens, num_kv_heads, head_dim] + value : [1, total_prefill_tokens, num_kv_heads, head_dim] + Returns: + [1, total_prefill_tokens, num_heads, head_dim] + """ + import torch.nn.functional as F + + assert attn_metadata.seq_lens is not None + orig_dtype = query.dtype + total_tokens = query.shape[1] + + # ── 块对角 causal mask [T, T] ───────────────────────────────────── + mask = torch.full( + (total_tokens, total_tokens), + float("-inf"), + dtype=orig_dtype, + device=query.device, + ) + start = 0 + for seq_len in attn_metadata.seq_lens: + end = start + seq_len + mask[start:end, start:end] = torch.tril( + torch.zeros(seq_len, seq_len, dtype=orig_dtype, device=query.device) + ) + start = end + + # ── [1, H, T, D] ────────────────────────────────────────────────── + q_all = query.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + k_all = key.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + v_all = value.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + + # ── GQA：展开 KV heads ──────────────────────────────────────────── + if k_all.shape[1] != q_all.shape[1]: + n = q_all.shape[1] // k_all.shape[1] + k_all = k_all.repeat_interleave(n, dim=1).contiguous() + v_all = v_all.repeat_interleave(n, dim=1).contiguous() + + # ── F.scaled_dot_product_attention（可走硬件 kernel）───────────── + # is_causal=False：避免 cudnnFlashAttnForward "not support causal mode" + # attn_mask 传 additive float mask（非 bool），SDPA 选择 math/kernel 路径 + out = F.scaled_dot_product_attention( + q_all, k_all, v_all, + attn_mask=mask, + dropout_p=0.0, + is_causal=False, + scale=self.scale, + ) + # [1, H, T, D] → [1, T, H, D] + return out.squeeze(0).permute(1, 0, 2).contiguous().unsqueeze(0) + +''' + +OLD_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op = self.attn_op + ) + return out.view_as(original_query)\ +""" + +NEW_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + if self.head_size > 128: + out = self._run_sdpa_fallback(query, key, value, attn_metadata) + else: + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op=self.attn_op, + ) + return out.view_as(original_query)\ +""" + +INJECT_ANCHOR = " def _run_memory_efficient_xformers_forward(" + + +def patch_file(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "_run_sdpa_fallback" in content: + print(" [skip] _run_sdpa_fallback already present") + elif INJECT_ANCHOR not in content: + print(" [warn] inject anchor not found") + else: + content = content.replace(INJECT_ANCHOR, FALLBACK_METHOD + INJECT_ANCHOR, 1) + print(" [ok] injected _run_sdpa_fallback (batch, F.sdpa kernel)") + changed = True + + if NEW_XFORMER_BLOCK in content: + print(" [skip] dispatch block already patched") + elif OLD_XFORMER_BLOCK in content: + content = content.replace(OLD_XFORMER_BLOCK, NEW_XFORMER_BLOCK, 1) + print(" [ok] patched dispatch block") + changed = True + else: + print(" [warn] dispatch block anchor not found") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def main(): + print("=== patch_xformers_sdpa_batch_kernel (batch, F.sdpa + kernel dispatch) ===") + print(f"Target: {XFORMERS_PATH}") + patch_file(XFORMERS_PATH) + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_xformers_sdpa_seq.py b/qwen3_6_scripts/patch_xformers_sdpa_seq.py new file mode 100644 index 00000000..496abc1f --- /dev/null +++ b/qwen3_6_scripts/patch_xformers_sdpa_seq.py @@ -0,0 +1,321 @@ +""" +策略：顺序（per-sequence）fallback — 纯 PyTorch 数学实现 +========================================================== +逐条序列用 matmul + softmax 手写 attention，完全绕开所有硬件 +flash attention kernel（ixformer / cudnnFlashAttnForward）。 + +背景： + Iluvatar cudnnFlashAttnForward 存在两个已知问题： + 1. 不支持 is_causal=True（报错） + 2. 使用 attn_mask 路径时数值结果不正确（静默错误，输出全为"!"） + 与华为昇腾 910B4 上 llama.cpp --flash-attn off 修复同类问题的原理相同。 + 纯数学路径（matmul + softmax）在任何 PyTorch 后端上结果都正确。 + +优点： + 数值正确，不依赖任何硬件特定 attention kernel。 + 峰值显存 = max(seq_len)² × H × dtype_size，由 --max-model-len 控制。 + +缺点： + 并发请求的 prefill attention 串行执行。 + O(L²) 显存（无 flash attention 的 O(L) 优化）。 + +内存参考（fp16，H_local=6）： + max-model-len=4096 → 峰值 ~200 MB + max-model-len=8192 → 峰值 ~800 MB + max-model-len=16384 → 峰值 ~3.2 GB + +额外 patch（arg_utils.py）： + vllm 0.6.3 在 max_model_len > 32K 时会自动开启 chunked prefill（无命令行 + 关闭选项），原意是防止 profiling OOM。但 _run_sdpa_fallback 已通过 Q-tiling + 解决了该问题，chunked prefill 反而会把推理路径从 _run_sdpa_fallback 切换到 + _forward_prefix_pytorch，属于不必要的行为变更，因此一并禁用该自动逻辑。 + +Deploy: + python3 modified_scripts/patch_xformers_sdpa_seq.py +""" + +XFORMERS_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/attention/backends/xformers.py" +) + +ARG_UTILS_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/engine/arg_utils.py" +) + +LOGITS_PROC_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/model_executor/layers/logits_processor.py" +) + +# _apply_logits_processors crashes when seq_groups is None (intermediate +# chunked-prefill chunks on the driver rank). Add an early-return guard. +_LP_OLD_BLOCK = """\ +def _apply_logits_processors( + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> torch.Tensor: + found_logits_processors = False\ +""" + +_LP_NEW_BLOCK = """\ +def _apply_logits_processors( + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> torch.Tensor: + if sampling_metadata.seq_groups is None: # intermediate chunked-prefill chunk + return logits + found_logits_processors = False\ +""" + +# vllm 0.6.3 自动开启 chunked prefill 的原始块 +_ARG_OLD_BLOCK = """\ + if (is_gpu and not use_sliding_window and not use_spec_decode + and not self.enable_lora + and not self.enable_prompt_adapter): + self.enable_chunked_prefill = True + logger.warning( + "Chunked prefill is enabled by default for models with " + "max_model_len > 32K. Currently, chunked prefill might " + "not work with some features or models. If you " + "encounter any issues, please disable chunked prefill " + "by setting --enable-chunked-prefill=False.")\ +""" + +_ARG_NEW_BLOCK = """\ + if (is_gpu and not use_sliding_window and not use_spec_decode + and not self.enable_lora + and not self.enable_prompt_adapter): + pass # skip auto-enable: Q-tiling in _run_sdpa_fallback + # handles long-context memory without chunked prefill\ +""" + +FALLBACK_METHOD = ''' + def _run_sdpa_fallback( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: "XFormersMetadata", + ) -> torch.Tensor: + """纯数学 causal attention fallback，带 Q-tiling 内存优化。 + + 调用时机：kv_cache.numel()==0（profiling 阶段）。 + 此路径无 KV 缓存前缀，KV 长度 == query 长度。 + + 内存优化（Q-tiling，与 Flash Attention 同思路）： + 将 Q 分成 _Q_CHUNK 大小的子块逐块计算，每块峰值内存 + O(_Q_CHUNK × q_len) 而非 O(q_len²)。 + profiling 阶段序列可能达到 max_model_len（如 20K tokens）， + 不加 Q-tiling 会产生 9.6 GB 矩阵直接 OOM。 + + softmax 在 float32 下计算以防止 float16 溢出，结果转回原始 dtype。 + + Args: + query : [1, total_query_tokens, num_heads, head_dim] + key : [1, total_query_tokens, num_kv_heads, head_dim] + value : [1, total_query_tokens, num_kv_heads, head_dim] + Returns: + [1, total_query_tokens, num_heads, head_dim] + """ + _Q_CHUNK = 256 # 与 _forward_prefix_pytorch 的 _ATTN_Q_CHUNK 保持一致 + + assert attn_metadata.seq_lens is not None + orig_dtype = query.dtype + num_seqs = len(attn_metadata.seq_lens) + + # 推导每条序列的实际 query 长度。 + # 正常 prefill 时 q_len == seq_len；如果将来遇到 chunked 场景， + # query_start_loc 记录的是真实 query token 数（非全序列长度）。 + if (attn_metadata.query_start_loc is not None + and len(attn_metadata.query_start_loc) == num_seqs + 1): + q_lens = [ + int(attn_metadata.query_start_loc[i + 1].item()) - + int(attn_metadata.query_start_loc[i].item()) + for i in range(num_seqs) + ] + else: + q_lens = list(attn_metadata.seq_lens) + + q_flat = query.squeeze(0) # [T, H, D] + k_flat = key.squeeze(0) # [T, Hkv, D] + v_flat = value.squeeze(0) + + output = torch.empty_like(q_flat) + seq_start = 0 + for q_len in q_lens: + seq_end = seq_start + q_len + + # 当前序列的完整 K/V（此路径无前缀，KV == Q） + k_s = k_flat[seq_start:seq_end].permute(1, 0, 2).float() # [Hkv, q_len, D] + v_s = v_flat[seq_start:seq_end].permute(1, 0, 2).float() # [Hkv, q_len, D] + + # GQA：展开 KV heads 至与 query heads 一致 + if k_s.shape[0] != self.num_heads: + n = self.num_heads // k_s.shape[0] + k_s = k_s.repeat_interleave(n, dim=0).contiguous() + v_s = v_s.repeat_interleave(n, dim=0).contiguous() + + # k_pos 用于因果掩码 + k_pos = torch.arange(q_len, device=query.device) + + # Q-tiling：分块处理 query，峰值内存 O(_Q_CHUNK × q_len) + for qc_start in range(0, q_len, _Q_CHUNK): + qc_end = min(qc_start + _Q_CHUNK, q_len) + + # [H, qc, D] + q_c = q_flat[seq_start + qc_start:seq_start + qc_end] \ + .permute(1, 0, 2).float() + + # [H, qc, q_len] + attn_w = torch.matmul(q_c, k_s.transpose(-2, -1)) * self.scale + + # 因果掩码：q_c 里位置 j 只能看 k_pos <= j（相对位置） + qc_q_pos = torch.arange(qc_start, qc_end, device=query.device) + mask = k_pos.unsqueeze(0) > qc_q_pos.unsqueeze(1) + attn_w = attn_w.masked_fill(mask.unsqueeze(0), float("-inf")) + + attn_w = torch.softmax(attn_w, dim=-1) + out_c = torch.matmul(attn_w, v_s).to(orig_dtype) # [H, qc, D] + + output[seq_start + qc_start:seq_start + qc_end] = ( + out_c.permute(1, 0, 2)) + + seq_start = seq_end + + return output.unsqueeze(0) # [1, T, H, D] + +''' + +OLD_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op = self.attn_op + ) + return out.view_as(original_query)\ +""" + +NEW_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + if self.head_size > 128: + out = self._run_sdpa_fallback(query, key, value, attn_metadata) + else: + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op=self.attn_op, + ) + return out.view_as(original_query)\ +""" + +INJECT_ANCHOR = " def _run_memory_efficient_xformers_forward(" + + +def patch_file(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "_run_sdpa_fallback" in content: + print(" [skip] _run_sdpa_fallback already present") + elif INJECT_ANCHOR not in content: + print(" [warn] inject anchor not found") + else: + content = content.replace(INJECT_ANCHOR, FALLBACK_METHOD + INJECT_ANCHOR, 1) + print(" [ok] injected _run_sdpa_fallback (sequential, pure-math)") + changed = True + + if NEW_XFORMER_BLOCK in content: + print(" [skip] dispatch block already patched") + elif OLD_XFORMER_BLOCK in content: + content = content.replace(OLD_XFORMER_BLOCK, NEW_XFORMER_BLOCK, 1) + print(" [ok] patched dispatch block") + changed = True + else: + print(" [warn] dispatch block anchor not found") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def patch_arg_utils(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "skip auto-enable: Q-tiling" in content: + print(" [skip] chunked-prefill auto-enable already disabled") + elif _ARG_OLD_BLOCK in content: + content = content.replace(_ARG_OLD_BLOCK, _ARG_NEW_BLOCK, 1) + print(" [ok] disabled chunked-prefill auto-enable for 32K+") + changed = True + else: + print(" [warn] target block not found — check arg_utils.py version") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def patch_logits_processor(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "intermediate chunked-prefill chunk" in content: + print(" [skip] seq_groups=None guard already present") + elif _LP_OLD_BLOCK in content: + content = content.replace(_LP_OLD_BLOCK, _LP_NEW_BLOCK, 1) + print(" [ok] added seq_groups=None guard in _apply_logits_processors") + changed = True + else: + print(" [warn] target block not found — check logits_processor.py version") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def main(): + print("=== patch_xformers_sdpa_seq (sequential, pure-math) ===") + print(f"Target: {XFORMERS_PATH}") + patch_file(XFORMERS_PATH) + + print("\n=== patch_arg_utils (disable chunked-prefill auto-enable) ===") + print(f"Target: {ARG_UTILS_PATH}") + patch_arg_utils(ARG_UTILS_PATH) + + print("\n=== patch_logits_processor (seq_groups=None guard for chunked prefill) ===") + print(f"Target: {LOGITS_PROC_PATH}") + patch_logits_processor(LOGITS_PROC_PATH) + + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/patch_xformers_sdpa_seq_kernel.py b/qwen3_6_scripts/patch_xformers_sdpa_seq_kernel.py new file mode 100644 index 00000000..82df8d09 --- /dev/null +++ b/qwen3_6_scripts/patch_xformers_sdpa_seq_kernel.py @@ -0,0 +1,181 @@ +""" +策略：顺序（per-sequence）— F.scaled_dot_product_attention，可走硬件 kernel +============================================================================= +逐条序列调用 F.scaled_dot_product_attention，is_causal=False + 显式因果 mask。 +与 patch_xformers_sdpa_seq.py（纯 matmul）的区别： + SDPA 可分发到 Flash Attention / mem-efficient attention kernel， + 而纯 matmul 固定走 cublas。 + +硬件限制（BI-V100）： + cudnnFlashAttnForward 不支持 is_causal=True（直接报错）。 + 必须使用 is_causal=False + 显式 additive causal mask。 + 每条序列单独构造上三角 -inf mask，peak 显存 = max(seq_len)² × dtype， + 比 batch 版的 total_tokens² 小得多。 + +与 batch_kernel 的对比： + seq_kernel: 显存小，peak = max_single_seq²；并发 prefill 串行排队 + batch_kernel: 显存大，peak = total_tokens²；并发 prefill 一次并行处理， + 通过 --max-num-batched-tokens 控制 total_tokens 上限 + +Deploy: + python3 modified_scripts/patch_xformers_sdpa_seq_kernel.py +""" + +XFORMERS_PATH = ( + "/usr/local/corex/lib64/python3/dist-packages/" + "vllm/attention/backends/xformers.py" +) + +FALLBACK_METHOD = ''' + def _run_sdpa_fallback( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: "XFormersMetadata", + ) -> torch.Tensor: + """顺序 F.scaled_dot_product_attention fallback（可走硬件 kernel）。 + + 逐条序列调用 SDPA，is_causal=False + 显式上三角 additive mask。 + cudnnFlashAttnForward 不支持 is_causal=True，必须用显式 mask。 + 逐序列构造 mask，peak 显存 = max(seq_len)² × dtype（远小于 batch 版）。 + + Args: + query : [1, total_prefill_tokens, num_heads, head_dim] + key : [1, total_prefill_tokens, num_kv_heads, head_dim] + value : [1, total_prefill_tokens, num_kv_heads, head_dim] + Returns: + [1, total_prefill_tokens, num_heads, head_dim] + """ + import torch.nn.functional as F + + assert attn_metadata.seq_lens is not None + orig_dtype = query.dtype + + q_flat = query.squeeze(0) # [T, H, D] + k_flat = key.squeeze(0) # [T, Hkv, D] + v_flat = value.squeeze(0) + + output = torch.empty_like(q_flat) + start = 0 + for seq_len in attn_metadata.seq_lens: + end = start + seq_len + # [1, H, L, D] + q_s = q_flat[start:end].permute(1, 0, 2).contiguous().unsqueeze(0) + k_s = k_flat[start:end].permute(1, 0, 2).contiguous().unsqueeze(0) + v_s = v_flat[start:end].permute(1, 0, 2).contiguous().unsqueeze(0) + + # GQA：展开 KV heads + if k_s.shape[1] != q_s.shape[1]: + n = q_s.shape[1] // k_s.shape[1] + k_s = k_s.repeat_interleave(n, dim=1).contiguous() + v_s = v_s.repeat_interleave(n, dim=1).contiguous() + + # 逐序列因果 mask [L, L]，上三角 -inf + causal_mask = torch.tril( + torch.zeros(seq_len, seq_len, dtype=orig_dtype, device=q_s.device) + ) + causal_mask = causal_mask.masked_fill( + torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, + device=q_s.device), diagonal=1), + float("-inf"), + ) + + # is_causal=False + 显式 mask，规避 cudnnFlashAttnForward 不支持 is_causal=True + out_s = F.scaled_dot_product_attention( + q_s, k_s, v_s, + attn_mask=causal_mask, + dropout_p=0.0, + is_causal=False, + scale=self.scale, + ) + # [1, H, L, D] → [L, H, D] + output[start:end] = out_s.squeeze(0).permute(1, 0, 2).to(orig_dtype) + start = end + + return output.unsqueeze(0) # [1, T, H, D] + +''' + +OLD_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op = self.attn_op + ) + return out.view_as(original_query)\ +""" + +NEW_XFORMER_BLOCK = """\ + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + if self.head_size > 128: + out = self._run_sdpa_fallback(query, key, value, attn_metadata) + else: + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op=self.attn_op, + ) + return out.view_as(original_query)\ +""" + +INJECT_ANCHOR = " def _run_memory_efficient_xformers_forward(" + + +def patch_file(path): + with open(path, "r") as f: + content = f.read() + changed = False + + if "_run_sdpa_fallback" in content: + print(" [skip] _run_sdpa_fallback already present") + elif INJECT_ANCHOR not in content: + print(" [warn] inject anchor not found") + else: + content = content.replace(INJECT_ANCHOR, FALLBACK_METHOD + INJECT_ANCHOR, 1) + print(" [ok] injected _run_sdpa_fallback (seq, F.sdpa kernel)") + changed = True + + if NEW_XFORMER_BLOCK in content: + print(" [skip] dispatch block already patched") + elif OLD_XFORMER_BLOCK in content: + content = content.replace(OLD_XFORMER_BLOCK, NEW_XFORMER_BLOCK, 1) + print(" [ok] patched dispatch block") + changed = True + else: + print(" [warn] dispatch block anchor not found") + + if changed: + with open(path, "w") as f: + f.write(content) + print(f" Written: {path}") + + +def main(): + print("=== patch_xformers_sdpa_seq_kernel (seq, F.sdpa + kernel dispatch) ===") + print(f"Target: {XFORMERS_PATH}") + patch_file(XFORMERS_PATH) + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/qwen3_6_scripts/protocol.py b/qwen3_6_scripts/protocol.py new file mode 100644 index 00000000..1efccdd3 --- /dev/null +++ b/qwen3_6_scripts/protocol.py @@ -0,0 +1,1043 @@ +# Adapted from +# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py +import time +from argparse import Namespace +from typing import Any, Dict, List, Literal, Optional, Union + +import torch +from openai.types.chat import ChatCompletionContentPartParam +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Annotated, Required, TypedDict + +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +from vllm.pooling_params import PoolingParams +from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams, + RequestOutputKind, SamplingParams) +from vllm.sequence import Logprob +from vllm.utils import random_uuid + +# torch is mocked during docs generation, +# so we have to provide the values as literals +_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807) +_LONG_INFO: Union["torch.iinfo", Namespace] + +try: + from sphinx.ext.autodoc.mock import _MockModule + + if isinstance(torch, _MockModule): + _LONG_INFO = _MOCK_LONG_INFO + else: + _LONG_INFO = torch.iinfo(torch.long) +except ModuleNotFoundError: + _LONG_INFO = torch.iinfo(torch.long) + +assert _LONG_INFO.min == _MOCK_LONG_INFO.min +assert _LONG_INFO.max == _MOCK_LONG_INFO.max + + +class CustomChatCompletionMessageParam(TypedDict, total=False): + """Enables custom roles in the Chat Completion API.""" + role: Required[str] + """The role of the message's author.""" + + content: Union[str, List[ChatCompletionContentPartParam]] + """The contents of the message.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the + same role. + """ + + tool_call_id: Optional[str] + + tool_calls: Optional[List[dict]] + + +class OpenAIBaseModel(BaseModel): + # OpenAI API does not allow extra fields + model_config = ConfigDict(extra="forbid") + + +class ErrorResponse(OpenAIBaseModel): + object: str = "error" + message: str + type: str + param: Optional[str] = None + code: int + + +class ModelPermission(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}") + object: str = "model_permission" + created: int = Field(default_factory=lambda: int(time.time())) + allow_create_engine: bool = False + allow_sampling: bool = True + allow_logprobs: bool = True + allow_search_indices: bool = False + allow_view: bool = True + allow_fine_tuning: bool = False + organization: str = "*" + group: Optional[str] = None + is_blocking: bool = False + + +class ModelCard(OpenAIBaseModel): + id: str + object: str = "model" + created: int = Field(default_factory=lambda: int(time.time())) + owned_by: str = "vllm" + root: Optional[str] = None + parent: Optional[str] = None + max_model_len: Optional[int] = None + permission: List[ModelPermission] = Field(default_factory=list) + + +class ModelList(OpenAIBaseModel): + object: str = "list" + data: List[ModelCard] = Field(default_factory=list) + + +class PromptTokensDetails(OpenAIBaseModel): + cached_tokens: int = 0 + + +class UsageInfo(OpenAIBaseModel): + prompt_tokens: int = 0 + total_tokens: int = 0 + completion_tokens: Optional[int] = 0 + reasoning_tokens: Optional[int] = None + prompt_tokens_details: Optional[PromptTokensDetails] = None + + +class RequestResponseMetadata(BaseModel): + request_id: str + final_usage_info: Optional[UsageInfo] = None + + +class JsonSchemaResponseFormat(OpenAIBaseModel): + name: str + description: Optional[str] = None + # schema is the field in openai but that causes conflicts with pydantic so + # instead use json_schema with an alias + json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema') + strict: Optional[bool] = None + + +class ResponseFormat(OpenAIBaseModel): + # type must be "json_schema", "json_object" or "text" + type: Literal["text", "json_object", "json_schema"] + json_schema: Optional[JsonSchemaResponseFormat] = None + + +class StreamOptions(OpenAIBaseModel): + include_usage: Optional[bool] = True + continuous_usage_stats: Optional[bool] = True + + +class FunctionDefinition(OpenAIBaseModel): + name: str + description: Optional[str] = None + parameters: Optional[Dict[str, Any]] = None + + +class ChatCompletionToolsParam(OpenAIBaseModel): + type: Literal["function"] = "function" + function: FunctionDefinition + + +class ChatCompletionNamedFunction(OpenAIBaseModel): + name: str + + +class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel): + function: ChatCompletionNamedFunction + type: Literal["function"] = "function" + + +class ChatCompletionRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/chat/create + messages: List[ChatCompletionMessageParam] + model: str + frequency_penalty: Optional[float] = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: Optional[bool] = False + top_logprobs: Optional[int] = 0 + max_tokens: Optional[int] = None + n: Optional[int] = 1 + presence_penalty: Optional[float] = 0.0 + response_format: Optional[ResponseFormat] = None + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max) + stop: Optional[Union[str, List[str]]] = Field(default_factory=list) + stream: Optional[bool] = False + stream_options: Optional[StreamOptions] = None + temperature: Optional[float] = 0.7 + top_p: Optional[float] = 1.0 + tools: Optional[List[ChatCompletionToolsParam]] = None + tool_choice: Optional[Union[Literal["none"], Literal["auto"], + ChatCompletionNamedToolChoiceParam]] = "none" + + # NOTE this will be ignored by VLLM -- the model determines the behavior + parallel_tool_calls: Optional[bool] = False + user: Optional[str] = None + + # doc: begin-chat-completion-sampling-params + best_of: Optional[int] = None + use_beam_search: bool = False + top_k: int = -1 + min_p: float = 0.0 + repetition_penalty: float = 1.0 + length_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = Field(default_factory=list) + include_stop_str_in_output: bool = False + ignore_eos: bool = False + min_tokens: int = 0 + skip_special_tokens: bool = True + spaces_between_special_tokens: bool = True + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + prompt_logprobs: Optional[int] = None + # doc: end-chat-completion-sampling-params + + # doc: begin-chat-completion-extra-params + echo: bool = Field( + default=False, + description=( + "If true, the new message will be prepended with the last message " + "if they belong to the same role."), + ) + add_generation_prompt: bool = Field( + default=True, + description= + ("If true, the generation prompt will be added to the chat template. " + "This is a parameter used by chat template in tokenizer config of the " + "model."), + ) + continue_final_message: bool = Field( + default=False, + description= + ("If this is set, the chat will be formatted so that the final " + "message in the chat is open-ended, without any EOS tokens. The " + "model will continue this message rather than starting a new one. " + "This allows you to \"prefill\" part of the model's response for it. " + "Cannot be used at the same time as `add_generation_prompt`."), + ) + add_special_tokens: bool = Field( + default=False, + description=( + "If true, special tokens (e.g. BOS) will be added to the prompt " + "on top of what is added by the chat template. " + "For most models, the chat template takes care of adding the " + "special tokens so this should be set to false (as is the " + "default)."), + ) + documents: Optional[List[Dict[str, str]]] = Field( + default=None, + description= + ("A list of dicts representing documents that will be accessible to " + "the model if it is performing RAG (retrieval-augmented generation)." + " If the template does not support RAG, this argument will have no " + "effect. We recommend that each document should be a dict containing " + "\"title\" and \"text\" keys."), + ) + chat_template: Optional[str] = Field( + default=None, + description=( + "A Jinja template to use for this conversion. " + "As of transformers v4.44, default chat template is no longer " + "allowed, so you must provide a chat template if the tokenizer " + "does not define one."), + ) + chat_template_kwargs: Optional[Dict[str, Any]] = Field( + default=None, + description=("Additional kwargs to pass to the template renderer. " + "Will be accessible by the chat template."), + ) + guided_json: Optional[Union[str, dict, BaseModel]] = Field( + default=None, + description=("If specified, the output will follow the JSON schema."), + ) + guided_regex: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the regex pattern."), + ) + guided_choice: Optional[List[str]] = Field( + default=None, + description=( + "If specified, the output will be exactly one of the choices."), + ) + guided_grammar: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the context free grammar."), + ) + guided_decoding_backend: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default guided decoding backend " + "of the server for this specific request. If set, must be either " + "'outlines' / 'lm-format-enforcer'")) + guided_whitespace_pattern: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default whitespace pattern " + "for guided json decoding.")) + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-chat-completion-extra-params + + def to_beam_search_params(self, + default_max_tokens: int) -> BeamSearchParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + n = self.n if self.n is not None else 1 + temperature = self.temperature if self.temperature is not None else 0.0 + + return BeamSearchParams( + beam_width=n, + max_tokens=max_tokens, + ignore_eos=self.ignore_eos, + temperature=temperature, + length_penalty=self.length_penalty, + ) + + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + prompt_logprobs = self.prompt_logprobs + if prompt_logprobs is None and self.echo: + prompt_logprobs = self.top_logprobs + + guided_json_object = None + guided_json_from_schema = None + if self.response_format is not None: + if self.response_format.type == "json_object": + guided_json_object = True + elif (self.response_format.type == "json_schema" + and self.response_format.json_schema is not None + and self.response_format.json_schema.json_schema is not None): + guided_json_from_schema = \ + self.response_format.json_schema.json_schema + + guided_decoding = GuidedDecodingParams.from_optional( + json=(self._get_guided_json_from_tool() + or self.guided_json + or guided_json_from_schema), + regex=self.guided_regex, + choice=self.guided_choice, + grammar=self.guided_grammar, + json_object=guided_json_object, + backend=self.guided_decoding_backend, + whitespace_pattern=self.guided_whitespace_pattern) + + return SamplingParams.from_optional( + n=self.n, + best_of=self.best_of, + presence_penalty=self.presence_penalty, + frequency_penalty=self.frequency_penalty, + repetition_penalty=self.repetition_penalty, + temperature=self.temperature, + top_p=self.top_p, + top_k=self.top_k, + min_p=self.min_p, + seed=self.seed, + stop=self.stop, + stop_token_ids=self.stop_token_ids, + logprobs=self.top_logprobs if self.logprobs else None, + prompt_logprobs=prompt_logprobs, + ignore_eos=self.ignore_eos, + max_tokens=max_tokens, + min_tokens=self.min_tokens, + skip_special_tokens=self.skip_special_tokens, + spaces_between_special_tokens=self.spaces_between_special_tokens, + include_stop_str_in_output=self.include_stop_str_in_output, + truncate_prompt_tokens=self.truncate_prompt_tokens, + output_kind=RequestOutputKind.DELTA if self.stream \ + else RequestOutputKind.FINAL_ONLY, + guided_decoding=guided_decoding, + logit_bias=self.logit_bias) + + def _get_guided_json_from_tool( + self) -> Optional[Union[str, dict, BaseModel]]: + # user has chosen to not use any tool + if self.tool_choice == "none" or self.tools is None: + return None + + # user has chosen to use a named tool + if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam: + tool_name = self.tool_choice.function.name + tools = {tool.function.name: tool.function for tool in self.tools} + if tool_name not in tools: + raise ValueError( + f"Tool '{tool_name}' has not been passed in `tools`.") + tool = tools[tool_name] + return tool.parameters + + return None + + @model_validator(mode="before") + @classmethod + def normalize_messages(cls, data): + """Normalize incoming messages before pydantic union validation. + + Real-world clients (e.g. from other providers) send assistant tool_call + messages with content=null, which fails the strict Union type check. + Replace null content with "" so validation passes. + reasoning_content is intentionally kept — chat_utils.py wraps it as + ... for multi-turn reasoning history. + """ + messages = data.get("messages") + if not isinstance(messages, list): + return data + normalized = [] + for msg in messages: + if not isinstance(msg, dict): + normalized.append(msg) + continue + if msg.get("content") is None: + if msg.get("reasoning_content") is None: + raise ValueError( + "Each message must have at least one of 'content' or " + "'reasoning_content'.") + msg = {**msg, "content": ""} + normalized.append(msg) + data = {**data, "messages": normalized} + return data + + @model_validator(mode="before") + @classmethod + def validate_stream_options(cls, data): + if data.get("stream_options") and not data.get("stream"): + raise ValueError( + "Stream options can only be defined when `stream=True`.") + + return data + + @model_validator(mode="before") + @classmethod + def check_logprobs(cls, data): + if (prompt_logprobs := data.get("prompt_logprobs")) is not None: + if data.get("stream") and prompt_logprobs > 0: + raise ValueError( + "`prompt_logprobs` are not available when `stream=True`.") + + if prompt_logprobs < 0: + raise ValueError("`prompt_logprobs` must be a positive value.") + + if (top_logprobs := data.get("top_logprobs")) is not None: + if top_logprobs < 0: + raise ValueError("`top_logprobs` must be a positive value.") + + if not data.get("logprobs"): + raise ValueError( + "when using `top_logprobs`, `logprobs` must be set to true." + ) + + return data + + @model_validator(mode="before") + @classmethod + def check_guided_decoding_count(cls, data): + if isinstance(data, ValueError): + raise data + + guide_count = sum([ + "guided_json" in data and data["guided_json"] is not None, + "guided_regex" in data and data["guided_regex"] is not None, + "guided_choice" in data and data["guided_choice"] is not None + ]) + # you can only use one kind of guided decoding + if guide_count > 1: + raise ValueError( + "You can only use one kind of guided decoding " + "('guided_json', 'guided_regex' or 'guided_choice').") + # you can only either use guided decoding or tools, not both + if guide_count > 1 and data.get("tool_choice", + "none") not in ("none", "auto"): + raise ValueError( + "You can only either use guided decoding or tools, not both.") + return data + + @model_validator(mode="before") + @classmethod + def check_tool_usage(cls, data): + + # if "tool_choice" is not specified but tools are provided, + # default to "auto" tool_choice + if "tool_choice" not in data and data.get("tools"): + data["tool_choice"] = "auto" + + # if "tool_choice" is specified -- validation + if "tool_choice" in data: + + # ensure that if "tool choice" is specified, tools are present + if "tools" not in data or data["tools"] is None: + raise ValueError( + "When using `tool_choice`, `tools` must be set.") + + # make sure that tool choice is either a named tool + # OR that it's set to "auto" + if data["tool_choice"] != "auto" and not isinstance( + data["tool_choice"], dict): + raise ValueError( + "`tool_choice` must either be a named tool or \"auto\". " + "`tool_choice=\"none\" is not supported.") + + # ensure that if "tool_choice" is specified as an object, + # it matches a valid tool + if isinstance(data["tool_choice"], dict): + valid_tool = False + specified_function = data["tool_choice"]["function"] + if not specified_function: + raise ValueError( + "Incorrectly formatted `tool_choice`. Should be like " + "`{\"type\": \"function\"," + " \"function\": {\"name\": \"my_function\"}}`") + specified_function_name = specified_function["name"] + if not specified_function_name: + raise ValueError( + "Incorrectly formatted `tool_choice`. Should be like " + "`{\"type\": \"function\", " + "\"function\": {\"name\": \"my_function\"}}`") + for tool in data["tools"]: + if tool["function"]["name"] == specified_function_name: + valid_tool = True + break + if not valid_tool: + raise ValueError( + "The tool specified in `tool_choice` does not match any" + " of the specified `tools`") + return data + + @model_validator(mode="before") + @classmethod + def check_generation_prompt(cls, data): + if data.get("continue_final_message") and data.get( + "add_generation_prompt"): + raise ValueError("Cannot set both `continue_final_message` and " + "`add_generation_prompt` to True.") + return data + + +class CompletionRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/completions/create + model: str + prompt: Union[List[int], List[List[int]], str, List[str]] + best_of: Optional[int] = None + echo: Optional[bool] = False + frequency_penalty: Optional[float] = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: Optional[int] = None + max_tokens: Optional[int] = 16 + n: int = 1 + presence_penalty: Optional[float] = 0.0 + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max) + stop: Optional[Union[str, List[str]]] = Field(default_factory=list) + stream: Optional[bool] = False + stream_options: Optional[StreamOptions] = None + suffix: Optional[str] = None + temperature: Optional[float] = 1.0 + top_p: Optional[float] = 1.0 + user: Optional[str] = None + + # doc: begin-completion-sampling-params + use_beam_search: bool = False + top_k: int = -1 + min_p: float = 0.0 + repetition_penalty: float = 1.0 + length_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = Field(default_factory=list) + include_stop_str_in_output: bool = False + ignore_eos: bool = False + min_tokens: int = 0 + skip_special_tokens: bool = True + spaces_between_special_tokens: bool = True + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + allowed_token_ids: Optional[List[int]] = None + prompt_logprobs: Optional[int] = None + # doc: end-completion-sampling-params + + # doc: begin-completion-extra-params + add_special_tokens: bool = Field( + default=True, + description=( + "If true (the default), special tokens (e.g. BOS) will be added to " + "the prompt."), + ) + response_format: Optional[ResponseFormat] = Field( + default=None, + description= + ("Similar to chat completion, this parameter specifies the format of " + "output. Only {'type': 'json_object'} or {'type': 'text' } is " + "supported."), + ) + guided_json: Optional[Union[str, dict, BaseModel]] = Field( + default=None, + description="If specified, the output will follow the JSON schema.", + ) + guided_regex: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the regex pattern."), + ) + guided_choice: Optional[List[str]] = Field( + default=None, + description=( + "If specified, the output will be exactly one of the choices."), + ) + guided_grammar: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the context free grammar."), + ) + guided_decoding_backend: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default guided decoding backend " + "of the server for this specific request. If set, must be one of " + "'outlines' / 'lm-format-enforcer'")) + guided_whitespace_pattern: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default whitespace pattern " + "for guided json decoding.")) + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-completion-extra-params + + def to_beam_search_params(self, + default_max_tokens: int) -> BeamSearchParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + n = self.n if self.n is not None else 1 + temperature = self.temperature if self.temperature is not None else 0.0 + + return BeamSearchParams( + beam_width=n, + max_tokens=max_tokens, + ignore_eos=self.ignore_eos, + temperature=temperature, + length_penalty=self.length_penalty, + ) + + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + prompt_logprobs = self.prompt_logprobs + if prompt_logprobs is None and self.echo: + prompt_logprobs = self.logprobs + + echo_without_generation = self.echo and self.max_tokens == 0 + + guided_json_object = None + guided_json_from_schema = None + if self.response_format is not None: + if self.response_format.type == "json_object": + guided_json_object = True + elif (self.response_format.type == "json_schema" + and self.response_format.json_schema is not None + and self.response_format.json_schema.json_schema is not None): + guided_json_from_schema = \ + self.response_format.json_schema.json_schema + + guided_decoding = GuidedDecodingParams.from_optional( + json=self.guided_json or guided_json_from_schema, + regex=self.guided_regex, + choice=self.guided_choice, + grammar=self.guided_grammar, + json_object=guided_json_object, + backend=self.guided_decoding_backend, + whitespace_pattern=self.guided_whitespace_pattern) + + return SamplingParams.from_optional( + n=self.n, + best_of=self.best_of, + presence_penalty=self.presence_penalty, + frequency_penalty=self.frequency_penalty, + repetition_penalty=self.repetition_penalty, + temperature=self.temperature, + top_p=self.top_p, + top_k=self.top_k, + min_p=self.min_p, + seed=self.seed, + stop=self.stop, + stop_token_ids=self.stop_token_ids, + logprobs=self.logprobs, + ignore_eos=self.ignore_eos, + max_tokens=max_tokens if not echo_without_generation else 1, + min_tokens=self.min_tokens, + prompt_logprobs=prompt_logprobs, + skip_special_tokens=self.skip_special_tokens, + spaces_between_special_tokens=self.spaces_between_special_tokens, + include_stop_str_in_output=self.include_stop_str_in_output, + truncate_prompt_tokens=self.truncate_prompt_tokens, + output_kind=RequestOutputKind.DELTA if self.stream \ + else RequestOutputKind.FINAL_ONLY, + guided_decoding=guided_decoding, + logit_bias=self.logit_bias, + allowed_token_ids=self.allowed_token_ids) + + @model_validator(mode="before") + @classmethod + def check_guided_decoding_count(cls, data): + guide_count = sum([ + "guided_json" in data and data["guided_json"] is not None, + "guided_regex" in data and data["guided_regex"] is not None, + "guided_choice" in data and data["guided_choice"] is not None + ]) + if guide_count > 1: + raise ValueError( + "You can only use one kind of guided decoding " + "('guided_json', 'guided_regex' or 'guided_choice').") + return data + + @model_validator(mode="before") + @classmethod + def check_logprobs(cls, data): + if (prompt_logprobs := data.get("prompt_logprobs")) is not None: + if data.get("stream") and prompt_logprobs > 0: + raise ValueError( + "`prompt_logprobs` are not available when `stream=True`.") + + if prompt_logprobs < 0: + raise ValueError("`prompt_logprobs` must be a positive value.") + + if (logprobs := data.get("logprobs")) is not None and logprobs < 0: + raise ValueError("`logprobs` must be a positive value.") + + return data + + @model_validator(mode="before") + @classmethod + def validate_stream_options(cls, data): + if data.get("stream_options") and not data.get("stream"): + raise ValueError( + "Stream options can only be defined when `stream=True`.") + + return data + + +class EmbeddingRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/embeddings + model: str + input: Union[List[int], List[List[int]], str, List[str]] + encoding_format: Literal["float", "base64"] = "float" + dimensions: Optional[int] = None + user: Optional[str] = None + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + + # doc: begin-embedding-pooling-params + additional_data: Optional[Any] = None + + # doc: end-embedding-pooling-params + + # doc: begin-embedding-extra-params + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-embedding-extra-params + + def to_pooling_params(self): + return PoolingParams(additional_data=self.additional_data) + + +class CompletionLogProbs(OpenAIBaseModel): + text_offset: List[int] = Field(default_factory=list) + token_logprobs: List[Optional[float]] = Field(default_factory=list) + tokens: List[str] = Field(default_factory=list) + top_logprobs: List[Optional[Dict[str, + float]]] = Field(default_factory=list) + + +class CompletionResponseChoice(OpenAIBaseModel): + index: int + text: str + logprobs: Optional[CompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = Field( + default=None, + description=( + "The stop string or token id that caused the completion " + "to stop, None if the completion finished for some other reason " + "including encountering the EOS token"), + ) + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None + + +class CompletionResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseChoice] + usage: UsageInfo + + +class CompletionResponseStreamChoice(OpenAIBaseModel): + index: int + text: str + logprobs: Optional[CompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = Field( + default=None, + description=( + "The stop string or token id that caused the completion " + "to stop, None if the completion finished for some other reason " + "including encountering the EOS token"), + ) + + +class CompletionStreamResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseStreamChoice] + usage: Optional[UsageInfo] = Field(default=None) + + +class EmbeddingResponseData(OpenAIBaseModel): + index: int + object: str = "embedding" + embedding: Union[List[float], str] + + +class EmbeddingResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "list" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + data: List[EmbeddingResponseData] + usage: UsageInfo + + +class FunctionCall(OpenAIBaseModel): + name: str + arguments: str + + +class ToolCall(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}") + type: Literal["function"] = "function" + function: FunctionCall + + +class DeltaFunctionCall(BaseModel): + name: Optional[str] = None + arguments: Optional[str] = None + + +# a tool call delta where everything is optional +class DeltaToolCall(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}") + type: Literal["function"] = "function" + index: int + function: Optional[DeltaFunctionCall] = None + + +class ExtractedToolCallInformation(BaseModel): + # indicate if tools were called + tools_called: bool + + # extracted tool calls + tool_calls: List[ToolCall] + + # content - per OpenAI spec, content AND tool calls can be returned rarely + # But some models will do this intentionally + content: Optional[str] = None + + +class ChatMessage(OpenAIBaseModel): + role: str + reasoning_content: Optional[str] = None + content: Optional[str] = None + tool_calls: List[ToolCall] = Field(default_factory=list) + + +class ChatCompletionLogProb(OpenAIBaseModel): + token: str + logprob: float = -9999.0 + bytes: Optional[List[int]] = None + + +class ChatCompletionLogProbsContent(ChatCompletionLogProb): + top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list) + + +class ChatCompletionLogProbs(OpenAIBaseModel): + content: Optional[List[ChatCompletionLogProbsContent]] = None + + +class ChatCompletionResponseChoice(OpenAIBaseModel): + index: int + message: ChatMessage + logprobs: Optional[ChatCompletionLogProbs] = None + # per OpenAI spec this is the default + finish_reason: Optional[str] = "stop" + # not part of the OpenAI spec but included in vLLM for legacy reasons + stop_reason: Optional[Union[int, str]] = None + + +class ChatCompletionResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") + object: Literal["chat.completion"] = "chat.completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseChoice] + usage: UsageInfo + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None + + +class DeltaMessage(OpenAIBaseModel): + role: Optional[str] = None + reasoning_content: Optional[str] = None + content: Optional[str] = None + tool_calls: List[DeltaToolCall] = Field(default_factory=list) + + +class ChatCompletionResponseStreamChoice(OpenAIBaseModel): + index: int + delta: DeltaMessage + logprobs: Optional[ChatCompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = None + + +class ChatCompletionStreamResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") + object: Literal["chat.completion.chunk"] = "chat.completion.chunk" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseStreamChoice] + usage: Optional[UsageInfo] = Field(default=None) + + +class BatchRequestInput(OpenAIBaseModel): + """ + The per-line object of the batch input file. + + NOTE: Currently only the `/v1/chat/completions` endpoint is supported. + """ + + # A developer-provided per-request id that will be used to match outputs to + # inputs. Must be unique for each request in a batch. + custom_id: str + + # The HTTP method to be used for the request. Currently only POST is + # supported. + method: str + + # The OpenAI API relative URL to be used for the request. Currently + # /v1/chat/completions is supported. + url: str + + # The parameters of the request. + body: Union[ChatCompletionRequest, EmbeddingRequest] + + +class BatchResponseData(OpenAIBaseModel): + # HTTP status code of the response. + status_code: int = 200 + + # An unique identifier for the API request. + request_id: str + + # The body of the response. + body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None + + +class BatchRequestOutput(OpenAIBaseModel): + """ + The per-line object of the batch output and error files + """ + + id: str + + # A developer-provided per-request id that will be used to match outputs to + # inputs. + custom_id: str + + response: Optional[BatchResponseData] + + # For requests that failed with a non-HTTP error, this will contain more + # information on the cause of the failure. + error: Optional[Any] + + +class TokenizeCompletionRequest(OpenAIBaseModel): + model: str + prompt: str + + add_special_tokens: bool = Field(default=True) + + +class TokenizeChatRequest(OpenAIBaseModel): + model: str + messages: List[ChatCompletionMessageParam] + + add_generation_prompt: bool = Field(default=True) + continue_final_message: bool = Field(default=False) + add_special_tokens: bool = Field(default=False) + + @model_validator(mode="before") + @classmethod + def check_generation_prompt(cls, data): + if data.get("continue_final_message") and data.get( + "add_generation_prompt"): + raise ValueError("Cannot set both `continue_final_message` and " + "`add_generation_prompt` to True.") + return data + + +TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest] + + +class TokenizeResponse(OpenAIBaseModel): + count: int + max_model_len: int + tokens: List[int] + + +class DetokenizeRequest(OpenAIBaseModel): + model: str + tokens: List[int] + + +class DetokenizeResponse(OpenAIBaseModel): + prompt: str + + +class LoadLoraAdapterRequest(BaseModel): + lora_name: str + lora_path: str + + +class UnloadLoraAdapterRequest(BaseModel): + lora_name: str + lora_int_id: Optional[int] = Field(default=None) diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py new file mode 100644 index 00000000..ca427609 --- /dev/null +++ b/qwen3_6_scripts/qwen3_5.py @@ -0,0 +1,1369 @@ +# Inference-only Qwen3.6-27B (Qwen3_5 architecture) for Iluvatar BI-V100. +# Pure-PyTorch DeltaNet (no fla / causal_conv1d dependency). +# Text-only (no VL, no MTP). + +from collections import OrderedDict +from typing import Dict, Iterable, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, sharded_weight_loader) +from vllm.model_executor.models.mamba_cache import MambaCacheManager +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors +from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE, + _get_graph_batch_size) +from vllm.logger import init_logger + +from vllm.model_executor.models.interfaces import HasInnerState, SupportsLoRA + +logger = init_logger(__name__) + + +# --------------------------------------------------------------------------- +# Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0) +# --------------------------------------------------------------------------- + +def _l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6) -> torch.Tensor: + return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps) + + +def _torch_causal_conv1d_update( + hidden_states: torch.Tensor, # (batch, channels, seq=1) + conv_state: torch.Tensor, # (batch, channels, state_len) modified in-place + weight: torch.Tensor, # (channels, kernel_size) + bias: Optional[torch.Tensor] = None, + activation: Optional[str] = None, +) -> torch.Tensor: + _, channels, seq_len = hidden_states.shape + state_len = conv_state.shape[-1] + cat = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype) + conv_state.copy_(cat[:, :, -state_len:]) + out = F.conv1d(cat, weight.unsqueeze(1), bias, padding=0, groups=channels) + out = out[:, :, -seq_len:] + if activation is not None: + out = F.silu(out) + return out.to(hidden_states.dtype) + + +def _torch_chunk_gated_delta_rule( + query: torch.Tensor, # (batch, seq, num_heads, head_k_dim) + key: torch.Tensor, + value: torch.Tensor, # (batch, seq, num_heads, head_v_dim) + g: torch.Tensor, # (batch, seq, num_heads) + beta: torch.Tensor, # (batch, seq, num_heads) + chunk_size: int = 64, + initial_state: Optional[torch.Tensor] = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + initial_dtype = query.dtype + if use_qk_l2norm_in_kernel: + query = _l2norm(query) + key = _l2norm(key) + # Transpose to (batch, num_heads, seq, dim) + query, key, value, beta, g = [ + x.transpose(1, 2).contiguous().to(torch.float32) + for x in (query, key, value, beta, g) + ] + batch, num_heads, seq_len, k_dim = key.shape + v_dim = value.shape[-1] + pad = (chunk_size - seq_len % chunk_size) % chunk_size + query = F.pad(query, (0, 0, 0, pad)) + key = F.pad(key, (0, 0, 0, pad)) + value = F.pad(value, (0, 0, 0, pad)) + beta = F.pad(beta, (0, pad)) + g = F.pad(g, (0, pad)) + total_len = seq_len + pad + scale = 1.0 / (query.shape[-1] ** 0.5) + query = query * scale + + v_beta = value * beta.unsqueeze(-1) + k_beta = key * beta.unsqueeze(-1) + query, key, value, k_beta, v_beta = [ + x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) + for x in (query, key, value, k_beta, v_beta) + ] + g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size) + mask_upper = torch.triu( + torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), + diagonal=0) + + g = g.cumsum(dim=-1) + decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril() + attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask_upper, 0) + for i in range(1, chunk_size): + row = attn[..., i, :i].clone() + sub = attn[..., :i, :i].clone() + attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2) + attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device) + value = attn @ v_beta + k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1)) + + last_state = ( + torch.zeros(batch, num_heads, k_dim, v_dim, dtype=value.dtype, device=value.device) + if initial_state is None + else initial_state.to(value) + ) + core_out = torch.zeros_like(value) + mask_upper2 = torch.triu( + torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), + diagonal=1) + + for i in range(total_len // chunk_size): + q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i] + attn_i = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]).masked_fill_(mask_upper2, 0) + v_prime = k_cumdecay[:, :, i] @ last_state + v_new = v_i - v_prime + attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state + core_out[:, :, i] = attn_inter + attn_i @ v_new + last_state = ( + last_state * g[:, :, i, -1, None, None].exp() + + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]) + .transpose(-1, -2) @ v_new + ) + + if not output_final_state: + last_state = None + core_out = core_out.reshape(batch, num_heads, -1, v_dim)[:, :, :seq_len] + core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype) + return core_out, last_state + +def _torch_recurrent_gated_delta_rule( + query: torch.Tensor, # (batch, 1, num_heads, head_k_dim) + key: torch.Tensor, + value: torch.Tensor, + g: torch.Tensor, # (batch, 1, num_heads) + beta: torch.Tensor, + initial_state: Optional[torch.Tensor] = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + initial_dtype = query.dtype + if use_qk_l2norm_in_kernel: + query = _l2norm(query) + key = _l2norm(key) + query, key, value, beta, g = [ + x.transpose(1, 2).contiguous().to(torch.float32) + for x in (query, key, value, beta, g) + ] + batch, num_heads, seq_len, k_dim = key.shape + v_dim = value.shape[-1] + scale = 1.0 / (query.shape[-1] ** 0.5) + query = query * scale + + core_out = torch.zeros(batch, num_heads, seq_len, v_dim, + dtype=value.dtype, device=value.device) + last_state = ( + torch.zeros(batch, num_heads, k_dim, v_dim, + dtype=value.dtype, device=value.device) + if initial_state is None + else initial_state.to(value) + ) + for t in range(seq_len): + q_t = query[:, :, t] + k_t = key[:, :, t] + v_t = value[:, :, t] + g_t = g[:, :, t].exp().unsqueeze(-1).unsqueeze(-1) + beta_t = beta[:, :, t].unsqueeze(-1) + last_state = last_state * g_t + kv_mem = (last_state * k_t.unsqueeze(-1)).sum(dim=-2) + delta = (v_t - kv_mem) * beta_t + last_state = last_state + k_t.unsqueeze(-1) * delta.unsqueeze(-2) + core_out[:, :, t] = (last_state * q_t.unsqueeze(-1)).sum(dim=-2) + + if not output_final_state: + last_state = None + core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype) + return core_out, last_state + + +# --------------------------------------------------------------------------- +# Gated RMSNorm (for DeltaNet output normalisation) +# --------------------------------------------------------------------------- + +class Qwen3_5RMSNormGated(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor, + gate: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hs = hidden_states.to(torch.float32) + variance = hs.pow(2).mean(-1, keepdim=True) + hs = hs * torch.rsqrt(variance + self.variance_epsilon) + hs = self.weight * hs.to(input_dtype) + return (hs * F.silu(gate.to(torch.float32))).to(input_dtype) + + +# --------------------------------------------------------------------------- +# Gated DeltaNet (linear_attention layers) +# --------------------------------------------------------------------------- + +class GatedDeltaNet(nn.Module): + def __init__( + self, + text_cfg, + layer_idx: int, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.hidden_size = text_cfg.hidden_size + self.num_v_heads = text_cfg.linear_num_value_heads # 48 + self.num_k_heads = text_cfg.linear_num_key_heads # 16 + self.head_k_dim = text_cfg.linear_key_head_dim # 128 + self.head_v_dim = text_cfg.linear_value_head_dim # 128 + self.key_dim = self.num_k_heads * self.head_k_dim # 2048 + self.value_dim = self.num_v_heads * self.head_v_dim # 6144 + self.conv_dim = self.key_dim * 2 + self.value_dim # 10240 + self.conv_kernel_size = text_cfg.linear_conv_kernel_dim # 4 + self.head_expand_ratio = self.num_v_heads // self.num_k_heads # 3 + + tp_size = get_tensor_model_parallel_world_size() + + # Sharded projections — MergedColumnParallelLinear shards each of q/k/v + # independently so each TP rank gets [q_shard, k_shard, v_shard]. + # Plain ColumnParallelLinear would shard contiguously, giving rank 0 + # [q_all, k_partial] — completely wrong Q/K/V after the split below. + self.in_proj_qkv = MergedColumnParallelLinear( + self.hidden_size, [self.key_dim, self.key_dim, self.value_dim], + bias=False, quant_config=quant_config) + self.in_proj_z = ColumnParallelLinear( + self.hidden_size, self.value_dim, + bias=False, quant_config=quant_config) + self.in_proj_b = ColumnParallelLinear( + self.hidden_size, self.num_v_heads, + bias=False, quant_config=quant_config) + self.in_proj_a = ColumnParallelLinear( + self.hidden_size, self.num_v_heads, + bias=False, quant_config=quant_config) + self.out_proj = RowParallelLinear( + self.value_dim, self.hidden_size, + bias=False, quant_config=quant_config) + + # Depthwise conv weight — sharded along channel dim (dim 0) + local_conv_dim = self.conv_dim // tp_size + self.conv1d_weight = nn.Parameter( + torch.empty(local_conv_dim, 1, self.conv_kernel_size)) + set_weight_attrs(self.conv1d_weight, { + "weight_loader": self._conv1d_weight_loader}) + + # Per-head scalar parameters — sharded along dim 0 + local_num_v = self.num_v_heads // tp_size + self.A_log = nn.Parameter(torch.zeros(local_num_v)) + self.dt_bias = nn.Parameter(torch.zeros(local_num_v)) + set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)}) + set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)}) + + # Gated RMSNorm on head_v_dim — replicated (head_v_dim=128 is small) + self.norm = Qwen3_5RMSNormGated(self.head_v_dim, + eps=text_cfg.rms_norm_eps) + + def _conv1d_weight_loader(self, param: torch.Tensor, + loaded_weight: torch.Tensor) -> None: + # loaded_weight: (conv_dim=10240, 1, kernel) ordered as [q, k, v] channels + # Must gather channels in the same non-contiguous pattern that + # MergedColumnParallelLinear uses for in_proj_qkv, so that each rank's + # conv1d_weight[i] applies to the correct in_proj_qkv output channel. + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + key_local = self.key_dim // tp_size # 512 with TP=4 + val_local = self.value_dim // tp_size # 1536 with TP=4 + q_s = loaded_weight[tp_rank * key_local : (tp_rank + 1) * key_local] + k_s = loaded_weight[self.key_dim + tp_rank * key_local : + self.key_dim + (tp_rank + 1) * key_local] + v_s = loaded_weight[2 * self.key_dim + tp_rank * val_local : + 2 * self.key_dim + (tp_rank + 1) * val_local] + param.data.copy_(torch.cat([q_s, k_s, v_s], dim=0)) + + def forward( + self, + hidden_states: torch.Tensor, # (total_tokens, hidden_size) + attn_metadata: AttentionMetadata, + conv_state: torch.Tensor, # (batch, local_conv_dim, kernel-1) in-place + temporal_state: torch.Tensor, # (batch, local_v_heads, k_dim, v_dim) in-place + ) -> torch.Tensor: + tp_size = get_tensor_model_parallel_world_size() + local_key_dim = self.key_dim // tp_size + local_val_dim = self.value_dim // tp_size + local_num_v = self.num_v_heads // tp_size + local_num_k = self.num_k_heads // tp_size + local_conv_dim = self.conv_dim // tp_size + + is_prefill = attn_metadata.num_prefill_tokens > 0 + + # Compute all projections for every token at once (batched, efficient) + mixed_qkv_all, _ = self.in_proj_qkv(hidden_states) # (total, local_conv_dim) + z_all, _ = self.in_proj_z(hidden_states) # (total, local_val_dim) + b_all, _ = self.in_proj_b(hidden_states) # (total, local_num_v) + a_all, _ = self.in_proj_a(hidden_states) # (total, local_num_v) + + if is_prefill: + seq_starts = attn_metadata.query_start_loc.tolist() + outputs = [] + state_len = self.conv_kernel_size - 1 + weight_2d = self.conv1d_weight.squeeze(1) # (local_conv_dim, kernel) + + for si in range(len(seq_starts) - 1): + s, e = int(seq_starts[si]), int(seq_starts[si + 1]) + seq_len = e - s + + # Shape: (1, local_conv_dim, seq_len) + mixed_qkv = (mixed_qkv_all[s:e] + .transpose(0, 1).unsqueeze(0) + .to(weight_2d.dtype)) + + # Load prev conv state BEFORE overwriting (needed for causal conv padding). + # For first prefill of a request: mamba_cache is zeros → correct. + # For chunked prefill chunk 2+: carries last state_len tokens from prev chunk. + prev_conv = conv_state[si:si + 1].clone().to(weight_2d.dtype) # [1, local_conv_dim, state_len] + + # Save conv state (last state_len positions) + if seq_len >= state_len: + conv_state[si].copy_(mixed_qkv[0, :, -state_len:]) + else: + conv_state[si, :, state_len - seq_len:].copy_( + mixed_qkv[0]) + conv_state[si, :, :state_len - seq_len] = 0 + + # Causal conv: left-pad with previous conv state (not zeros). + padded = torch.cat([prev_conv, mixed_qkv], dim=2) + mixed_qkv_conv = F.conv1d( + padded, self.conv1d_weight, + bias=None, padding=0, groups=local_conv_dim) + mixed_qkv_conv = F.silu(mixed_qkv_conv) + # (1, seq_len, local_conv_dim) + mixed_qkv_conv = mixed_qkv_conv.squeeze(0).transpose(0, 1).unsqueeze(0) + + q, k, v = torch.split( + mixed_qkv_conv, + [local_key_dim, local_key_dim, local_val_dim], dim=-1) + q = q.reshape(1, seq_len, local_num_k, self.head_k_dim) + k = k.reshape(1, seq_len, local_num_k, self.head_k_dim) + v = v.reshape(1, seq_len, local_num_v, self.head_v_dim) + + beta = b_all[s:e].sigmoid().unsqueeze(0) # (1, seq_len, local_num_v) + g = (-self.A_log.float().exp() + * F.softplus(a_all[s:e].float() + self.dt_bias) + ).unsqueeze(0) # (1, seq_len, local_num_v) + + # Expand k/q to match num_v_heads + q = q.repeat_interleave(self.head_expand_ratio, dim=2) + k = k.repeat_interleave(self.head_expand_ratio, dim=2) + + # Sub-sequence chunking: call _torch_chunk_gated_delta_rule + # on _DNN_CHUNK tokens at a time to cap peak memory. + # Full 18K: tensors [1,6,282,64,64]=220 MB each → ~990 MB/call. + # With _DNN_CHUNK=4096: [1,6,64,64,64]=6 MB each → ~137 MB/call. + # State is chained via initial_state / output_final_state. + _DNN_CHUNK = 4096 + cur_state = temporal_state[si:si + 1].clone() + core_out_parts = [] + for sc_start in range(0, seq_len, _DNN_CHUNK): + sc_end = min(sc_start + _DNN_CHUNK, seq_len) + c_out, cur_state = _torch_chunk_gated_delta_rule( + q[:, sc_start:sc_end], + k[:, sc_start:sc_end], + v[:, sc_start:sc_end], + g[:, sc_start:sc_end], + beta[:, sc_start:sc_end], + initial_state=cur_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + core_out_parts.append(c_out) + if cur_state is not None: + temporal_state[si].copy_(cur_state[0]) + # [1, seq_len, num_v_heads, head_v_dim] + core_out = torch.cat(core_out_parts, dim=1) + + # Gate + norm + output proj + z = z_all[s:e].reshape(seq_len, local_num_v, self.head_v_dim) + core_out = core_out.reshape(seq_len, local_num_v, self.head_v_dim) + normed = self.norm( + core_out.reshape(-1, self.head_v_dim), + z.reshape(-1, self.head_v_dim)) + normed = normed.reshape(seq_len, -1) + out, _ = self.out_proj(normed) + outputs.append(out) + + result = torch.cat(outputs, dim=0) + if torch.isnan(result).any(): + logger.warning("NaN in prefill GatedDeltaNet layer %d (frac=%.4f), replacing with zeros", + self.layer_idx, torch.isnan(result).float().mean().item()) + result = torch.nan_to_num(result, nan=0.0) + return result + + else: + # Decode: one token per sequence + num_seqs = hidden_states.shape[0] + weight_2d = self.conv1d_weight.squeeze(1) + + # (num_seqs, local_conv_dim, 1) + mixed_qkv = (mixed_qkv_all + .to(weight_2d.dtype) + .unsqueeze(-1)) + + mixed_qkv_conv = _torch_causal_conv1d_update( + mixed_qkv, conv_state, weight_2d, + bias=None, activation='silu') + # (num_seqs, local_conv_dim, 1) → (num_seqs, 1, local_conv_dim) + mixed_qkv_conv = mixed_qkv_conv.squeeze(-1).unsqueeze(1) + + q, k, v = torch.split( + mixed_qkv_conv, + [local_key_dim, local_key_dim, local_val_dim], dim=-1) + q = q.reshape(num_seqs, 1, local_num_k, self.head_k_dim) + k = k.reshape(num_seqs, 1, local_num_k, self.head_k_dim) + v = v.reshape(num_seqs, 1, local_num_v, self.head_v_dim) + + beta = b_all.sigmoid().unsqueeze(1) # (num_seqs, 1, local_num_v) + g = (-self.A_log.float().exp() + * F.softplus(a_all.float() + self.dt_bias) + ).unsqueeze(1) # (num_seqs, 1, local_num_v) + + q = q.repeat_interleave(self.head_expand_ratio, dim=2) + k = k.repeat_interleave(self.head_expand_ratio, dim=2) + + # Inlined decode recurrent step (seq_len=1). + # Replaces _torch_recurrent_gated_delta_rule to avoid 5 transpose+ + # contiguous+float32 copies, core_out allocation, and Python loop. + # Uses bmm/baddbmm_ to eliminate 3 large (B,H,k,v) intermediate tensors. + # temporal_state: (B, H_v, k_dim, v_dim) float32 — updated in-place. + orig_dtype = q.dtype + _scale = self.head_k_dim ** -0.5 + + q_t = _l2norm(q.squeeze(1)).float() * _scale # (B, H_v, k_dim) + k_t = _l2norm(k.squeeze(1)).float() # (B, H_v, k_dim) + v_t = v.squeeze(1).float() # (B, H_v, v_dim) + g_t = g.squeeze(1).float().exp_() # (B, H_v) + bt = beta.squeeze(1).float() # (B, H_v) + + # Decay state in-place: (B, H_v, k_dim, v_dim) *= scalar per head + temporal_state.mul_(g_t[:, :, None, None]) + + # Reshape to batched-matmul layout: (B*H_v, k_dim, v_dim) + ts_flat = temporal_state.view(-1, self.head_k_dim, self.head_v_dim) + BH = ts_flat.shape[0] + + # kv_mem = k_t @ temporal_state shape: (B*H_v, 1, k_dim) @ (B*H_v, k_dim, v_dim) + kv_mem = torch.bmm( + k_t.view(BH, 1, self.head_k_dim), ts_flat + ).view(num_seqs, local_num_v, self.head_v_dim) # (B, H_v, v_dim) + + delta = (v_t - kv_mem) * bt[:, :, None] # (B, H_v, v_dim) + + # State update: temporal_state += outer(k_t, delta) fused, no intermediate + ts_flat.baddbmm_( + k_t.view(BH, self.head_k_dim, 1), + delta.view(BH, 1, self.head_v_dim), + ) + + # Output: core_out = q_t @ updated temporal_state + core_out = torch.bmm( + q_t.view(BH, 1, self.head_k_dim), ts_flat + ).view(num_seqs, local_num_v, self.head_v_dim).to(orig_dtype) + # core_out: (B, H_v, v_dim) = (num_seqs, local_num_v, head_v_dim) already + + z = z_all.reshape(num_seqs, local_num_v, self.head_v_dim) + normed = self.norm( + core_out.reshape(-1, self.head_v_dim), + z.reshape(-1, self.head_v_dim)) + normed = normed.reshape(num_seqs, -1) + out, _ = self.out_proj(normed) + if torch.isnan(out).any(): + logger.warning("NaN in decode GatedDeltaNet layer %d (frac=%.4f), replacing with zeros", + self.layer_idx, torch.isnan(out).float().mean().item()) + out = torch.nan_to_num(out, nan=0.0) + return out + + +# --------------------------------------------------------------------------- +# Full Attention (with gated q — unique to Qwen3.5) +# --------------------------------------------------------------------------- + +class Qwen3_5FullAttention(nn.Module): + def __init__( + self, + text_cfg, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.hidden_size = text_cfg.hidden_size # 5120 + self.num_heads = text_cfg.num_attention_heads # 24 + self.num_kv_heads = text_cfg.num_key_value_heads # 4 + self.head_dim = text_cfg.head_dim # 256 + self.rms_norm_eps = text_cfg.rms_norm_eps + + tp_size = get_tensor_model_parallel_world_size() + self.local_num_heads = self.num_heads // tp_size + self.scaling = self.head_dim ** -0.5 + + # When num_kv_heads < tp_size we cannot shard KV further (would give + # fractional heads per rank). Use ReplicatedLinear so every rank holds + # all KV heads; local_num_kv_heads equals the full count. + # When num_kv_heads >= tp_size standard ColumnParallel sharding applies. + if tp_size > self.num_kv_heads: + # GQA-aware TP sharding: ixformer kernel only supports num_kv_heads=1 + # per rank. With num_kv_heads=2 < tp_size=4 we cannot shard KV + # evenly, but we CAN assign each rank the ONE KV head that serves + # its Q heads: + # q_per_kv = num_heads // num_kv_heads (e.g. 16//2 = 8) + # Rank r uses KV head r * local_num_heads // q_per_kv + # e.g. ranks 0,1 → KV head 0; ranks 2,3 → KV head 1. + # We replicate all KV heads to every rank and select in forward(). + self.proj_kv_heads = self.num_kv_heads # heads available from projection + self.local_num_kv_heads = 1 # heads after rank-local selection + self.q_per_kv_global = self.num_heads // self.num_kv_heads + self.k_proj = ReplicatedLinear( + self.hidden_size, self.num_kv_heads * self.head_dim, + bias=False, quant_config=quant_config) + self.v_proj = ReplicatedLinear( + self.hidden_size, self.num_kv_heads * self.head_dim, + bias=False, quant_config=quant_config) + else: + # Standard sharding: each rank gets num_kv_heads // tp_size heads. + self.local_num_kv_heads = self.num_kv_heads // tp_size + self.proj_kv_heads = self.local_num_kv_heads # already sharded + self.q_per_kv_global = None + self.k_proj = ColumnParallelLinear( + self.hidden_size, self.num_kv_heads * self.head_dim, + bias=False, quant_config=quant_config, + prefix=f"{prefix}.k_proj") + self.v_proj = ColumnParallelLinear( + self.hidden_size, self.num_kv_heads * self.head_dim, + bias=False, quant_config=quant_config, + prefix=f"{prefix}.v_proj") + + self.local_q_dim = self.local_num_heads * self.head_dim + self.local_kv_dim = self.local_num_kv_heads * self.head_dim + + # q_proj includes gate: output = num_heads * head_dim * 2 + self.q_proj = ColumnParallelLinear( + self.hidden_size, self.num_heads * self.head_dim * 2, + bias=False, quant_config=quant_config, + prefix=f"{prefix}.q_proj") + self.o_proj = RowParallelLinear( + self.num_heads * self.head_dim, self.hidden_size, + bias=False, quant_config=quant_config, + prefix=f"{prefix}.o_proj") + + self.q_norm = GemmaRMSNorm(self.head_dim, eps=self.rms_norm_eps) + self.k_norm = GemmaRMSNorm(self.head_dim, eps=self.rms_norm_eps) + + # Partial RoPE: rotary_dim = head_dim * partial_rotary_factor = 256 * 0.25 = 64 + rope_params = getattr(text_cfg, "rope_parameters", {}) or {} + rope_theta = rope_params.get("rope_theta", 10_000_000) + partial_factor = rope_params.get("partial_rotary_factor", 0.25) + rotary_dim = int(self.head_dim * partial_factor) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=rotary_dim, + max_position=text_cfg.max_position_embeddings, + base=rope_theta, + ) + + self.attn = Attention( + self.local_num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.local_num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + total_tokens = hidden_states.shape[0] + + # q_proj output includes gate (dim doubled) + qg, _ = self.q_proj(hidden_states) # (total, local_num_heads * head_dim * 2) + qg = qg.view(total_tokens, self.local_num_heads, self.head_dim * 2) + q = qg[:, :, :self.head_dim].reshape(total_tokens, -1) + gate = qg[:, :, self.head_dim:].reshape(total_tokens, -1) + + k, _ = self.k_proj(hidden_states) # (total, proj_kv_heads * head_dim) + v, _ = self.v_proj(hidden_states) + + # q_norm on local Q heads + q = self.q_norm.forward_cuda( + q.view(total_tokens, self.local_num_heads, self.head_dim) + .contiguous()).view(total_tokens, -1) + + # GQA-aware TP: select rank-local KV head BEFORE k_norm and rope so + # that ixformer kernels always see num_kv_heads=1 (same as 27B path). + # Doing k_norm/rope on 2 KV heads (proj_kv_heads=2) triggers ixformer + # paths that can produce NaN; restricting to 1 head avoids the issue. + if self.q_per_kv_global is not None: + tp_rank = get_tensor_model_parallel_rank() + kv_idx = (tp_rank * self.local_num_heads) // self.q_per_kv_global + k = (k.view(total_tokens, self.proj_kv_heads, self.head_dim) + [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head + v = (v.view(total_tokens, self.proj_kv_heads, self.head_dim) + [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head + + # k_norm on the (now always 1) rank-local KV head + k = self.k_norm.forward_cuda( + k.view(total_tokens, self.local_num_kv_heads, self.head_dim) + .contiguous()).view(total_tokens, -1) + + # rope: q=(T, local_num_heads*head_dim), k=(T, 1*head_dim) — mirrors 27B + q, k = self.rotary_emb(positions, q, k) + + attn_out = self.attn(q, k, v, kv_cache, attn_metadata) + + # Multiply by sigmoid gate before output projection + attn_out = attn_out * torch.sigmoid(gate.float()).to(attn_out.dtype) + output, _ = self.o_proj(attn_out) + return output + + +# --------------------------------------------------------------------------- +# MLP (SwiGLU, same as Qwen2/Qwen3) +# --------------------------------------------------------------------------- + +class Qwen3_5MLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, quant_config=quant_config) + self.down_proj = RowParallelLinear( + intermediate_size, hidden_size, + bias=False, quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}") + self.act_fn = SiluAndMul() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +# --------------------------------------------------------------------------- +# MoE sparse block (Qwen3.5-MoE / Qwen3.6-35B-A3B) +# --------------------------------------------------------------------------- + +class Qwen3_5MoeSparseBlock(nn.Module): + """Replaces Qwen3_5MLP for qwen3_5_moe_text layers. + + FusedMoE is used ONLY for weight storage and loading (create_weights / + weight_loader are pure PyTorch). Its forward kernel is bypassed because + ixformer on BI-V100 lacks vllm_moe_topk_softmax / vllm_invoke_fused_moe_kernel. + Routing and expert computation use a pure-PyTorch loop instead. + + Shared expert uses RowParallelLinear(reduce_results=False) so both paths + produce partial (pre-all-reduce) outputs that are combined before a single + all-reduce. + """ + + def __init__( + self, + text_cfg, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + hidden_size = text_cfg.hidden_size + self.num_experts = text_cfg.num_experts + self.top_k = text_cfg.num_experts_per_tok + + # Router: replicated (small: num_experts outputs) + self.gate = ReplicatedLinear(hidden_size, text_cfg.num_experts, + bias=False, quant_config=quant_config) + + # FusedMoE: only used for weight storage + weight_loader. + # Forward is bypassed — see _pure_pytorch_experts(). + self.experts = FusedMoE( + num_experts=text_cfg.num_experts, + top_k=text_cfg.num_experts_per_tok, + hidden_size=hidden_size, + intermediate_size=text_cfg.moe_intermediate_size, + reduce_results=False, # we do the all-reduce ourselves below + renormalize=True, + quant_config=quant_config, + ) + + # Shared expert: defer all-reduce to combine with routed output first + shared_size = text_cfg.shared_expert_intermediate_size + self.shared_expert_gate_up = MergedColumnParallelLinear( + hidden_size, [shared_size] * 2, bias=False, + quant_config=quant_config) + self.shared_expert_down = RowParallelLinear( + shared_size, hidden_size, bias=False, reduce_results=False, + quant_config=quant_config) + self.act_fn = SiluAndMul() + # Scalar sigmoid gate on shared expert output (same as Qwen2-MoE / Qwen3.5-MoE): + # shared_out *= sigmoid(shared_expert_gate(hidden_states)) + # Without this, shared expert is always fully active → wrong logits. + self.shared_expert_gate = ReplicatedLinear( + hidden_size, 1, bias=False, quant_config=quant_config) + + def _pure_pytorch_experts( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + ) -> torch.Tensor: + """Pure-PyTorch MoE (ixformer has no MoE kernels on BI-V100). + + w13_weight: (num_experts, 2*inter_per_partition, hidden) [TP-sharded] + w2_weight: (num_experts, hidden, inter_per_partition) [TP-sharded] + Output is partial (pre-all-reduce), same contract as FusedMoE + with reduce_results=False. + """ + # Routing: softmax → topk → renormalise + routing_weights = torch.softmax(router_logits.float(), dim=-1) + topk_weights, topk_ids = torch.topk( + routing_weights, self.top_k, dim=-1) # (T, top_k) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_weights = topk_weights.to(hidden_states.dtype) + + w13 = self.experts.w13_weight # (E, 2*I, H) + w2 = self.experts.w2_weight # (E, H, I) + + T = hidden_states.shape[0] + if T == 1: + # Fast path: single token (decode). + # Batched GEMM: replace top_k separate F.linear calls with 2 fused ops. + # gate_up: 1 large GEMM (1,H) × (K*2*I,H)^T → (1, K*2*I) + # down: 1 bmm (K,H,I) @ (K,I,1) → (K,H) + # Total: 3 kernel launches vs previous 16 (top_k*2). + eids = topk_ids[0] # (K,) + ws = topk_weights[0].to(hidden_states.dtype) # (K,) + w13_sel = w13[eids] # (K, 2*I, H) + w2_sel = w2[eids] # (K, H, I) + + H = hidden_states.shape[-1] + + gate_up = F.linear( + hidden_states, + w13_sel.reshape(-1, H), # (K*2*I, H) — contiguous after indexing + ) # (1, K*2*I) + gate_up = gate_up.view(self.top_k, -1) # (K, 2*I) + gate, up = gate_up.chunk(2, dim=-1) # (K, I) each + act = F.silu(gate) * up # (K, I) + + # bmm: (K,H,I) @ (K,I,1) → (K,H,1) → (K,H) + expert_out = torch.bmm(w2_sel, act.unsqueeze(-1)).squeeze(-1) # (K, H) + + out = (expert_out * ws.unsqueeze(-1)).sum(0, keepdim=True).to( + hidden_states.dtype) # (1, H) + else: + # General path (prefill / multi-seq): loop over unique active experts. + # At most T*top_k unique experts, always <= num_experts. + out = torch.zeros_like(hidden_states) + unique_eids = topk_ids.view(-1).unique().tolist() + for eid in unique_eids: + eid = int(eid) + mask = (topk_ids == eid) # (T, top_k) + tok_ids, topk_pos = mask.nonzero(as_tuple=True) + tokens = hidden_states[tok_ids] # (n, H) + gate_up = F.linear(tokens, w13[eid]) # (n, 2*I) + gate, up = gate_up.chunk(2, dim=-1) + act = F.silu(gate) * up # (n, I) + expert_out = F.linear(act, w2[eid]) # (n, H) + weights = topk_weights[tok_ids, topk_pos].unsqueeze(-1) + out.index_add_(0, tok_ids, (expert_out * weights).to(out.dtype)) + + return out # partial, all-reduce done in forward() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + router_logits, _ = self.gate(hidden_states) + routed_out = self._pure_pytorch_experts(hidden_states, router_logits) + + gate_up, _ = self.shared_expert_gate_up(hidden_states) + shared_out = self.act_fn(gate_up) + shared_out, _ = self.shared_expert_down(shared_out) + # Scalar sigmoid gate (Qwen2-MoE / Qwen3.5-MoE style) + gate_score, _ = self.shared_expert_gate(hidden_states) # (T, 1) + shared_out = shared_out * torch.sigmoid(gate_score) + + out = routed_out + shared_out + if self.experts.tp_size > 1: + out = tensor_model_parallel_all_reduce(out) + return out + + +# --------------------------------------------------------------------------- +# Decoder layer (dispatches to GatedDeltaNet or Qwen3_5FullAttention) +# --------------------------------------------------------------------------- + + +class Qwen3_5DecoderLayer(nn.Module): + def __init__( + self, + text_cfg, + layer_idx: int, + layer_type: str, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.layer_type = layer_type + self.input_layernorm = GemmaRMSNorm(text_cfg.hidden_size, + eps=text_cfg.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm(text_cfg.hidden_size, + eps=text_cfg.rms_norm_eps) + + if layer_type == "linear_attention": + self.linear_attn = GatedDeltaNet(text_cfg, layer_idx, + quant_config=quant_config) + else: + self.self_attn = Qwen3_5FullAttention( + text_cfg, layer_idx, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"layers.{layer_idx}.self_attn", + ) + + if getattr(text_cfg, 'model_type', '') == 'qwen3_5_moe_text': + self.mlp = Qwen3_5MoeSparseBlock(text_cfg, quant_config=quant_config) + else: + self.mlp = Qwen3_5MLP( + hidden_size=text_cfg.hidden_size, + intermediate_size=text_cfg.intermediate_size, + hidden_act=text_cfg.hidden_act, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: Optional[torch.Tensor], + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + # Only for linear_attention layers: + conv_state: Optional[torch.Tensor] = None, + temporal_state: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + if self.layer_type == "linear_attention": + hidden_states = self.linear_attn( + hidden_states, attn_metadata, conv_state, temporal_state) + else: + hidden_states = self.self_attn( + positions, hidden_states, kv_cache, attn_metadata) + + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + +# --------------------------------------------------------------------------- +# Full transformer model +# --------------------------------------------------------------------------- + +class Qwen3_5Model(nn.Module): + def __init__( + self, + text_cfg, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.text_cfg = text_cfg + self.embed_tokens = VocabParallelEmbedding( + text_cfg.vocab_size, text_cfg.hidden_size) + self.layers = nn.ModuleList([ + Qwen3_5DecoderLayer( + text_cfg, i, text_cfg.layer_types[i], + cache_config=cache_config, quant_config=quant_config) + for i in range(text_cfg.num_hidden_layers) + ]) + self.norm = GemmaRMSNorm(text_cfg.hidden_size, eps=text_cfg.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + conv_states: torch.Tensor, # (num_linear_layers, batch, ...) + temporal_states: torch.Tensor, # (num_linear_layers, batch, ...) + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + residual = None + + attn_idx = 0 + linear_idx = 0 + for layer in self.layers: + if layer.layer_type == "linear_attention": + hidden_states, residual = layer( + positions, hidden_states, + kv_cache=None, + attn_metadata=attn_metadata, + residual=residual, + conv_state=conv_states[linear_idx], + temporal_state=temporal_states[linear_idx], + ) + linear_idx += 1 + else: + kv_cache = kv_caches[attn_idx] + hidden_states, residual = layer( + positions, hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + residual=residual, + ) + attn_idx += 1 + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +# --------------------------------------------------------------------------- +# Top-level CausalLM wrapper with MambaCacheManager +# --------------------------------------------------------------------------- + +class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA): + + has_inner_state = True + supports_lora = True + + packed_modules_mapping = { + "gate_up_proj": ["gate_proj", "up_proj"], + } + + supported_lora_modules = [ + "gate_up_proj", + "down_proj", + "o_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config, # Qwen3_5Config (top-level) + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + scheduler_config: Optional[SchedulerConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.scheduler_config = scheduler_config + + # The text config holds all architecture parameters + text_cfg = config.text_config + self.text_cfg = text_cfg + + # Pre-compute counts + self.num_linear_layers = sum( + 1 for lt in text_cfg.layer_types if lt == "linear_attention") + self.num_attn_layers = sum( + 1 for lt in text_cfg.layer_types if lt == "full_attention") + + # DeltaNet state dimensions (per layer, per sequence, TP-sharded) + tp_size = get_tensor_model_parallel_world_size() + self.conv_dim = (text_cfg.linear_num_key_heads * text_cfg.linear_key_head_dim * 2 + + text_cfg.linear_num_value_heads * text_cfg.linear_value_head_dim) + self.num_v_heads = text_cfg.linear_num_value_heads + self.head_k_dim = text_cfg.linear_key_head_dim + self.head_v_dim = text_cfg.linear_value_head_dim + self.conv_kernel_size = text_cfg.linear_conv_kernel_dim + + self.model = Qwen3_5Model( + text_cfg, + cache_config=cache_config, + quant_config=quant_config, + ) + + self.lm_head = ParallelLMHead( + text_cfg.vocab_size, text_cfg.hidden_size, + quant_config=quant_config, + ) + + self.logits_processor = LogitsProcessor(text_cfg.vocab_size) + self.sampler = Sampler() + + # Lazy initialised in first forward call + self.mamba_cache: Optional[MambaCacheManager] = None + + # GDN prefix state cache (align mode): stores (conv_states, temporal_states) snapshots + # at KV-block boundaries so that prefix-cache-hit requests can restore correct GDN state. + # Key: tuple of physical block IDs covering the cached prefix + # Value: (conv_states_cpu, temporal_states_cpu) each of shape (num_gdn_layers, ...) + self._gdn_prefix_cache: OrderedDict = OrderedDict() + self._gdn_prefix_cache_max: int = 16 # ~16 × 16 MB ≈ 256 MB CPU RAM + self._block_size: int = (cache_config.block_size + if cache_config is not None else 16) + + def _get_mamba_cache_shape(self): + tp_size = get_tensor_model_parallel_world_size() + # Each sequence's state is stored in float32 + conv_state_shape = (self.conv_dim // tp_size, self.conv_kernel_size - 1) + temporal_state_shape = ( + self.num_v_heads // tp_size, self.head_k_dim, self.head_v_dim) + return conv_state_shape, temporal_state_shape + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs, + ) -> torch.Tensor: + if self.mamba_cache is None: + if self.scheduler_config is not None: + max_batch_size = _get_graph_batch_size( + self.scheduler_config.max_num_seqs) + else: + max_batch_size = max(_BATCH_SIZES_TO_CAPTURE) + 2 + self.mamba_cache = MambaCacheManager( + torch.float32, + self.num_linear_layers, + max_batch_size, + *self._get_mamba_cache_shape(), + ) + + mamba_tensors = self.mamba_cache.current_run_tensors( + input_ids, attn_metadata, **kwargs) + # conv_states: (num_linear_layers, batch, local_conv_dim, kernel-1) + # temporal_states: (num_linear_layers, batch, local_num_v, k_dim, v_dim) + conv_states, temporal_states = mamba_tensors + + # ── GDN prefix-cache align mode: inject saved state on prefix hit ───── + # Conditions: prefill pass, batch=1, context_len > 0 (prefix cached or + # previous chunk already processed), block_tables available. + # We always attempt a lookup: for subsequent chunked-prefill chunks the + # key matches our own saved state (same data already in slot → no-op). + # For a true cross-request prefix hit the key matches a previous request. + _is_single_seq_prefill = ( + attn_metadata is not None + and attn_metadata.num_prefill_tokens > 0 + and conv_states.shape[1] == 1 # batch == 1 + and getattr(attn_metadata, 'context_lens_tensor', None) is not None + and getattr(attn_metadata, 'block_tables', None) is not None + and attn_metadata.block_tables.numel() > 0 + ) + if _is_single_seq_prefill: + context_len = int(attn_metadata.context_lens_tensor[0].item()) + if context_len > 0: + num_prefix_blocks = context_len // self._block_size + if (num_prefix_blocks > 0 + and attn_metadata.block_tables.shape[1] >= num_prefix_blocks): + lookup_key = tuple( + attn_metadata.block_tables[0, :num_prefix_blocks] + .cpu().tolist()) + if lookup_key in self._gdn_prefix_cache: + saved_conv, saved_temporal = self._gdn_prefix_cache[lookup_key] + conv_states[:, 0].copy_( + saved_conv.to(conv_states.device), non_blocking=True) + temporal_states[:, 0].copy_( + saved_temporal.to(temporal_states.device), non_blocking=True) + self._gdn_prefix_cache.move_to_end(lookup_key) + logger.debug("GDN prefix cache hit: prefix_len=%d blocks=%d", + context_len, num_prefix_blocks) + # ── End inject ────────────────────────────────────────────────────────── + + hidden_states = self.model( + input_ids, positions, kv_caches, attn_metadata, + conv_states, temporal_states) + + # ── GDN prefix-cache align mode: save state after this prefill chunk ─── + # Save state keyed by ALL complete KV blocks processed so far. + # Next requests reusing this prefix will restore from here. + if _is_single_seq_prefill: + context_len = int(attn_metadata.context_lens_tensor[0].item()) + query_len = attn_metadata.num_prefill_tokens + total_processed = context_len + query_len + num_complete_blocks = total_processed // self._block_size + if (num_complete_blocks > 0 + and attn_metadata.block_tables.shape[1] >= num_complete_blocks): + save_key = tuple( + attn_metadata.block_tables[0, :num_complete_blocks] + .cpu().tolist()) + # Move to end (LRU: most recent = last) and update value + if save_key in self._gdn_prefix_cache: + self._gdn_prefix_cache.move_to_end(save_key) + self._gdn_prefix_cache[save_key] = ( + conv_states[:, 0].cpu().clone(), + temporal_states[:, 0].cpu().clone(), + ) + # Evict oldest entries beyond max + while len(self._gdn_prefix_cache) > self._gdn_prefix_cache_max: + self._gdn_prefix_cache.popitem(last=False) + # ── End save ──────────────────────────────────────────────────────────── + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + # All TP ranks must call logits_processor to participate in the NCCL + # gather inside lm_head. Non-driver ranks return None after the gather. + # With chunked prefill, intermediate chunks have seq_groups=None on all + # ranks; _apply_logits_processors is guarded against this in + # logits_processor.py (patched by patch_xformers_sdpa_seq.py). + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.sampler(logits, sampling_metadata) + + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + return self.mamba_cache.copy_inputs_before_cuda_graphs( + input_buffers, **kwargs) + + def get_seqlen_agnostic_capture_inputs(self, batch_size: int): + return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, weight_name, shard_id) + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + + for name, loaded_weight in weights: + # Skip vision and MTP branches + if (name.startswith("model.visual") + or name.startswith("mtp.") + or name.startswith("model.mtp")): + continue + + # Prefix remapping: checkpoint may wrap under language_model + if name.startswith("model.language_model."): + name = "model." + name[len("model.language_model."):] + + # Skip positional embedding caches + if "rotary_emb.inv_freq" in name: + continue + + # Remap conv1d.weight → conv1d_weight + # The conv has depth (1) dim in the checkpoint that we handle separately + if ".linear_attn.conv1d.weight" in name: + name = name.replace(".linear_attn.conv1d.weight", + ".linear_attn.conv1d_weight") + + # Stacked param loading (gate_up_proj) + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if name.endswith(".bias") and name not in params_dict: + break + if name not in params_dict: + break + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if name.endswith(".bias") and name not in params_dict: + continue + if name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + +# --------------------------------------------------------------------------- +# Qwen3.6-35B-A3B (Qwen3_5-MoE architecture) +# --------------------------------------------------------------------------- + +class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): + """Qwen3.6-35B-A3B: same hybrid-attention backbone as 27B, dense MLP + replaced by Qwen3_5MoeSparseBlock (256 routed experts + shared expert). + Only load_weights differs from the dense variant. + """ + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + # Checkpoint key format for this model (transformers Qwen3_5MoeExperts): + # mlp.experts.gate_up_proj shape (num_experts, 2*intermediate, hidden) + # mlp.experts.down_proj shape (num_experts, hidden, intermediate) + # mlp.gate.weight shape (num_experts, hidden) [router] + # mlp.shared_expert.{gate,up,down}_proj.weight [shared MLP] + # Our FusedMoE stores: + # mlp.experts.w13_weight shape (num_experts, 2*intermediate//tp, hidden) + # mlp.experts.w2_weight shape (num_experts, hidden, intermediate//tp) + # Our shared expert stores: + # mlp.shared_expert_gate_up.weight (merged gate+up) + # mlp.shared_expert_down.weight + + stacked_params_mapping = [ + # (param_name, weight_name, shard_id) + # shared expert + ("shared_expert_gate_up", "shared_expert.gate_proj", 0), + ("shared_expert_gate_up", "shared_expert.up_proj", 1), + # linear_attention dense proj (same as 27B) + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + params_dict = dict(self.named_parameters()) + + for name, loaded_weight in weights: + # Skip vision and MTP branches + if (name.startswith("model.visual") + or name.startswith("mtp.") + or name.startswith("model.mtp")): + continue + + # Prefix remapping for VL checkpoint (Qwen3_5MoeForConditionalGeneration): + # model.language_model.model.{layers,embed_tokens,norm} -> model.{...} + # model.language_model.lm_head -> lm_head + # Prefix remapping: checkpoint may wrap under language_model + if name.startswith("model.language_model."): + name = "model." + name[len("model.language_model."):] + + if "rotary_emb.inv_freq" in name: + continue + + if ".linear_attn.conv1d.weight" in name: + name = name.replace(".linear_attn.conv1d.weight", + ".linear_attn.conv1d_weight") + + # --- Fused routed-expert weights (all experts in one tensor) --- + + if "mlp.experts.gate_up_proj" in name: + # loaded_weight: (num_experts, 2*intermediate, hidden) + w13_name = name.replace("mlp.experts.gate_up_proj", + "mlp.experts.w13_weight") + if w13_name not in params_dict: + continue + param = params_dict[w13_name] + n_exp = loaded_weight.shape[0] + inter = loaded_weight.shape[1] // 2 + gate_w = loaded_weight[:, :inter, :].contiguous() + up_w = loaded_weight[:, inter:, :].contiguous() + for eid in range(n_exp): + param.weight_loader(param, gate_w[eid], "w1_weight", "w1", eid) + param.weight_loader(param, up_w[eid], "w3_weight", "w3", eid) + continue + + if "mlp.experts.down_proj" in name: + # loaded_weight: (num_experts, hidden, intermediate) + w2_name = name.replace("mlp.experts.down_proj", + "mlp.experts.w2_weight") + if w2_name not in params_dict: + continue + param = params_dict[w2_name] + n_exp = loaded_weight.shape[0] + for eid in range(n_exp): + param.weight_loader(param, loaded_weight[eid], "w2_weight", "w2", eid) + continue + + # --- Shared expert down_proj rename --- + if "mlp.shared_expert.down_proj" in name: + name = name.replace("mlp.shared_expert.down_proj", + "mlp.shared_expert_down") + if name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + continue + + # --- Individual expert weights (FT checkpoint: experts.{i}.{proj}.weight) --- + # Standard transformers fine-tuning saves each expert separately instead of + # the pre-merged (num_experts, ...) tensors in the original checkpoint. + if ".mlp.experts." in name: + parts = name.split(".mlp.experts.", 1) + expert_rest = parts[1] # e.g. "0.gate_proj.weight" + dot_pos = expert_rest.find(".") + if dot_pos > 0 and expert_rest[:dot_pos].isdigit(): + eid = int(expert_rest[:dot_pos]) + proj_raw = expert_rest[dot_pos + 1:] + proj = proj_raw[:-7] if proj_raw.endswith(".weight") else proj_raw + prefix = parts[0] # e.g. "model.layers.0" + if proj == "gate_proj": + w13_name = f"{prefix}.mlp.experts.w13_weight" + if w13_name in params_dict: + param = params_dict[w13_name] + param.weight_loader(param, loaded_weight, "w1_weight", "w1", eid) + elif proj == "up_proj": + w13_name = f"{prefix}.mlp.experts.w13_weight" + if w13_name in params_dict: + param = params_dict[w13_name] + param.weight_loader(param, loaded_weight, "w3_weight", "w3", eid) + elif proj == "down_proj": + w2_name = f"{prefix}.mlp.experts.w2_weight" + if w2_name in params_dict: + param = params_dict[w2_name] + param.weight_loader(param, loaded_weight, "w2_weight", "w2", eid) + continue + + # --- Stacked / standard weights --- + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if name not in params_dict: + break + param = params_dict[name] + param.weight_loader(param, loaded_weight, shard_id) + break + else: + if name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/qwen3_6_scripts/qwen3_5/__init__.py b/qwen3_6_scripts/qwen3_5/__init__.py new file mode 100644 index 00000000..168e15ff --- /dev/null +++ b/qwen3_6_scripts/qwen3_5/__init__.py @@ -0,0 +1,3 @@ +from .configuration_qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig, Qwen3_5VisionConfig + +__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"] diff --git a/qwen3_6_scripts/qwen3_5/configuration_qwen3_5.py b/qwen3_6_scripts/qwen3_5/configuration_qwen3_5.py new file mode 100644 index 00000000..afe21f79 --- /dev/null +++ b/qwen3_6_scripts/qwen3_5/configuration_qwen3_5.py @@ -0,0 +1,188 @@ +# Adapted from transformers 5.2.0 for compatibility with transformers 4.55.3 + torch 2.1.0 +# Stubs layer_type_validation and RopeParameters which do not exist in 4.55.3 + +from typing import Optional, List + +from ...configuration_utils import PretrainedConfig as PreTrainedConfig + +# --- Local stubs for APIs not present in transformers 4.55.3 --- +# Always use these definitions; do NOT import from the older transformers +# as same-named functions there have incompatible signatures. + +def layer_type_validation(layer_types, num_hidden_layers=None, attention=True): + allowed = {"full_attention", "linear_attention"} + if not all(lt in allowed for lt in layer_types): + raise ValueError(f"layer_types entries must be in {allowed}, got {layer_types}") + if num_hidden_layers is not None and num_hidden_layers != len(layer_types): + raise ValueError( + f"num_hidden_layers ({num_hidden_layers}) != len(layer_types) ({len(layer_types)})" + ) + +try: + from typing import TypedDict + class RopeParameters(TypedDict, total=False): + rope_theta: float + rope_type: str + partial_rotary_factor: float + factor: float +except Exception: + RopeParameters = dict + +# --- End stubs --- + + +class Qwen3_5TextConfig(PreTrainedConfig): + r""" + Configuration for the text backbone of Qwen3.5 / Qwen3.6-27B models. + model_type is "qwen3_5_text" (used internally by the nested config). + """ + + model_type = "qwen3_5_text" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=248320, + hidden_size=4096, + intermediate_size=12288, + num_hidden_layers=32, + num_attention_heads=16, + num_key_value_heads=4, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + tie_word_embeddings=False, + rope_parameters=None, + attention_bias=False, + attention_dropout=0.0, + head_dim=256, + linear_conv_kernel_dim=4, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=32, + layer_types=None, + pad_token_id=None, + bos_token_id=None, + eos_token_id=None, + **kwargs, + ): + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.tie_word_embeddings = tie_word_embeddings + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.head_dim = head_dim + self.rope_parameters = rope_parameters + kwargs.setdefault("partial_rotary_factor", 0.25) + + self.layer_types = layer_types + if self.layer_types is None: + interval_pattern = kwargs.get("full_attention_interval", 4) + self.layer_types = [ + "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention" + for i in range(self.num_hidden_layers) + ] + layer_type_validation(self.layer_types, self.num_hidden_layers) + + self.linear_conv_kernel_dim = linear_conv_kernel_dim + self.linear_key_head_dim = linear_key_head_dim + self.linear_value_head_dim = linear_value_head_dim + self.linear_num_key_heads = linear_num_key_heads + self.linear_num_value_heads = linear_num_value_heads + super().__init__(**kwargs) + + +class Qwen3_5VisionConfig(PreTrainedConfig): + model_type = "qwen3_5_vision" + + def __init__( + self, + depth=27, + hidden_size=1152, + hidden_act="gelu_pytorch_tanh", + intermediate_size=4304, + num_heads=16, + in_channels=3, + patch_size=16, + spatial_merge_size=2, + temporal_patch_size=2, + out_hidden_size=3584, + num_position_embeddings=2304, + initializer_range=0.02, + **kwargs, + ): + super().__init__(**kwargs) + self.depth = depth + self.hidden_size = hidden_size + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.num_heads = num_heads + self.in_channels = in_channels + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.out_hidden_size = out_hidden_size + self.num_position_embeddings = num_position_embeddings + self.initializer_range = initializer_range + + +class Qwen3_5Config(PreTrainedConfig): + r""" + Top-level configuration for Qwen3.5 / Qwen3.6-27B. + model_type = "qwen3_5" matches the model card / config.json. + Wraps Qwen3_5TextConfig (and optionally Qwen3_5VisionConfig for multimodal use). + For vLLM text-only inference only text_config is consumed. + """ + + model_type = "qwen3_5" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config=None, + vision_config=None, + image_token_id=248056, + video_token_id=248057, + vision_start_token_id=248053, + vision_end_token_id=248054, + tie_word_embeddings=False, + **kwargs, + ): + if isinstance(text_config, dict): + self.text_config = Qwen3_5TextConfig(**text_config) + elif text_config is None: + self.text_config = Qwen3_5TextConfig() + else: + self.text_config = text_config + + if isinstance(vision_config, dict): + self.vision_config = Qwen3_5VisionConfig(**vision_config) + elif vision_config is None: + self.vision_config = Qwen3_5VisionConfig() + else: + self.vision_config = vision_config + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.vision_end_token_id = vision_end_token_id + self.tie_word_embeddings = tie_word_embeddings + super().__init__(**kwargs) + + +__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"] diff --git a/qwen3_6_scripts/qwen3_5_moe/__init__.py b/qwen3_6_scripts/qwen3_5_moe/__init__.py new file mode 100644 index 00000000..6376ee8b --- /dev/null +++ b/qwen3_6_scripts/qwen3_5_moe/__init__.py @@ -0,0 +1,3 @@ +from .configuration_qwen3_5_moe import Qwen3_5MoeConfig, Qwen3_5MoeTextConfig + +__all__ = ["Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig"] diff --git a/qwen3_6_scripts/qwen3_5_moe/configuration_qwen3_5_moe.py b/qwen3_6_scripts/qwen3_5_moe/configuration_qwen3_5_moe.py new file mode 100644 index 00000000..50734aff --- /dev/null +++ b/qwen3_6_scripts/qwen3_5_moe/configuration_qwen3_5_moe.py @@ -0,0 +1,198 @@ +# Adapted from transformers 5.2.0 for compatibility with transformers 4.55.3 + torch 2.1.0 +# Source: transformers/models/qwen3_5_moe/configuration_qwen3_5_moe.py +# Stubs layer_type_validation and RopeParameters which do not exist in 4.55.3 +# Removes ignore_keys_at_rope_validation / base_model_tp_plan / base_model_pp_plan +# which are 5.x-only and irrelevant for vLLM inference. + +from typing import Optional + +from ...configuration_utils import PretrainedConfig as PreTrainedConfig + +# --- Local stubs for APIs not present in transformers 4.55.3 --- +def layer_type_validation(layer_types, num_hidden_layers=None, attention=True): + allowed = {"full_attention", "linear_attention"} + if not all(lt in allowed for lt in layer_types): + raise ValueError(f"layer_types entries must be in {allowed}, got {layer_types}") + if num_hidden_layers is not None and num_hidden_layers != len(layer_types): + raise ValueError( + f"num_hidden_layers ({num_hidden_layers}) != len(layer_types) ({len(layer_types)})" + ) + +try: + from typing import TypedDict + class RopeParameters(TypedDict, total=False): + rope_theta: float + rope_type: str + partial_rotary_factor: float + factor: float +except Exception: + RopeParameters = dict + +# --- End stubs --- + + +class Qwen3_5MoeTextConfig(PreTrainedConfig): + r""" + Configuration for the text backbone of Qwen3.5-MoE / Qwen3.6-35B-A3B models. + model_type is "qwen3_5_moe_text" (used internally by the nested config). + """ + + model_type = "qwen3_5_moe_text" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=248320, + hidden_size=2048, + num_hidden_layers=40, + num_attention_heads=16, + num_key_value_heads=2, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + tie_word_embeddings=False, + rope_parameters=None, + attention_bias=False, + attention_dropout=0.0, + head_dim=256, + linear_conv_kernel_dim=4, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=32, + moe_intermediate_size=512, + shared_expert_intermediate_size=512, + num_experts_per_tok=8, + num_experts=256, + output_router_logits=False, + router_aux_loss_coef=0.001, + layer_types=None, + pad_token_id=None, + bos_token_id=None, + eos_token_id=None, + **kwargs, + ): + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.tie_word_embeddings = tie_word_embeddings + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.head_dim = head_dim + self.rope_parameters = rope_parameters + kwargs.setdefault("partial_rotary_factor", 0.25) + + self.layer_types = layer_types + if self.layer_types is None: + interval_pattern = kwargs.get("full_attention_interval", 4) + self.layer_types = [ + "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention" + for i in range(self.num_hidden_layers) + ] + layer_type_validation(self.layer_types, self.num_hidden_layers) + + self.linear_conv_kernel_dim = linear_conv_kernel_dim + self.linear_key_head_dim = linear_key_head_dim + self.linear_value_head_dim = linear_value_head_dim + self.linear_num_key_heads = linear_num_key_heads + self.linear_num_value_heads = linear_num_value_heads + self.moe_intermediate_size = moe_intermediate_size + self.shared_expert_intermediate_size = shared_expert_intermediate_size + self.num_experts_per_tok = num_experts_per_tok + self.num_experts = num_experts + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + super().__init__(**kwargs) + + +class Qwen3_5MoeVisionConfig(PreTrainedConfig): + model_type = "qwen3_5_moe" + + def __init__( + self, + depth=27, + hidden_size=1152, + hidden_act="gelu_pytorch_tanh", + intermediate_size=4304, + num_heads=16, + in_channels=3, + patch_size=16, + spatial_merge_size=2, + temporal_patch_size=2, + out_hidden_size=3584, + num_position_embeddings=2304, + initializer_range=0.02, + **kwargs, + ): + super().__init__(**kwargs) + self.depth = depth + self.hidden_size = hidden_size + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.num_heads = num_heads + self.in_channels = in_channels + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.out_hidden_size = out_hidden_size + self.num_position_embeddings = num_position_embeddings + self.initializer_range = initializer_range + + +class Qwen3_5MoeConfig(PreTrainedConfig): + r""" + Top-level configuration for Qwen3.5-MoE / Qwen3.6-35B-A3B. + model_type = "qwen3_5_moe" matches the model card / config.json. + Wraps Qwen3_5MoeTextConfig (and optionally Qwen3_5MoeVisionConfig). + For vLLM text-only inference only text_config is consumed. + """ + + model_type = "qwen3_5_moe" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config=None, + vision_config=None, + image_token_id=248056, + video_token_id=248057, + vision_start_token_id=248053, + vision_end_token_id=248054, + tie_word_embeddings=False, + **kwargs, + ): + if isinstance(text_config, dict): + self.text_config = Qwen3_5MoeTextConfig(**text_config) + elif text_config is None: + self.text_config = Qwen3_5MoeTextConfig() + else: + self.text_config = text_config + + if isinstance(vision_config, dict): + self.vision_config = Qwen3_5MoeVisionConfig(**vision_config) + elif vision_config is None: + self.vision_config = Qwen3_5MoeVisionConfig() + else: + self.vision_config = vision_config + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.vision_end_token_id = vision_end_token_id + self.tie_word_embeddings = tie_word_embeddings + super().__init__(**kwargs) + + +__all__ = ["Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig"] diff --git a/qwen3_6_scripts/qwen3coder_tool_parser.py b/qwen3_6_scripts/qwen3coder_tool_parser.py new file mode 100644 index 00000000..e839e853 --- /dev/null +++ b/qwen3_6_scripts/qwen3coder_tool_parser.py @@ -0,0 +1,509 @@ +import ast +import json +import uuid +from typing import Any, Dict, List, Optional, Sequence, Union + +import regex as re + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + ChatCompletionToolsParam, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer + +logger = init_logger(__name__) + + +@ToolParserManager.register_module("qwen3_coder") +class Qwen3CoderToolParser(ToolParser): + """ + Tool parser for Qwen3 models using XML-style tool call format: + + value + + + Port of vllm-original qwen3coder_tool_parser.py to vllm 0.6.3 API. + """ + + def __init__(self, tokenizer: AnyTokenizer): + super().__init__(tokenizer) + + self.current_tool_name_sent: bool = False + self.prev_tool_call_arr: List[Dict] = [] + # Base class uses int; we override with string IDs + self.current_tool_id: Optional[str] = None # type: ignore[assignment] + self.streamed_args_for_tool: List[str] = [] + + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + self.tool_call_prefix: str = "(.*?)", re.DOTALL) + self.tool_call_regex = re.compile( + r"(.*?)|(.*?)$", re.DOTALL) + self.tool_call_function_regex = re.compile( + r"||(?=)|$)", + re.DOTALL) + + if not self.model_tokenizer: + raise ValueError( + "The model tokenizer must be passed to the ToolParser " + "constructor during construction.") + + self.tool_call_start_token_id = self.vocab.get( + self.tool_call_start_token) + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) + + if (self.tool_call_start_token_id is None + or self.tool_call_end_token_id is None): + raise RuntimeError( + "Qwen3 XML Tool parser could not locate tool call start/end " + "tokens in the tokenizer!") + + logger.debug("vLLM Successfully imported tool parser %s !", + self.__class__.__name__) + + + def _generate_tool_call_id(self) -> str: + return f"call_{uuid.uuid4().hex[:24]}" + + def _reset_streaming_state(self) -> None: + self.current_tool_index = 0 + self.is_tool_call_started = False + self.header_sent = False + self.current_tool_id = None + self.current_function_name: Optional[str] = None + self.current_param_name: Optional[str] = None + self.current_param_value: str = "" + self.param_count = 0 + self.in_param = False + self.in_function = False + self.accumulated_text: str = "" + self.json_started = False + self.json_closed = False + self.accumulated_params: Dict[str, Any] = {} + self.streaming_request: Optional[ChatCompletionRequest] = None + + def _get_arguments_config( + self, func_name: str, + tools: Optional[List[ChatCompletionToolsParam]]) -> Dict: + if tools is None: + return {} + for config in tools: + if not hasattr(config, "type") or not ( + hasattr(config, "function") + and hasattr(config.function, "name")): + continue + if config.type == "function" and config.function.name == func_name: + if not hasattr(config.function, "parameters"): + return {} + params = config.function.parameters + if isinstance(params, dict) and "properties" in params: + return params["properties"] + elif isinstance(params, dict): + return params + else: + return {} + logger.debug("Tool '%s' is not defined in the tools list.", func_name) + return {} + + def _convert_param_value(self, param_value: str, param_name: str, + param_config: Dict, func_name: str) -> Any: + if param_value.lower() == "null": + return None + + if param_name not in param_config: + if param_config != {}: + logger.debug( + "Parsed parameter '%s' is not defined in tool '%s', " + "returning string value.", param_name, func_name) + return param_value + + if (isinstance(param_config[param_name], dict) + and "type" in param_config[param_name]): + param_type = str( + param_config[param_name]["type"]).strip().lower() + else: + param_type = "string" + + if param_type in ["string", "str", "text", "varchar", "char", "enum"]: + return param_value + elif (param_type.startswith("int") or param_type.startswith("uint") + or param_type.startswith("long") + or param_type.startswith("short") + or param_type.startswith("unsigned")): + try: + return int(param_value) + except (ValueError, TypeError): + return param_value + elif param_type.startswith("num") or param_type.startswith("float"): + try: + v = float(param_value) + return int(v) if v - int(v) == 0 else v + except (ValueError, TypeError): + return param_value + elif param_type in ["boolean", "bool", "binary"]: + lower = param_value.lower() + if lower not in ["true", "false"]: + logger.debug( + "Parameter '%s' value '%s' is not boolean in tool '%s'.", + param_name, param_value, func_name) + return lower == "true" + else: + if (param_type in ["object", "array", "arr"] + or param_type.startswith("dict") + or param_type.startswith("list")): + try: + return json.loads(param_value) + except (json.JSONDecodeError, TypeError, ValueError): + pass + try: + return ast.literal_eval(param_value) + except (ValueError, SyntaxError, TypeError): + pass + return param_value + + def _parse_xml_function_call( + self, function_call_str: str, + tools: Optional[List[ChatCompletionToolsParam]]) -> ToolCall: + end_index = function_call_str.index(">") + function_name = function_call_str[:end_index] + param_config = self._get_arguments_config(function_name, tools) + parameters = function_call_str[end_index + 1:] + param_dict: Dict[str, Any] = {} + for match_text in self.tool_call_parameter_regex.findall(parameters): + idx = match_text.index(">") + param_name = match_text[:idx] + param_value = str(match_text[idx + 1:]) + if param_value.startswith("\n"): + param_value = param_value[1:] + if param_value.endswith("\n"): + param_value = param_value[:-1] + param_dict[param_name] = self._convert_param_value( + param_value, param_name, param_config, function_name) + return ToolCall( + type="function", + function=FunctionCall( + name=function_name, + arguments=json.dumps(param_dict, ensure_ascii=False))) + + def _get_function_calls(self, model_output: str) -> List[str]: + matched_ranges = self.tool_call_regex.findall(model_output) + raw_tool_calls = [ + match[0] if match[0] else match[1] for match in matched_ranges + ] + if not raw_tool_calls: + raw_tool_calls = [model_output] + raw_function_calls: List[tuple] = [] + for tool_call in raw_tool_calls: + raw_function_calls.extend( + self.tool_call_function_regex.findall(tool_call)) + return [match[0] if match[0] else match[1] + for match in raw_function_calls] + + def extract_tool_calls( + self, model_output: str, + request: ChatCompletionRequest) -> ExtractedToolCallInformation: + if self.tool_call_prefix not in model_output: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + try: + function_calls = self._get_function_calls(model_output) + if not function_calls: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + tool_calls = [ + self._parse_xml_function_call(fc, request.tools) + for fc in function_calls + ] + + self.prev_tool_call_arr.clear() + for tc in tool_calls: + self.prev_tool_call_arr.append({ + "name": tc.function.name, + "arguments": tc.function.arguments, + }) + + content_index = model_output.find(self.tool_call_start_token) + idx = model_output.find(self.tool_call_prefix) + content_index = content_index if content_index >= 0 else idx + content = model_output[:content_index] + + return ExtractedToolCallInformation( + tools_called=bool(tool_calls), + tool_calls=tool_calls, + content=content if content else None, + ) + except Exception: + logger.exception("Error extracting tool call from response.") + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + if not previous_text: + self._reset_streaming_state() + self.streaming_request = request + + if not delta_text: + if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids: + complete_calls = len( + self.tool_call_complete_regex.findall(current_text)) + if complete_calls > 0 and self.prev_tool_call_arr: + open_calls = ( + current_text.count(self.tool_call_start_token) - + current_text.count(self.tool_call_end_token)) + if open_calls == 0: + return DeltaMessage(content="") + elif not self.is_tool_call_started and current_text: + return DeltaMessage(content="") + return None + + self.accumulated_text = current_text + + if self.json_closed and not self.in_function: + tool_ends = current_text.count(self.tool_call_end_token) + if tool_ends > self.current_tool_index: + self.current_tool_index += 1 + self.header_sent = False + self.param_count = 0 + self.json_started = False + self.json_closed = False + self.accumulated_params = {} + tool_starts = current_text.count(self.tool_call_start_token) + if self.current_tool_index >= tool_starts: + self.is_tool_call_started = False + return None + + if not self.is_tool_call_started: + if (self.tool_call_start_token_id in delta_token_ids + or self.tool_call_start_token in delta_text): + self.is_tool_call_started = True + if self.tool_call_start_token in delta_text: + content_before = delta_text[:delta_text.index( + self.tool_call_start_token)] + if content_before: + return DeltaMessage(content=content_before) + return None + else: + if (current_text.rstrip().endswith(self.tool_call_end_token) + and delta_text.strip() == ""): + return None + return DeltaMessage(content=delta_text) + + tool_starts_count = current_text.count(self.tool_call_start_token) + if self.current_tool_index >= tool_starts_count: + return None + + # Locate the current tool call's text slice + tool_start_positions: List[int] = [] + search = 0 + while True: + search = current_text.find(self.tool_call_start_token, search) + if search == -1: + break + tool_start_positions.append(search) + search += len(self.tool_call_start_token) + + if self.current_tool_index >= len(tool_start_positions): + return None + + tool_start_idx = tool_start_positions[self.current_tool_index] + tool_end_idx = current_text.find(self.tool_call_end_token, + tool_start_idx) + if tool_end_idx == -1: + tool_text = current_text[tool_start_idx:] + else: + tool_text = current_text[tool_start_idx:tool_end_idx + + len(self.tool_call_end_token)] + + if not self.header_sent: + if self.tool_call_prefix in tool_text: + func_start = (tool_text.find(self.tool_call_prefix) + + len(self.tool_call_prefix)) + func_end = tool_text.find(">", func_start) + if func_end != -1: + self.current_function_name = tool_text[func_start:func_end] + self.current_tool_id = self._generate_tool_call_id() + self.header_sent = True + self.in_function = True + self.prev_tool_call_arr.append({ + "name": self.current_function_name, + "arguments": "{}", + }) + self.streamed_args_for_tool.append("") + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + id=self.current_tool_id, + function=DeltaFunctionCall( + name=self.current_function_name, + arguments=""), + type="function", + ) + ]) + return None + + if self.in_function: + if not self.json_started: + self.json_started = True + self.streamed_args_for_tool[self.current_tool_index] += "{" + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="{"), + ) + ]) + + # Collect all complete parameters in one pass (speculative-decode safe) + param_starts: List[int] = [] + search = 0 + while True: + search = tool_text.find(self.parameter_prefix, search) + if search == -1: + break + param_starts.append(search) + search += len(self.parameter_prefix) + + json_fragments: List[str] = [] + while not self.in_param and self.param_count < len(param_starts): + param_idx = param_starts[self.param_count] + param_start = param_idx + len(self.parameter_prefix) + remaining = tool_text[param_start:] + + if ">" not in remaining: + break + + name_end = remaining.find(">") + current_param_name = remaining[:name_end] + value_start = param_start + name_end + 1 + value_text = tool_text[value_start:] + if value_text.startswith("\n"): + value_text = value_text[1:] + + param_end_idx = value_text.find(self.parameter_end_token) + if param_end_idx == -1: + next_param = value_text.find(self.parameter_prefix) + func_end = value_text.find(self.function_end_token) + if next_param != -1 and (func_end == -1 + or next_param < func_end): + param_end_idx = next_param + elif func_end != -1: + param_end_idx = func_end + else: + tool_end_in_value = value_text.find( + self.tool_call_end_token) + if tool_end_in_value != -1: + param_end_idx = tool_end_in_value + else: + break + + if param_end_idx == -1: + break + + param_value = value_text[:param_end_idx] + if param_value.endswith("\n"): + param_value = param_value[:-1] + + self.accumulated_params[current_param_name] = param_value + param_config = self._get_arguments_config( + self.current_function_name or "", + self.streaming_request.tools + if self.streaming_request else None) + converted = self._convert_param_value( + param_value, current_param_name, param_config, + self.current_function_name or "") + serialized = json.dumps(converted, ensure_ascii=False) + + sep = "" if self.param_count == 0 else ", " + json_fragments.append( + f'{sep}"{current_param_name}": {serialized}') + self.param_count += 1 + + if json_fragments: + combined = "".join(json_fragments) + if self.current_tool_index < len(self.streamed_args_for_tool): + self.streamed_args_for_tool[ + self.current_tool_index] += combined + else: + logger.warning( + "streamed_args_for_tool out of sync: index=%d len=%d", + self.current_tool_index, + len(self.streamed_args_for_tool)) + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments=combined), + ) + ]) + + # Emit closing brace when is seen (after params are done) + if not self.json_closed and self.function_end_token in tool_text: + self.json_closed = True + func_start = (tool_text.find(self.tool_call_prefix) + + len(self.tool_call_prefix)) + func_content_end = tool_text.find(self.function_end_token, + func_start) + if func_content_end != -1: + try: + parsed_tool = self._parse_xml_function_call( + tool_text[func_start:func_content_end], + self.streaming_request.tools + if self.streaming_request else None) + if self.current_tool_index < len( + self.prev_tool_call_arr): + self.prev_tool_call_arr[ + self.current_tool_index]["arguments"] = ( + parsed_tool.function.arguments) + except Exception: + logger.debug("Failed to parse tool call during " + "streaming: %s", + tool_text, + exc_info=True) + + if self.current_tool_index < len(self.streamed_args_for_tool): + self.streamed_args_for_tool[ + self.current_tool_index] += "}" + else: + logger.warning( + "streamed_args_for_tool out of sync: index=%d len=%d", + self.current_tool_index, + len(self.streamed_args_for_tool)) + + result = DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="}"), + ) + ]) + self.in_function = False + self.accumulated_params = {} + return result + + return None diff --git a/qwen3_6_scripts/reasoning/__init__.py b/qwen3_6_scripts/reasoning/__init__.py new file mode 100644 index 00000000..5f2e50de --- /dev/null +++ b/qwen3_6_scripts/reasoning/__init__.py @@ -0,0 +1,16 @@ +""" +Reasoning parser module for vLLM 0.6.3 (BI-V100 / Qwen3.6-27B adaptation). + +Usage: --reasoning-parser qwen3 +""" + +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser, ReasoningParserManager + +__all__ = ["ReasoningParser", "ReasoningParserManager"] + +# Lazy-register Qwen3 parser; imported on first get_reasoning_parser("qwen3"). +ReasoningParserManager.register_lazy( + "qwen3", + "vllm.reasoning.qwen3_reasoning_parser", + "Qwen3ReasoningParser", +) diff --git a/qwen3_6_scripts/reasoning/abs_reasoning_parsers.py b/qwen3_6_scripts/reasoning/abs_reasoning_parsers.py new file mode 100644 index 00000000..c6141074 --- /dev/null +++ b/qwen3_6_scripts/reasoning/abs_reasoning_parsers.py @@ -0,0 +1,243 @@ +""" +Abstract reasoning parser base classes for vLLM 0.6.3. +Adapted from vllm-original/vllm/reasoning/abs_reasoning_parsers.py: + - Removed vllm.entrypoints.mcp, vllm.utils.collection_utils, import_utils + - DeltaMessage from vllm 0.6.3 protocol path + - TokenizerLike -> AnyTokenizer + - ReasoningParserManager: simplified eager + lazy registration +""" + +import importlib +from abc import abstractmethod +from collections.abc import Iterable, Sequence +from functools import cached_property +from typing import Any, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from vllm.entrypoints.openai.protocol import DeltaMessage + from vllm.transformers_utils.tokenizer import AnyTokenizer +else: + DeltaMessage = Any + AnyTokenizer = Any + + +class ReasoningParser: + """Abstract base for all reasoning parsers.""" + + def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs): + self.model_tokenizer = tokenizer + + @cached_property + def vocab(self) -> dict: + return self.model_tokenizer.get_vocab() + + @abstractmethod + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + """Return True once the reasoning block has closed in input_ids.""" + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + return self.is_reasoning_end(input_ids) + + @abstractmethod + def extract_content_ids(self, input_ids: list) -> list: + """Return token ids that belong to the content (post-reasoning) part.""" + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + return 0 + + @abstractmethod + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + """ + Split a complete model output into (reasoning_text, content_text). + Either part may be None. + """ + + @abstractmethod + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> Optional["DeltaMessage"]: + """ + Extract reasoning from a streaming delta. + Returns a DeltaMessage with reasoning_content and/or content set, + or None if this delta should be suppressed (control token). + """ + + +class BaseThinkingReasoningParser(ReasoningParser): + """ + Base for parsers that use ... delimiters. + Subclasses define start_token / end_token properties. + """ + + @property + @abstractmethod + def start_token(self) -> str: + raise NotImplementedError + + @property + @abstractmethod + def end_token(self) -> str: + raise NotImplementedError + + def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + + if not self.model_tokenizer: + raise ValueError("Tokenizer must be passed to ReasoningParser.") + if not self.start_token or not self.end_token: + raise ValueError("start_token and end_token must be defined.") + + self.start_token_id: Optional[int] = self.vocab.get(self.start_token) + self.end_token_id: Optional[int] = self.vocab.get(self.end_token) + if self.start_token_id is None or self.end_token_id is None: + raise RuntimeError( + f"{self.__class__.__name__}: could not find think tokens " + f"'{self.start_token}'/'{self.end_token}' in tokenizer vocab." + ) + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + for token_id in reversed(input_ids): + if token_id == self.start_token_id: + return False + if token_id == self.end_token_id: + return True + return False + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + return self.end_token_id in delta_ids + + def extract_content_ids(self, input_ids: list) -> list: + if self.end_token_id not in input_ids[:-1]: + return [] + return input_ids[input_ids.index(self.end_token_id) + 1:] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + count = 0 + depth = 0 + for tid in token_ids: + if tid == self.start_token_id: + depth += 1 + elif tid == self.end_token_id: + if depth > 0: + depth -= 1 + elif depth > 0: + count += 1 + return count + + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + # Strip if the model generated it (old-style template). + parts = model_output.partition(self.start_token) + model_output = parts[2] if parts[1] else parts[0] + + if self.end_token not in model_output: + return model_output, None + reasoning, _, content = model_output.partition(self.end_token) + return reasoning, content or None + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> Optional["DeltaMessage"]: + from vllm.entrypoints.openai.protocol import DeltaMessage as _DeltaMessage + + # Suppress lone control tokens. + if len(delta_token_ids) == 1 and delta_token_ids[0] in ( + self.start_token_id, self.end_token_id + ): + return None + + start_in_prev = self.start_token_id in previous_token_ids + start_in_delta = self.start_token_id in delta_token_ids + end_in_prev = self.end_token_id in previous_token_ids + end_in_delta = self.end_token_id in delta_token_ids + + if start_in_prev: + if end_in_delta: + end_idx = delta_text.find(self.end_token) + reasoning = delta_text[:end_idx] if end_idx >= 0 else "" + content = delta_text[end_idx + len(self.end_token):] if end_idx >= 0 else None + return _DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + elif end_in_prev: + return _DeltaMessage(content=delta_text) + else: + return _DeltaMessage(reasoning_content=delta_text) + + elif start_in_delta: + if end_in_delta: + start_idx = delta_text.find(self.start_token) + end_idx = delta_text.find(self.end_token) + reasoning = delta_text[start_idx + len(self.start_token):end_idx] + content = delta_text[end_idx + len(self.end_token):] + return _DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + else: + return _DeltaMessage(reasoning_content=delta_text) + + else: + return _DeltaMessage(content=delta_text) + + +class ReasoningParserManager: + """ + Registry for ReasoningParser implementations. + Supports eager and lazy registration. + """ + + _parsers: dict = {} # name -> class (eager) + _lazy: dict = {} # name -> (module_path, class_name) + + @classmethod + def register_module(cls, name: str, parser_cls: type) -> None: + """Eagerly register a ReasoningParser class.""" + if not issubclass(parser_cls, ReasoningParser): + raise TypeError(f"{parser_cls} is not a ReasoningParser subclass.") + cls._parsers[name] = parser_cls + + @classmethod + def register_lazy(cls, name: str, module_path: str, class_name: str) -> None: + """Register a parser for deferred import.""" + cls._lazy[name] = (module_path, class_name) + + @classmethod + def get_reasoning_parser(cls, name: str) -> type: + if name in cls._parsers: + return cls._parsers[name] + if name in cls._lazy: + module_path, class_name = cls._lazy[name] + mod = importlib.import_module(module_path) + parser_cls = getattr(mod, class_name) + cls._parsers[name] = parser_cls + return parser_cls + registered = sorted(set(cls._parsers) | set(cls._lazy)) + raise KeyError( + f"Reasoning parser '{name}' not found. " + f"Available: {registered}" + ) + + @classmethod + def list_registered(cls) -> list: + return sorted(set(cls._parsers) | set(cls._lazy)) diff --git a/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py b/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py new file mode 100644 index 00000000..d90fc919 --- /dev/null +++ b/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py @@ -0,0 +1,108 @@ +""" +Reasoning parser for Qwen3 / Qwen3.5 / Qwen3.6 model family. +Adapted from vllm-original/vllm/reasoning/qwen3_reasoning_parser.py. + +The model uses ... to wrap chain-of-thought output. +For Qwen3.5+ the chat template injects into the prompt, so only + appears in the generated tokens; older templates generate +themselves. Both styles are handled. +""" + +from typing import Optional, Sequence, Any + +from vllm.reasoning.abs_reasoning_parsers import ( + BaseThinkingReasoningParser, + ReasoningParserManager, +) + + +class Qwen3ReasoningParser(BaseThinkingReasoningParser): + + def __init__(self, tokenizer: Any, *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + self.thinking_enabled = chat_kwargs.get("enable_thinking", True) + + @property + def start_token(self) -> str: + return "" + + @property + def end_token(self) -> str: + return "" + + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + # Strip if the model generated it (old template / edge case). + parts = model_output.partition(self.start_token) + model_output = parts[2] if parts[1] else parts[0] + + if self.end_token not in model_output: + if not self.thinking_enabled: + return None, model_output + # Thinking enabled but output truncated before . + return model_output, None + + reasoning, _, content = model_output.partition(self.end_token) + return reasoning, content or None + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + token_ids = list(token_ids) + if self.start_token_id in token_ids: + # Old-style template: model generates itself. + # Use depth-counting from the base class. + return super().count_reasoning_tokens(token_ids) + elif self.end_token_id in token_ids: + # New-style template (Qwen3.5+): is injected into the + # prompt, so output starts already inside the thinking block. + # Every token before is a reasoning token. + return token_ids.index(self.end_token_id) + else: + # No in output: either truncated (all reasoning) + # or thinking disabled (none). + return len(token_ids) if self.thinking_enabled else 0 + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ): + from vllm.entrypoints.openai.protocol import DeltaMessage + + if not self.thinking_enabled: + return DeltaMessage(content=delta_text) if delta_text else None + + # Strip from delta if the model generates it itself. + if self.start_token_id in delta_token_ids: + start_idx = delta_text.find(self.start_token) + if start_idx >= 0: + delta_text = delta_text[start_idx + len(self.start_token):] + + if self.end_token_id in delta_token_ids: + end_idx = delta_text.find(self.end_token) + if end_idx >= 0: + reasoning = delta_text[:end_idx] + content = delta_text[end_idx + len(self.end_token):] + if not reasoning and not content: + return None + return DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + return None + + if not delta_text: + return None + elif self.end_token_id in previous_token_ids: + return DeltaMessage(content=delta_text) + else: + return DeltaMessage(reasoning_content=delta_text) + + +# Register immediately when this module is imported. +ReasoningParserManager.register_module("qwen3", Qwen3ReasoningParser) diff --git a/qwen3_6_scripts/scheduler.py b/qwen3_6_scripts/scheduler.py new file mode 100644 index 00000000..3c06fd71 --- /dev/null +++ b/qwen3_6_scripts/scheduler.py @@ -0,0 +1,1656 @@ +import enum +import os +import random +import time +from collections import deque +from dataclasses import dataclass, field +from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set, + Tuple, Union) + +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.core.interfaces import AllocStatus, BlockSpaceManager +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import (Sequence, SequenceData, SequenceGroup, + SequenceGroupMetadata, SequenceGroupMetadataDelta, + SequenceStatus) +from vllm.utils import Device, PyObjectCache + +logger = init_logger(__name__) + +# Test-only. If configured, decode is preempted with +# ARTIFICIAL_PREEMPTION_PROB% probability. +ENABLE_ARTIFICIAL_PREEMPT = bool( + os.getenv("VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT", False)) # noqa +ARTIFICIAL_PREEMPTION_PROB = 0.5 +ARTIFICIAL_PREEMPTION_MAX_CNT = 500 + + +class PreemptionMode(enum.Enum): + """Preemption modes. + + 1. Swapping: Swap out the blocks of the preempted sequences to CPU memory + and swap them back in when the sequences are resumed. + 2. Recomputation: Discard the blocks of the preempted sequences and + recompute them when the sequences are resumed, treating the sequences as + new prompts. + """ + SWAP = enum.auto() + RECOMPUTE = enum.auto() + + +@dataclass +class SchedulingBudget: + """The available slots for scheduling. + + TODO(sang): Right now, the budget is request_id-aware meaning it can ignore + budget update from the same request_id. It is because in normal scheduling + path, we update RUNNING num_seqs ahead of time, meaning it could be + updated more than once when scheduling RUNNING requests. Since this won't + happen if we only have chunked prefill scheduling, we can remove this + feature from the API when chunked prefill is enabled by default. + """ + token_budget: int + max_num_seqs: int + _request_ids_num_batched_tokens: Set[str] = field(default_factory=set) + _request_ids_num_curr_seqs: Set[str] = field(default_factory=set) + _num_batched_tokens: int = 0 + _num_curr_seqs: int = 0 + + def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int): + assert num_new_tokens != 0 + assert num_new_seqs != 0 + return (self.num_batched_tokens + num_new_tokens <= self.token_budget + and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs) + + def remaining_token_budget(self): + return self.token_budget - self.num_batched_tokens + + def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int): + if req_id in self._request_ids_num_batched_tokens: + return + + self._request_ids_num_batched_tokens.add(req_id) + self._num_batched_tokens += num_batched_tokens + + def subtract_num_batched_tokens(self, req_id: str, + num_batched_tokens: int): + if req_id in self._request_ids_num_batched_tokens: + self._request_ids_num_batched_tokens.remove(req_id) + self._num_batched_tokens -= num_batched_tokens + + def add_num_seqs(self, req_id: str, num_curr_seqs: int): + if req_id in self._request_ids_num_curr_seqs: + return + + self._request_ids_num_curr_seqs.add(req_id) + self._num_curr_seqs += num_curr_seqs + + def subtract_num_seqs(self, req_id: str, num_curr_seqs: int): + if req_id in self._request_ids_num_curr_seqs: + self._request_ids_num_curr_seqs.remove(req_id) + self._num_curr_seqs -= num_curr_seqs + + @property + def num_batched_tokens(self): + return self._num_batched_tokens + + @property + def num_curr_seqs(self): + return self._num_curr_seqs + + +@dataclass +class ScheduledSequenceGroup: + # A sequence group that's scheduled. + seq_group: SequenceGroup + # The total chunk size (number of tokens) to process for next iteration. + # 1 for decoding. Same as prompt tokens for prefill, but if prefill is + # chunked, it can be smaller than that. + token_chunk_size: int + + +@dataclass +class SchedulerOutputs: + """The scheduling decision made from a scheduler.""" + # Scheduled sequence groups. + scheduled_seq_groups: Iterable[ScheduledSequenceGroup] + # Number of prefill groups scheduled. + num_prefill_groups: int + # Total number of batched tokens. + num_batched_tokens: int + # Blocks to swap in. List of CPU -> GPU block number. + blocks_to_swap_in: List[Tuple[int, int]] + # Blocks to swap out. List of GPU -> CPU block number. + blocks_to_swap_out: List[Tuple[int, int]] + # Blocks to copy. Source to dest block. + blocks_to_copy: List[Tuple[int, int]] + # Sequence groups that are going to be ignored. + ignored_seq_groups: List[SequenceGroup] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + # The number of requests in the running queue + running_queue_size: int + preempted: int + + def __post_init__(self): + # Swap in and swap out should never happen at the same time. + assert not (self.blocks_to_swap_in and self.blocks_to_swap_out) + + self.num_loras: int = len(self.lora_requests) + if self.num_loras > 0: + self._sort_by_lora_ids() + + self.num_prompt_adapters: int = len(self.prompt_adapter_requests) + + def is_empty(self) -> bool: + # NOTE: We do not consider the ignored sequence groups. + return (not self.scheduled_seq_groups and not self.blocks_to_swap_in + and not self.blocks_to_swap_out and not self.blocks_to_copy) + + def _sort_by_lora_ids(self): + self.scheduled_seq_groups = sorted( + self.scheduled_seq_groups, + key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id)) + + @property + def lora_requests(self) -> Set[LoRARequest]: + return { + g.seq_group.lora_request + for g in self.scheduled_seq_groups + if g.seq_group.lora_request is not None + } + + @property + def prompt_adapter_requests(self) -> Set[PromptAdapterRequest]: + return { + g.seq_group.prompt_adapter_request + for g in self.scheduled_seq_groups + if g.seq_group.prompt_adapter_request is not None + } + + +@dataclass +class SchedulerRunningOutputs: + """The requests that are scheduled from a running queue. + + Could contain prefill (prefill that's chunked) or decodes. If there's not + enough memory, it can be preempted (for recompute) or swapped out. + """ + # Selected sequences that are running and in a decoding phase. + decode_seq_groups: List[ScheduledSequenceGroup] + # Selected sequences that are running and in a prefill phase. + # I.e., it means the prefill has been chunked. + prefill_seq_groups: List[ScheduledSequenceGroup] + # The preempted sequences. + preempted: List[SequenceGroup] + # Sequences that are swapped out. + swapped_out: List[SequenceGroup] + # The blocks to swap out. + blocks_to_swap_out: List[Tuple[int, int]] + # The blocks to copy. + blocks_to_copy: List[Tuple[int, int]] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + + # Optimization for fast-access to seq_group lists + decode_seq_groups_list: List[SequenceGroup] + prefill_seq_groups_list: List[SequenceGroup] + + @classmethod + def create_empty(cls) -> "SchedulerRunningOutputs": + return SchedulerRunningOutputs( + decode_seq_groups=[], + prefill_seq_groups=[], + preempted=[], + swapped_out=[], + blocks_to_swap_out=[], + blocks_to_copy=[], + num_lookahead_slots=0, + decode_seq_groups_list=[], + prefill_seq_groups_list=[], + ) + + +@dataclass +class SchedulerSwappedInOutputs: + """The requests that are scheduled from a swap queue. + + Could contain prefill (prefill that's chunked) or decodes. + """ + # Selected sequences that are going to be swapped in and is in a + # decoding phase. + decode_seq_groups: List[ScheduledSequenceGroup] + # Selected sequences that are going to be swapped in and in a prefill + # phase. I.e., it means the prefill has been chunked. + prefill_seq_groups: List[ScheduledSequenceGroup] + # The blocks to swap in. + blocks_to_swap_in: List[Tuple[int, int]] + # The blocks to copy. + blocks_to_copy: List[Tuple[int, int]] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + # Infeasible sequence groups. + infeasible_seq_groups: List[SequenceGroup] + + @classmethod + def create_empty(cls) -> "SchedulerSwappedInOutputs": + return SchedulerSwappedInOutputs( + decode_seq_groups=[], + prefill_seq_groups=[], + blocks_to_swap_in=[], + blocks_to_copy=[], + num_lookahead_slots=0, + infeasible_seq_groups=[], + ) + + +@dataclass +class SchedulerPrefillOutputs: + """The requests that are scheduled from a waiting queue. + + Could contain a fresh prefill requests or preempted requests that need + to be recomputed from scratch. + """ + # Selected sequences for prefill. + seq_groups: List[ScheduledSequenceGroup] + # Ignored sequence groups. + ignored_seq_groups: List[SequenceGroup] + num_lookahead_slots: int + + @classmethod + def create_empty(cls) -> "SchedulerPrefillOutputs": + return SchedulerPrefillOutputs( + seq_groups=[], + ignored_seq_groups=[], + num_lookahead_slots=0, + ) + + +def seq_group_metadata_builder(): + return SequenceGroupMetadata(request_id="", + is_prompt=False, + seq_data={}, + sampling_params=None, + block_tables={}) + + +def scheduler_running_outputs_builder(): + return SchedulerRunningOutputs(decode_seq_groups=[], + prefill_seq_groups=[], + preempted=[], + swapped_out=[], + blocks_to_swap_out=[], + blocks_to_copy=[], + num_lookahead_slots=0, + prefill_seq_groups_list=[], + decode_seq_groups_list=[]) + + +def scheduled_seq_group_builder(): + return ScheduledSequenceGroup(SequenceGroup("", [], -1), + token_chunk_size=0) + # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0) + + +class Scheduler: + + def __init__( + self, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig, + lora_config: Optional[LoRAConfig], + pipeline_parallel_size: int = 1, + output_proc_callback: Optional[Callable] = None, + ) -> None: + self.scheduler_config = scheduler_config + self.cache_config = cache_config + # Note for LoRA scheduling: the current policy is extremely + # simple and NOT fair. It can lead to starvation of some + # LoRAs. This should be improved in the future. + self.lora_config = lora_config + + version = "v1" + if self.scheduler_config.use_v2_block_manager: + version = "v2" + if (self.scheduler_config.embedding_mode + or self.cache_config.is_attention_free): + version = "placeholder" + + BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class( + version) + + num_gpu_blocks = cache_config.num_gpu_blocks + if num_gpu_blocks: + num_gpu_blocks //= pipeline_parallel_size + + num_cpu_blocks = cache_config.num_cpu_blocks + if num_cpu_blocks: + num_cpu_blocks //= pipeline_parallel_size + + # Create the block space manager. + self.block_manager = BlockSpaceManagerImpl( + block_size=self.cache_config.block_size, + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks, + sliding_window=self.cache_config.sliding_window, + enable_caching=self.cache_config.enable_prefix_caching) + + # Sequence groups in the WAITING state. + # Contain new prefill or preempted requests. + self.waiting: Deque[SequenceGroup] = deque() + # Sequence groups in the RUNNING state. + # Contain decode requests. + self.running: Deque[SequenceGroup] = deque() + # Sequence groups in the SWAPPED state. + # Contain decode requests that are swapped out. + self.swapped: Deque[SequenceGroup] = deque() + # Sequence groups finished requests ids since last step iteration. + # It lets the model know that any state associated with these requests + # can and must be released after the current step. + # This is used to evict the finished requests from the Mamba cache. + self._finished_requests_ids: List[str] = list() + # Time at previous scheduling step + self.prev_time = 0.0 + # Did we schedule a prompt at previous step? + self.prev_prompt = False + # Latency of the last prompt step + self.last_prompt_latency = 0.0 + # preemption mode, RECOMPUTE or SWAP + self.user_specified_preemption_mode = scheduler_config.preemption_mode + + # The following field is test-only. It is used to inject artificial + # preemption. + self.enable_artificial_preemption = ENABLE_ARTIFICIAL_PREEMPT + self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT + if self.enable_artificial_preemption + else 0) + self.num_cumulative_preemption: int = 0 + + # Used to cache python objects + self._seq_group_metadata_cache: List[PyObjectCache] = [] + self._scheduler_running_outputs_cache: List[PyObjectCache] = [] + self._scheduled_seq_group_cache: List[PyObjectCache] = [] + + # For async output processing, we need to swap cache buffers between + # iterations. I.e. since the output processing is lagged one step, + # we cannot reuse the cached objects immediately when the schedule() + # is called again, but only when schedule() is called the second time. + self.output_proc_callback = output_proc_callback + self.use_async_output_proc = self.output_proc_callback is not None + self.num_cache_iters = 2 if self.use_async_output_proc else 1 + + self.cache_id = 0 + for i in range(self.num_cache_iters): + self._seq_group_metadata_cache.append( + PyObjectCache(seq_group_metadata_builder)) + self._scheduler_running_outputs_cache.append( + PyObjectCache(scheduler_running_outputs_builder)) + self._scheduled_seq_group_cache.append( + PyObjectCache(scheduled_seq_group_builder)) + + # For async postprocessor, the extra decode run cannot be done + # when the request reaches max_model_len. In this case, the request + # will be stopped during schedule() call and added to this stop list + # for processing and deallocation by the free_finished_seq_groups() + self._async_stopped: List[SequenceGroup] = [] + + @property + def next_cache_id(self): + return (self.cache_id + 1) % self.num_cache_iters + + @property + def lora_enabled(self) -> bool: + return bool(self.lora_config) + + @property + def num_decoding_tokens_per_seq(self) -> int: + """The number of new tokens.""" + return 1 + + def add_seq_group(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the waiting queue. + self.waiting.append(seq_group) + + def _add_seq_group_to_running(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the running queue. + # Only for testing purposes. + self.running.append(seq_group) + + def _add_seq_group_to_swapped(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the swapped queue. + # Only for testing purposes. + self.swapped.append(seq_group) + + def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None: + """Aborts a sequence group with the given ID. + + Check if the sequence group with the given ID + is present in any of the state queue. + If present, remove the sequence group from the state queue. + Also, if any of the sequences in the sequence group is not finished, + free the sequence with status `FINISHED_ABORTED`. + Otherwise, do nothing. + + Args: + request_id: The ID(s) of the sequence group to abort. + """ + if isinstance(request_id, str): + request_id = (request_id, ) + request_ids = set(request_id) + for state_queue in [self.waiting, self.running, self.swapped]: + aborted_groups: List[SequenceGroup] = [] + for seq_group in state_queue: + if not request_ids: + # Using 'break' here may add two extra iterations, + # but is acceptable to reduce complexity. + break + if seq_group.request_id in request_ids: + # Appending aborted group into pending list. + aborted_groups.append(seq_group) + request_ids.remove(seq_group.request_id) + for aborted_group in aborted_groups: + # Remove the sequence group from the state queue. + state_queue.remove(aborted_group) + # Remove the aborted request from the Mamba cache. + self._finished_requests_ids.append(aborted_group.request_id) + for seq in aborted_group.get_seqs(): + if seq.is_finished(): + continue + seq.status = SequenceStatus.FINISHED_ABORTED + self.free_seq(seq) + + self._free_seq_group_cross_attn_blocks(aborted_group) + + def _free_seq_group_cross_attn_blocks( + self, + seq_group: SequenceGroup, + ) -> None: + """ + Free a sequence group from a cross-attention block table. + Has no effect on decoder-only models. + """ + if seq_group.is_encoder_decoder(): + self.block_manager.free_cross(seq_group) + + def has_unfinished_seqs(self) -> bool: + return len(self.waiting) != 0 or len(self.running) != 0 or len( + self.swapped) != 0 + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + return self.block_manager.get_prefix_cache_hit_rate(device) + + def get_num_unfinished_seq_groups(self) -> int: + return len(self.waiting) + len(self.running) + len(self.swapped) + + def get_and_reset_finished_requests_ids(self) -> List[str]: + """Flushes the list of request ids of previously finished seq_groups.""" + finished_requests_ids = self._finished_requests_ids + self._finished_requests_ids = list() + return finished_requests_ids + + def _schedule_running( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerRunningOutputs: + """Schedule sequence groups that are running. + + Running queue should include decode and chunked prefill requests. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any decodes are preempted. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any decodes are preempted. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerRunningOutputs. + """ + ret: SchedulerRunningOutputs = \ + self._scheduler_running_outputs_cache[self.cache_id].get_object() + ret.blocks_to_swap_out.clear() + ret.blocks_to_copy.clear() + ret.decode_seq_groups.clear() + ret.prefill_seq_groups.clear() + ret.preempted.clear() + ret.swapped_out.clear() + + ret.num_lookahead_slots = self._get_num_lookahead_slots( + is_prefill=False, enable_chunking=enable_chunking) + + ret.decode_seq_groups_list.clear() + ret.prefill_seq_groups_list.clear() + + # Blocks that need to be swapped or copied before model execution. + blocks_to_swap_out: List[Tuple[int, int]] = ret.blocks_to_swap_out + blocks_to_copy: List[Tuple[int, int]] = ret.blocks_to_copy + + decode_seq_groups: List[ScheduledSequenceGroup] = ret.decode_seq_groups + prefill_seq_groups: List[ + ScheduledSequenceGroup] = ret.prefill_seq_groups + preempted: List[SequenceGroup] = ret.preempted + swapped_out: List[SequenceGroup] = ret.swapped_out + + running_queue = self.running + assert len(self._async_stopped) == 0 + while running_queue: + seq_group = running_queue[0] + num_running_tokens = self._get_num_new_tokens( + seq_group, SequenceStatus.RUNNING, enable_chunking, budget) + + if num_running_tokens == 0: + # No budget => Stop + break + + running_queue.popleft() + + # With async postprocessor, an extra decode run is done + # to process the final tokens. The check below avoids this extra + # decode run when the model max len is reached, in order to avoid + # a memory overflow. + if self.use_async_output_proc and seq_group.seqs[0].get_len( + ) > self.scheduler_config.max_model_len: + self._async_stopped.append(seq_group) + continue + + # NOTE(woosuk): Preemption happens only when there is no available + # slot to keep all the sequence groups in the RUNNING state. + while not self._can_append_slots(seq_group, enable_chunking): + budget.subtract_num_batched_tokens(seq_group.request_id, + num_running_tokens) + num_running_seqs = seq_group.get_max_num_running_seqs() + budget.subtract_num_seqs(seq_group.request_id, + num_running_seqs) + + if (curr_loras is not None and seq_group.lora_int_id > 0 + and seq_group.lora_int_id in curr_loras): + curr_loras.remove(seq_group.lora_int_id) + + # Determine victim sequence + cont_loop = True + if running_queue: + # Preempt the lowest-priority sequence group. + victim_seq_group = running_queue.pop() + else: + # No other sequence group can be preempted. + # Preempt the current sequence group. + # Note: This is also where we stop this loop + # (since there is nothing else to preempt) + victim_seq_group = seq_group + cont_loop = False + + # With async postprocessor, before preempting a sequence + # we need to ensure it has no pending async postprocessor + do_preempt = True + if self.use_async_output_proc: + assert self.output_proc_callback is not None + self.output_proc_callback( + request_id=victim_seq_group.request_id) + + # It may be that the async pending "victim_seq_group" + # becomes finished, in which case we simply free it. + if victim_seq_group.is_finished(): + self._free_finished_seq_group(victim_seq_group) + do_preempt = False + + # Do preemption + if do_preempt: + preempted_mode = self._preempt(victim_seq_group, + blocks_to_swap_out) + if preempted_mode == PreemptionMode.RECOMPUTE: + preempted.append(victim_seq_group) + else: + swapped_out.append(victim_seq_group) + + if not cont_loop: + break + else: + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + is_prefill = seq_group.is_prefill() + + scheduled_seq_group: ScheduledSequenceGroup = \ + self._scheduled_seq_group_cache[self.cache_id].get_object() + scheduled_seq_group.seq_group = seq_group + if is_prefill: + scheduled_seq_group.token_chunk_size = num_running_tokens + prefill_seq_groups.append(scheduled_seq_group) + ret.prefill_seq_groups_list.append(seq_group) + else: + scheduled_seq_group.token_chunk_size = 1 + decode_seq_groups.append(scheduled_seq_group) + ret.decode_seq_groups_list.append(seq_group) + + budget.add_num_batched_tokens(seq_group.request_id, + num_running_tokens) + # OPTIMIZATION: Note that get_max_num_running_seqs is + # expensive. For the default scheduling chase where + # enable_chunking is False, num_seqs are updated before running + # this method, so we don't have to update it again here. + if enable_chunking: + num_running_seqs = seq_group.get_max_num_running_seqs() + budget.add_num_seqs(seq_group.request_id, num_running_seqs) + if curr_loras is not None and seq_group.lora_int_id > 0: + curr_loras.add(seq_group.lora_int_id) + + self._scheduler_running_outputs_cache[self.next_cache_id].reset() + self._scheduled_seq_group_cache[self.next_cache_id].reset() + + return ret + + def _schedule_swapped( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerSwappedInOutputs: + """Schedule sequence groups that are swapped out. + + It schedules swapped requests as long as it fits `budget` and + curr_loras <= max_lora from the scheduling config. The input arguments + `budget` and `curr_loras` are updated based on scheduled seq_groups. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are swapped in. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any requests are swapped in. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerSwappedInOutputs. + """ + # Blocks that need to be swapped or copied before model execution. + blocks_to_swap_in: List[Tuple[int, int]] = [] + blocks_to_copy: List[Tuple[int, int]] = [] + decode_seq_groups: List[ScheduledSequenceGroup] = [] + prefill_seq_groups: List[ScheduledSequenceGroup] = [] + infeasible_seq_groups: List[SequenceGroup] = [] + + swapped_queue = self.swapped + + leftover_swapped: Deque[SequenceGroup] = deque() + while swapped_queue: + seq_group = swapped_queue[0] + + # If the sequence group cannot be swapped in, stop. + is_prefill = seq_group.is_prefill() + alloc_status = self.block_manager.can_swap_in( + seq_group, + self._get_num_lookahead_slots(is_prefill, enable_chunking)) + if alloc_status == AllocStatus.LATER: + break + elif alloc_status == AllocStatus.NEVER: + logger.warning( + "Failing the request %s because there's not enough kv " + "cache blocks to run the entire sequence.", + seq_group.request_id) + for seq in seq_group.get_seqs(): + seq.status = SequenceStatus.FINISHED_IGNORED + infeasible_seq_groups.append(seq_group) + swapped_queue.popleft() + continue + + lora_int_id = 0 + if self.lora_enabled: + lora_int_id = seq_group.lora_int_id + assert curr_loras is not None + assert self.lora_config is not None + if (lora_int_id > 0 and (lora_int_id not in curr_loras) + and len(curr_loras) >= self.lora_config.max_loras): + # We don't have a space for another LoRA, so + # we ignore this request for now. + leftover_swapped.appendleft(seq_group) + swapped_queue.popleft() + continue + + # The total number of sequences in the RUNNING state should not + # exceed the maximum number of sequences. + num_new_seqs = seq_group.get_max_num_running_seqs() + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.SWAPPED, + enable_chunking, budget) + + if (num_new_tokens == 0 + or not budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + if lora_int_id > 0 and curr_loras is not None: + curr_loras.add(lora_int_id) + swapped_queue.popleft() + self._swap_in(seq_group, blocks_to_swap_in) + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + is_prefill = seq_group.is_prefill() + if is_prefill: + prefill_seq_groups.append( + ScheduledSequenceGroup(seq_group, + token_chunk_size=num_new_tokens)) + else: + decode_seq_groups.append( + ScheduledSequenceGroup(seq_group, token_chunk_size=1)) + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens) + budget.add_num_seqs(seq_group.request_id, num_new_seqs) + + swapped_queue.extendleft(leftover_swapped) + + return SchedulerSwappedInOutputs( + decode_seq_groups=decode_seq_groups, + prefill_seq_groups=prefill_seq_groups, + blocks_to_swap_in=blocks_to_swap_in, + blocks_to_copy=blocks_to_copy, + num_lookahead_slots=self._get_num_lookahead_slots( + is_prefill=False, enable_chunking=enable_chunking), + infeasible_seq_groups=infeasible_seq_groups, + ) + + def _get_prompt_limit(self, seq_group: SequenceGroup) -> int: + if self.scheduler_config.chunked_prefill_enabled and \ + not self.scheduler_config.is_multi_step: + prompt_limit = self.scheduler_config.max_model_len + else: + prompt_limit = min(self.scheduler_config.max_model_len, + self.scheduler_config.max_num_batched_tokens) + + # Model is fine tuned with long context. Return the fine tuned max_len. + if (seq_group.lora_request + and seq_group.lora_request.long_lora_max_len): + assert prompt_limit <= seq_group.lora_request.long_lora_max_len + return seq_group.lora_request.long_lora_max_len + else: + return prompt_limit + + def _get_priority(self, + seq_group: SequenceGroup) -> Tuple[Optional[int], float]: + """ Get the priority of the sequence group. + Highest preference to user-defined priority, followed by arrival time. + Args: + seq_group: The sequence group input. + Returns: + The priority of the sequence group. + """ + return seq_group.priority, seq_group.arrival_time + + def _schedule_priority_preemption( + self, + budget: SchedulingBudget, + ) -> int: + """Sorts waiting and running queue. Also, force preempt requests + from the running queue if their priority is lower. + Priority-based preemption is used with the priority policy. + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are scheduled. + Returns: + A count of priority-based preemptions. + """ + + waiting_queue = self.waiting + + running_queue = deque(sorted(self.running, key=self._get_priority)) + + blocks_to_swap_out: List[Tuple[int, int]] = [] + force_preemption_count = 0 + + if waiting_queue: + seq_group = waiting_queue.popleft() + num_new_seqs = seq_group.get_max_num_running_seqs() + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.WAITING, + False, budget) + + #Only preempt if priority inversion exists + while running_queue and self._get_priority( + running_queue[-1]) > self._get_priority(seq_group): + #Only preempt if waiting sequence cannot be allocated + can_allocate = self.block_manager.can_allocate(seq_group) + if (num_new_tokens and can_allocate == AllocStatus.OK + and budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + #Adjust budget to remove the victim sequence group + vseq_group = running_queue.pop() + num_running_tokens = self._get_num_new_tokens( + vseq_group, SequenceStatus.RUNNING, False, budget) + budget.subtract_num_batched_tokens(vseq_group.request_id, + num_running_tokens) + num_running_seqs = vseq_group.get_max_num_running_seqs() + budget.subtract_num_seqs(vseq_group.request_id, + num_running_seqs) + + #Preempt out the victim sequence group + self._preempt(vseq_group, blocks_to_swap_out, + PreemptionMode.RECOMPUTE) + waiting_queue.appendleft(vseq_group) + force_preemption_count += 1 + #Put the sequence back into the waiting queue + waiting_queue.appendleft(seq_group) + + waiting_queue = deque(sorted(waiting_queue, key=self._get_priority)) + + self.waiting = waiting_queue + self.running = running_queue + return force_preemption_count + + def _schedule_prefills( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerPrefillOutputs: + """Schedule sequence groups that are in prefill stage. + + Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE + as a new prefill (that starts from beginning -> most recently generated + tokens). + + It schedules waiting requests as long as it fits `budget` and + curr_loras <= max_lora from the scheduling config. The input arguments + `budget` and `curr_loras` are updated based on scheduled seq_groups. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are scheduled. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any requests are scheduled. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerPrefillOutputs. + """ + ignored_seq_groups: List[SequenceGroup] = [] + seq_groups: List[ScheduledSequenceGroup] = [] + + waiting_queue = self.waiting + + leftover_waiting_sequences: Deque[SequenceGroup] = deque() + while self._passed_delay(time.time()) and waiting_queue: + seq_group = waiting_queue[0] + + waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING) + assert len(waiting_seqs) == 1, ( + "Waiting sequence group should have only one prompt " + "sequence.") + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.WAITING, + enable_chunking, budget) + if not enable_chunking: + num_prompt_tokens = waiting_seqs[0].get_len() + assert num_new_tokens == num_prompt_tokens + + prompt_limit = self._get_prompt_limit(seq_group) + if num_new_tokens > prompt_limit: + logger.warning( + "Input prompt (%d tokens) is too long" + " and exceeds limit of %d", num_new_tokens, prompt_limit) + for seq in waiting_seqs: + seq.status = SequenceStatus.FINISHED_IGNORED + ignored_seq_groups.append(seq_group) + waiting_queue.popleft() + continue + + num_lookahead_slots: int = 0 + if self.scheduler_config.is_multi_step and enable_chunking: + num_lookahead_slots = self._get_num_lookahead_slots( + True, enable_chunking) + + # If the sequence group cannot be allocated, stop. + can_allocate = self.block_manager.can_allocate( + seq_group, num_lookahead_slots=num_lookahead_slots) + if can_allocate == AllocStatus.LATER: + break + elif can_allocate == AllocStatus.NEVER: + logger.warning( + "Input prompt (%d tokens) + lookahead slots (%d) is " + "too long and exceeds the capacity of block_manager", + num_new_tokens, num_lookahead_slots) + for seq in waiting_seqs: + seq.status = SequenceStatus.FINISHED_IGNORED + ignored_seq_groups.append(seq_group) + waiting_queue.popleft() + continue + + lora_int_id = 0 + if self.lora_enabled: + lora_int_id = seq_group.lora_int_id + assert curr_loras is not None + assert self.lora_config is not None + if (self.lora_enabled and lora_int_id > 0 + and lora_int_id not in curr_loras + and len(curr_loras) >= self.lora_config.max_loras): + # We don't have a space for another LoRA, so + # we ignore this request for now. + leftover_waiting_sequences.appendleft(seq_group) + waiting_queue.popleft() + continue + + num_new_seqs = seq_group.get_max_num_running_seqs() + if (num_new_tokens == 0 + or not budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + # Can schedule this request. + if curr_loras is not None and lora_int_id > 0: + curr_loras.add(lora_int_id) + waiting_queue.popleft() + self._allocate_and_set_running(seq_group) + + if enable_chunking and self.scheduler_config.is_multi_step: + blocks_to_copy: List[Tuple[int, int]] = [] + # init_multi_step_from_lookahead_slots happens in append_slots + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + # This assert will trip when a copy-on-write happens. This is + # not a concern as the very first sequence-group block + # allocation happens above. Still, we have the assert to + # catch any edge-cases. + assert not blocks_to_copy + else: + seq_group.init_multi_step_from_lookahead_slots( + num_lookahead_slots, + num_scheduler_steps=self.scheduler_config. + num_scheduler_steps, + is_multi_step=self.scheduler_config.is_multi_step, + enable_chunking=enable_chunking) + + seq_groups.append( + ScheduledSequenceGroup(seq_group=seq_group, + token_chunk_size=num_new_tokens)) + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens) + budget.add_num_seqs(seq_group.request_id, num_new_seqs) + + # Queue requests that couldn't be scheduled. + waiting_queue.extendleft(leftover_waiting_sequences) + if len(seq_groups) > 0: + self.prev_prompt = True + + return SchedulerPrefillOutputs( + seq_groups=seq_groups, + ignored_seq_groups=ignored_seq_groups, + num_lookahead_slots=self._get_num_lookahead_slots( + is_prefill=True, enable_chunking=enable_chunking)) + + def _schedule_default(self) -> SchedulerOutputs: + """Schedule queued requests. + + The current policy is designed to optimize the throughput. First, + it batches as many prefill requests as possible. And it schedules + decodes. If there's a pressure on GPU memory, decode requests can + be swapped or preempted. + """ + # Include running requests to the budget. + budget = SchedulingBudget( + token_budget=self.scheduler_config.max_num_batched_tokens, + max_num_seqs=self.scheduler_config.max_num_seqs, + ) + # Make sure we include num running seqs before scheduling prefill, + # so that we don't schedule beyond max_num_seqs for prefill. + for seq_group in self.running: + budget.add_num_seqs(seq_group.request_id, + seq_group.get_max_num_running_seqs()) + curr_loras = set( + seq_group.lora_int_id for seq_group in self.running + if seq_group.lora_int_id > 0) if self.lora_enabled else None + + prefills = SchedulerPrefillOutputs.create_empty() + running_scheduled = SchedulerRunningOutputs.create_empty() + swapped_in = SchedulerSwappedInOutputs.create_empty() + + # If any requests are swapped, prioritized swapped requests. + if not self.swapped: + prefills = self._schedule_prefills(budget, + curr_loras, + enable_chunking=False) + + if len(prefills.seq_groups + ) == 0 and self.scheduler_config.policy == "priority": + self._schedule_priority_preemption(budget) + + # Don't schedule decodes if prefills are scheduled. + # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running + # only contains decode requests, not chunked prefills. + if len(prefills.seq_groups) == 0: + running_scheduled = self._schedule_running(budget, + curr_loras, + enable_chunking=False) + + # If any sequence group is preempted, do not swap in any sequence + # group. because it means there's no slot for new running requests. + if len(running_scheduled.preempted) + len( + running_scheduled.swapped_out) == 0: + swapped_in = self._schedule_swapped(budget, curr_loras) + + assert (budget.num_batched_tokens <= + self.scheduler_config.max_num_batched_tokens) + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs + + # Update waiting requests. + self.waiting.extendleft(running_scheduled.preempted) + # Update new running requests. + if len(prefills.seq_groups) > 0: + self.running.extend([s.seq_group for s in prefills.seq_groups]) + + self.running.extend(running_scheduled.decode_seq_groups_list) + + if len(swapped_in.decode_seq_groups) > 0: + self.running.extend( + [s.seq_group for s in swapped_in.decode_seq_groups]) + + # Update swapped requests. + self.swapped.extend(running_scheduled.swapped_out) + preempted = (len(running_scheduled.preempted) + + len(running_scheduled.swapped_out)) + + # There should be no prefill from running queue because this policy + # doesn't allow chunked prefills. + assert len(running_scheduled.prefill_seq_groups) == 0 + assert len(swapped_in.prefill_seq_groups) == 0 + + # Merge lists + num_prefill_groups = len(prefills.seq_groups) + if num_prefill_groups > 0: + scheduled_seq_groups = prefills.seq_groups + scheduled_seq_groups.extend(running_scheduled.decode_seq_groups) + else: + scheduled_seq_groups = running_scheduled.decode_seq_groups + scheduled_seq_groups.extend(swapped_in.decode_seq_groups) + + blocks_to_copy = running_scheduled.blocks_to_copy + blocks_to_copy.extend(swapped_in.blocks_to_copy) + + ignored_seq_groups = prefills.ignored_seq_groups + ignored_seq_groups.extend(swapped_in.infeasible_seq_groups) + + return SchedulerOutputs( + scheduled_seq_groups=scheduled_seq_groups, + num_prefill_groups=num_prefill_groups, + num_batched_tokens=budget.num_batched_tokens, + blocks_to_swap_in=swapped_in.blocks_to_swap_in, + blocks_to_swap_out=running_scheduled.blocks_to_swap_out, + blocks_to_copy=blocks_to_copy, + ignored_seq_groups=ignored_seq_groups, + num_lookahead_slots=running_scheduled.num_lookahead_slots, + running_queue_size=len(self.running), + preempted=preempted, + ) + + def _schedule_chunked_prefill(self) -> SchedulerOutputs: + """Schedule queued requests. + + Chunked prefill allows to chunk prefill requests, batch them together + with decode requests. This policy 1. schedule as many decoding requests + as possible. 2. schedule chunked prefill requests that are not + finished. 3. schedule swapped request. 4. schedule new prefill + requests. + + The policy can sustain the high GPU utilization because it can put + prefill and decodes requests to the same batch, while it improves + inter token latency because decodes requests don't need to be blocked + by prefill requests. + """ + budget = SchedulingBudget( + token_budget=self.scheduler_config.max_num_batched_tokens, + max_num_seqs=self.scheduler_config.max_num_seqs, + ) + curr_loras: Set[int] = set() + + prefills = SchedulerPrefillOutputs.create_empty() + swapped_in = SchedulerSwappedInOutputs.create_empty() + + # Decoding should be always scheduled first by fcfs. + running_scheduled = self._schedule_running(budget, + curr_loras, + enable_chunking=True) + + # Schedule swapped out requests. + # If preemption happens, it means we don't have space for swap-in. + if len(running_scheduled.preempted) + len( + running_scheduled.swapped_out) == 0: + swapped_in = self._schedule_swapped(budget, curr_loras) + + # Schedule new prefills. + prefills = self._schedule_prefills(budget, + curr_loras, + enable_chunking=True) + + assert (budget.num_batched_tokens <= + self.scheduler_config.max_num_batched_tokens) + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs + + # Update waiting requests. + self.waiting.extendleft(running_scheduled.preempted) + + # Update new running requests. + # By default, vLLM scheduler prioritizes prefills. + # Once chunked prefill is enabled, + # the policy is changed to prioritize decode requests. + self.running.extend( + [s.seq_group for s in swapped_in.decode_seq_groups]) + self.running.extend( + [s.seq_group for s in swapped_in.prefill_seq_groups]) + self.running.extend( + [s.seq_group for s in running_scheduled.decode_seq_groups]) + self.running.extend( + [s.seq_group for s in running_scheduled.prefill_seq_groups]) + self.running.extend([s.seq_group for s in prefills.seq_groups]) + + # Update swapped requests. + self.swapped.extend(running_scheduled.swapped_out) + return SchedulerOutputs( + scheduled_seq_groups=(prefills.seq_groups + + running_scheduled.prefill_seq_groups + + swapped_in.prefill_seq_groups + + running_scheduled.decode_seq_groups + + swapped_in.decode_seq_groups), + num_prefill_groups=(len(prefills.seq_groups) + + len(swapped_in.prefill_seq_groups) + + len(running_scheduled.prefill_seq_groups)), + num_batched_tokens=budget.num_batched_tokens, + blocks_to_swap_in=swapped_in.blocks_to_swap_in, + blocks_to_swap_out=running_scheduled.blocks_to_swap_out, + blocks_to_copy=running_scheduled.blocks_to_copy + + swapped_in.blocks_to_copy, + ignored_seq_groups=prefills.ignored_seq_groups + + swapped_in.infeasible_seq_groups, + num_lookahead_slots=running_scheduled.num_lookahead_slots, + running_queue_size=len(self.running), + preempted=(len(running_scheduled.preempted) + + len(running_scheduled.swapped_out)), + ) + + def _schedule(self) -> SchedulerOutputs: + """Schedule queued requests.""" + if self.scheduler_config.chunked_prefill_enabled: + return self._schedule_chunked_prefill() + else: + return self._schedule_default() + + def _can_append_slots(self, seq_group: SequenceGroup, + enable_chunking: bool) -> bool: + """Determine whether or not we have enough space in the KV cache to + continue generation of the sequence group. + """ + # It is True only for testing case to trigger artificial preemption. + if (self.enable_artificial_preemption + and random.uniform(0, 1) < ARTIFICIAL_PREEMPTION_PROB + and self.artificial_preempt_cnt > 0): + self.artificial_preempt_cnt -= 1 + return False + + is_prefill = seq_group.is_prefill() + num_lookahead_slots = self._get_num_lookahead_slots( + is_prefill, enable_chunking) + + if is_prefill and num_lookahead_slots > 0: + # Appending prefill slots only happens multi-step and + # chunked-prefill are enabled together. + assert self.scheduler_config.is_multi_step and enable_chunking + + return self.block_manager.can_append_slots( + seq_group=seq_group, num_lookahead_slots=num_lookahead_slots) + + def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool: + # async_output_proc is allowed only when we have a single sequence + # in the sequence group + no_single_seq = seq_group.sampling_params is None or ( + seq_group.sampling_params.n == 1) + return no_single_seq + + def schedule( + self + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, bool]: + # Schedule sequence groups. + # This function call changes the internal states of the scheduler + # such as self.running, self.swapped, and self.waiting. + scheduler_start_time = time.perf_counter() + + scheduler_outputs: SchedulerOutputs = self._schedule() + now = time.time() + + if not self.cache_config.enable_prefix_caching: + common_computed_block_nums = [] + + allow_async_output_proc: bool = self.use_async_output_proc + + # Create input data structures. + seq_group_metadata_list: List[SequenceGroupMetadata] = [] + for i, scheduled_seq_group in enumerate( + scheduler_outputs.scheduled_seq_groups): + seq_group = scheduled_seq_group.seq_group + token_chunk_size = scheduled_seq_group.token_chunk_size + seq_group.maybe_set_first_scheduled_time(now) + + seq_group_metadata = self._seq_group_metadata_cache[ + self.cache_id].get_object() + seq_group_metadata.seq_data.clear() + seq_group_metadata.block_tables.clear() + + # seq_id -> SequenceData + seq_data: Dict[int, SequenceData] = {} + # seq_id -> physical block numbers + block_tables: Dict[int, List[int]] = {} + + if seq_group.is_encoder_decoder(): + # Encoder associated with SequenceGroup + encoder_seq = seq_group.get_encoder_seq() + assert encoder_seq is not None + encoder_seq_data = encoder_seq.data + # Block table for cross-attention + # Also managed at SequenceGroup level + cross_block_table = self.block_manager.get_cross_block_table( + seq_group) + else: + encoder_seq_data = None + cross_block_table = None + + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + seq_id = seq.seq_id + seq_data[seq_id] = seq.data + block_tables[seq_id] = self.block_manager.get_block_table(seq) + self.block_manager.access_all_blocks_in_seq(seq, now) + + if self.cache_config.enable_prefix_caching: + common_computed_block_nums = ( + self.block_manager.get_common_computed_block_ids( + seq_group.get_seqs(status=SequenceStatus.RUNNING))) + + do_sample = True + is_prompt = seq_group.is_prefill() + # We should send the metadata to workers when the first prefill + # is sent. Subsequent requests could be chunked prefill or decode. + is_first_prefill = False + if is_prompt: + seqs = seq_group.get_seqs() + # Prefill has only 1 sequence. + assert len(seqs) == 1 + num_computed_tokens = seqs[0].data.get_num_computed_tokens() + is_first_prefill = num_computed_tokens == 0 + if (is_first_prefill + and self.cache_config.enable_prefix_caching + and seq_group.metrics is not None): + seq_group.metrics.num_cached_tokens = ( + len(common_computed_block_nums) + * self.cache_config.block_size) + # In the next iteration, all prompt tokens are not computed. + # It means the prefill is chunked, and we don't need sampling. + # NOTE: We use get_len instead of get_prompt_len because when + # a sequence is preempted, prefill includes previous generated + # output tokens. + if (token_chunk_size + num_computed_tokens < + seqs[0].data.get_len()): + do_sample = False + + # It assumes the scheduled_seq_groups is ordered by + # prefill < decoding. + if is_first_prefill or not self.scheduler_config.send_delta_data: + seq_group_metadata = SequenceGroupMetadata( + request_id=seq_group.request_id, + is_prompt=is_prompt, + seq_data=seq_data, + sampling_params=seq_group.sampling_params, + block_tables=block_tables, + do_sample=do_sample, + pooling_params=seq_group.pooling_params, + token_chunk_size=token_chunk_size, + lora_request=seq_group.lora_request, + computed_block_nums=common_computed_block_nums, + encoder_seq_data=encoder_seq_data, + cross_block_table=cross_block_table, + state=seq_group.state, + # `multi_modal_data` will only be present for the 1st comm + # between engine and worker. + # the subsequent comms can still use delta, but + # `multi_modal_data` will be None. + multi_modal_data=seq_group.multi_modal_data + if scheduler_outputs.num_prefill_groups > 0 else None, + mm_processor_kwargs=seq_group.mm_processor_kwargs, + prompt_adapter_request=seq_group.prompt_adapter_request, + ) + else: + # When SPMD mode is enabled, we only send delta data except for + # the first request to reduce serialization cost. + seq_data_delta = {} + for id, data in seq_data.items(): + seq_data_delta[id] = data.get_delta_and_reset() + seq_group_metadata = SequenceGroupMetadataDelta( + seq_data_delta, + seq_group.request_id, + block_tables, + is_prompt, + do_sample=do_sample, + token_chunk_size=token_chunk_size, + computed_block_nums=common_computed_block_nums, + ) + seq_group_metadata_list.append(seq_group_metadata) + + if allow_async_output_proc: + allow_async_output_proc = self._allow_async_output_proc( + seq_group) + + # Now that the batch has been created, we can assume all blocks in the + # batch will have been computed before the next scheduling invocation. + # This is because the engine assumes that a failure in model execution + # will crash the vLLM instance / will not retry. + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups: + self.block_manager.mark_blocks_as_computed( + scheduled_seq_group.seq_group, + scheduled_seq_group.token_chunk_size) + + self._seq_group_metadata_cache[self.next_cache_id].reset() + + scheduler_time = time.perf_counter() - scheduler_start_time + # Add this to scheduler time to all the sequences that are currently + # running. This will help estimate if the scheduler is a significant + # component in the e2e latency. + for seq_group in self.running: + if seq_group is not None and seq_group.metrics is not None: + if seq_group.metrics.scheduler_time is not None: + seq_group.metrics.scheduler_time += scheduler_time + else: + seq_group.metrics.scheduler_time = scheduler_time + + # Move to next cache (if exists) + self.cache_id = self.next_cache_id + + # Return results + return (seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc) + + def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None: + self.block_manager.fork(parent_seq, child_seq) + + def free_seq(self, seq: Sequence) -> None: + """Free a sequence from a block table.""" + self.block_manager.free(seq) + + def _free_finished_seqs(self, seq_group: SequenceGroup) -> None: + """Free finished seqs in a sequence group.""" + for seq in seq_group.get_seqs(): + if seq.is_finished(): + self.free_seq(seq) + + def _free_finished_seq_group(self, seq_group: SequenceGroup) -> None: + if seq_group.is_finished(): + # Free cross-attention block table, if it exists + self._free_seq_group_cross_attn_blocks(seq_group) + + # Add the finished requests to the finished requests list. + # This list will be used to update the Mamba cache in the + # next step. + self._finished_requests_ids.append(seq_group.request_id) + + # Free finished seqs + self._free_finished_seqs(seq_group) + + def free_finished_seq_groups(self) -> None: + remaining: Deque[SequenceGroup] = deque() + for seq_group in self.running: + self._free_finished_seq_group(seq_group) + if not seq_group.is_finished(): + remaining.append(seq_group) + + self.running = remaining + + # Handle async stopped sequence groups + # (ones that reached max model len) + if self._async_stopped: + for seq_group in self._async_stopped: + self._free_seq_group_cross_attn_blocks(seq_group) + self._finished_requests_ids.append(seq_group.request_id) + + # Free finished seqs + self._free_finished_seqs(seq_group) + + self._async_stopped.clear() + + def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None: + self.block_manager.allocate(seq_group) + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING): + seq.status = SequenceStatus.RUNNING + + def _append_slots(self, + seq_group: SequenceGroup, + blocks_to_copy: List[Tuple[int, int]], + enable_chunking: bool = False) -> None: + """Appends new slots to the sequences in the given sequence group. + + Args: + seq_group (SequenceGroup): The sequence group containing the + sequences to append slots to. + blocks_to_copy (List[Tuple[int, int]]): A list of tuple of two + ints, the first int is the source block index, and the second + int is the destination block index. This list is updated with + the new source and destination block indices for the appended + slots. + enable_chunking (bool): True if chunked prefill is enabled. + """ + is_prefill: bool = seq_group.is_prefill() + num_lookahead_slots: int = self._get_num_lookahead_slots( + is_prefill, enable_chunking) + + seq_group.init_multi_step_from_lookahead_slots( + num_lookahead_slots, + num_scheduler_steps=self.scheduler_config.num_scheduler_steps, + is_multi_step=self.scheduler_config.is_multi_step, + enable_chunking=enable_chunking) + + seq_status: Optional[SequenceStatus] = SequenceStatus.RUNNING + if self.scheduler_config.is_multi_step and enable_chunking: + # In multi-step chunked-prefill any sequence type can have + # slots appended. + seq_status = None + + for seq in seq_group.get_seqs(status=seq_status): + cows = self.block_manager.append_slots(seq, num_lookahead_slots) + if len(cows) > 0: + blocks_to_copy.extend(cows) + + def _preempt( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + preemption_mode: Optional[PreemptionMode] = None, + ) -> PreemptionMode: + # If preemption mode is not specified, we determine the mode as follows: + # We use recomputation by default since it incurs lower overhead than + # swapping. However, when the sequence group has multiple sequences + # (e.g., beam search), recomputation is not currently supported. In + # such a case, we use swapping instead. + # FIXME(woosuk): This makes our scheduling policy a bit bizarre. + # As swapped sequences are prioritized over waiting sequences, + # sequence groups with multiple sequences are implicitly prioritized + # over sequence groups with a single sequence. + # TODO(woosuk): Support recomputation for sequence groups with multiple + # sequences. This may require a more sophisticated CUDA kernel. + if self.user_specified_preemption_mode is None: + if seq_group.get_max_num_running_seqs() == 1: + preemption_mode = PreemptionMode.RECOMPUTE + else: + preemption_mode = PreemptionMode.SWAP + + elif self.user_specified_preemption_mode == "swap": + preemption_mode = PreemptionMode.SWAP + else: + preemption_mode = PreemptionMode.RECOMPUTE + + if self.num_cumulative_preemption % 50 == 0: + logger.warning( + "Sequence group %s is preempted by %s mode because there is " + "not enough KV cache space. This can affect the end-to-end " + "performance. Increase gpu_memory_utilization or " + "tensor_parallel_size to provide more KV cache memory. " + "total_num_cumulative_preemption=%d", seq_group.request_id, + preemption_mode, self.num_cumulative_preemption + 1) + self.num_cumulative_preemption += 1 + + if preemption_mode == PreemptionMode.RECOMPUTE: + self._preempt_by_recompute(seq_group) + elif preemption_mode == PreemptionMode.SWAP: + self._preempt_by_swap(seq_group, blocks_to_swap_out) + else: + raise AssertionError("Invalid preemption mode.") + return preemption_mode + + def _preempt_by_recompute( + self, + seq_group: SequenceGroup, + ) -> None: + seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING) + assert len(seqs) == 1 + for seq in seqs: + seq.status = SequenceStatus.WAITING + self.free_seq(seq) + seq.reset_state_for_recompute() + + def _preempt_by_swap( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + ) -> None: + self._swap_out(seq_group, blocks_to_swap_out) + + def _swap_in( + self, + seq_group: SequenceGroup, + blocks_to_swap_in: List[Tuple[int, int]], + ) -> None: + mapping = self.block_manager.swap_in(seq_group) + blocks_to_swap_in.extend(mapping) + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED): + seq.status = SequenceStatus.RUNNING + + def _swap_out( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + ) -> None: + if not self.block_manager.can_swap_out(seq_group): + # FIXME(woosuk): Abort the sequence group instead of aborting the + # entire engine. + raise RuntimeError( + "Aborted due to the lack of CPU swap space. Please increase " + "the swap space to avoid this error.") + mapping = self.block_manager.swap_out(seq_group) + blocks_to_swap_out.extend(mapping) + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + seq.status = SequenceStatus.SWAPPED + + def _passed_delay(self, now: float) -> bool: + if self.prev_prompt: + self.last_prompt_latency = now - self.prev_time + self.prev_time, self.prev_prompt = now, False + # Delay scheduling prompts to let waiting queue fill up + if self.scheduler_config.delay_factor > 0 and self.waiting: + earliest_arrival_time = min( + [e.metrics.arrival_time for e in self.waiting]) + passed_delay = ( + (now - earliest_arrival_time) > + (self.scheduler_config.delay_factor * self.last_prompt_latency) + or not self.running) + else: + passed_delay = True + return passed_delay + + def _get_num_lookahead_slots(self, is_prefill: bool, + enable_chunking: bool) -> int: + """The number of slots to allocate per sequence per step, beyond known + token ids. Speculative decoding uses these slots to store KV activations + of tokens which may or may not be accepted. + + Speculative decoding does not yet support prefill, so we do not perform + lookahead allocation for prefill. + + When chunking is enabled with multi-step, we allocate lookahead slots + for the prefills for when the prefills turn into decodes in the first + step. + """ + if is_prefill: + if self.scheduler_config.is_multi_step and enable_chunking: + # num_lookahead_slots was introduced in the context of decodes, + # in Speculative Decoding. + # When the num_scheduler_steps is 8, say, then the + # num_lookahead_slots is 7. Meaning, we are doing a 1-step of + # decode anyways and we wish to do 7 more. + # + # "lookaheads" for prefills, is introduced in support for + # Chunked-Prefill in Multi-Step. + return self.scheduler_config.num_lookahead_slots + 1 + else: + return 0 + + return self.scheduler_config.num_lookahead_slots + + def _get_num_new_tokens(self, seq_group: SequenceGroup, + status: SequenceStatus, enable_chunking: bool, + budget: SchedulingBudget) -> int: + """Get the next new tokens to compute for a given sequence group + that's in a given `status`. + + The API could chunk the number of tokens to compute based on `budget` + if `enable_chunking` is True. If a sequence group has multiple + sequences (e.g., running beam search), it means it is in decoding + phase, so chunking doesn't happen. + + Returns 0 if the new token cannot be computed due to token budget. + """ + num_new_tokens = 0 + seqs = seq_group.get_seqs(status=status) + for seq in seqs: + num_new_tokens += seq.get_num_new_tokens() + assert num_new_tokens > 0 + # Chunk if a running request cannot fit in the given budget. + # If number of seq > 1, it means it is doing beam search + # in a decode phase. Do not chunk. + if enable_chunking and len(seqs) == 1: + remaining_token_budget = budget.remaining_token_budget() + if self.scheduler_config.is_multi_step: + # The current multi-step + chunked prefill capability does + # not actually support chunking prompts. + # + # Therefore, `num_new_tokens` is computed in the same fashion + # for both multi-step+chunked-prefill & + # multi-step+chunked-prefill+APC + # + # Prompts with more tokens than the current remaining budget + # are postponed to future scheduler steps + if num_new_tokens > self._get_prompt_limit(seq_group): + # If the seq_group is in prompt-stage, pass the + # num_new_tokens as-is so the caller can ignore + # the sequence. + pass + else: + num_new_tokens = 0 \ + if num_new_tokens > remaining_token_budget \ + else num_new_tokens + elif self.cache_config.enable_prefix_caching: + # When prefix caching is enabled, we always allocate + # the number of new tokens that is dividable by the block + # size to avoid partial block matching. + block_size = self.cache_config.block_size + remainder = budget.token_budget % block_size + if remainder != 0: + raise ValueError("When enabling chunked prefill and " + "prefix caching, max_num_batched_tokens " + "(chunk size) must be dividable by " + "block size, but got chunk_size " + f"({budget.token_budget}) % block_size " + f"({block_size}) = {remainder}") + if remaining_token_budget < num_new_tokens: + num_new_tokens = (remaining_token_budget // + block_size) * block_size + else: + num_new_tokens = min(num_new_tokens, remaining_token_budget) + return num_new_tokens diff --git a/qwen3_6_scripts/sequence.py b/qwen3_6_scripts/sequence.py new file mode 100644 index 00000000..6c08613e --- /dev/null +++ b/qwen3_6_scripts/sequence.py @@ -0,0 +1,1386 @@ +"""Sequence and its related classes.""" +import copy +import enum +from abc import ABC, abstractmethod +from array import array +from collections import defaultdict +from dataclasses import dataclass +from functools import cached_property, reduce +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional +from typing import Sequence as GenericSequence +from typing import Set, Tuple, Union, cast + +import msgspec +import torch + +from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs +from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs +from vllm.lora.request import LoRARequest +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import SamplingParams +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics + +if TYPE_CHECKING: + from vllm.multimodal.base import MultiModalDataDict + +VLLM_TOKEN_ID_ARRAY_TYPE = "l" + +VLLM_INVALID_TOKEN_ID = -1 + + +# We use dataclass for now because it is used for +# openai server output, and msgspec is not serializable. +# TODO(sang): Fix it. +@dataclass +class Logprob: + """Infos for supporting OpenAI compatible logprobs and token ranks. + + Attributes: + logprob: The logprob of chosen token + rank: The vocab rank of chosen token (>=1) + decoded_token: The decoded chosen token index + """ + logprob: float + rank: Optional[int] = None + decoded_token: Optional[str] = None + + +# {token_id -> logprob} per each sequence group. None if the corresponding +# sequence group doesn't require prompt logprob. +PromptLogprobs = List[Optional[Dict[int, Logprob]]] +# {token_id -> logprob} for each sequence group. +SampleLogprobs = List[Dict[int, Logprob]] + + +class SequenceStatus(enum.IntEnum): + """Status of a sequence.""" + WAITING = 0 + RUNNING = 1 + SWAPPED = 2 + # Note: anything after SWAPPED (2) will be considered + # as a finished status. + FINISHED_STOPPED = 3 + FINISHED_LENGTH_CAPPED = 4 + FINISHED_ABORTED = 5 + FINISHED_IGNORED = 6 + + @staticmethod + def is_finished(status: "SequenceStatus") -> bool: + return status > SequenceStatus.SWAPPED + + @staticmethod + def get_finished_reason(status: "SequenceStatus") -> Union[str, None]: + if status == SequenceStatus.FINISHED_STOPPED: + finish_reason = "stop" + elif status == SequenceStatus.FINISHED_LENGTH_CAPPED: + finish_reason = "length" + elif status == SequenceStatus.FINISHED_ABORTED: + finish_reason = "abort" + elif status == SequenceStatus.FINISHED_IGNORED: + # The ignored sequences are the sequences whose prompt lengths + # are longer than the model's length cap. Therefore, the stop + # reason should also be "length" as in OpenAI API. + finish_reason = "length" + else: + finish_reason = None + return finish_reason + + +class SequenceStage(enum.Enum): + PREFILL = enum.auto() + DECODE = enum.auto() + + +@dataclass +class RequestMetrics: + """Metrics associated with a request. + + Attributes: + arrival_time: The time when the request arrived. + first_scheduled_time: The time when the request was first scheduled. + first_token_time: The time when the first token was generated. + time_in_queue: The time the request spent in the queue. + finished_time: The time when the request was finished. + scheduler_time: The time spent in the scheduler when this request was + being considered by the scheduler. + model_forward_time: The time spent in the model forward pass when this + request was in the batch. + model_execute_time: The time spent in the model execute function. This + will include model forward, block/sync across + workers, cpu-gpu sync time and sampling time. + """ + arrival_time: float + last_token_time: float + first_scheduled_time: Optional[float] + first_token_time: Optional[float] + time_in_queue: Optional[float] + finished_time: Optional[float] = None + scheduler_time: Optional[float] = None + model_forward_time: Optional[float] = None + model_execute_time: Optional[float] = None + num_cached_tokens: Optional[int] = None + + +class SequenceDataDelta( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Delta SequenceData to send to workers per step.""" + # A new token to be appended to existing SequenceData. + new_output_token_ids: List[int] + # Overwriting existing `cumulative_logprob` + new_cumulative_logprob: float + # Overwriting existing `num_computed_tokens`. + new_num_computed_tokens: int + # Overwriting existing `stage`. + new_stage: SequenceStage + + +class SequenceData(msgspec.Struct, + omit_defaults=True): # type: ignore[call-arg] + """Data associated with a sequence. + + Args: + prompt_token_ids: The token IDs of the prompt. + output_token_ids: The token IDs of the output. Set to an empty list if + None. + + Attributes: + prompt_token_ids: The token IDs of the prompt. + output_token_ids: The token IDs of the output. + cumulative_logprob: The cumulative log probability of the output. + """ + # NOTE: we cannot use Union[List, array] because msgspec cannot support + # union of 2 list types. + _prompt_token_ids: array + _output_token_ids: array = msgspec.field( + default_factory=lambda: array(VLLM_TOKEN_ID_ARRAY_TYPE, [])) + + ### The below fields should not be passed as an argument ### + _cumulative_logprob: float = 0.0 + _prompt_token_ids_tuple: Tuple[int, + ...] = msgspec.field(default_factory=tuple) + # The number of tokens that are computed (that run against the model). + _num_computed_tokens: int = 0 + _stage: SequenceStage = SequenceStage.PREFILL + _cached_all_token_ids: List[int] = msgspec.field(default_factory=list) + + # It is used to get delta input. It is reset when `get_delta_and_reset` + # is called. + _new_appended_tokens: List[int] = msgspec.field(default_factory=list) + + # It is used to compute mrope_position_ids. + _mrope_position_delta: Optional[int] = None + + @staticmethod + def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData": + if len(token_counts) == 0: + return SequenceData.from_seqs([]) + + arrs = [ + array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count + for token_id, count in token_counts + ] + + return SequenceData(reduce(array.__add__, arrs)) + + @staticmethod + def from_seqs( + prompt_token_ids: GenericSequence[int], + output_token_ids: Optional[GenericSequence[int]] = None, + ) -> "SequenceData": + prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE, + prompt_token_ids) + + if output_token_ids is None: + return SequenceData(prompt_token_ids_arr) + + output_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE, + output_token_ids) + + return SequenceData(prompt_token_ids_arr, + _output_token_ids=output_token_ids_arr) + + def __post_init__(self) -> None: + assert self._prompt_token_ids.typecode == "l" + assert self._output_token_ids.typecode == "l" + self._prompt_token_ids_tuple: Tuple[int, ...] = tuple( + self._prompt_token_ids) + self._update_cached_all_tokens() + + def _update_cached_all_tokens(self): + assert isinstance(self._prompt_token_ids, array) + assert isinstance(self._output_token_ids, array) + self._cached_all_token_ids: List[int] = list(self._prompt_token_ids + + self._output_token_ids) + + @property + def cumulative_logprob(self) -> float: + return self._cumulative_logprob + + @property + def prompt_token_ids(self) -> Tuple[int, ...]: + return self._prompt_token_ids_tuple + + @prompt_token_ids.setter + def prompt_token_ids(self, new_prompt_token_ids) -> None: + raise NotImplementedError + + @property + def prompt_token_ids_array(self) -> array: + """Return the prompt token ids in array type. + + Note that the array is in "I" type, and it is not compatible + with torch.long (2 bytes vs 4 bytes). So beware of the usage. + """ + return self._prompt_token_ids + + @property + def output_token_ids(self) -> Tuple[int, ...]: + return tuple(self._output_token_ids) + + @output_token_ids.setter + def output_token_ids(self, new_output_token_ids: List[int]) -> None: + self._output_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + new_output_token_ids) + self._update_cached_all_tokens() + + @property + def output_token_ids_array(self) -> array: + """Return the prompt token ids in array type. + + Note that the array is in "I" type, and it is not compatible + with torch.long (2 bytes vs 4 bytes). So beware of the usage. + """ + assert isinstance(self._output_token_ids, array) + return self._output_token_ids + + @property + def mrope_position_delta(self) -> Optional[int]: + return self._mrope_position_delta + + @mrope_position_delta.setter + def mrope_position_delta(self, new_mrope_position_delta): + self._mrope_position_delta = new_mrope_position_delta + + def append_token_id(self, token_id: int, logprob: float) -> None: + self._output_token_ids.append(token_id) + self._new_appended_tokens.append(token_id) + self._cached_all_token_ids.append(token_id) + self._cumulative_logprob += logprob + + def get_len(self) -> int: + return len(self._output_token_ids) + len(self._prompt_token_ids) + + def get_prompt_len(self) -> int: + return len(self._prompt_token_ids) + + def get_output_len(self) -> int: + return len(self._output_token_ids) + + def get_token_ids(self) -> List[int]: + return self._cached_all_token_ids + + def get_prefix_token_ids( + self, num_tokens: int + ) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]: + """Get prefix tokens, and make the return value hashable""" + prompt_length = self.get_prompt_len() + if num_tokens > prompt_length: + return (self._prompt_token_ids_tuple, + tuple(self._output_token_ids[:num_tokens - prompt_length])) + else: + return (self._prompt_token_ids_tuple[:num_tokens], None) + + def get_num_computed_tokens(self) -> int: + """Return the number of prefill tokens that are already computed.""" + return self._num_computed_tokens + + def update_num_computed_tokens(self, num_new_computed_tokens: int): + """Update number of tokens computed so far.""" + self._num_computed_tokens += num_new_computed_tokens + assert self._num_computed_tokens <= self.get_len(), ( + self._num_computed_tokens, self.get_len()) + # If all tokens are computed, it means it is in decoding phase. + if self.get_num_uncomputed_tokens() == 0: + self._stage = SequenceStage.DECODE + + def reset_state_for_recompute(self) -> None: + """Reset the number of computed tokens from this sequence. It is + supposed to be called when a sequence needs to be started from + the beginning again (e.g., sequence is preempted). + """ + self._num_computed_tokens = 0 + self._stage = SequenceStage.PREFILL + self._new_appended_tokens = [] + + def get_num_uncomputed_tokens(self) -> int: + """Return the number of prefill tokens that are not computed.""" + # we use `get_len()` which includes prompt_len + output_len instead + # of prompt_len here. This is because during recompute we need to + # prefill for both prompt and output. + return self.get_len() - self.get_num_computed_tokens() + + def get_last_token_id(self) -> int: + if not self._output_token_ids: + return self._prompt_token_ids[-1] + return self._output_token_ids[-1] + + def get_prompt_token_ids(self) -> Tuple[int, ...]: + return self.prompt_token_ids + + def get_output_token_ids(self) -> Tuple[int, ...]: + return self.output_token_ids + + def get_delta_and_reset(self) -> SequenceDataDelta: + delta = SequenceDataDelta(self._new_appended_tokens, + self._cumulative_logprob, + self.get_num_computed_tokens(), self.stage) + # Reset delta state. + self._new_appended_tokens = [] + return delta + + def apply_delta(self, delta: SequenceDataDelta): + self._num_computed_tokens = delta.new_num_computed_tokens + self._cumulative_logprob = delta.new_cumulative_logprob + self._stage = delta.new_stage + self._output_token_ids.extend(delta.new_output_token_ids) + self._cached_all_token_ids.extend(delta.new_output_token_ids) + + @property + def stage(self) -> SequenceStage: + return self._stage + + def __repr__(self) -> str: + return (f"SequenceData(" + f"prompt_token_ids={self._prompt_token_ids}, " + f"output_token_ids={self.output_token_ids}, " + f"cumulative_logprob={self.cumulative_logprob}, " + f"get_num_computed_tokens={self.get_num_computed_tokens()}") + + +class Sequence: + """Stores the data, status, and block information of a sequence. + + The sequence is constructed from the LLMInputs instance passed + in through the `inputs` constructor argument. + + For encoder/decoder models, LLMInputs encapsulates both a + decoder and encoder prompt, creating an ambiguity about which + prompt to construct the sequence from. The `from_decoder_prompt` + constructor argument signals whether to construct the Sequence + from the LLMInputs decoder prompt, or encoder prompt. + + Args: + seq_id: The ID of the sequence. + inputs: The inputs of the sequence. + block_size: The block size of the sequence. Should be the same as the + block size used by the block manager and cache engine. + eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM. + lora_request: LoRA request. + prompt_adapter_request: Prompt Adapter request. + from_decoder_prompt: Construct Sequence from LLMInputs decoder prompt + (True) or encoder prompt (False.) Must be True + for decoder-only model. + + """ + + def __init__( + self, + seq_id: int, + inputs: "LLMInputs", + block_size: int, + eos_token_id: Optional[int] = None, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + from_decoder_prompt: bool = True, + ) -> None: + self.seq_id = seq_id + self.inputs = inputs + self.block_size = block_size + self.eos_token_id = eos_token_id + self.lora_request = lora_request + self.prompt_adapter_request = prompt_adapter_request + self.from_decoder_prompt = from_decoder_prompt + + # For decoder-only models, a Sequence is constructed + # from an LLMInputs instance (the `inputs` arg.) + # + # For encoder/decoder models the same `inputs` + # instance could be utilized to construct either an + # encoder sequence or a decoder sequence, because + # `LLMInputs` has both decoder- and encoder-oriented + # member variables (i.e. it encapsulates both an encoder + # and a decoder prompt.) The decision of which type of sequence + # to generate is determined by the `from_decoder_prompt` argument. + # + # When constructing a encoder sequence + # (`from_decoder_prompt` False) it matters that + # the `LLMInputs` instance stored in `inputs` is valid + # in the sense that its encoder-related member variables are + # populated; below, an exception is raised if this is + # not the case. + # + # When constructing a decoder sequence (`from_decoder_prompt` True) + # it does not matter whether `inputs` has its encoder-related + # member variables populated. + if not (from_decoder_prompt + or is_valid_encoder_decoder_llm_inputs(inputs)): + raise ValueError("Cannot extract encoder input prompt from " + f"invalid input {inputs}; did you forget the " + "encoder input prompt fields?") + + self.data = SequenceData.from_seqs(self.prompt_token_ids) + self.output_logprobs: SampleLogprobs = [] + self.output_text = "" + + self.status = SequenceStatus.WAITING + self.stop_reason: Union[int, str, None] = None + + # These are used to keep track of delta outputs + self._last_output_token_ids_offset: int = 0 + self._last_output_text_offset: int = 0 + + # Used for incremental detokenization + self.prefix_offset = 0 + self.read_offset = 0 + # Input + output tokens + self.tokens: Optional[List[str]] = None + + @property + def n_blocks(self) -> int: + return (self.get_len() + self.block_size - 1) // self.block_size + + @cached_property + def prompt(self) -> Optional[str]: + # Select decoder or encoder input prompt str, as appropriate + prompt_key: str = ("prompt" + if self.from_decoder_prompt else "encoder_prompt") + + return cast(Optional[str], self.inputs.get(prompt_key)) + + @cached_property + def prompt_token_ids(self) -> List[int]: + # Select decoder or encoder input prompt token ids, as appropriate + prompt_token_ids_key: str = ("prompt_token_ids" + if self.from_decoder_prompt else + "encoder_prompt_token_ids") + + # Cache computed prompt token ids + return cast(List[int], self.inputs.get(prompt_token_ids_key)) + + @property + def multi_modal_data(self) -> "MultiModalDataDict": + if self.inputs.get("multi_modal_data") and self.inputs.get( + "encoder_multi_modal_data"): + raise ValueError( + "Multi-modal data in both encoder and decoder is not supported." + ) + inputs = self.inputs + return self.inputs.get("multi_modal_data") or (cast( + EncoderDecoderLLMInputs, + inputs).get("encoder_multi_modal_data")) or {} + + @property + def mm_processor_kwargs(self) -> Dict[str, Any]: + return self.inputs.get("mm_processor_kwargs") or {} + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + def get_output_text_to_return(self, buffer_length: int, + delta: bool) -> str: + """If delta is True, only new text since the last call to + this method is returned""" + + # We return the full output text if the sequence is finished. + truncate = buffer_length and not self.is_finished() + if not delta: + return self.output_text[:-buffer_length] if truncate else ( + self.output_text) + length = len(self.output_text) + if truncate: + length -= buffer_length + last_offset = self._last_output_text_offset + if last_offset < length: + self._last_output_text_offset = length + return self.output_text[last_offset:length] + return "" + + def get_output_token_ids_to_return( + self, delta: bool) -> Union[GenericSequence[int], int]: + """If delta is True, only new tokens since the last call to + this method are returned""" + if not delta: + return self.get_output_token_ids() + + output_len = self.get_output_len() + + # Get the number of new tokens + num_new_tokens = output_len - self._last_output_token_ids_offset + self._last_output_token_ids_offset = output_len + + # Return new tokens + if num_new_tokens == 0: + # During chunked prefill steps with no output yet, num_new_tokens=0. + # Python's [-0:] == [0:] returns the ENTIRE list — guard against this. + return [] + + if num_new_tokens == 1: + # Optimization for single decode token case + # (which is what we have most of the time) + return self.data._cached_all_token_ids[-1] + + return self.data._cached_all_token_ids[-num_new_tokens:] + + def hash_of_block(self, logical_idx: int) -> int: + # TODO This can produce incorrect hash when block size > prompt size + + # Compute the number of tokens in the sequence + # TODO: The current hashing function is O(L^2). We should optimize + # this in the future. + num_tokens = self.num_hashed_tokens_of_block(logical_idx) + hashed_tokens = self.data.get_prefix_token_ids(num_tokens) + return hash((hashed_tokens, self.lora_int_id)) + + def num_hashed_tokens_of_block(self, logical_idx: int): + return logical_idx * self.block_size + self.block_size + + def reset_state_for_recompute(self): + """Reset the sequence states for recomputation.""" + self.data.reset_state_for_recompute() + + def append_token_id(self, token_id: int, logprobs: Dict[int, + Logprob]) -> None: + assert token_id in logprobs + self.output_logprobs.append(logprobs) + self.data.append_token_id(token_id, logprobs[token_id].logprob) + + def get_len(self) -> int: + return self.data.get_len() + + def get_prompt_len(self) -> int: + return self.data.get_prompt_len() + + def get_output_len(self) -> int: + return self.data.get_output_len() + + def get_token_ids(self) -> List[int]: + return self.data.get_token_ids() + + def get_prompt_token_ids(self) -> Tuple[int, ...]: + return self.data.get_prompt_token_ids() + + def get_last_token_id(self) -> int: + return self.data.get_last_token_id() + + def get_output_token_ids(self) -> Tuple[int, ...]: + return self.data.get_output_token_ids() + + def get_cumulative_logprob(self) -> float: + return self.data.cumulative_logprob + + def is_finished(self) -> bool: + return SequenceStatus.is_finished(self.status) + + def fork(self, new_seq_id: int) -> "Sequence": + new_seq = copy.deepcopy(self) + new_seq.seq_id = new_seq_id + return new_seq + + def get_num_new_tokens(self) -> int: + """Get the number of new tokens to be computed. + + Returns: + The new number of tokens to be computed. I.e., 1 for decode, or + the remaining prompt size for prefill. + """ + if self.data.stage == SequenceStage.DECODE: + return 1 + return self.data.get_num_uncomputed_tokens() + + def is_prefill(self) -> bool: + return self.data.stage == SequenceStage.PREFILL + + def __repr__(self) -> str: + return (f"Sequence(seq_id={self.seq_id}, " + f"status={self.status.name}, " + f"num_blocks={self.n_blocks}, ") + + +class SequenceGroupState(msgspec.Struct, + omit_defaults=True): # type: ignore[call-arg] + """Mutable state tied to a specific sequence group""" + + # for multi-step decoding + num_steps: int = 1 + current_step: int = 0 + + @property + def remaining_steps(self) -> int: + return self.num_steps - self.current_step + + +class SequenceGroup: + """A group of sequences that are generated from the same prompt. + + Args: + request_id: The ID of the request. + seqs: The list of sequences. + sampling_params: The sampling parameters used to generate the outputs. + arrival_time: The arrival time of the request. + lora_request: LoRA request. + embeddings: The embeddings vectors of the prompt of the sequence group + for an embedding model. + pooling_params: The pooling parameters used to generate the pooling + for an embedding model. + encoder_seq: Optional, the single encoder sequence. Should be None + unless you are working with an encoder/decoder model. + trace_headers: OpenTelemetry trace headers. + prompt_adapter_request: Prompt Adapter request. + priority: User-defined priority of the request. + """ + + def __init__( + self, + request_id: str, + seqs: List[Sequence], + arrival_time: float, + sampling_params: Optional[SamplingParams] = None, + lora_request: Optional[LoRARequest] = None, + embeddings: Optional[List[float]] = None, + pooling_params: Optional[PoolingParams] = None, + encoder_seq: Optional[Sequence] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + self.request_id = request_id + self.seqs = seqs + self.arrival_time = arrival_time + self.is_single_seq = len(seqs) == 1 + self.seqs_dict = {seq.seq_id: seq for seq in seqs} + + self.sampling_params = sampling_params + self.metrics = RequestMetrics(arrival_time=arrival_time, + last_token_time=arrival_time, + first_scheduled_time=None, + first_token_time=None, + time_in_queue=None) + self.lora_request = lora_request + self.prompt_logprobs: Optional[PromptLogprobs] = None + self.state = SequenceGroupState() + self.embeddings = embeddings + self.pooling_params = pooling_params + self.prompt_adapter_request = prompt_adapter_request + self.encoder_seq = encoder_seq + self.trace_headers = trace_headers + self.priority = priority + + self.cached_request_output = None + + @property + def prompt(self) -> Optional[str]: + # All sequences in the group should have the same prompt. + # We use the prompt of an arbitrary sequence. + return self.seqs[0].prompt + + @property + def prompt_token_ids(self) -> List[int]: + # All sequences in the group should have the same prompt. + # We use the prompt of an arbitrary sequence. + return self.seqs[0].prompt_token_ids + + @property + def encoder_prompt(self) -> Optional[str]: + # There are either 0 or 1 encoder sequences + # If one is present, its prompt is distinct + # from the decoder's. + return (self.encoder_seq.prompt + if self.encoder_seq is not None else None) + + @property + def encoder_prompt_token_ids(self) -> Optional[List[int]]: + # There are either 0 or 1 encoder sequences + # If one is present, its prompt token ids are + # distinct from the decoder's. + return (self.encoder_seq.prompt_token_ids + if self.encoder_seq is not None else None) + + @property + def multi_modal_data(self) -> "MultiModalDataDict": + # All sequences in the group should have the same multi-modal data. + # We use the multi-modal data of an arbitrary sequence. + return self.seqs[0].multi_modal_data + + @property + def mm_processor_kwargs(self) -> Dict[str, Any]: + # As with multi-modal data, all sequences in the group should have the + # same processor kwargs (i.e., mm_processor_kwargs are optionally + # provided per request; note that are independent of whether the model + # decoder-only or an encoder-decoder). + return self.seqs[0].mm_processor_kwargs + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + @property + def prompt_adapter_num_virtual_tokens(self) -> int: + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\ + if self.prompt_adapter_request else 0 + + def init_multi_step(self, num_steps: int) -> None: + self.state.num_steps = num_steps + self.state.current_step = 0 + + def init_multi_step_from_lookahead_slots(self, num_lookahead_slots: int, + num_scheduler_steps: int, + is_multi_step: bool, + enable_chunking: bool) -> None: + + if not is_multi_step: + self.init_multi_step(num_steps=num_scheduler_steps) + return + + # Multi-Step case + is_prefill = self.is_prefill() + + # The asserts below reflect the expectations of the current system. + if is_prefill and enable_chunking: + assert num_lookahead_slots == num_scheduler_steps + self.init_multi_step(num_steps=num_lookahead_slots) + else: + is_decode: bool = not is_prefill + # If it is a prefill, num_lookahead_slots must be 0 + assert num_lookahead_slots == 0 or is_decode + # If it is a decode, num_lookahead_slots + 1 must match + # the scheduler steps. + assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill + self.init_multi_step(num_steps=num_lookahead_slots + 1) + + def get_last_latency(self, now: float) -> Optional[float]: + """Sets the last token time for Request level timings.""" + # If still in prefill phase, raise Error. + if self.is_prefill(): + raise ValueError( + "seq_group.get_last_latency() should not be called " + "if the seq_group is in prefill phase.") + + # Otherwise return token latency. + latency = now - self.metrics.last_token_time + self.metrics.last_token_time = now + return latency + + def maybe_set_first_token_time(self, time: float) -> None: + """Sets the first token time for Request level timings.""" + # Note: in a case where a sequence_group is swapped and + # recomputed, the time between iterations is counted + # in TPOT, rather than recalculating TTFT (since from the ) + # POV of the user, there is simply a long generation delay. + if (self.metrics.first_token_time is None + and self.seqs[0].get_output_len() == 1): + self.metrics.first_token_time = time + + def maybe_set_first_scheduled_time(self, time: float) -> None: + """Sets the first scheduled time and time in queue for Request + level timings.""" + if self.metrics.first_scheduled_time is None: + self.metrics.first_scheduled_time = time + self.metrics.time_in_queue = time - self.metrics.arrival_time + + def set_finished_time(self, time: Optional[float]) -> None: + """Sets the finished time for Request level timings.""" + self.metrics.finished_time = time + + def get_max_num_running_seqs(self) -> int: + """The maximum number of sequences running in parallel in the remaining + lifetime of the request.""" + if self.sampling_params: + n = self.sampling_params.n + assert isinstance(n, int) + if n > self.num_seqs(): + # At prompt stage, the sequence group is not yet filled up + # and only have one sequence running. However, in the + # generation stage, we will have `n` sequences + # running. + return n + # At sampling stages, return the number of actual sequences + # that are not finished yet. + return self.num_unfinished_seqs() + + def get_seqs( + self, + status: Optional[SequenceStatus] = None, + ) -> List[Sequence]: + if status is None: + return self.seqs + + if self.is_single_seq: + return self.seqs if self.seqs[0].status == status else [] + + return [seq for seq in self.seqs if seq.status == status] + + def is_encoder_decoder(self) -> bool: + return self.encoder_seq is not None + + def get_encoder_seq(self) -> Optional[Sequence]: + return self.encoder_seq + + def get_unfinished_seqs(self) -> List[Sequence]: + if self.is_single_seq: + return self.seqs if not self.seqs[0].is_finished() else [] + + return [seq for seq in self.seqs if not seq.is_finished()] + + def get_finished_seqs(self) -> List[Sequence]: + if self.is_single_seq: + return self.seqs if self.seqs[0].is_finished() else [] + + return [seq for seq in self.seqs if seq.is_finished()] + + def update_num_computed_tokens(self, num_new_computed_tokens: int): + """Update number of tokens computed so far.""" + for seq in self.seqs: + if not seq.is_finished(): + seq.data.update_num_computed_tokens(num_new_computed_tokens) + + def get_num_uncomputed_tokens(self) -> int: + num_uncomputed_tokens = 0 + for seq in self.seqs: + if not seq.is_finished(): + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens() + return num_uncomputed_tokens + + def num_seqs(self, status: Optional[SequenceStatus] = None) -> int: + # Optimization. We don't need to call get_seqs if we don't need to + # filter by states. + if status is None: + return len(self.seqs) + + if self.is_single_seq: + return 1 if self.seqs[0].status == status else 0 + + return len(self.get_seqs(status)) + + def num_unfinished_seqs(self) -> int: + if self.is_single_seq: + return 1 if not self.seqs[0].is_finished() else 0 + + return len(self.get_unfinished_seqs()) + + def num_finished_seqs(self) -> int: + if self.is_single_seq: + return 1 if self.seqs[0].is_finished() else 0 + + return len(self.get_finished_seqs()) + + def find(self, seq_id: int) -> Sequence: + if seq_id not in self.seqs_dict: + raise ValueError(f"Sequence {seq_id} not found.") + return self.seqs_dict[seq_id] + + def add(self, seq: Sequence) -> None: + if seq.seq_id in self.seqs_dict: + raise ValueError(f"Sequence {seq.seq_id} already exists.") + self.seqs_dict[seq.seq_id] = seq + self.seqs.append(seq) + self.is_single_seq = len(self.seqs) == 1 + + def remove(self, seq_id: int) -> None: + seq = self.seqs_dict.pop(seq_id, None) + if seq is None: + raise ValueError(f"Sequence {seq_id} not found.") + self.seqs.remove(seq) + self.is_single_seq = len(self.seqs) == 1 + + def is_finished(self) -> bool: + if self.is_single_seq: + return self.seqs[0].is_finished() + + return all(seq.is_finished() for seq in self.seqs) + + def is_prefill(self) -> bool: + # Every sequence should be in the same stage. + return self.seqs[0].is_prefill() + + def __repr__(self) -> str: + return (f"SequenceGroup(request_id={self.request_id}, " + f"sampling_params={self.sampling_params}, " + f"num_seqs={len(self.seqs)})") + + +class SequenceGroupMetadataDelta( + msgspec.Struct, + tag=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Delta of SequenceGroupMetadata. + + After sending the first SequenceGroupMetadata, vLLM scheduler + only sends delta to reduce the data payload size. + """ + seq_data_delta: Dict[int, SequenceDataDelta] + request_id: str + block_tables: Dict[int, List[int]] + is_prompt: bool + do_sample: bool = True + token_chunk_size: Optional[int] = None + computed_block_nums: Optional[List[int]] = None + state: Optional[SequenceGroupState] = msgspec.field( + default_factory=lambda: SequenceGroupState()) + + +class SequenceGroupMetadata( + msgspec.Struct, + tag=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Metadata for a sequence group. Used to create `AttentionMetadata`. + + Args: + request_id: The ID of the request. + is_prompt: Whether the request is at prompt stage. + seq_data: The sequence data. (Seq id -> sequence data) + sampling_params: The sampling parameters used to generate the outputs. + block_tables: The block tables. (Seq id -> list of physical block + numbers) + do_sample: True if sampling is required. Sampling is not required when + e.g., prefill is chunked, and the current iteration only computes + query tokens for prefill, we don't need sampling. + token_chunk_size: The number of tokens to be processed (per sequence). + None if chunking is not required. + lora_request: LoRA request. + computed_block_nums: The block numbers that are already computed, + used in prefix caching. + state: Internal state tied to this sequence group. + multi_modal_data: Multi modal data. + mm_processor_kwargs: Multimodal input processor / mapper overrides. + encoder_seq_data: Optional sequence data for encoder prompt + (SequenceGroup.encoder_seq). Should be None + unless you are working with an encoder/decoder + model. + cross_block_table: Optional cross-attention block table associated + with the encoder prompt + (SequenceGroup.encoder_seq). Should be None + unless you are working with an encoder/decoder + model. + prompt_adapter_request: Prompt Adapter request. + """ + + request_id: str + is_prompt: bool + seq_data: Dict[int, SequenceData] + sampling_params: Optional[SamplingParams] + block_tables: Dict[int, List[int]] + do_sample: bool = True + pooling_params: Optional[PoolingParams] = None + lora_request: Optional[LoRARequest] = None + computed_block_nums: Optional[List[int]] = None + state: Optional[SequenceGroupState] = msgspec.field( + default_factory=lambda: SequenceGroupState()) + # "MultiModalDataDict" types. We have to use Any due to msgspec + # doesn't allow to have union of 2 different dicts. + multi_modal_data: Optional[Any] = None + mm_processor_kwargs: Optional[Dict[str, Any]] = None + encoder_seq_data: Optional[SequenceData] = None + cross_block_table: Optional[List[int]] = None + prompt_adapter_request: Optional[PromptAdapterRequest] = None + token_chunk_size: Optional[int] = None + + ### Stateful fields that are lazily defined. ### + # The number of speculative tokens adopted in this request. + # None means specuative decoding is not used. + # Zero means speculative decoding is disabled for some reasons. + # TODO: We should maintain this states out of the sequence group. + num_speculative_tokens: Optional[int] = None + + def __post_init__(self): + if self.seq_data is not None and self.token_chunk_size is None: + if self.is_prompt: + self.token_chunk_size = next(iter( + self.seq_data.values())).get_len() + else: + self.token_chunk_size = 1 + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + @property + def prompt_adapter_num_virtual_tokens(self) -> int: + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \ + if self.prompt_adapter_request else 0 + + # Multi-Step Chunked-Prefill property + @property + def is_single_step_prompt(self) -> bool: + # do_sample is true, only when the token_chunk_size matches the + # num_uncomputed_tokens of the sequence. This indicates that + # the prompt will finish processing in a single `execute_model` + # step. + return self.is_prompt and self.do_sample + + def get_first_seq_id(self) -> int: + # This is an efficient way of fetching the seq_id when + # we know this SequenceGroup has only one sequence. + return next(iter(self.seq_data)) + + def apply_delta(self, + sequence_group_metadata_delta: SequenceGroupMetadataDelta): + for id, delta in sequence_group_metadata_delta.seq_data_delta.items(): + self.seq_data[id].apply_delta(delta) + assert self.request_id == sequence_group_metadata_delta.request_id + self.block_tables = sequence_group_metadata_delta.block_tables + self.token_chunk_size = sequence_group_metadata_delta.token_chunk_size + self.do_sample = sequence_group_metadata_delta.do_sample + self.is_prompt = sequence_group_metadata_delta.is_prompt + + def finish_step(self) -> None: + assert self.state is not None + assert self.state.current_step < self.state.num_steps, \ + f"current step {self.state.current_step}, num_steps {self.state.num_steps}" # noqa + self.state.current_step += 1 + + +class SequenceOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """The model output associated with a sequence. + + Args: + parent_seq_id: The ID of the parent sequence (for forking in beam + search). + output_token: The output token ID. + logprobs: The logprobs of the output token. + (Token id -> logP(x_i+1 | x_0, ..., x_i)) + """ + parent_seq_id: int + output_token: int + logprobs: Dict[int, Logprob] + + def __repr__(self) -> str: + return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, " + f"output_token={self.output_token}, " + f"logprobs={self.logprobs})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SequenceOutput): + raise NotImplementedError() + equal = (self.parent_seq_id == other.parent_seq_id + and self.output_token == other.output_token) + log_probs_equal = other.logprobs == self.logprobs + return equal and log_probs_equal + + +class SequenceGroupOutput(ABC): + """The base class for model outputs associated with a sequence group.""" + + @abstractmethod + def __repr__(self) -> str: + pass + + @abstractmethod + def __eq__(self, other: object) -> bool: + pass + + +class CompletionSequenceGroupOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + __metaclass__ = SequenceGroupOutput + """The model output associated with a completion sequence group.""" + samples: List[SequenceOutput] + # Prompt logprob for each prompt query token. + prompt_logprobs: Optional[PromptLogprobs] + + def __repr__(self) -> str: + return (f"CompletionSequenceGroupOutput(samples={self.samples}, " + f"prompt_logprobs={self.prompt_logprobs})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CompletionSequenceGroupOutput): + raise NotImplementedError() + return (self.samples == other.samples + and self.prompt_logprobs == other.prompt_logprobs) + + +class EmbeddingSequenceGroupOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] +): + """The model output associated with an embedding sequence group.""" + __metaclass__ = SequenceGroupOutput + embeddings: List[int] + + def __repr__(self) -> str: + return (f"EmbeddingSequenceGroupOutput(" + f"embeddings_shape={len(self.embeddings)})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, EmbeddingSequenceGroupOutput): + raise NotImplementedError() + return self.embeddings == other.embeddings + + +# cannot use msgspec.Struct here because Dynamo does not support it +@dataclass +class IntermediateTensors: + """For all pipeline stages except the last, we need to return the hidden + states and residuals to be sent to the next stage. This data structure + contains the hidden states and residuals for a request. + """ + + tensors: Dict[str, torch.Tensor] + + def __getitem__(self, key: Union[str, slice]): + if isinstance(key, str): + return self.tensors[key] + elif isinstance(key, slice): + return self.__class__({k: v[key] for k, v in self.tensors.items()}) + + def __setitem__(self, key: str, value): + self.tensors[key] = value + + def __len__(self): + return len(self.tensors) + + def __eq__(self, other: object): + return isinstance(other, self.__class__) and self + + def __repr__(self) -> str: + return f"IntermediateTensors(tensors={self.tensors})" + + +class PoolerOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """The output from a pooling operation in the embedding model.""" + outputs: List[EmbeddingSequenceGroupOutput] + + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None + + def __getitem__(self, idx: int): + return self.outputs[idx] + + def __setitem__(self, idx: int, value): + self.outputs[idx] = value + + def __len__(self): + return len(self.outputs) + + def __eq__(self, other: object): + return isinstance(other, + self.__class__) and self.outputs == other.outputs + + +def get_all_seq_ids( + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]: + """Given a list of SequenceGroupMetadata, create a list of all + sequence ids. + """ + return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data] + + +def get_all_seq_ids_and_request_ids( + seq_group_metadata_list: List[SequenceGroupMetadata] +) -> Tuple[List[int], Dict[str, Set[int]]]: + """Given a list of SequenceGroupMetadata, create a list of all + sequence ids. + """ + seq_ids: List[int] = [] + request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set) + for sg in seq_group_metadata_list: + for seq_id in sg.seq_data: + seq_ids.append(seq_id) + request_id_seq_ids_mapping[sg.request_id].add(seq_id) + return seq_ids, request_id_seq_ids_mapping + + +class HiddenStates(msgspec.Struct, array_like=True, + omit_defaults=True): # type: ignore[call-arg] + """Hidden states corresponding to in-progress sequences. + Used in speculative decoding to pass hidden states from + the target model to the proposer model. + + seq_ids are the sequence ids of each entry of the batch + dimension of the hidden_states tensor""" + # Scorer hidden states. For prefill step, it is used for hidden states of + # all tokens, whereas for decode step, it use used for last accepted tokens. + hidden_states: torch.Tensor + # The sequence group metadata list. Only needed for decode step. + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None + # Scorer hidden states of the 2nd last token proposed by the proposer ( + # irrespective of whether it was accepted or not). Only used for cases when + # last proposed token is accepted (i.e., in case of bonus tokens). For the + # case of no bonus tokens, these are ignored. + second_last_token_hidden_states: Optional[torch.Tensor] = None + + _seq_ids: List[int] = msgspec.field(default_factory=list) + + def __post_init__(self): + if self.seq_group_metadata_list is not None: + assert len(self.seq_group_metadata_list) == len(self.hidden_states) + self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list) + + @property + def seq_ids(self) -> List[int]: + return self._seq_ids + + def update(self, + hidden_states: torch.Tensor, + seq_group_metadata_list: List[SequenceGroupMetadata], + second_last_token_hidden_states: Optional[torch.Tensor] = None): + """Update hidden states from target model invocation. Only used for + decode steps""" + assert len(seq_group_metadata_list) == len(hidden_states) + self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list)) + self.hidden_states = torch.cat([self.hidden_states, hidden_states]) + + if self.second_last_token_hidden_states is not None: + # Adding dummy hidden_states to this to maintain same shape + self.second_last_token_hidden_states = torch.cat([ + self.second_last_token_hidden_states, + torch.zeros_like(hidden_states) + if second_last_token_hidden_states is None else + second_last_token_hidden_states + ]) + + def prune(self, + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None: + """Prune to provided list of sequence ids. Only used for decode steps. + """ + # Currently this prunes all seq_ids not present in + # seq_group_metadata_list which might cause problems where a sequence + # may be "paused" then "resumed" later. This should only prune sequences + # which are confirmed to be aborted. + seq_ids = get_all_seq_ids(seq_group_metadata_list) + if seq_ids != self._seq_ids: + # Batch contents changed - prune removed sequences. + index = [self._seq_ids.index(seq_id) for seq_id in seq_ids] + self.hidden_states = self.hidden_states[index] + if self.second_last_token_hidden_states is not None: + self.second_last_token_hidden_states = self\ + .second_last_token_hidden_states[index] + self._seq_ids = seq_ids + + def expand_with_bonus_tokens( + self, seq_with_bonus_token_in_last_step: set) -> None: + """Expand hidden states for sequences with bonus tokens. This is in + alignment with `MultiStepWorker._expand_execute_model_request`.""" + if self.second_last_token_hidden_states is None \ + or not seq_with_bonus_token_in_last_step: + return + + index = [] + for seq_id in self._seq_ids: + i = self._seq_ids.index(seq_id) + if seq_id in seq_with_bonus_token_in_last_step: + index.append(i + len(self._seq_ids)) + index.append(i) + + self.hidden_states = torch.cat( + [self.hidden_states, self.second_last_token_hidden_states])[index] + + +class ExecuteModelRequest( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """The model execution request, containing CPU metadata only. The LLM + engine should create an instance of this class for each request batch.""" + # The sequence group metadata list. + seq_group_metadata_list: List[Union[SequenceGroupMetadata, + SequenceGroupMetadataDelta]] + # Blocks to swap in. List of CPU -> GPU block number. + blocks_to_swap_in: List[Tuple[int, + int]] = msgspec.field(default_factory=list) + # Blocks to swap out. List of GPU -> CPU block number. + blocks_to_swap_out: List[Tuple[int, + int]] = msgspec.field(default_factory=list) + # Blocks to copy. Source to dest block. + blocks_to_copy: List[Tuple[int, int]] = msgspec.field(default_factory=list) + # Virtual engine ID for pipeline parallel. + virtual_engine: int = 0 + # The number of slots for lookahead decoding. + num_lookahead_slots: int = 0 + # The number of requests in the running queue. + running_queue_size: int = 0 + # Optional hidden states from prior step. + previous_hidden_states: Optional[HiddenStates] = None + # The number of forward steps to run. + num_steps: int = 1 + # Finished request ids since last step. + finished_requests_ids: List[str] = msgspec.field(default_factory=list) + # The last sampled token ids for multi step decoding. + last_sampled_token_ids: Optional[torch.Tensor] = None + # Async callback + async_callback: Optional[Callable] = None + + @property + def is_first_multi_step(self) -> bool: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + first_seq_group = self.seq_group_metadata_list[0] + assert first_seq_group.state is not None + return first_seq_group.state.current_step == 0 + + @property + def is_last_step(self) -> bool: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + first_seq_group = self.seq_group_metadata_list[0] + assert first_seq_group.state is not None + return first_seq_group.state.remaining_steps == 1 + + @property + def current_step(self) -> int: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + state = self.seq_group_metadata_list[0].state + assert state is not None + return state.current_step + + def clone( + self, seq_group_metadata_list: List[Union[SequenceGroupMetadata, + SequenceGroupMetadataDelta]] + ) -> "ExecuteModelRequest": + """Clone the request with a new sequence group metadata list.""" + return ExecuteModelRequest( + seq_group_metadata_list=seq_group_metadata_list, + blocks_to_swap_in=self.blocks_to_swap_in.copy(), + blocks_to_swap_out=self.blocks_to_swap_out.copy(), + blocks_to_copy=self.blocks_to_copy.copy(), + virtual_engine=self.virtual_engine, + num_lookahead_slots=self.num_lookahead_slots, + running_queue_size=self.running_queue_size, + previous_hidden_states=self.previous_hidden_states, + num_steps=self.num_steps, + finished_requests_ids=self.finished_requests_ids, + last_sampled_token_ids=self.last_sampled_token_ids.clone() + if self.last_sampled_token_ids is not None else None, + async_callback=self.async_callback) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py new file mode 100644 index 00000000..988905da --- /dev/null +++ b/qwen3_6_scripts/serving_chat.py @@ -0,0 +1,1090 @@ +import asyncio +import json +import time +from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List, + Optional) +from typing import Sequence as GenericSequence +from typing import Union + +from fastapi import Request + +from vllm.config import ModelConfig +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.multiprocessing.client import MQLLMEngineClient +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.chat_utils import (ConversationMessage, + apply_hf_chat_template, + apply_mistral_chat_template, + load_chat_template, + parse_chat_messages_futures) +from vllm.entrypoints.logger import RequestLogger +from vllm.entrypoints.openai.protocol import ( + ChatCompletionLogProb, ChatCompletionLogProbs, + ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam, + ChatCompletionRequest, ChatCompletionResponse, + ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage, + DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata, + PromptTokensDetails, ToolCall, UsageInfo) +from vllm.entrypoints.openai.serving_engine import (BaseModelPath, + LoRAModulePath, + OpenAIServing, + PromptAdapterPath, + TextTokensPrompt) +from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager +from vllm.inputs import TokensPrompt +from vllm.logger import init_logger +from vllm.outputs import CompletionOutput, RequestOutput +from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.sequence import Logprob +from vllm.tracing import (contains_trace_headers, extract_trace_headers, + log_tracing_disabled_warning) +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.utils import iterate_with_cancellation, random_uuid + +logger = init_logger(__name__) + + +class OpenAIServingChat(OpenAIServing): + + def __init__(self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + response_role: str, + *, + lora_modules: Optional[List[LoRAModulePath]], + prompt_adapters: Optional[List[PromptAdapterPath]], + request_logger: Optional[RequestLogger], + chat_template: Optional[str], + return_tokens_as_token_ids: bool = False, + enable_auto_tools: bool = False, + tool_parser: Optional[str] = None, + reasoning_parser: Optional[str] = None): + super().__init__(engine_client=engine_client, + model_config=model_config, + base_model_paths=base_model_paths, + lora_modules=lora_modules, + prompt_adapters=prompt_adapters, + request_logger=request_logger, + return_tokens_as_token_ids=return_tokens_as_token_ids) + + self.response_role = response_role + self.use_tool_use_model_template = False + self.chat_template = load_chat_template(chat_template) + + # set up tool use + self.enable_auto_tools: bool = enable_auto_tools + if self.enable_auto_tools: + logger.info( + "\"auto\" tool choice has been enabled please note that while" + " the parallel_tool_calls client option is preset for " + "compatibility reasons, it will be ignored.") + + self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None + if self.enable_auto_tools: + try: + self.tool_parser = ToolParserManager.get_tool_parser( + tool_parser) + except Exception as e: + raise TypeError("Error: --enable-auto-tool-choice requires " + f"tool_parser:'{tool_parser}' which has not " + "been registered") from e + + # set up reasoning parser + self.reasoning_parser_cls = None + if reasoning_parser: + try: + from vllm.reasoning import ReasoningParserManager + self.reasoning_parser_cls = \ + ReasoningParserManager.get_reasoning_parser(reasoning_parser) + logger.info("Reasoning parser '%s' enabled.", reasoning_parser) + except Exception as e: + raise TypeError( + f"Error: --reasoning-parser '{reasoning_parser}' could not " + "be loaded. Make sure vllm/reasoning/ is installed." + ) from e + + async def create_chat_completion( + self, + request: ChatCompletionRequest, + raw_request: Optional[Request] = None, + ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, + ErrorResponse]: + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/chat/create + for the API specification. This API mimics the OpenAI + ChatCompletion API. + + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + logger.error("Error with model %s", error_check_ret) + return error_check_ret + + # If the engine is dead, raise the engine's DEAD_ERROR. + # This is required for the streaming case, where we return a + # success status before we actually start generating text :). + if self.engine_client.errored: + raise self.engine_client.dead_error + + try: + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + model_config = self.model_config + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + conversation, mm_data_future = parse_chat_messages_futures( + request.messages, model_config, tokenizer) + + tool_dicts = None if request.tools is None else [ + tool.model_dump() for tool in request.tools + ] + + prompt: Union[str, List[int]] + is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer) + if is_mistral_tokenizer: + prompt = apply_mistral_chat_template( + tokenizer, + messages=request.messages, + chat_template=request.chat_template or self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + tools=tool_dicts, + documents=request.documents, + **(request.chat_template_kwargs or {}), + ) + else: + prompt = apply_hf_chat_template( + tokenizer, + conversation=conversation, + chat_template=request.chat_template or self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + tools=tool_dicts, + documents=request.documents, + **(request.chat_template_kwargs or {}), + ) + except Exception as e: + logger.exception("Error in applying chat template from request") + return self.create_error_response(str(e)) + + try: + mm_data = await mm_data_future + except Exception as e: + logger.exception("Error in loading multi-modal data") + return self.create_error_response(str(e)) + + # n > max_num_seqs deadlock guard: scheduler uses break (not continue) + # when can_schedule(num_new_seqs=n) fails, so an n that exceeds + # max_num_seqs permanently blocks the entire waiting queue with no error. + _sched_cfg = await self.engine_client.get_scheduler_config() + _max_seqs = _sched_cfg.max_num_seqs + if request.n is not None and request.n > _max_seqs: + return self.create_error_response( + f"n={request.n} exceeds max_num_seqs={_max_seqs}. " + f"Use n<={_max_seqs} or omit n.") + + # validation for OpenAI tools + # tool_choice = "required" is not supported + if request.tool_choice == "required": + return self.create_error_response( + "tool_choice = \"required\" is not supported!") + + if not is_mistral_tokenizer and request.tool_choice == "auto" and not ( + self.enable_auto_tools and self.tool_parser is not None): + # for hf tokenizers, "auto" tools requires + # --enable-auto-tool-choice and --tool-call-parser + return self.create_error_response( + "\"auto\" tool choice requires " + "--enable-auto-tool-choice and --tool-call-parser to be set") + + request_id = f"chat-{random_uuid()}" + + request_metadata = RequestResponseMetadata(request_id=request_id) + if raw_request: + raw_request.state.request_metadata = request_metadata + + try: + if self.enable_auto_tools and self.tool_parser: + request = self.tool_parser(tokenizer).adjust_request( + request=request) + + if isinstance(prompt, str): + prompt_inputs = self._tokenize_prompt_input( + request, + tokenizer, + prompt, + truncate_prompt_tokens=request.truncate_prompt_tokens, + add_special_tokens=request.add_special_tokens, + ) + else: + assert isinstance(prompt, list) and isinstance( + prompt[0], int + ), "Prompt has to be either a string or a list of token ids" + prompt_inputs = TextTokensPrompt( + prompt=tokenizer.decode(prompt), prompt_token_ids=prompt) + + assert prompt_inputs is not None + + sampling_params: Union[SamplingParams, BeamSearchParams] + default_max_tokens = self.max_model_len - len( + prompt_inputs["prompt_token_ids"]) + if request.use_beam_search: + sampling_params = request.to_beam_search_params( + default_max_tokens) + else: + sampling_params = request.to_sampling_params( + default_max_tokens) + + self._log_inputs(request_id, + prompt_inputs, + params=sampling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + engine_inputs = TokensPrompt( + prompt_token_ids=prompt_inputs["prompt_token_ids"]) + if mm_data is not None: + engine_inputs["multi_modal_data"] = mm_data + + is_tracing_enabled = (await + self.engine_client.is_tracing_enabled()) + trace_headers = None + if is_tracing_enabled and raw_request: + trace_headers = extract_trace_headers(raw_request.headers) + if (not is_tracing_enabled and raw_request + and contains_trace_headers(raw_request.headers)): + log_tracing_disabled_warning() + + if isinstance(sampling_params, BeamSearchParams): + assert isinstance(self.engine_client, + (AsyncLLMEngine, + MQLLMEngineClient)), \ + "Beam search is only supported with" \ + "AsyncLLMEngine and MQLLMEngineClient." + result_generator = self.engine_client.beam_search( + engine_inputs['prompt_token_ids'], + request_id, + sampling_params, + ) + else: + result_generator = self.engine_client.generate( + engine_inputs, + sampling_params, + request_id, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=request.priority, + ) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + if raw_request: + result_generator = iterate_with_cancellation( + result_generator, raw_request.is_disconnected) + + # Streaming response + if request.stream: + return self.chat_completion_stream_generator( + request, result_generator, request_id, conversation, tokenizer, + request_metadata, raw_request=raw_request) + + try: + return await self.chat_completion_full_generator( + request, result_generator, request_id, conversation, tokenizer, + request_metadata, raw_request=raw_request) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + def get_chat_request_role(self, request: ChatCompletionRequest) -> str: + if request.add_generation_prompt: + return self.response_role + return request.messages[-1]["role"] + + async def chat_completion_stream_generator( + self, + request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], + request_id: str, + conversation: List[ConversationMessage], + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + raw_request: Optional[Request] = None, + ) -> AsyncGenerator[str, None]: + model_name = self.base_model_paths[0].name + created_time = int(time.time()) + chunk_object_type: Final = "chat.completion.chunk" + first_iteration = True + + # Send response for each token for each request.n (index) + num_choices = 1 if request.n is None else request.n + previous_num_tokens = [0] * num_choices + finish_reason_sent = [False] * num_choices + num_prompt_tokens = 0 + num_cached_tokens: Optional[int] = None + + if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): + tool_choice_function_name = request.tool_choice.function.name + else: + tool_choice_function_name = None + + # Determine whether tools are in use with "auto" tool choice + tool_choice_auto = ( + not tool_choice_function_name + and self._should_stream_with_auto_tool_parsing(request)) + + use_reasoning = self.reasoning_parser_cls is not None + + all_previous_token_ids: Optional[List[List[int]]] + # previous_texts / all_previous_token_ids are needed for both tool + # parsing and reasoning parsing (both require full-history context). + if tool_choice_auto or use_reasoning: + previous_texts = [""] * num_choices + all_previous_token_ids = [[]] * num_choices + else: + previous_texts, all_previous_token_ids = None, None + + # Prepare the tool parser if it's needed + try: + if tool_choice_auto and self.tool_parser: + tool_parsers: List[Optional[ToolParser]] = [ + self.tool_parser(tokenizer) + ] * num_choices + else: + tool_parsers = [None] * num_choices + except RuntimeError as e: + logger.error("Error in tool parser creation: %s", e) + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + yield "data: [DONE]\n\n" + return + + # Prepare reasoning parsers (one instance per choice for state isolation) + reasoning_parsers: List[Optional[object]] = [None] * num_choices + reasoning_end_arr: List[bool] = [False] * num_choices + reasoning_token_counts: List[int] = [0] * num_choices + if use_reasoning: + try: + reasoning_parsers = [ + self.reasoning_parser_cls( + tokenizer, + chat_template_kwargs=request.chat_template_kwargs) + for _ in range(num_choices) + ] + # If thinking is disabled per-request, mark reasoning as + # already ended so the tool-auto branch is reachable. + for idx, rp in enumerate(reasoning_parsers): + if hasattr(rp, 'thinking_enabled') and not rp.thinking_enabled: + reasoning_end_arr[idx] = True + except RuntimeError as e: + logger.error("Error in reasoning parser creation: %s", e) + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + yield "data: [DONE]\n\n" + return + + # Background task: poll is_disconnected() every 300 ms and abort the + # engine request as soon as the client goes away. This catches the + # case where the HTTP layer (Starlette/uvicorn) does not actively read + # the receive channel during streaming, so is_disconnected() in + # iterate_with_cancellation never fires during fast decode. + _disconnect_watcher: Optional[asyncio.Task] = None + if raw_request is not None: + async def _watch_disconnect() -> None: + try: + while True: + if await raw_request.is_disconnected(): + logger.info( + "Client disconnected (decode watcher), " + "aborting request %s", request_id) + await self.engine_client.abort(request_id) + return + await asyncio.sleep(0.3) + except asyncio.CancelledError: + pass + _disconnect_watcher = asyncio.ensure_future(_watch_disconnect()) + + try: + async for res in result_generator: + if res.prompt_token_ids is not None: + num_prompt_tokens = len(res.prompt_token_ids) + if res.encoder_prompt_token_ids is not None: + num_prompt_tokens += len(res.encoder_prompt_token_ids) + if (num_cached_tokens is None + and res.metrics is not None + and res.metrics.num_cached_tokens is not None): + num_cached_tokens = res.metrics.num_cached_tokens + + # We need to do it here, because if there are exceptions in + # the result_generator, it needs to be sent as the FIRST + # response (by the try...catch). + if first_iteration: + # Send first response for each request.n (index) with + # the role + role = self.get_chat_request_role(request) + + # NOTE num_choices defaults to 1 so this usually executes + # once per request + for i in range(num_choices): + tool_parser = tool_parsers[i] + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage( + role=role, + content="", + ), + logprobs=None, + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + + # if usage should be included + if (request.stream_options + and request.stream_options.include_usage): + # if continuous usage stats are requested, add it + if request.stream_options.continuous_usage_stats: + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=0, + total_tokens=num_prompt_tokens) + chunk.usage = usage + # otherwise don't + else: + chunk.usage = None + + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + + # Send response to echo the input portion of the + # last message + if request.echo or request.continue_final_message: + last_msg_content: str = "" + if conversation and "content" in conversation[ + -1] and conversation[-1].get("role") == role: + last_msg_content = conversation[-1]["content"] or "" + + if last_msg_content: + for i in range(num_choices): + choice_data = ( + ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage( + content=last_msg_content), + logprobs=None, + finish_reason=None)) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if (request.stream_options and + request.stream_options.include_usage): + if (request.stream_options. + continuous_usage_stats): + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=0, + total_tokens=num_prompt_tokens) + chunk.usage = usage + else: + chunk.usage = None + + data = chunk.model_dump_json( + exclude_unset=True) + yield f"data: {data}\n\n" + first_iteration = False + + for output in res.outputs: + i = output.index + tool_parser = tool_parsers[i] + + if finish_reason_sent[i]: + continue + + if request.logprobs and request.top_logprobs is not None: + assert output.logprobs is not None, ( + "Did not output logprobs") + logprobs = self._create_chat_logprobs( + token_ids=output.token_ids, + top_logprobs=output.logprobs, + tokenizer=tokenizer, + num_output_top_logprobs=request.top_logprobs, + ) + else: + logprobs = None + + delta_text = output.text + delta_message: Optional[DeltaMessage] + + # Maintain text/token history when either reasoning or + # auto-tool parsing is active. + assert previous_texts is not None or not ( + tool_choice_auto or use_reasoning) + if previous_texts is not None: + assert all_previous_token_ids is not None + previous_text = previous_texts[i] + previous_token_ids = all_previous_token_ids[i] + current_text = previous_text + delta_text + current_token_ids = previous_token_ids + list( + output.token_ids) + previous_texts[i] = current_text + all_previous_token_ids[i] = current_token_ids + else: + previous_text = "" + previous_token_ids = [] + current_text = delta_text + current_token_ids = list(output.token_ids) + + # handle streaming deltas for tools with named tool_choice + if tool_choice_function_name: + delta_message = DeltaMessage(tool_calls=[ + DeltaToolCall(function=DeltaFunctionCall( + name=tool_choice_function_name, + arguments=delta_text), + index=i) + ]) + + # handle reasoning: route through reasoning parser while + # has not yet been seen. + elif use_reasoning and not reasoning_end_arr[i]: + r_parser = reasoning_parsers[i] + delta_message = r_parser.extract_reasoning_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=output.token_ids, + ) + # Mark reasoning as ended when end token appears. + if r_parser.end_token_id in current_token_ids: + reasoning_end_arr[i] = True + + # handle streaming deltas for tools with "auto" tool choice + # (only reached after reasoning block, if any, has ended) + elif tool_choice_auto: + assert tool_parser is not None + delta_message = ( + tool_parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=output.token_ids, + request=request)) + + # handle streaming just a content delta + else: + delta_message = DeltaMessage(content=delta_text) + + # set the previous values for the next iteration + previous_num_tokens[i] += len(output.token_ids) + + # if the message delta is None (e.g. because it was a + # "control token" for tool calls or the parser otherwise + # wasn't ready to send a token, then + # get the next token without streaming a chunk. + # However, if this is the finish token we must NOT skip — + # the finish block updates reasoning_token_counts, sets + # finish_reason_sent, and flushes the final usage chunk. + if delta_message is None: + if output.finish_reason is None: + continue + delta_message = DeltaMessage() + + if output.finish_reason is None: + # Send token-by-token response for each request.n + + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=delta_message, + logprobs=logprobs, + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + + # handle usage stats if requested & if continuous + if (request.stream_options + and request.stream_options.include_usage): + if request.stream_options.continuous_usage_stats: + completion_tokens = len(output.token_ids) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + + completion_tokens, + ) + chunk.usage = usage + else: + chunk.usage = None + + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + + # if the model is finished generating + else: + # check to make sure we haven't "forgotten" to stream + # any tokens that were generated but previously + # matched by partial json parsing + # only happens if we are NOT using guided decoding + auto_tools_called = False + if tool_parser: + auto_tools_called = len( + tool_parser.prev_tool_call_arr) > 0 + index = len(tool_parser.prev_tool_call_arr + ) - 1 if auto_tools_called else 0 + else: + index = 0 + + if self._should_check_for_unstreamed_tool_arg_tokens( + delta_message, output) and tool_parser: + # get the expected call based on partial JSON + # parsing which "autocompletes" the JSON + expected_call = json.dumps( + tool_parser.prev_tool_call_arr[index].get( + "arguments", {})) + + # get what we've streamed so far for arguments + # for the current tool + actual_call = tool_parser.streamed_args_for_tool[ + index] + + # check to see if there's anything left to stream + remaining_call = expected_call.replace( + actual_call, "", 1) + + # set that as a delta message + delta_message = DeltaMessage(tool_calls=[ + DeltaToolCall(index=index, + function=DeltaFunctionCall( + arguments=remaining_call). + model_dump(exclude_none=True)) + ]) + + # Count reasoning tokens for this choice at finish time. + if use_reasoning and all_previous_token_ids is not None: + r_parser = reasoning_parsers[i] + reasoning_token_counts[i] = \ + r_parser.count_reasoning_tokens( + all_previous_token_ids[i]) + + # Send the finish response for each request.n only once + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=delta_message, + logprobs=logprobs, + finish_reason=output.finish_reason + if not auto_tools_called else "tool_calls", + stop_reason=output.stop_reason) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if (request.stream_options + and request.stream_options.include_usage): + if request.stream_options.continuous_usage_stats: + completion_tokens = len(output.token_ids) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + + completion_tokens, + ) + chunk.usage = usage + else: + chunk.usage = None + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + finish_reason_sent[i] = True + + # once the final token is handled, if stream_options.include_usage + # is sent, send the usage + if (request.stream_options + and request.stream_options.include_usage): + completion_tokens = previous_num_tokens[i] + total_reasoning = sum(reasoning_token_counts) if use_reasoning else None + final_usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + completion_tokens, + reasoning_tokens=total_reasoning, + prompt_tokens_details=( + PromptTokensDetails(cached_tokens=num_cached_tokens) + if num_cached_tokens is not None else None), + ) + + final_usage_chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[], + model=model_name, + usage=final_usage) + final_usage_data = (final_usage_chunk.model_dump_json( + exclude_unset=True, exclude_none=True)) + yield f"data: {final_usage_data}\n\n" + + # report to FastAPI middleware aggregate usage across all choices + num_completion_tokens = sum(previous_num_tokens) + total_reasoning = sum(reasoning_token_counts) if use_reasoning else None + request_metadata.final_usage_info = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_completion_tokens, + total_tokens=num_prompt_tokens + num_completion_tokens, + reasoning_tokens=total_reasoning) + + except asyncio.CancelledError: + # Client disconnected via CancelledError path; abort engine request. + await self.engine_client.abort(request_id) + return + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + logger.error("error in chat completion stream generator: %s", e) + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + finally: + # Stop the disconnect watcher (it may already be done if it fired). + if _disconnect_watcher is not None and not _disconnect_watcher.done(): + _disconnect_watcher.cancel() + try: + await _disconnect_watcher + except asyncio.CancelledError: + pass + # Covers GeneratorExit when Starlette calls aclose() on disconnect + # during decode (tokens arrive fast so CancelledError path is not + # always triggered). abort() is a no-op for already-finished requests. + await self.engine_client.abort(request_id) + # Send the final done message after all response.n are finished + yield "data: [DONE]\n\n" + + async def chat_completion_full_generator( + self, + request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], + request_id: str, + conversation: List[ConversationMessage], + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + raw_request: Optional[Request] = None, + ) -> Union[ErrorResponse, ChatCompletionResponse]: + + model_name = self.base_model_paths[0].name + created_time = int(time.time()) + final_res: Optional[RequestOutput] = None + + # Background watcher: same logic as the streaming path — polls + # is_disconnected() every 300 ms so that a client disconnect during + # non-streaming decode is caught even when uvicorn isn't actively + # reading the receive channel. + _disconnect_watcher: Optional[asyncio.Task] = None + if raw_request is not None: + async def _watch_disconnect() -> None: + try: + while True: + if await raw_request.is_disconnected(): + logger.info( + "Client disconnected (non-stream watcher), " + "aborting request %s", request_id) + await self.engine_client.abort(request_id) + return + await asyncio.sleep(0.3) + except asyncio.CancelledError: + pass + _disconnect_watcher = asyncio.ensure_future(_watch_disconnect()) + + try: + async for res in result_generator: + final_res = res + except asyncio.CancelledError: + await self.engine_client.abort(request_id) + return self.create_error_response("Client disconnected") + finally: + if _disconnect_watcher is not None and not _disconnect_watcher.done(): + _disconnect_watcher.cancel() + try: + await _disconnect_watcher + except asyncio.CancelledError: + pass + await self.engine_client.abort(request_id) + + assert final_res is not None + + choices: List[ChatCompletionResponseChoice] = [] + + role = self.get_chat_request_role(request) + for output in final_res.outputs: + token_ids = output.token_ids + out_logprobs = output.logprobs + + if request.logprobs and request.top_logprobs is not None: + assert out_logprobs is not None, "Did not output logprobs" + logprobs = self._create_chat_logprobs( + token_ids=token_ids, + top_logprobs=out_logprobs, + num_output_top_logprobs=request.top_logprobs, + tokenizer=tokenizer, + ) + else: + logprobs = None + + # In the OpenAI API the finish_reason is "tools_called" + # if the tool choice is auto and the model produced a tool + # call. The same is not true for named function calls + auto_tools_called = False + + # Extract reasoning content if parser is configured. + # output_text is what remains after stripping .... + reasoning_text: Optional[str] = None + output_text: str = output.text + if self.reasoning_parser_cls: + r_parser = self.reasoning_parser_cls( + tokenizer, + chat_template_kwargs=request.chat_template_kwargs) + reasoning_text, extracted = r_parser.extract_reasoning( + output.text, request) + output_text = extracted or "" + + # if auto tools are not enabled, and a named tool choice using + # outlines is not being used + if (not self.enable_auto_tools + or not self.tool_parser) and not isinstance( + request.tool_choice, + ChatCompletionNamedToolChoiceParam): + message = ChatMessage(role=role, + reasoning_content=reasoning_text, + content=output_text) + + # if the request uses tools and specified a tool choice + elif request.tool_choice and type( + request.tool_choice) is ChatCompletionNamedToolChoiceParam: + + message = ChatMessage( + role=role, + reasoning_content=reasoning_text, + content="", + tool_calls=[ + ToolCall(function=FunctionCall( + name=request.tool_choice.function.name, + arguments=output_text)) + ]) + + # if the request doesn't use tool choice + # OR specifies to not use a tool + elif not request.tool_choice or request.tool_choice == "none": + + message = ChatMessage(role=role, + reasoning_content=reasoning_text, + content=output_text) + + # handle when there are tools and tool choice is auto + elif request.tools and ( + request.tool_choice == "auto" + or request.tool_choice is None) and self.enable_auto_tools \ + and self.tool_parser: + + try: + tool_parser = self.tool_parser(tokenizer) + except RuntimeError as e: + logger.error("Error in tool parser creation: %s", e) + return self.create_error_response(str(e)) + + # Parse tool calls from the post-reasoning content. + tool_call_info = tool_parser.extract_tool_calls( + output_text, request=request) + auto_tools_called = tool_call_info.tools_called + if tool_call_info.tools_called: + message = ChatMessage( + role=role, + reasoning_content=reasoning_text, + content=tool_call_info.content, + tool_calls=tool_call_info.tool_calls) + else: + message = ChatMessage(role=role, + reasoning_content=reasoning_text, + content=output_text) + + # undetermined case that is still important to handle + else: + logger.error( + "Error in chat_completion_full_generator - cannot determine" + " if tools should be extracted. Returning a standard chat " + "completion.") + message = ChatMessage(role=role, + reasoning_content=reasoning_text, + content=output_text) + + choice_data = ChatCompletionResponseChoice( + index=output.index, + message=message, + logprobs=logprobs, + finish_reason="tool_calls" if auto_tools_called else + output.finish_reason if output.finish_reason else "stop", + stop_reason=output.stop_reason) + choices.append(choice_data) + + if request.echo or request.continue_final_message: + last_msg_content = "" + if conversation and "content" in conversation[-1] and conversation[ + -1].get("role") == role: + last_msg_content = conversation[-1]["content"] or "" + + for choice in choices: + full_message = last_msg_content + (choice.message.content + or "") + choice.message.content = full_message + + assert final_res.prompt_token_ids is not None + num_prompt_tokens = len(final_res.prompt_token_ids) + if final_res.encoder_prompt_token_ids is not None: + num_prompt_tokens += len(final_res.encoder_prompt_token_ids) + num_generated_tokens = sum( + len(output.token_ids) for output in final_res.outputs) + total_reasoning_tokens: Optional[int] = None + if self.reasoning_parser_cls: + rp = self.reasoning_parser_cls( + tokenizer, + chat_template_kwargs=request.chat_template_kwargs) + total_reasoning_tokens = sum( + rp.count_reasoning_tokens(list(output.token_ids)) + for output in final_res.outputs) + num_cached_tokens = (final_res.metrics.num_cached_tokens + if final_res.metrics is not None else None) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + reasoning_tokens=total_reasoning_tokens, + prompt_tokens_details=( + PromptTokensDetails(cached_tokens=num_cached_tokens) + if num_cached_tokens is not None else None), + ) + + request_metadata.final_usage_info = usage + + response = ChatCompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=usage, + prompt_logprobs=final_res.prompt_logprobs, + ) + + return response + + def _get_top_logprobs( + self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int], + tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]: + return [ + ChatCompletionLogProb(token=(token := self._get_decoded_token( + p[1], + p[0], + tokenizer, + return_as_token_id=self.return_tokens_as_token_ids)), + logprob=max(p[1].logprob, -9999.0), + bytes=list( + token.encode("utf-8", errors="replace"))) + for i, p in enumerate(logprobs.items()) + if top_logprobs and i < top_logprobs + ] + + def _create_chat_logprobs( + self, + token_ids: GenericSequence[int], + top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]], + tokenizer: AnyTokenizer, + num_output_top_logprobs: Optional[int] = None, + ) -> ChatCompletionLogProbs: + """Create OpenAI-style logprobs.""" + logprobs_content: List[ChatCompletionLogProbsContent] = [] + + for i, token_id in enumerate(token_ids): + step_top_logprobs = top_logprobs[i] + if step_top_logprobs is None: + token = tokenizer.decode(token_id) + if self.return_tokens_as_token_ids: + token = f"token_id:{token_id}" + + logprobs_content.append( + ChatCompletionLogProbsContent( + token=token, + bytes=list(token.encode("utf-8", errors="replace")), + )) + else: + step_token = step_top_logprobs[token_id] + step_decoded = step_token.decoded_token + + logprobs_content.append( + ChatCompletionLogProbsContent( + token=self._get_decoded_token( + step_token, + token_id, + tokenizer, + self.return_tokens_as_token_ids, + ), + logprob=max(step_token.logprob, -9999.0), + bytes=None if step_decoded is None else list( + step_decoded.encode("utf-8", errors="replace")), + top_logprobs=self._get_top_logprobs( + step_top_logprobs, + num_output_top_logprobs, + tokenizer, + ), + )) + + return ChatCompletionLogProbs(content=logprobs_content) + + def _should_stream_with_auto_tool_parsing(self, + request: ChatCompletionRequest): + """ + Utility function to check if streamed tokens should go through the tool + call parser that was configured. + + We only want to do this IF user-provided tools are set, a tool parser + is configured, "auto" tool choice is enabled, and the request's tool + choice field indicates that "auto" tool choice should be used. + """ + return (request.tools and self.tool_parser and self.enable_auto_tools + and request.tool_choice in ['auto', None]) + + def _should_check_for_unstreamed_tool_arg_tokens( + self, + delta_message: Optional[DeltaMessage], + output: CompletionOutput, + ) -> bool: + """ + Check to see if we should check for unstreamed tool arguments tokens. + This is only applicable when auto tool parsing is enabled, the delta + is a tool call with arguments. + """ + + # yapf: disable + return bool( + # if there is a delta message that includes tool calls which + # include a function that has arguments + output.finish_reason is not None + and self.enable_auto_tools and self.tool_parser and delta_message + and delta_message.tool_calls and delta_message.tool_calls[0] + and delta_message.tool_calls[0].function + and delta_message.tool_calls[0].function.arguments is not None + ) diff --git a/vllm/__init__.py b/vllm/__init__.py new file mode 100644 index 00000000..8f477ea8 --- /dev/null +++ b/vllm/__init__.py @@ -0,0 +1,36 @@ +"""vLLM: a high-throughput and memory-efficient inference engine for LLMs""" + +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.llm_engine import LLMEngine +from vllm.entrypoints.llm import LLM +from vllm.executor.ray_utils import initialize_ray_cluster +from vllm.inputs import PromptType, TextPrompt, TokensPrompt +from vllm.model_executor.models import ModelRegistry +from vllm.outputs import (CompletionOutput, EmbeddingOutput, + EmbeddingRequestOutput, RequestOutput) +from vllm.pooling_params import PoolingParams +from vllm.sampling_params import SamplingParams + +from .version import __version__, __version_tuple__ + +__all__ = [ + "__version__", + "__version_tuple__", + "LLM", + "ModelRegistry", + "PromptType", + "TextPrompt", + "TokensPrompt", + "SamplingParams", + "RequestOutput", + "CompletionOutput", + "EmbeddingOutput", + "EmbeddingRequestOutput", + "LLMEngine", + "EngineArgs", + "AsyncLLMEngine", + "AsyncEngineArgs", + "initialize_ray_cluster", + "PoolingParams", +] diff --git a/vllm/__pycache__/__init__.cpython-310.pyc b/vllm/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba64550a2c6a8a7041378751ce1af0fcef7174af GIT binary patch literal 1148 zcmZ8f%TC)s6t(jpPv=2GNK+`@W|2fll`2(LRaDUxvH5@XS3|cwXIs=kV8`{?J}$K8md`)i`97pHLP7>P3|F&w@{0>QQLA= z*5O^$wRVkd^B(H)KI-!ww8IBzVEH=RIc7wg(BQyfL8G7s`-$Q$RAMLwNpnUoUowt=Pr&p^vS*+9_%tfeE( z9?FV=4#4N}K~}*y6myKx&NX9vnhQ1!WQ0>qnaV3n1GM)f!3c<<0_PRBJsNDLYy{0I zP=6hSQ73}NT1_FtUJ32yp&K$!wUASzX8v_^RMfCh8iO9{*~QVw#<Dmy6D zvV$E4>u)%p72bvDs&gr^za-zJn{IjJ!8HqDBBEr&DOU O`L7S}q1yx>xWa!fIYk`+ literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/_core_ext.cpython-310.pyc b/vllm/__pycache__/_core_ext.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c7a3da722c8b7075830a0dd6a521b1f342ff5a5 GIT binary patch literal 9612 zcmcgy>x&#$cCS}=*Yx!CJT;n;Mx&A~$!c5bS+DK2SAMK*Wu(<^JoZ}S*zQeIs_A=Y zrpMh?omY9EEi9mX&({=0Iy617vJ+Gc%W~Qj&^Xi}fbpJ0eXxh(d(ElOq`?$iFbxjkR(3@IK z$G6cmT4v4E>6zKITDe-Tm9OPng<7Fy*X-6*ZHjSvvsf!)j@6uQm1-qjlO;KGU$2#= zelYuRPUJ-XksaFwnKys;+fQvi$IrD9hDHhOT|C zeTjLs)gQ0=x#atftv~>RmTa#?k3T`BCPz_40 znZl@9!mQU(eauo z#@hOLYcsZQZf~{b1|Hru)$cU$`IL+0l9&?32O6wzMoh~xFtfYHshlp{Hdr`@pKr z<7pmq=EOWk3+ae#$*L`;qa|?yqos7TEKXu{A|0)WRg9L?(J65nqm$`qO`O4K1*26^ zSQlq8I+czbaSo%?7+Ju*AYR01O`Z|wr6nx6F0EBfObcx{FD{6S4=hM^b}aTKafz`G zum;|RKGXLq;$_C1>%+Vv^El1PjH0+KuJDW(L>1<^5f^XOy{4zO58KlF0SvL|q@UyG zPTlhz1(TJb^drY}?ly#Ul;`hBC)jm%n}HWK{5_`~Gzjer2j3EBCR;1Q>fvQPcG{kb z8ubn+I*o`I-jPlxl){OEJ{n`m+oUqew2P&#motu%<2pz8(2H9G5ukGzLS&U=qyfSeNL8E=hyVLN9 z`>L#7&E9mJ+aV}R;ok9@9qGL5B;T*_?Aw>KiQqIOx)5@=({y$N<={B`QKL&jGnq@W zx>t3=2&(gxaBh6E{Uhh~9q0$|fMH%STMrt43&L#gOQ+F-yJ|sR=s)mzX{N?Uv!~qM zpu;|teS=)tM#x+55);+^cKb5)i7jTPoUpOydr=2_?u7e6rzsq4)K{_%9r{jq z=-0FLSV_DSyN#x-X3t|=-fIUw*o*BJ2^@xAY>_VGJgqLp^?1?UB_AsjC5CR`H{pf{}JAG$YjUBeYF0~~1wbm=Fx$iTOGA9uI3$NE;qQYV2|QtCm4q0iCINd-zLsUZY$R<_sI>^L`l_)m_I6|cklmAN_C#Z884G} z@5h9E31Cnn@7_d-LfSh@A1X%o9|rV!la-wu3+OwsUFQ<{17{zb;qfDEOjNn-gb3gc z9^Ou--5yxU69R{xcCjA#VIzzXHV@g$^g9;E*^i=jcy)uqGds6H?QMvl9&W%#z+G+Z zNna{2l42vlZ%aQ6RJ9Sh3DxR;)M^e;-byxpoVup6*LI)*J^)sJ-~eP3MwCZZ!-h}! z0v$-9e`T@?&kt$M8;%p?bvOrltS85H9B=a=-HR_S$Cm>TVq+~D-L*br9ho?h*RYRQ zgLz#a=GE^-Fy3_LQoRDgW6iAy(rLi+Y0edZ?ukLxQv5`o)XR7YhIh7~lf<_Cz`xR! zD&T|9tW|>>e9bu&B!8qTejV!^J;(Wi>&k&5iw;6rGJ`J#N zNmzIseli=q5R|fE1Vu1qrD_{Id_y`MdlZsfW;bXygS(+a z>Ai#We&7tlXWT(FMW_!4o>y{xlFyU)-zG3|52ndrrIb`Z{3gkt2Po(Ay78pD+-EF( z(?-7(z^?NbW-&Hi3Un13a_sFdnEdI00y?RhkXVb zRNrjFH%*S2kf$kq*ZoOGYET+Rs&l~BO@Q8q?sK@vUxw=dN7|7t^cBD(LrotU!n}_X zQKa7$tNY}khYO-1^(NX5zo>4`)VHLd(mJ^pZ zWEuHH4}S(cJGD~5(5)xk`4LTRRbEsxvDe^bb5!#NLebu-cL=;o;ClozN6YkQH-Lric6cKW*K(L(Cr+q zJT54wWElFKW#~q(s4tt#g>s>6m#y^A#JyS08HF|7u;^ORjY3(+>?e5Z39sIL`MQUy zkIQ$IqbRf;w1Zj>b(DGyv!sl^pwQn!&q;aeFkV!0kJ4Y|?s{KImj&!DrVX~MSC99r z4(_f6MamKrcxYr zGp$-tPN!q;7NHAM!gvO~rE=O^O8Q2#Lr;_b(Hx%A%1QrdJ{=|fqlI*o^p6(PQPMwJ zN=Hfm=tMe7`bW#@DCr-a#Aqe!AFZaN^Wp+{KZOx#;G%d5qth6z;crErA97YS>OVqX)lGxkMb-@v=y z!IixCrg)1n=ld|#_{-0XFKikY0G6!q@OLE6>c9Q6VwO-v=erj{us zs0Sz}lho8;J|<`F)0DBh=+5Fq{8f(uTkoD4VxVu@MDprmEHFTpW6QYeYPQ#)NlE`T zz4;@6?)nhnp^48yoSHCWQ}EedPbek+Hxr2uO?(dGl#z@jK9i&$De1q>NUsgaJuvMV z2tQ7Dnv%beBoI1Ly_iV**tk#dHV22z|=phunbL!N^qCmrKR65?kW z$<EH{~WY9w;4x^5I@go4SU#${fWtY#tGCoN`&~Ar&4-qpJ8;%I7=Hxi4gQt zV`AsyWO`~VVU!Z3g5%r==ZOEBQ5p^pPb=6bE0D9zabgkT-!fv$LuN6lYM)GqGum;4 z2=VV3p<(?^tlcM5qO^Ie{s{4(8KqH?xOA_M*tv;UILV;YIznKl&eKB+LEe0n1^HZ@ z<8EwR;B;1oC@GR(j5+#KNfGQE;P`q7^JpK8Wsv&T|AmXomm+?3MfoxujEB~Vr)Ww3 z6^9AcNiV$*-BV+mxnIJ~Q9u9sBmMp{|LP0i)t3Zn1gLt5=M!ojKjRze6ql^ON^_2@ z4oaicuL1lIF7B}!R*{PZh%oHNMixD8NQ@NkJ@&sYVc`e?h|2dz=-dv(2QG=mbPKpv zvQR5L`XRC5?sP0xD(SlTSt4|1gO5hfOO#<5leucnw0_O-&ajk9;xJcj` zOKbIgn#|6Q9TEEoVp!2;D>``UCIPbH=PR>+8E^aruJAU1t=T5Z=Jd}n=cusG#=YnC zy>1#Py-PSF))SP}M(G`c zJe7@OOX9y>6dnCIk8Txm1w|e#5tJJUEH~?s&U|f}%k|_RU6f5zsW}qolP9s25I|Q+ zxM5q?YjcTTbG^1++Nq ld}+!y@Xt%XWzCb$%le9L|F-d@Xci54DSR*K)_>Uc{{o~XA&CG0 literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/_custom_ops.cpython-310.pyc b/vllm/__pycache__/_custom_ops.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc17fb1a445bad3351367ff5ae3c208ad14c21fe GIT binary patch literal 31179 zcmchA37j0ob@y~n&$+X+hgPe@AO;MW#X1mRkOc;WK->a@FlGRcXQz91M?06QXSLe( z3JZh{#>U{=_+0sxZG6Uw6DLmMIEkI)bK)f5_a&Xnxg*~W*cby+{{L6qy>sk>iu?*c|440RepJHo>Z;o6{2D)gZS4hS#E8^R%AagRoFQl3K_h>P zoyg~$sJhcm+Nt{@`BUpJoJ*qgbhWQ`YW`Fsa$BUn)`~c%&E9PwuMv^AeR%sq|7}vo z+Zmnz_Q>s#x-k>6`|T|7oo){}>D_18g93+aqa3$~?G^V~`7@VzGh(mAn-?wdX4GDV zH)k#JX0^QrZ^o8*v(|nA-kiO}o0II5@#e)TBV{~d*bDYL`;_~P{Q6Wx{&J}ZVh{QG zP8G^e6Zk@brwcqo;F$tn{RgeSIr0uKiYmnw8scRwk57|1+ zbzLd1ha5LTip-I*Z?JblN*CBSqO_ZEQQ9w;95)NRMWC*UT@u!H!`vhGt@bNW-i0Te z`!>P!N`bcv%nKyelk7X}JArkJeV4?&N?<|Yt3h26*Ep^r#+0*m$+nUDr6~tIf7s`_ zJ!e^RPJ}r!_M}|_&s*)?@@7h4RbWkEU0?$>)(mqE+Iwt;axQW%#@miL>$e_ZI{VcKw+<3hV9?9e@pW3MEBOZww$1y9&N1s|_` zPVN7h{Up-d?rV>>jI{Z*{nU6}OBws;_ESig_tW*JNZY@#pGJy1o)^YM+1_5)rq!v6~miX|NpWBC@+pH0Zxj( zOxtV#-Xw9QK+$;6^Wr&;zBjJk;+%?bAKIMOQC<7K^}w49u6}79rDalNi3_-GpIi36 zANMu&tU#v`H-Ip#8$yF^u|r}};?!tfWfL-FnZvkNfFDP&Ch+u$Zk%@5V(@mP6Hib= zISPDfykCiH6!^*=HGKm8Ci1VswHlmGEZ_Sh#Tr~|k!sRUwPeqaG%vt)64F(kC*8?N zw+`ucKcDSJx>Ewa?dP%HNSniTD)3JEcz-F&4IHQ8dLi&s|GQQjc%Ga7e-RB?i*%>s zIs;|aa6gl=xN}s8aRowW`7ttv!1LLB*5e{Py%^Ve zLYj+kU5xZiC-+EFJ?6LNJ&1i7+A?Fpf8 zfR_v0F7OKB>`G~Yz*IgP?|0z3O6-QRFY?Q|TFQ}ju4~QKW4-o~Hf6)Me+{l{!K?dA z<-QKNuSf3IBSz{5zZX7yBAAeWr?Uz0MxXYZgmzN58nG|Obu+BVH14;+0|nPo$Gh6X zw}v@reP4m=Hc+$|cNvl5E@M*MJ5%Nna|cq+fc86a-G%2*;?n-E_MI_%ftUhjC`b|Q z+pBRELD4?kF>VDcNzc8Z%$~u<3G`yM8J~l#C z!CaG?-cY9Huj6U}!)tJF`Z@PVD%F{^tfWUxnax{n3ck9$54{}C&<`Mr|crP2u?5m0IZp0kGU6=maQZT(5%;yb<9~`*ym0%!T;|ym=G#fV@c0`4D2>3{7~w?vIiC!Qe`4u#kOfzD~dM9#y0M`fai{Q<>7T(Q#2aSV9ftJ0_!t6pr^bhGMsv0LkS_N*%^rIr`nR-YX+y!0)l zVzsDt%{Co8^;PPXR-xLMm~a#x2TRjRIrUbdSuM884OK&2reFiPs*F#!9D7#@RD?f< zqyF`H27&t{Ww^Swd0+CD$XIk|Phw_Rf$MP#;DN|3k;BH$!vLODFV-B~&2n80;@vMU z+|;TyH#IBG4XvV@a9SHKtW8hWHrHl0RV(A0nzOCRMt#%XYPGhhP?~mIjas46bT>9< zz2wE!Vr|?mUb33ym;hGf7pEv^0!%onP%ch68!xUlAO!ak1aw*{Q<1W^&`Lad{T(@1 z1voQ?T7&RdaP#^rwPv+Y9EYTerB)B)fs|A~b3KP-R2B);00HHoh6ol3BTWkH2$546 zAUTSBwoxs)B!i?Sl{uzZ3(!H97LBezmv38<1>=FpA!D%;4;uT8dFUPqMuPCi+T)_f zDY)DN0CUL)BJ+_-WZw9Q@n+*TRICMbvu7ER{U(+NMr>n$v}G)q^U(()yQA~wp{RlP z!Bg2BMRS-p?=@|tzuTNKklUKQ)X$x<&EDKdje(U;JwpQL{`5+v8Y)+*=TNuRN_`@y z6(iSZ=FV`p;!5vqv~sK`rv@E&>>QGhPgkm~^|@NZZB=J;)ykBE+qhFIPPeZs20*z+}xCdNz`iYtg7R>0kT~;-n{+l!uDIQ*jCua z(Dm2cvMV=JbaSo)YU-s1@RwNl%w(nN2thWeADjF3^D}f)!*wg;$UEDZ&P^8g>g+8t zR;jx!$N^jy>$%3b+o(D%C%4y8E=)qMQO>dPY)s!&b&BLv&E|@^5+3TB0_3s-96{>! zK&zZu1I09|wiaEk45GoybZIgNo}@*dUDECI?2T?i@KDJ1rrgG{n70CTaLr;B2#a1^ z8z3*{unN6op+Iv`D0tazE#LA;!ZFLUU=3U^J~bnE!||duccQC-c!?QRY&IQrCiQb2 zfN2gISrflwgRv1Kc_3>@IBp?q{(NrTWCb>_Q7=gOi{`;cGiG|J@^rn_YBZ{@7h|kD zW~vb&R%Z}&wmGlA1KsH)A`Xx=vxYUQ&O*?OHl|yivCoSur!}qWB{s(37c|W6hi{ zjpl5@Gj`-H(zFyAUP`Df7v0tkC`9_R5yLXnmggc!orkyTe1Z!ImXM*B6vTCB=4PgN zDZy5NWtlu4oN2+iXS!0Fx-GBU@_bjQn#qu-t?w2I>}Ir&Vf0VBQ--iuRry|jMk zR_2`iK&`m1(Dv5L6yY>f3U0O0bX+eBl7zT|-GUd?rNrKFvp4~|L!t|NH{V5xu#2>^ zrm@tY^;1Q?KXMs>mvHtq3+{ByOOp_s({_(e@V-Mvcxn-pTFpQZa6|!afQaoo0#dNbqCWz$ML(?XVR&7`fLF zJ95}iCn08RR9mR45LQ?a92ko8RiqP25{|qZ#6pwUkq-vCE z??m~U<`s&K+DkPXZiTJP^`dB1UP{*#E3m)RaM?Ju+IxwLTc|sYeI-^<@RJ@24^ne9 zoZYSeP`sn}(d~Y%w0qTnlRjH;YU7SAdO_@CarH`|(F%V%LwSNb2<{~4;&QMX%VS6+ z9cLzqe)^bt6`s8WQei++^Hz57RA`jT@Tc5T5{ZHz!*o-HC8}OGOEhSQuvew6Q`Kyp zENqPzTVlex2^&j34`&6GrCv?Y#pCMbQ2sg+yQDW0i^IYskE(Gzpog#1SjK0Z%EV*~ zmZgclzELmH0S|r*2R)ukpTu_;%y~H2hpb-iyril@0XVv|wEmWmOW6btK_}6wjJU7k z61QQ*l;SjQN?F3NjnPz>Uw<)vG=KOT*qFrYH#wr#Wb7V8*E6mztq zIXLB4-E?aXgYKNO4`WrBSNPwkeOKa&IXHK%$_7cRrt5W#*L}Ziw5TzOH7=B^MRyXN zSA8s{YM?~b3B+_Z7-|yiArPCY7!niPB}Er6{uGy+1xP1!uV+Cwt)~^=d4t+2q3^2H zZ48MZtM-^?OL?PV1c<_<0yhM<#iZ&j~2EA_IY{vK&XrIKm24`DB{ z$8VN#>4Pycm~&La_4?gr6_YAcg$gDdV2oRn`v_)x%DwE#HA-xV;b%W686wT1s0iWTe%xU z4k|AvbmpzKuqN{{JNi%p>TB)HM^WmL9CAPn@5N0W)Zq=dd4mBlb^#-Cr>M{wl$dCd zG3LqlHH&cQ%9t_1D3}Pkc}tSF38z+j94|%sCzZu%RDQWKVf)BR~@&kPTL4pN>Lj)fp z=rURu&?uM&Y1^xSSRG)Zj{tb_@j}gM6-n!!0!L^)z3g&?q}poqqr|sJ&Byq-K(H)H zD@1ZWUFV)Jq^(0y+L&7E6UaTcD%9W3ggrEdnyh_kXfm8l(O%JD+o1LyHg`g2sg0tS zhb>W9^=?LlvFhV|?2_Lat%#jOh)TW)B#Wh79aoQ$1x%BPZr(6T+XS&RJ`Hd<+g43d zL!aTR&k}r&K-6cc3i~|Yeu3bN1YIH-?GQ=WL^p-<7*JnAwz(A|kBM3}Fo@88&$!J? zi@_9)^NgBj^t#!-jACY_Y{E`;)~Vv)ti3Y5M7i21wl-fNwTd^tO8S3|;L8Mb|GU{) zS(xagQEK$&@hZ@Jm~Ty-ul6+~OIp^rEUU(=%4+VZugd*2aV>W;MD~ zEUIv#zJh`RqJNvD3gy4UP*)v5Eqh53!mkVoPpGdUXInJ{gcrAT5KBTsVo$>Z`T+FC zhm8G}^um^qZTP)GNH)re2-z_Ik21_YKsTx1WQgqwucQXLn0Y(jcX8xb!y=8s%}#Pd z8=g+AHq`GSp!xCJ`uFLTS=VxCQaPj@MRn>XrHpQYTf>Zs)$v*xx7k z1A^l)y(^(5VtWHBg`-!JBf21I%!*4-_6Di=uI)$)?LOX|zinsAbW6Jeb2SJQ6RZC{UWQgN>)_A4p zcIXHl{7^PcJOpNC-h{3|BG3(L3nU{t6Q$Ehy=j?{Q9|p}d|diB3v!E5YoJZYvpazm z)6{%wA$bT>hPbC}ZdF<-xqm?-?nG5oZ->Jv%l6PQX7_kPiCEsET# zaxMN5Wb~rVM$?OPovUuc^XGy;Jo*w|DURFKpApCy^4kpkIe?d;=CL}aQAj?Gxq=o( z@0euB_emJ9`M_kjYY}!=+(03$2~pA4vgSsBeFRHugT8{&@lP9BLwR^HHzG?j9g|{Ed zv=%5W8jo}~jX|yYcLe8F>Ml}Ctf?HC)4056!H(X7*kjg<9@?^l$3TFS0On4tL-yCO zx{ir-?z}0sFw~<7@X|7T#YD=KlON);R%K$kG41-PFn)LdsWi7;RX!Rp5K<*vd?DZ> z9z|zzOnn#mH5c6WfQ2FyvUjsxD9^yK(0&%YK1N~ej|Eb!wUwrWwJ0ntP*uVyz#)zq zxTmNBG?LsK(96@MR%LII%M(3Q^idQ~!8}f#(L-VBeg^Oe?!*lK2f~@ZzgL z3_VU#z@qvIsUu_RKLIdu7LJ}^giRo92@5~v;5N#wW_21B@4A>o z7bWPKutnKgy{YHnf{GA7)~}u-BflVcn&26NX9wuekvyN_Yj zp3W@@ndi>D4hNUM?U<=9#!Y2FDdxoW?-SZZ*~%Wu3MlvcNG{qHPpUzLJ1ULc0^*jW!=L?L z^h7%SvTnJgLJ#ZbB=9h>Do*8fxJ|Sv%as@Ha(cQ8$VdUchjR>AL{;Oa8fGc%tAiiI z%an3BrhA^xA&K@l4lyLINS8Q=+o#ra4;h2gp$;&^N|woH7yKArrpz%*mnja;vc(N? zB;^lUb!i!vhHoKY^CU|6g${nVH`l7!4h#r;#tKB-s3eVvIN-8)IgV4eIfNgTI?xAO+teo!@Q^m-ImH& z(t&$H;K2D^3;LPB!*(x4LAu!*D#xN|HiI9-%N6A>`+NsUj@w7>s>w`17sa_f6lX@A zMnWjs;K%SXgs3BU*DcP_=MG)gn}I?%-4_Cpm*F%uwxYvHt-5-&(jL=OIL)L8yvy}6 zb=M%;+{N9nEZGEQh+lDgz+*b9&O8ww&p_fX9@XiHE0C*G$OkT+L8EpO% zM`3#0Tg|z5@E7bzC>QKm?2WCIIt)yYksdRO-vHz39DXnN*M;?Nf*iKux!7aSYUFm@ zv_*F3>Mf`_nf=C8xE&O2>TR~UO$&@A|##l>f;dMcjhllkGG+K$Le4cJhxy-{y^$J zN^GVNsx!nw2AD}y7JKboCw$eT$x`nwR`4q2ies;*P(-@rigH^yY~zA5LYYDY#NAZ^ zsI69xo7DX+_OeTKW0+&P_OUg08)h&jg-&eNC&dL{^`ZS?i^_$FSTxwB00WlWg(!^r zm3g!5*>0;(Ic;-?EWU5tCGBfH#Dk@H|LHDW*Bv^=!u%H4K|^N^Xh!u`7k>xTWwe@H zFJzRK3(T>j^0_2AJl|2z;IBLZ4ub`B*t$#)bbx@g3kw zWcTvG#Ig3eTNJkTci1_m@oRPoTen^ePOUhLh6s=UzuSCS_@c5NX9E_B+m&w-1GN;2 zU6-RLz9B`snjzbJv576v-3`$24Z4LcjR?H8nE@^YZ)i49aA-xIhlm?+=!(WFph0g^ z3$-ecuo$3RrJQ@U@c!x?x{x5tB3EIyj#ew3L7=9c2+tC1*fUq1MTN~ZbCn6~XLRhb zrCaFpwa@jTP3>sJciokPjASYt;)NDz1F~NCJFZrhpaa1n9MKon5O(d*jovwT0copI zR3!Cy<+V}$C35R2ZJF@QEKY)PDXFzZ?^0c@d);6Lf-iJ)m(|XE4CdIWYA2sVcz+T4 zymPs&v~}LO`8B?w>qUS$fD%c#j6ep-FQ%E{96{dSGAT9S=2Oz#I5_S$LPL_m#++C{ z*6V47&H|3Xkt7e`a4Y?>_%YM8j%K5nPsMB`7q_x9E15OZIH7`4dsa&kx0DO5{*c6W zSwb8I=veCf4BamA&bck1!}dMLs8jvcQmDI@=YN%`Wr%ssiJ<=7B~Y(eViEl-K(F2X zxZ3>REnuLdfZ~>~llF6gRhX&{Q-L18D6G@6P?cNVt=XW z%;DbY%J=vTJ!o=!v-n%+qn} zt=%(O#c2r{8Qq0I$F{RIvh9pRbok&Cb$xI|R(!DB`%=tqfg6a4|tw`x{vb z$@eIDmY%k_p%|OpUxut+Vw*NZUgB~;kP)W!?nTT#YYv6M(IptP9ork?J`L<}rEL72 zq1Y7JEKGTEOw*Su6JCZMBON17WqMgYiG{-B<_k(BKlm}6#U;+rnFs{i^f1W+CnPuv zHQzU%;oR(@KE2f*OF8+><=5Z1{kp;}*W|A(+HgG-oE+4`HbwvXxmPvx6b?>(u|GhDU0hVh8qYL~_O+vubP-^d&{ z5xkt>W`Ztl8R-(AOoyzapxJ^m8Gl2{Ey(J{WN_BgThLldBRUO%d22z=W6Cxzjuh)o7Fg!e|?WOAtx1FK%<4S#?DEKiP(CeOr zhc=VOBcA$j=Tax4*6E`hi>D$*+y%Y;1dey#amzhV=INz2Ggj$qtC`LOL+ z{oNKS`Ezv_(0S=lCOF*FCJi=XIX&v1DeYNG37X^M!yRJbpfg;}3@R3@;W!|yPeo#t z{#E3oK=A5bj?_5f91;%m8X1ntGqz5tk{7kz*0OY{B5}2=Q=dCiY%Z2s;(ME5sZt%h zm7}4U`fDPxu-G$Ui!{1D?ILnnE58*X$bL5qlEuib(nj!skYJ5B#6H7f zPi2-V07@(TG0bZ_E(NQ@?H85E0Xz^fMsG_Mat;fTW4Mz=3^lPwUG{2lDoJaWzsMXC z@EQ_;wiHoS0MC+5?I|A4EnrUpR)Hrmw~3R?Ucx`T=vi`dw~xE83-!I2{+#M@+Cs9s zq!U*)mQ*Kb5Htz)5GaB!iDl)xFaDQPis}GK1^*G{BC!`m2j(S(SMH5Yu+GkZQ_kANwi~@C8ZHn7{>_+o+6x!F&oV87#sxTR#)CL+NryWdh&Q>p8Bhe+=R0?%ljsAOJ? zY;DTeMLYZ@|C>Bo?(Yi)bAUXM=HSOLukEUbHbHJgi|;oxo>0CjB}sRGK9@ zRB|Zf=^o@eQRP}}SY5n~hPV7n2>0Xu=uoK+bAzX$_x-=)k{|1mZ&N&&_`oLJ3c=OC%JH5E}v3i!-LSL3{NIUsB z5NJLW6N1&7fC~N9L&SIx>3qK-ACo~Q_Aj1IGG+;6BM8&z)LnX&K_7tAlN_sVzThh$ zSxqqMn}HY~xhb?TKCgPltT%vTT{w~#PS|)^|639MnuoWd+EB${ds+~g(5KPyMU1wu zOH8yTy|fdb*tm3X&9zZ2ZoOE_iY9Hx7-gkuYRhawA^Ly+gfeBm0{y@q*hucr z69hZO3>`K-bK^PEtPUIbOuMpcGo`z>izl!`yRWG(`5q{cZ}yl6)<)(|amOcWdvPqd z35RU)>rY{7u?iS{b5386NPiO{oKFl6b**Q6?8nJD%rW`9bG?|nRJZb#g(;Gn?C!_b zKg_KB1gKv6xmhJA(Dnor;@Z4GS%+I3Reb(`%G3$sGUE?BTIYb0z+_!J2n zIH*CZa=z@146Moq{~b34!-o1ZRe$HaQX7E-V&X|XBVlI&^qi6x2K zi8f2J2a1avb`qAP0ZYQ%*y_HJ5j)aPJ2IcbDTlz4Fx^L6+15apB{{vrk}%bWmbE0b z9Kbw?11KZh5Ej}SSdqJZE28;`FP0=$Bo|r{mhf%giVWJZ4l5GkUr6wcUrGkpKQ9&} zH8-Ix$O)Se+8tRn+M1Kr6WA0nDcYjs&ekcqEQ@*7@zzH^Ybs2Ggin%{c@EE~o2tG^jIfBm<2(LYjTeE0^bC-w)hevU0zr9hX z(Xf|l(kFE_>9x8^e-TB&ecQQnS+`YOMShG;5A#VD2H#@gghT;SC)^)+1?yxL0`dXO z4y5ckXedg!#Icl~Bg3A9KH7AonL*?0=s(8HIm{g|?$Ou@Cyld>UR>|sz~lSD0QMWJ z2`#WiYN#RCQ%-asVZI`!vG$l^EG80m!oNdEE20kV2p+*5`$639;mzE!V=Msy@u#PH zEQEGYXuB}4bUV-h9lp4OfSLe6Bwz$dOua@do+eS~$G>Dj1iJ)pSES$>dygDu;Fr_z zQ8aZX5~CZM<#*N;#i%Gp1?Dg_LO>g===3QvDOBbpVsg@_|>c z4NDw2=u$OmTR>n-xJ6Zv!o=ZjoUE~&LF`~?K>VL|UF_z1x#2p%K& z62VsqzDDqS1m7U|Cc&Q&e2c&%;5jxq{3Yj^U>(6J1UZ7!2wq5VCc)_hXA^8B*hFv+!DfQ<2wp;PA;A`cml0e|a2-HC zt`~9CEsVL1;FSb-5pc#;COj1<9A)}Xrr>09OU*EDmVgs5YCpl<1P2K2A>e3T29d0 zro)TtGTZD{XuR-&WIeLK1eCynp=~B3N#k@QX(s8miOuJI7%>+9(x!>c*cq%w3}T>+ zm5>AGGs)!Ilh<1E7h4-MhfE{$gn3RdCm;VV)-JGh+dOJT&1Wske8!5IPg`;G7goZ2 z%1oL+H&f=5R@(fT)o1<`p5hZ`zxjAFYyPJ>VE)7$G=B_?KQf2SADS!7ADAQN_sx~& z_smiAyXLC&5%UHAZ?LC-Xs&9<1XhJk6Lgx1^ix?ZZJEcoMTMMTyUJ11=f$byV4$jD5kpVX;!X(*9WMG0nN zou&;G_@#AI8cjbtn2GbFvjHtW55h+@^K3R1@UVzMVuTA_PBtOAyKobO#)2{uVHrUx z|1BjdQnfPA#1p7vBlCDjOP6MtZJGg<`vHk-hA62Tj_pIpiXs@OIL`L#OzF4q07 zH?_I;m)Fbe<5>)WS(a{@otl$j`Qh?sE(M4QtWr%l{%KU zaErvWWm|01*=GkywUlh)`x`o)yqA`iecDVd6`M_K9kW8cVNGit14UTVCQf~krcMpZ zK4aoLpNlDi5<)!_y|+B^ZcW4Ww;ENY zVqJ63k+`Hvha7UrE%^z#rmC;0N|hZu@e;@JCgyuR05}9GmoFYve|*#Z`t|GAuV23& zhNGiJ1JB7n{AFo*-Z1`7FWHA$&!9xV0dRwx4WnsR%qFX_rd6>_s$&hiV&kvX$Tgjc zV;aH{`47#CyXolGLZx`oXwBJ%7}+>&8WqnpHb;fKHOg(_Etk5?G%gzNK~BYc-)rU4 zziZ~2an|5D?p!sv^C7Do;Ox9n&VK|wjIyQdSL;!%+}GQ2J!}OHmAlYsH^jNJsYdFp zdh9pC#YG`Er(SBdLmAHtDMLBmj20iAzay%WjO=qCUe2IIS1rR3M#bbtg$awB!d`|x zm=FF^sj)euW^;?%S8dEN4<23L=j2Y!;d$=rR(Idl0x#;;!oIB|+|#Ya%;+?(%}04j zw~Xv*Il#wsi?^@DaXz72M}<|G)W501B(L=Jp4Ky*(RfhftSIq0aeyDvZ-+G=LETZ5 zo5UEs&l&t?G0x|~bqmU^Xt@pLc9c6%?nGg*#4&ysKXx?-J5B66c3I25TjM<%e@<;q z@O$}jt<5CtbRsj}f~N0-)+aS}N=#|}OZ#H$^2cMYI&Oh04XD ztS+gXW-8}o9-Wlc>NJ(f6ol8-+~$z$eRF}p|c%-!qD<&lCjSKYFtZ-gVs{t7Pk+E zJ2aT-9;RJ>Pdw|90&PFRxaB?TTGWy+ck0!Z z{fsWl7>iE$bX6yJ51Mw<`}z!cRPJ&|$PHCk5gUFrs4fXLvKllxqW9uq2m33l{*nlI zq;gR;Xb9z9YJ}AlKMpQ6M5MfG*owtEQV?uKRZQC`IH&{7VBOEODlfnWulrFWY@=fd za`litk2erP@+*ESKfl^1;`Bs7>OJ5DFd1^eCsCtv^;UfU zDP^zL#hMzIqAh~hN1c!j5v;=IwMGylhAL~l-BAi*XZcFdQuf>0qcvMuoI~~ ziSdJ|Jd*~rduUMY-^YnML9jre-R~q(rwHyRc!1zRf`|4iWrh`^$%l_Q2HV+SW&OTOlOy}C$FL91H6N!*l{}~aP*fg6(O$F( zA}Q_TMKj4vl7|-^?0Hchf8k35FUt44=wQ!_W_P?OVRUY5U&_Ik9Qcxx%#shmlSUFR z8SRz^z9gT323c2lKbyzMmq`4}1g{Y6j>htJ)KyU;@-A=g$NeGogMGc?iy!lfQ>xGoqdM&zq)CXV zs*na43WlTAFhT@s^#W_Y9{DX1u2&r>)PE>xqFt_Cg$^eyYntLY8j_xk4CU;_pw)3RBe%Y z5Icu|)(%O&>!y~_r}CaNQ@)L^W%tJ?tCf=Pf^&GbvpsY9!}It(nDwkn4h0rwx<72* zGMM#^dQntvcVgwNiTdJFtn#9bWG`%0Er{zsL=V#a3B2r{dDrTtK6~b>T-k(gg|b<; zB;6C_`ve~l4DE_vpze<-5hb4!RwhB|i-0f@)p;jqR4F0uKcpBfjb4^JK@60`xtK6+ zB>0%nb#T)#;Iekt9JuKPxae;0tat(Eet*D9t{S#Cd~j*ie+k(;`YQ_oc4!i6qSlG% z(8>t;6M99Y-yCSe(Jjz_2hRk_A&E z(?~%-hq|1`LE^AGw+zLM{zG&> zehlugTlhGjFaG*VW

hI&n-7^%1@O(i|D_<1(a3Ne~eXO{Ignf1v0P=-BDGi4Zt7 z2!R=C%8FzaUk`Avx{;oelI+xDcj*+Z1?^tK51m$tX^mh%oz~It8H$c+Q&vWotdyKi zq%&|g$uj}vo0aSPt)MAM87xcFXNr6&M z6C5F!C%A=x0;MD;)mJpx1f_DdMi|7BECK_?QfIC@T~YR>Fl@++#Qz=vMQiyhfGV67 zEg=Kk2_)?*NiHQRhD%xzNhZiFm0q+ds&fDbP@=zLrmn#l<-=^dWP01Ic*JB@zs!QB zLho#>pBLMo61UCRb=zV;*f#r~<=A)3Vc*($_Koeb|FHu5nibi9-4XT`^VombDEpF? z*ca>o`y6sUW8>`KY=ZrZO|nnfl=lfc0jigG+aC{0N0$Db-ILCG(rrk$Gt%7~N@t(2 zY4%Sx!#-vQy^q)f>0JBTGiXopNqd&}zG0)3wyTB}O5KspmC6WxQ#+CVr$AC{kQ)Hyq&1bpbT4f9xXQzi>DyW@ls>-3 z(T6>KeyEZ3zYEXPKNpmV+v8dwTNq7Q_)mk@U_ZB0?&IkudXn~6M0cYp!-adeFtUfw Nb%#v%*KB*j{y*K65xoEa literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/beam_search.cpython-310.pyc b/vllm/__pycache__/beam_search.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e88877a9fd2f760fbf685d5cd677f797e88f465 GIT binary patch literal 2435 zcma)8PjA#l6!(n1yWWixLIQ}C7FngOx(Bi%1V|8~{5e!eC>&@nzF4+ryldir%*+xt zT24U%RqCbRf zH2S%17{B6Tek{P^4m|QZn3%yxYGlMC1ZA4i%=FC6@+@L-^0mQCX1z3+bwCe@*U~#J z)`p$7-f4qIhuN^U52)AacG(%$<0iM)47R|#%;7d$gx@9JjeG1ATYlN{dd+TZvD0kj zrR6QK)yGEv>|5~1=##3;0u@AQAZ4Fc*26@q%6?R;q{xFbLY?_x!r~4*@+M5&@CY+J z`YlA|ncU*7HG{Vqy#;Z29d3hcf!y-CyvG;xyEd-DdZ${s$Aj!K4@5M2%%6^V9`XHO z?z(a&yKy1h5L93Xb=G?>j0eiy<-C-x5<#@<7O|^F+*QRc&!rpWOfS=7=$4`ggD^=G zHEFb(o9mmt!E^k%a^r-j%uS?Q%wpE}>?LXH7I``WA3!BdIdk_$Jl6)Ao8Xl8DhP|Q z0xwWka5WY=Xu}eH8%r)~i&eY!a|p=kQ7uH{%m;7L*`3Nlclu6c`+go|-1n=V?`H)Y zrzktV|8yLrvo{^zXGP@u0`pOsNv^6>z8~ayp#sdDgjXGW)r!+1P{IaTSXx&q@x~CR zaRaA6ZVXgb4$7pwtO7CQ>hg_jJj$+QdxJCy2jxVKihS@aO|t=J!=GiNoQM{j#z!Lm zdxk}wwnNSkz5n05xd_T1;tg=as+Nzr@o22dvD*Iu^93k_jQRXGVnju*f&_Sgu%|%d zCh;K$dG5p!$K){;Ql1ahs5zle10{fqLE2{c3HREE{ZFnGe`W?}Qzyv}eu;PV- z6J|%_A(l`&0=<@2aBmeR6eRQxjpDMU_x>M(j6n5UYeq~NWhT^ii&=nRYreW3YLclt!2uYFTT=Cu$P zS#=}0JLQTq>>`qb1`F-NbV$0l<$86D;={<+lM`K|H8eacF;Jy-stZ1+wa4x zkzGUsYbUK}j5V@n;l%~Cg1onlb5!8YZ}7nQ{VN?rwE`tLPN8O=@um_hhZnL2$xm3d zR8zw*IrKv{soHggM|f*9undmi`w|uapK(B5SX1bXYsQrBQn56(7~L{pk4&wpxeC4k zoaTOYAKEOs0R0GvdxED?A-KDj05NC$oH?=H(|zYI3rYaojUlh~&Pb_J-W&{uaAq8? zLrw;x@o<>r!#Ife0J>%_p|NIM$U&GE;UJ2_?TwIcTxC4o+`JZV1To{m_GWbT%Es39 zcxyA<+UDW4a3Do=^7xSFTjJE{vfX|ExA@o}sb%ALSALE2H z@H$XPKD>UPEN7t|`3T6Y>-Z7@`3#OoY-qOQ5Sutud#>)q>wy=LMZotd^aMJQvuQBJ{NxC}U!=LM1W^lllbe;R$_hye0=&Xk-V<0L|BN zVatW`xCB3M=y8C@x0E@dXH2@gKo2e47@I{wr`P zuxY?YZBhcg>Q;xxYEKX}U2j4cDLPi;gcr```KVkg`&Rv}yN$R@EF#BPCe(z-Std(1 aU9{e{2|c8g{!LB#7q#d++UoszZuK8orcm1e literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/block.cpython-310.pyc b/vllm/__pycache__/block.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ec96641779c401381f56d0400f5c87333e78e35 GIT binary patch literal 3449 zcmZ`+OLH8z5guS3J3HLvV@WHdh3Zq-B+*jbqYIWvg7M;Q-Xm zY91?KRxVDhE+PLw&Z$%>M;-DP@;g#<^~qkJso67At7oNl&nEcYOq{gQYXEQc zT+tLBw>OM?26uSl*x-#L+M9uq%bPH2!l=c)T_cz&+k5$Ak@@>c9z9lTUxO+mu&Van zUq9JlTOaLgeSE)ruX67fLWV^yE9*h5ipu@du!!?4Oaf9l+u~^)iLZZzR$blR`S9*% z5BAuDySsbr?$*}M?(WVu+q?g8C&I0#TMGtvperQ8=n?Rca?&$76&5#z%`NCQbO(Ba zy9X`a1D{0WfbtpMf|1Kz@T6UJKItE-I0_T((Ff>IdGk{-ltPKDP=3)DzAC~( z`1ye!)*kzDroVsuxgUj5U#wLQ*X~p;jR?y|>AsMaSC3R&iV5EjRliyQIYUlxRv?h7 za-=9mGV4YTxHW0WQ|MdJ)$f3e$=G1Vm;xnZ6DS>9K+UlY)Z!?$3kPU};~bY8;G@}< z1lE2SG$n?PoI!#x$~KZ&AXQU}0~Y0@tdK5_+exSj7Dka!O7O~!@^m;V1P`3bQX)Bc z?M^QYxda=$ytQ7W!}Vc2yitVmktlAwmyY`Bt@O+FB;H>i9v1yPTYs7)>AJ>fZFpF@ zjKx`8Fow3Nc_5DKkdr^^5RaDEE*{p}WL1PtP8#y)oks3mU=W*u@ec!W84&md^ad1u z0l$F>zkuMtf?q&yV8d_YpKZX-?Uq+B>bzc`_GUS^S_j0*fp3;etNw+v-7H(H{wcnj z<@~CD2HnkaX4Rhny!ny@find!9*;yYwC+xzO!ui+t;92uML{GPZp+vvC3mC&G{%nzQx#+QJ9=a8jSHg0)-f& zl^H|apu);jg_JA6O&O=nL<=RUIWU210yH|I_aCD2_uisz{uk*-aa;Z zGomHec~i7;El6ki97t#RJYN9m9PjW&_?_q1_!55uBnvQC>w&pf_)p+VM}KMR)ysSZ zzASPI;qzwY>58%!?kA$W_fej31w|x*YKY}ThraT^jElZ5Bysi#KSRlhr5`GlM==y0 zeiCCbto3|o-O8y~Mpzss%>>4Ru0#GNFpmr<0W~FKD3c>1m&Rn<_@em?lHfTV(|-|S zJZWK(!2W`C1G8$ddT%&<%`i|~IpJ_9G9H+9;QbtSuWT)zI%j>v7zL})6~-;*u1j6g zA`pS3T>Jp5tH$CqYvr$D-Wfm8C-g=8@iSl`A&uwY0a^<@MSfu72(6aCfGL3`--aLg zDT5cjX+&E|e9t4{iGfi)n!tks~s(gH~oD zvf7wske^cv=rWKq_C9WaJ9(t+OesDQUjSsHRH0G?WK32j&OW3Mi#V0;%#6Nz^Z@y9&UyEc8v0! zGe@E(XSTKhnG+`JHZYTAfn9mAiZgheWRa+BmB4$dj=WbHVAmOd8Cls+!(=2RCa=!A z^Cm%%oGR34VhZCG;*n)yQIE`%b0Kp&{R~8S_nyWp-dxj#=vB(nDO)cK2OD2t#Y0%H za}@4>QqE7oqs6aPEPn$F>)Jr2kNn$6E>mPh`S9@1DAidpH>Cq57gV5sh|f28_hBxc zB)|ME?s2IGJhVMrg19)OoYtWa6!)AoIv1&jiE)*?Q9pKWBlrp* Qp^mXkTC_!H&CXl@2Xp}9=Kufz literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/config.cpython-310.pyc b/vllm/__pycache__/config.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f93840b6b46ff79d8d45ae39ce3d0a7a5c33dfd GIT binary patch literal 58835 zcmeIbdz@U?eIGV+=dm-hv$OBT;>in;ATS_yK!6XC5JBRx011g$fWU&3Mxq9@duJE3 zn8)(YEI>?VZCOy|*fDJLP3UZgUj#!U>HeS4xj#~59QKULn8L7O2_fJ$VTTkly zr}S^3^2$oqdfNKXXA{LM)-#pot!MFuxk_2gcY93V&b?U7s`s1{92JvSk{xvIlC6Yh;F6aG1KIWvYa=W}-E4NxsVx?NC zSx!=ZTKSAKviR{!XG>>ZI(z1$=NDddVy7CnozxkoeYI>m@%id<+ljwiZMB`D3+46o zYGc(&U0iQhn~ieKNi1%x*D6kY@%DPfNnC9pHXn7yF4>i~U9L7N)|qBwrMil^;cBDW zF4dZ=s}&oMQ?Fb-wXk@;cxv(d#f8#^vx_fXoWFwjkqb?$QoCGPMS=Eh=C!SDJP`Z#d)CR;k&p)Jo->bnwO(GWl-BL$a;4S6*ZTE@e%-FT>3)(=Iq4Uxr%Ow> z+m)8Hb9uumm#oUo>T(4iZJ?&Db{UmJHcn0oTHh$uEA^&*+exD0*Q@JJ8o!OohTUwS zU(lBIO5vk0x-74D+^mnP*SgxQea+hjoQ5~J}qTSqBe<$i>&%Rk% z-e@=N)8!UZjPh4{@a1ONl8&BW)ME2SrBUrvZ27Pn`E(>wjZ8%%%bXy=zvFl~iL3SF z0FIRi!C}#`qQ#hHZ1FE<#qp1C6P8hluOVESwvv^kHH0gLD_t3~G8Noa(rX!6Jj0b? zD_hA{l2|As`ZtH)5j>CLH;3Ob9Y2oWVLVUh-$@-mrGKaKJB0T$`u#TjJBu>5qvRd9 zb|Q}DkFrFy%&c9A8$;Z%e%_6Adk{B{xCtHiP$h1eJ#l-{zGnpZB#M)Lx$LcEtw&@L z@3$VsBA&ALR)$s#>wt9-Pt*D-ZymzZjP-HrG3zkyw&8BhD&THb07|5|9Wh6(V~E*- zyCxWp8#4_nV6W)I4dUPH+ztmhH)P$1?7 z>m*_TOxbqplyw?Ud-c;9YaUPg^wU}E9G)K5PcK?8;pq|USpX=08@J9|A3@CiK+H$2 zml5-5Am)O#fS3b;n2XjW#2mCfVZCBq#$CR02>G#>u2_qojbpob%!|2dy(%$>y_nam zk4em&WdOu|(ixWNu9kQCZ;nSI>UP5}6GWMS7-su=#jF#o6fzkTf2Zu#)^WMxle{@@ zE|lvP6N%T$?dxW9MLxQ|vAT-&w}Mrtl9}zMxzVZ=ypNf|`Li(qu}Z;jQBb4V zHe2hJ!^vC7et@7=bmj~bYc^o#&4J(w(zB>1Va&7R}e&H*2vu<9~ zi2;u5^=9LmKU{5lqt!0i0LEZwZ1PqnhK#Lwt5RLP-o_llm*wSl^`^|-R>54v z%++1qlr1+@yf+HxE**W^T(>bNTJD@bcj<{^lApYjVp`Iz=IL|6#GwqiSmd=z@7KLm zsaVI&%cvi^mk&~TyMpb~+E}hGRcqDuZ9ln<-Hwo7 z%%zH1Hka&jWBIztb#gdVGjthNjl^@2mN8QGdKC)~E8zT!xzX@e#NqJdv??p*jaoaD znXZ9fUk*EF#lT9t(R!(b0g9S1%|CZZkTW`*+k7Mp1HHOmHG zQEDxhYnY#yctGSZCT06JYTal9J9DS4PQqy5YCVuMRHe(!OJ~hWeW_w`&SQeUMnJXE zQaJ%Qt<=h^hfS;5IM6mX)`4|aLf>Djm)|U{H(OQKq~vD=pxBfZRCyDSt&REBsN6D< zxLV(+`{U%T^Q{KUOW<~f9<|UAglZI=aec$_L1tMfH|{ig#3S2;>v|b`5cV;Ec)~LG69w<34WVXh_!X=RR&%Sh zfi2amfDh&Difm%K9@xaZ%z{-bt(Wa`9bx>N2DRGUfF1GpsDBpY?K14i6ew^!)Uo(EcNq&zrb$QuYHvj z>;Tz@vQib7O@`=5Z~gpEsWbrQL90|i@7Tw=f{_)Fq^7CgnDWM~2x)q&s?29Wv<%g_3LUMQyaX>B{&U$`ltd}W4~JLTtM=3=p(-%!4G;e?x`J7 zsq43wZ0rP~Y9TMREI0wt$+l@#0aiTZ>Gok02%EcD(RwM_&x5N~P&m+iSfZtNv$R}Z z=ZN}CSHI^H#(V_)#cEl)9q5UkxhT*wu*$Dvi3Rc?7^szP3=3BvCs7r9Hu`*}40MZ> zY^a4CF^-|^tLDM^vzIQPJ#%XD?0kWfp3t)dc+yjObXTigDO>q)OZyZ0t@MD>{|}m3 zFi&CYQOt+hZP|6#=p4{B#J#b2wj6{|ZX6IU z8_B{O2)|$T!YH+BRRZ$Ttt!x`TcH)_t{t5OP=ZPP1s5t$FFwp;wcuE8_(OALqoFo= z;$toqBap$bmR@N5TEmQHMN<_U>tvtwWO`WUp^143a1MH=+6vXGy4q-B-uwBtZdBK? zS~NLOBHQOUS2l2OENs3-M`H+qmZ&{Q>glHM+vlu`msg;SUdl*enyNj-5*1htP+7#Y zuE~2vg@w^eFom!yD7r0>39ykSw~w1uAZqBZQ!#+L7aqnIAKpy7T~=EVjU9hiW%>axIm5)9H8 z=(KW8^+o{Z5hMaIS2VAS@DvCpW`jHaLgdVT-B-;dIJYKN@kr~H7P5U7bzGHBnEaBsEX_aLO^n; zZq{mbuOLhy?@Lui3B8C3sHU)Pyov%IQUy8Gx-Rg)ViHFPE;UY(c}3u0?l#Q>q|OiM zuk_ZYUR+-};IAIe4Uf-x158G+L&y}7^)2bq4AcN`P@D_mWL+GPnXOo)UNcw?ZzjyC zsle%V*i@9XuvR&T)OV^P3-5ye7&*7-BqU=e?JaXB=iLgv?2Px3?M^}v9Vbp&)JeHC znllojtzJcPAUfQ=M(WrvTRKjD%1v zXUEnkg)=jFw|26O2h$v#x72cG25%|Ob9H`QWc zbmItP-6X$dhRo~FAAdD`B^Q)x$cCqi6nS*hI{7E@;NJl1-Psn>Q{u^Wc1?srSg z-fHi$&|bXT{UZ@8@%h-DC*4yvSek7`N+I58h#x+vJ~u08wZj_7b2pmIdtWcEtu>EU*#fYXK2uJO8_4_c6+m8>ogu&wsjxspL;0Xp#Gx!jLXAn4< zSIf1H%2`UcY)WeE4>Mq~_6Y{t7`(_}I|3(%`UT`x_ADZc+qaf=O_vDj&?zPKb#>)- zNr0?9#|%$0;M#RE0WtvG!gj*w&kkz_3cg*`;M^8iMqE%Pw@LnWPVJ+0Uzd zi8q#fd_#A>Gu|IH<2J#E^Q^WrCHQE`n_8t6yHXK(Vo%hxN(c%KHw|D?BGfC6_C(lw zP&4}!tLGKzv9;9hHRV<%xJ@_{Zmft8g>3fYmKsaCZTVf?<7z7d=Z*}_gC~&8OHRL@ zsboLPxIuG>U2VU>;3NY!%|6Y5ga1y%{v>|hJ9ea9Uq7;5U7u6?$i^7 zs!K=KZ-a}}I6}1Nh#Kp{I%LWv)up96a%g=HL2_3z8vd6uQqiQL;!;unK7%+Tn!J~d zA&e&P=Oj%ghI=C$&Eg&6W6Ag9_mc_3h~p`abVfAw^>`9*?#B&=N#vd}GPv;mKFdns z+s@Pl%#7L@+13hfC7rPg#PRW}tY0ldBI^Cca2LlVyBSw$8!&Y(YDHHgcVe4H+qfI+ z8gEDK$#%ROT_3ALspKt)`(AsX2Lbzao|<(I`3XV@Xv9zvIugqOf(mbqetVh&cmVp9K2?ifdF`a>{z!mzeXSx z>ddHgX7p`z-#f;FeGTvPaeE0req)N;(3tQFR;<={4Al&N)!Twa#Qh4cud(c0G!yNZ z{WT9~x@EO(iv?5=^w(`#y0+8{Iac^ZPp7_$xL;%ZDuT}5{_6B(vSoc*1Phpvk-O2i zBb(7KX6DVPJ<*1op0U6PoCIEhGL(6y2S|3rp5C~E zw7-E%rdS4x)xdI%-_OK4`$9dVcPp=QGW1(_4MFzc83wc{&aO}cndq5h&IBe|7`WG* zTsXoWG^($mz~5mz)I=ML4z7PVr>*OMFi?Ms;D*CXqt6+&SjA}lcytq>)r#WRz^{Sdm=#-%ZbJD4Ay8ugUn2awd3VGxphT zW>XO!K+YS{H;(FZBKAcgK%$Lt#&m~n#O!PBA#12Rgrxi)e&Oo})wLy$Xs*%q_kM1+kcKUAS?TGzDk|K^2Kjfvz zsTA@ihBrUa9=@BkKoYSvBneQ9F%Tg?BW3(ddt@!w`WZ+Lt3*vk4djyJKOxd8kfs#LA_0&#OX&dIDS zkq9W^n_}YDGv%XC=QFmAaW5w8WgFN!6fLl4EPPE%Z;o8p>Q&6o?cUi^GQFB*Q^H9;E?8}LCo_J+|NX(NjgNg~TFmGMyP9@VF0wAq6C7 z&xsZ~M_rMf(t>#am4xO{epTo^^R0ZccwZGN(E&E|*qXz7RiVJnn zO_q7P(pD-B=mI=ysXzA#>>DES05@<>HxmhD{96Tko!zi}Riwhm+c=`MyGz*y?tryC$(gsJWp-I632@8vz4~H{ZL7S?(v2z^pUP&tGI)UXp3)m>t!|gwcTLL zZn9<3l9N=+tvDU(2)BEBHXHk6`07t^wLXi0XnQK2OeBG_WuvKB${34|#fb37f#}O` z)ObIW1hNl|f&UOM#6c#>7+xR=?k8jF38a8`mq9*AZ^Yjpi{nlR2c~3M{1Q3+`}hv* zYGG7SV<5q{g!VLIyCfN8PkR(MfUtn0*t5af!u|n9SX&S?J_&P2pzJ$QDDNsNO{n8o zz1nyusxaCfT6{2vhQD&QL9ho+NNdEwAB5Y8ky@Y)0Ft z;~JGbV|TFcZeSPg8a5eW###(}Dt6Vk4KS7xo3VBRD5XJ$-dZvnS%o^EU1$&8O<~vN zE<@&8(u!dh!Ie*SIvy8!5u9?(%5ilk2+fwN?G{)PqHJp(e7x}FlZD5jip5p`%p6{7 zDB(k2FGGMEmoy59WPLWclMUu=lPp~*X31m6E>U_0FVA%@1%%Ho81bkraHdXI+ou}V zX=(zS{DLqJs$<~SZy_CA%%>HA@Cg_eY~MzJy|h*nPLJJX+$I8$6Qsk6qW}iM>K3<= z6bMc@7qc5eN>-(J4Hc2iOPnDj4Ki3goZldp?-}vT{WxYeRtlcgiUDE$4U*2pw&Q;^ zcoz9wie=6m_mB79T>m?G;4Eq-h&6PhB$02#?J3f976@d_2(o@ z@Y{A`Eonb4Z`4c*e>VioW*#LC+4GVn#jp9CMX2+StflOasPZs-UzK#}a4D>1KC|#H z7i{O$DRRY4t&^$c7vvQNq&>2qV6T!135G}oC`tuNq|jBYRchx*QE|07F+%bfbh|sH zt#ZZu8bHo-0v z&ahVHCx6raHhfc@^)}ker#@b~a_Pc+>9vcOKYI4ETB7nbmuV^iohF^C_;~=TKPEh#O>M;WwJ$E`?Q^I&ItK20$ixq1u#Q!3J<`HS1!-GpuTL$@GSm?$kIAGn_42U%l zM?2ftHo67~op?8fFri7K=vsU;-i=#H0EHJMkAyWOJhVhRd3UG_AkaG2PFX2UNTuyl z?abX_OD$VUgwifKB1?A|OdM+%Y8;(D=DLeMdv0)T3(?xCOFzQI*1!e>Env0}V83gb0l z#!?eI4G!QmmncJNS8QmlRSKQrpcR6w2QLtJ;(Kji)jZKj7_Jn~<+F+x@DWag zDb@O(B}49w{oM>i;y|8aaTe2?5JwFoc@Iwsa4R&v zh$ppL{xCk&IWSvNf5R#$I<>P3t%Zk{472gIc;0#}{S{C#b zWm_9mgHahnKhYOa1^b7QO=o9s1p-<1)gXp1*cB}{J21Z?iRb(V)Hm!Lk$gPI>`5z* z+0J_q30MWfrdvatq!#wwT5<=GujGRLGIA_pAFiwe54lmgdF)+?qay;tofIK;$*R`# zL-r3Mo%FhVVPECrUuWPFhG{+Q$r)VhBPpw@NLjyWcRrW3`A>F|)N}}_Q zkygUj@tcqmt2+yZZaR&hHifjORo=j#)i()(6MU1jU$oL!Sepzvrrj8Lo#UHvRW7#O zVQkikZo(P{=k!zEgn@p(5_#hSzKpG*gnYKM|FWoZDAhOt%Ypq98q%_Hg0Wvrg6ILD zEoLqQYlPwFYn7$qsDCg0Se%7gKWrsMQL_}3d!2(RHRWxfXdvd7EfogXVEtuo+$=Rp zQc5vyE!l6%G|0z`X%$<#ans3iqd+>9O7{O7mD{D86qf7A=|JwKRa#GScLD!WZC;#l z{)Sf^2F!H1;%XjoZspY$Rh&>QRRSiFgSU#?dK*?!iJT0)6yIf8Vc6@; za7y}{p4*oHi*xfB(x;M~oyI+~C%}S;XJeFlQhEtAkeeCeA5eV!eku`WyARGj zgwb(5dwUjzZ~)U0Fv*Q})FMHTHN7KphcSOk=2hyo1K@jXCFkJ|wDd#7sBuHPy8kwRnF4h)y*k>aI zws)X=hMvWq##vMxSCuO*)tbvUlm<=On;D@E1=W`krU0dP7 z;A)K8e+#drJ75`a6dre?y{sQ6^IQ$XM2X+?kFaBy`)s81%%D}{)kpVa;W<tW|*t}Ti z6GoVT-Rk2|U$X1jV!>7vWCQaxO6DtIn~2powp{X^@AB}=7d0a8t4Me%!AhnAwg?gR zI)zl)u-L@k<#H9eY`%Fh<9xL(r3vqT=8|fD89sM%p|gt|QOSNlI^dg^(IzD`7=-1B zCY7JDW*Q3&mlD^WFl26UIbjO5SbW1&kj zM(3d*m_WIKDI&A;9dKUp$-+BvS@rh!<3Yt>lJbpc73B-m0HYa0spMoztn)^V_t_HJ z*fsrTs8q6=%cYV%gAbkfsjG_@ofx(lC%)8d)@%X`yUxg#k5x>{l5OzS`8!5LunbnQV$uY;w!(?_=@gH*Ajo&nO7{8I*Y5cmeGxO_)l>?rcG&1*cBe{4co=K(RnW4=2OwyQvpzOYJ z6ru#wCiDKpj)~!18Y?W88=oi`8L25!OeBo7d!GS%I~FyPxZ+b$n?1o<82X#W(@9*d z??n)FO_Da!G5j;qacj4gfXj|#1x!R-LFXfJxCj}B6A=EwVF=ugBrK4pD^u2}H3ok? zX=~h?_$=sm_#B$Frr?%h7?5peG8277)`p4lX)0jmdcjUz`T3@{YO(*iy9xkLWuEtZi03 zKQkp_VxEQXIl^43Q6oEGsA@ey>)OO z{R(mqJcC^M4pIc#yko{oK`>%A;V36yYA9_{YQ?>t5N_PmEp1_S!5f0zsmjZQ&Yt?GqlJGp)oXu*m>f4U1Gv2GqsL zEu*OB3W1$fT3r$_yG(fJ{_)=10G99&Hn(U4Sq1alHMXGP1^566lcLu|V+a~s(C`8m zb&e=ayF09Zv&ujKZ%4$KAV=oAVndDEKabMv|B1m41kRY>dgU_7{sq4KpBV_thP+Dq ze_`;yBEWv&iX~u|BU@_i%O z)ZMbCVx8Ho$_jM>ku47kHSr+G4du!?BJ_NMh)FBBidZPVu!>~(``s5h#{w`g04F`U z4;^;_vxh<7(Zb^e^F_!c?Kkl$kQtfOBKNq1h@Sqlf01!)43QM7i$EABiY8!Cgc{~f zX&T=3HO!k3Pa_xIFvA{VGvvttJ8-fM6-O&3qHDxMwnj@LG+HU?Etn;^&Iu+h1OHlD^fGvU0w_>`$7zV0neFZ*Vpx;19ALzT4X0t7B z%Ea6s*jWp*Vx|H3hlHz6yHe<;Rmu%m>eL{;psofa&C6wOJLP4_p32L&O$!*3NY5c= zH_L>}+&;qo8GHp~w_>LDrUNQKmTwpBbmE+qTT(>!|Bf_o`r`wtk}43<)cvt!=b_-l zbH`_y3s~(|eZ$1XaRyh*M8MA18V;m?sE3+^sKlXh`Rnb8}kaGpCcxsU+?;t{|bCJ2kBZq6#qk0aS6&I<)@}y!fo!BUA{X zWdudYn6di?^+*`r`sjX$b3$a>Tb^rpXh-gVj|lD)Sc*zXjf%HpJ3)v42)xI2 z4evho`8aq~;GvM8f+f8l{1j|W+W-uqtq$Z)$3M|d+=Y)hNQlAnVkb*S!(J^`IJVLm zBjQ1Y1tLA1l%1y~Mscu-uV^d?NCGr9z~P|N86}v7@XfY{ExyBNWmE=c~-wKUDOFqJE4X>Iam@AA$1tLt{Ae z2dUke1QY?Yh!YAW_rA@1mvM#uGI%oBTyzaB zE`jRRRK=bxsS3~X!CXi3fwTo*h`cL^Hv8Y*YT7s8m&;XdR@o3w=AV4a4FV?WbX=yF zR5%g*W7=iSWr8)i=cU87nRnM#_$y|=h~)`qaI#DxL89${$QU2?rC|6{If_ZbSUg%K zE_oDEUQA0>NkQ3Y<%&I%k1@|l12hL+iE#Ull+{ivlHGS-}yl)$KjMgM}S&dp~ zReZjSw;|1J?WT4M;gi6i_yUZ$Vo<8;OuDMutd`Uh3fudF74fi+G9>!0f35-~P5+wJ z&p-hMHnZGGoj6;eK!t;M_CH~8he3FeI3p!`*lGi}g0W=jZ*#d2ArVIS{V|AwX-ZUNrSPU;o8}S65?YD zRfGv?T$5|w%i$=(Go1Kq4+35g%+qLE>^Wwj_^*_ z`LEy(tOGh_R(kVl2mK8u@Rz^b$wv9F@Zv6OZ{j8sfnjEPilSk|7?SN$bbWK!{*tk= z_id(L#ufU*vWlF<)p`%JHf!`B4Akn7R z@zvFh75lTOja@~skRR#o)SpFk(YUl&9KLeu+}Xvm3s){)zET`oxOC_pWj7^zr=nfPDO;I_g^Y%GfiLb5+0t!C3etm@4CpvTB0b31<(d#1;|AuZN-34 z#EDE)Qt}K010H@ytmIqKVh*uDLnY`b57@bmPC?njW!!uf{vJRl3J=kJB~&0+sqEw0BOqKsD)%%1X?>fw z`${GNwm&yhpcethyFMRKcwImvJYPBL-bF{0q|9}tit?B~EQbqvuZ4GUR;stgE(R2s zq$CSaW}jy&f;wvYi)S|7Kohp_6i&cwLEKjGJF*o0Devai<3@{_QT^F-m+QBiulB9C zwd>ohD%@XiPY!`Pe3PsKDRpgoveJw80ok~<@w-lO7Q7k7M)9vq2JAYy0_1X&0U@?w z8|yNS*jQ{9J;e*C(1jDr5m#c$#h^R`Cz$=oUz=R6~n_gCtUtdw;D_OgB9Nf5|kw>tD8!JyL=aagL zexE20Y}n|aV`4Yp%8A=IW%XPcZY;Z2B}6{}4bp{6FU!jD^W(5+uTuPKU(>F_3wF77 zB-J>av?k08-WoPIqHypm>3 za4!I?bF2+BChBy6rH%JA3l{ScWjLG{GuY%EXhMM_k#DE&riB)OA+hE?Q2&F5P2fYg zdNiHq&eJs>aUGB?mJn4~Yzf+iJeiLwnrZ-|CEnx*e?R-$TuPyvY7FKK?rd zK8rlio%UDx`tLC4GWZt=z`zvtw`AmF4T3Nb!JyTS=$gP(KsaMb_{f-r_67Nvse9RY zXZB_Ez1knbw1PtNi88o<$V=GN{B5luQ2GEuN~PFM7=)-;XGiHGLmVx^!uz`-AgCht zk#6+QN5JSj`kj##bUL`=sdjuVNsc&_ZiZk=K$~f>h`{oMg_$C}r1KMDZn-=5ldQ%p z0$+~?1w?&48h?97HwmXay7Er6xJPUy0hj|i?ZIyLQLev@%(t-z$u~$wM78{WGTJ#7 z>@&?$3FHu9qI>E!oe?I`pw~Rm`OpDZ0mQScidHnBX_Bp*}$O~(BW z1K})j(kaAQ=DMtfG~+nUZC2JFZgnTE_-(xSHOv^1#Sm+roq<3a|90HpWu%N;&Sr{b zT%o^hcshxz^}PsOT#lGBMA8&ZtINQFIB{TfT0kvlUqDDL0%;|vPRKbH8Jw6g3>H!r zf@%nN#nhYeIX|ZC+Q9L|3VFZF|9>hbzG_}KKMH{31 zt3c+|y6qK62lV%X5?_JeVG`FhAo1-gl%#sWlft?wJQEYR1BRw;Kt!!}$*prx@3;P5 zv8ll1guC|gZD69F>#@Tk%nq~$8-zLe2q%251S2PvI}R!K|5pCiI=cUq2NTLz$ANVU2odXAR500P>;B~mvY;FK|QUtV6z<#M1V0SQN zz#AO_Mhm}vIg>ISq8T1Ql2m?DMi?RxED?{Q8QHm56N`z?S?raG=i zX`CN7Q2-5hT!mRZQ{0++$;1kZWxG{5h>+x1U^t@geI3R44YF9DAVX=_jm z!*z~H9gVoS$V`Z)Mp862hM0aWDmophZp?lPPlosx$U}J_UpHuWmTo=O&Isa4Lrg6n zrw15#l(~raDJu^2mUrsgbLv}pj>3rZBEEJ#+Ncyi=6&^Jly}r{4&G)4C~<0YxSP2< z3iE+1@*hL~o%Z-0;(VzaxbJ3Lw~$vFSYKvs;xh)=xxe&(NKXepVAJ&3r800 zUq%OaPO6zPhov&dyI1RVJ6pD-73=~e9^GcNF9izV9sa~aT5+Q;GmE+8JX;u(hZ)|`=qGXh^MCT~` z!@AJ99yYNv;ixZnr6{0CHwo9UQoUFdc@0{pP7aEkKJ9EFW+&BuwLvHkTML4&PC*ld zq*64Xfzy{Qo0#9}FGl+t=y&^%8T<(YE-Cvx246=|%wRxj$`s2P!XR*CS7Hw0?Dn5S z;Eaf=fxDsG_xRlyI}eTU7$!wJKA^3=y_4^A^4>cSPDbAbQ3rJhyb)UkJBml2K2l=H zpTftGWPSoafj0aRAGd0QY9VXo4C(n$l(AjGad1{~Lf16V-Ku0E9wZR-^Ha#+86wfw zaFZMfsud;a_zWnua_fP+dlN8GRZdALub+x#Y0!v&M7|-(hZZQ!T$u+gACwgYpO~8N zPCDDd*4-aKLun7Hh4l&H%MD{4nl`*AJ=dhkAIuUpm-(^Eq|IYP8OKJo$Q_u#>^gu&RNTF{aKm1l~b zt4#fj6GzqDZ|r<0#Ua~3Py0|*f9nI6s8~`fx0y(nhzM1OC^ClU5)^GAp(G0 zYFA3)_r3G-7ZG=d^O7=WFWo4N(UMuOHJSl_44uI!N$W~s^jki}DFMOu zj0GlKz`si3x#-J14F5k`PIC4wQ8aE)S6$QasP0ZZXkx@EE5Lgh|aM!N10rVHx z=CB+>HBov#_<(X57VIv7@L~(pA!G;@;1FN{vE`^dfGpQIaKu?X%PBt(RI`Hi(;F(6 zK_HW+tI*1R(`nd?R1y@=@yIY#e{Sq`oFt_989XjVc2GD1k|R6iL?@hTQ?cRAO-*)| z$g^COjG#xYewXS-wjw18^*>t-1m19fuc*J4ISPa9Iek9jcyN?}GUfQX&~8 zV~_}Ox_b*l7hds24ZFb=!dG zw2UdD&j2GdaO*213|i!mSqoe}81&WJIDZ+(jo_3x9DIlQ%X50p6TJn~SGtP9X(an5;D1#!6rjq7U7=-rT5RA@SfS5^P{eKztZ#hjb`CMQOGa z@HOAZH)o;#68RM>g)k>%D{oULj(SC`H*o+D*q@6WGJk=pXicBttk#v~l@=H}dZOfo za{`TR2$tHn_o;4N6k3$*bNPc+%ja<8#QCW@l%#)`Q-_i;o;#8pec((D0_@5aQxAlO zoG$>moSDG@L|H`=vjCUHTXwQERj%HkLOTS0LLeLz_Yr(R3>Zyy?K`XGjP}eEC#gEh zo=5h}WWBh5AvVlELacVI{AhQ`p2W!`l>UuDkOfV^JFvHCK=l)0!jI{u?1%6Kxgy_p z)ApnCB&mg^6GKkN+VQ*4o&n7e6J`ap3So9bcXmT^0!m5@ani!%$q18YSeQInIRRxv z*c~~Z4uX?RMi7q4=_hbRe0M^bSLvas~1YAPc1^jX1=s|@uSeX zaSoO)Tzz@*Jkp#(ng^t`!7K_=;KO_*vTbL12gVPuT?Edyo(%@}yt4GvJQ-uUaRw6% zCK*gIATI0@^HS7S1=yoZPEB8BpSfj@J(MeP@1KpiIQ1xL@N5F^qvAMmKd;>N!(yQ@rr`MHiKWIH3O|wA<{wJQgf9e z)0pRynrY)mn-8SooQw-C?6UdbEE{P)gfw(4lfb)__8Sl9GFBSz&f*f=n}F9%XYPfgFu;>XlYCPe1L!DDr8e~+~G~6A*eC0$;N!@<(DtMR`So5@Pb9UEIPHsH4~f;D%gC zDlLNJ$c+wu6r`zla35fPkniD)xIPO6RX7+&il0GzXLoM}Lh16u3OBM-DE{0zjTxOg zXQ$cCGY$s7iP_ucqbg2FZ1r_7zm$6xHH16*};7L?ZoHP4B^F(vZMKd@M#_i z@M+Ms>RI3C>d4;Z!U_Fv&1nCzwlG#hn2OZc+Yi`$_KiWf*B*A2o%8nJlX`c_4N(= zyKvXpc}^{304tH*U}5ucO403u>d|h_3R?PLM)ONJbEh1n$Vn`2yKzD%B=~;;aq8?T zGTTH-Ql!6l7GomD$-;6Fe)siZVP9mTUCdSMfbl*s`Gh~n>Gn8QN9!PNs4Px?rn)Fu z;6Fa64496^#-Q4hiB7~i)4{>^P!-hvI&y*A-5!%LSsh=*E2)kxiaY-57_=OAqN;PO z7O|ZS&J~$8Y{vCX+e3JN5?8B;x9+r6=315khG7g7H;ySwofTfa zjo*wgBUSt`^l!4bMyQ9Q(~Rn0XjjX(`@kk0vkqC0So?7|UYU@37}lfK0V!n?&qL5JI%wtbe#&|n3;Z!> z{EF|@SgrET53ApxdQpHiMvO>YGHYfk(THVHA8|jx4jEt&kXOR6}b!4sk;&hs{Z-ElhReo6{(TPC0Ac zoRoILN^jUt#+sy^zyCdK_ta|Yy{7Nakb|afJ z-RS1FM%l`3!gO?#LRn>D4c-EFDbjP4b|!J|DbF*-mW?pQlTX)p@C0|VVQciQQRFkT zmff7y{qm#jk?yQD25IS!f(tRrV}bBIE*=H9cehJFeL>Pj?4Ry#-%K_}k#0Nc`32N- zqPrcm;AA(6R!r$*jmGYdgNHGLvQhrqvF=PM)gHS$0iH;rJNtHQbDPfXm+<{G`?EVH zefP`O%;p%*a^QFUc>Vu8_gy3`!SC@lHu|TM}W-3Fkftfd# zgpd>lcdzj(CafF}E~<{&98nHQ2ViVeC?lc;0YIcW>|W{fBAs|qP<2F{&pdU_L`j(S zH52D{gSl%{S1R19ZYMrH$P6SGHH5MyW#RN;_{7PS5TF;a6j1~?Qmr+pN;@aj=HJ#M3TS$PT&*=~- zuC(>`8E7c%O59GiD&VVxC1=diosyzx>uDF$0lqs=q@Y35Wbh_ExOAk|Lf=3e`Z`ES z{lX(`zDAC$E78FKaLVcIz$)OA)UYoz)w5~=syR~uw;N!>(~czX!BrmHdM3GhsgGv_ znn=%k?BRKZG)Q$WymG3qG73TQ`ZUl9dft%J7Sb~oiC$kmIJK<|RT9VnIXui2OrNdm zPClIU1J7CxR68&)kVKe*s=Ju1**|mH%YAj0BdwtrC=xQrFUfvlLaiapEr5rw_qD;x z$ZbUvYr$tGhSCofftUx`@96f|aAG}A))ej}@nnrSE94kZDwG-LyDM9DS^elmEI8d1 z_NfggrECeMTfz==g^0qZ>dTKODN&*arV}|~!jEDD6(DQdPy^t>;^8b?(A4szVaRFs(&oz!o z&C%&=og#TSRB8uNz+Foy$Wm@Yuwb@tL5V=)nSEJkjlKg)?mz_uO(7i+2Ks@mNumRN zd2wxD7WoD8xuOUy$z@T|VW&imgcO>7*_Q(-F)VB)RtbaLt?My}lnV@>cx1)?(O1?f zEOt%#DI;Ax0}%li9=NU6V_2dJt;U|&hl*tE%Zdb`3zD%GfWdJxQrbWepfF6m39*l$ z*oZ;x(Xrud_4I6%jLEMb))MA`67{Ry;FgkvF#o>lY}MUrZDg#LXm*n=GqD}(slpg!Q4GP!Yg^6G~2t^*Kp^HO68%CroN>p=Lj}t!VE7Bj* zDnxTG(x_-341~2IrnQ6MXt+`wR=?D9U<-i?Qqu#)63f;z$1TP!qOV36vZgHui)jok z{5@2Hlzd9uD9Hi?U?3G!57=$7I{ZVGdenk2DK!Qtl%@BEC2ulC|noQOWBvJOcmgw?lUD6(9GIUi1+J2)7fJE2bVT^?AFM$idkpb_cP zD^Ph1VJZdltg;ZnOc5MxupG4T-q0sxM}A#IW1ZP8_Vg2-tml5}dGqlt(p4KLI!7oS zJw0kz4Nmt)`U!@QegP+)L`0Yyyq6AVO#`4sP-X7CIHdWW(< zjKG=V-11e4)IRIr5E$UN^qke17amPus5Q9Pn(N$ATq*!FXz)xu(b=tL%~r2-LM6L% zxzO1hoY3o5H>HnPdW>7)W^4U;uD1XMsxf^2K<3*ZqsR0b>T`?=tC@fK6YD23|L}#? zC*<${d|?qMGaLb*(|(-6Q3l5tJca;TJR%%Ez&_n_Q2e0?4}dx+=Lh-iQ_OLRcYAT? zJPZ!;?WKz6w^$SHA*#&TrTAp}oy)}0Joo$#dmrv>I?fy_XcOH?t@n;;i6?i>NN`5;IRqfH^|G4)IFZV%6~L7 zfHD#8@SCKkA(aNFAL9#NbUK^DI~XB|q7kL=?mzjC-|CBZ$=^**G7ma(7hjM#5&};# zRfaklPfmg5SMQ@#NcnMhj|U8?G%3jRSx;40#)bZAC8*?AJBI>r?~D+zg{`{6(W4Wd zK~tJd7>Zqvp~u`m-cxja23KnVK^IB_;PUUI_c-vTC%{{VEE!yBx{vU9=v@QvOz$0? zN_g+CMCc&|9+hcJpD6zajOct=bC*;Qi8SiqlCMDQvS&GKYCteZ*(|*M)=Ln?lrWw?9a(l!+$IH!kT)btzsGQC=04T$Sd)c}9%l0Xn%K?XZ0&LMDA4_PFX(iaZ z>J4ZDH3h@L&dAf=S*PVP6#Aijsy8^VC|NlgUhDn~qU?ckx2!gaUG2w5@N^jmk6#n2 z7xpr*Wb$r&`~Vs?-kXKajx7s2(Rpl(9Z|u6&>TI{IlM*ULBo0i5_$pCpTPu%fsz2c z6NDts_k$DrLx?M2&Wo42)IU#dj@I_1}) z1v7|~>q|QVN&73I)mo$%KM>~@0H!aOKF%%nvLKn|Gj79;*n<2LE5t#ZQq7C~0GU)9 z$Ye0PbzmkMIt|tW`7@3x85*Lf9`5=-K z(1!Si@DYKO!~75I!4+3#gcuq&Qf@%z8ezVxniXb;0jx~FC)*k(h^O4-CLUR z8)n&(<*O|e`m%95Y0CjodUd{h7EED1T(lUiLf>&2t%dN4eF>fun6Rt;vP~-K3d1n zyI!GN*KDBpks#v(eWFBZs?y+s17zqLi1VHgmWMB#IAL~|)P;ij^PXADbK>AZ+AfuToBcwb z5;qH2mqaIUN_T(%xXG36)pW06$rq4;lf@p}t}ZKxDaVMa(`d!QWRPLjbLa};qS>_l zw#jbjwbK$ix|4k2FFU!Dy>BmGTqN(F<%_YF;?{Gm$ziOnGo575VXW#nB4wBrJdD){ z9LAc&VXVoi>~BZD-T0VxJ>;2tWxrwU6P5?|RaKG^N8Dp>xnPKh6Q5wY{xL|dL0IN% znKQ<|vFJovggMJ-MCgx`?j)|(T?DR8jWPwJ5g1Lvcn}iLgTT>7IJFD+I1>WaC_J|S zr*+A*Sf{MPHU=?6h)ET*@?~1?M=Cj;IwPrRTQgP}-wvA@q#s5cPbrAtl!9Dk61FHj zu$8tbb-_S;YHsRn*t?}Ee zDp@;K%qQOkhbr>!7z2~RNd})rAbkeNfCZxN?HSy5M%`hMQGxaGxNpD=7qLW??KkmB zXWPKu2C_tZ&(5%$4UDAU8}+kz{sfL~RE2mWi9%*WDFWZxFQ8J8nx4Ur;$ObRyLkrZ z_)2-iE>5`mTOuvtglZM{cyR-(8h{r9Dt-m^j$)u&$8Zymk&{WIStZ!ZK<3&f{E6hUx_Y&{E$lz%PT?U&7-qFU4 zukafI$r7-$omq{zumG(cyewkJZ5)9KB%%FcHkyOrl%5hHH2Fs7;Xn^>^*zqBb=@;) zUT9y3dkkV0gU*WBLs&?m(;$yXn_ZM1_UTRA>yBQH6X_Yl-p*DMLv(F8uMTMTu3)u$ zEBY{-jths$Ltcy1{;yuQKol)Eia%G+z7x(S=!jm5ESxoy7o7Ff?$&k&&C3$V#eZ zBnbTx%AUm4dKJMp-O1GM#oa81-3*qZYp@W-Zj?ZbKG+X-m&4e9c$N;22&YhL*qOW} zZnREWWq9kd^+q&Fcmuu%jq1wkP0JA{7~x2TC@nbKn4Zup{JP~#g(J2&n$%nGFQKJs zy#tnSK{LbUFIRokbL<KZpD}(*qga`xB^!d0H_f zVSz|~jGqX}^KstMYImUH0nU)4-l~OS1Ie2VhMwye(|GTt{yq-_NShF%Z<%sNoWa$4 z27&4)z{O!KSb>Xdc>pr*2t!7%UjXzjBX=Cx_X##&OVD+NZ~ZwfI|66?huN|mYM>y# zY8_g2cu=c?t2+qu`&|)y2|212D*YS^+84|7LqfZME(@#$vb5ct>-q~Kl`5YHY3l|qlSFT>Vbn)^c z{3FdTe*Dr|C#iNKH45`6A~ZF*d_6kv9vHf5eLV*Zk$N^(+ke9oT;V2o$@1b7X8xswsV*nFIMXHvUyB- z)ob2rz~4&SltFmnm~hIZK#IZj|pu&-XzCSxt52ws}qP{%d58iz}IOLsZ7a z4glV{+ee=UmgyevDQR)E=j2tKW~Xd~fcAi8u^&Eir{{I8$OiVK$ItYpguAa5B z6*%o@4eeDT_b%`L1%pp9$tMwDuHlF}HM!t|3;!>lc}YDg_$tet7!#oGHGaj2VpdH< z-K}NGjrsb^|D$;P1lPgyxXC8z|ABCqke6&GvXoNs6g0%J@d`9fMso(0PjIwgG@j1< zvHT`-_u_`lyp$55D0)7}M{ZCyRX&$F;oU#d5@}oln|}+{f^HU&r_IDF6ij|Rx{2w$ zIn>2rK(BA#OwkQ>7dR;vN4kr1@}TdvznktRu;cCNrW(=i(B=@JIPA9JkuC8?wrkkW z!aken@GI$#S@A$RV8J|FRF#Qk^D329kos(wHsbKgfcuYj0kN#aW`?Imf$mCnGo0Yv z%w~f1fol}j2af3_^-p3#4`XMk;u+3sH?s}cfJ=;x7~mC%0oHA0grdXQiCBa`iF$q- z{-nis7A(?7P?PVZm3LzjPosGHerseiD=`^sjJdUb@FNk}O%GdR$Z-Uz#*ymB@jQWN zw5U7sEPvUWi2e6)q7^7eoPdR+1MaY8u*8PC&hJV6;bjH&&%u73t^Gy3%Yp0ui>T8t zpiWb)r~92XE%fM27d&xmTbCZfW?{b`v9^Ccy*WxBVD@iGFYV}#l4fwf1C=p~R68}0 z(Y46$>Ku9z#%AQ+s=ZFQegf}tsu(Z}G2>G%y$VhY9#rzFqToxQ#c&cFtY?WBCIGA;_j=Bo9%k-5P4_^ddjvskb%V6xiNn5dnf zA_vs1ty*3Mo?i-GI0$)2_aKrwy=bAQYvi8{cB67zz=kKmlGeKif%P<>fQ;-+DUL^m z70gpKxh0tK;`=&{^cP(E*+*IDe}e$05WrPU3H<6x=LxxWEk=aTiTxMGjwY>;D&l7R zfS3FLj*@{J?^w}jZ4^`Fb?{X&DQCbv(P__%UInWYKGohO0ezq1RM?VcP#OncSHWs{ zkeMcas0xIL;uH(|hOoeaHWdK-O)C1WLIznVChGiHP*rD0n^QPRxXl$EjuFjuQ2Bp~d=CRjZp!HjezTQ|brE7)>=w`rW&H>kbBo&YqLQlh@ zT!)%1D(B6&OuDqStIGa{j@-~ZLIa-iqcrm1a0zADX28kW%NOaZ5?(5m7s{8Sr}!qPDF@vUV+SN12dLj z2&a!W8*_B(Odpah*v)gEu2gN{kZP5P+Xl=}`TQD|BNmokgM0-95fB@*yirg|9#jGk zerl{jt|euQnzB0emM4Ts0YqqlL1M3f!LMF5US;)hO@Mf+!YVtiJ4DIneGP%2j0ciP zC6y5?nowIQNZYO{?E<0N!BhGQdeVur7w{mW9rk?hY`TQFN$_z*U=vT1iHQ-5+KgYu z75d{Soy6686hW98w2UHL{=<8f(0^6}{C%7xjnNu%a`RfN?u;<(I?v6+C5IeifnKRX z)~EMSMIw3*pwJ!I0am?j6>dM_UmK3tHrmi;skEGJa*!82Mk`-b?!jfq-Q;a{&i=LKTN;($~F#THFai;58{J(-09O7{0aHIUUk;@F#@cW%z z;eHOYOco%M9z;<~WquBsDe9T|2g>xJP7RfLm<78ebT*cSOC-c@Jxfj_v%#|jyf@&~ zOkND0;-;sOKCjMoI7XZ!bl6*A!I^5dp}>uK z1P46uFl@=0wdb52%Q(9g8=hYG`0^GGCNEKQr}v#Y%LM2Y-i78B8twxv(){hnok(jG z)Hx`7AlajB!-}>cVDXet;w`#qO4z1O1A+!=l)Ea-G?{I&h0b?fgrmVlcv6MX6E1@Q zj)-P@ry{}WgLkTw+Ss`p9~jio1GF!c%{rVVY&LE{b4iq~fG0r2%5&J@R7j&yYR5mG z>A*a8t9~Ms3cn5Zl<1$q$9fB%U0Jg^H3V2WY6!IEjPMPU_7vybIw!j@52yL~1|JE@ zw&Wnbz{$7=Y1NUQ+gjTM6`&<@uP@`coz4gau2aA?m=oI zK8pni_hVRq<071;sCFE%aKg!+6(Fov;P-6_$T6c}kxqDxQ!1%U2)kSD+nWqT*duZt>FisKyTfz?Hn3}4875w4Vq}lU@8x1SSSsS;ljs#vfQKz@M{vG{ zS8ZQ!lh4Z4)1sHab|i4pG|ryB6P?FJ#z;QaIi>9G19TkvhR7ogkHK<)tG2^p0YL5` zaK%H1ZeW904H*503Mc-`p{A*AYafE|z_7laCG!?iSQO6RJI0FfDvqmr0}S6Q5&Juk zEN?h*;+#Ub05AAy#3>H9-*FTxIO*qj{?420_VaCb2Zpgs1VUxV{6am{Zo(jbxg9u! z+Wivw{}blVUJ4A1lq7`=>2Kd)5g;!jSty+0#H*=61uSfFpz{Id{+kT`DTD84FvO{_ zlfiHEQ6MViexjfv7=%rrDDnp-J?!w&pEo`d(#DGg){k(XU{webAgugH`MUa-GP2_~ z)2eR%m=mu+cS%lIcZTT?PLR(^%jU?b9%r{WNh%qaa*#%0#NsPxP15?QxqydEM z_6P$aVj|2gX0gjPT0|0Y0JrEL71QcKw8qVrB0Y!Y89K{w;$x5D%i;lvQM8G7a2n41 zK^BFe<+4q`m&F}}Qp?9Jo5Ja0P7>g>$2RNHQoe?>63IQ7t&{DRn{0898}HRxonBc` z0K=X^B6}MHif8TZ{3avc$RU?D7n&T-Xz$|PZU!6~`ymDn|}$iGeJSIVKSn z`j>boD8&NrE-@fltrmsM;sHy+Nj-;t!GUMAB8{~ED+CaqN2j8XM^iDZ0-U+1{*6Uv zqX%PgoMxxuQo;tn+3w%2{^j-=ua8Bio-{^q{@e4pKNtD-+(d4ldS~Q;TRn+u7Ucej zfNB8flWsD)89E{p`PDTG; Dmq6ce literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/connections.cpython-310.pyc b/vllm/__pycache__/connections.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..977dbbab05833ea32dea4ccb61ac594e0d5074a2 GIT binary patch literal 4875 zcma)A-E$Pj5ucg;)amYY2oMN_I6iEA#*z?rjN>>C7@I%^Fl8{VTvckbx|u_#?0qn^ zC(PARArV!6vdKf95>h3(lB(z-|3Y5)FZMN0e#r|@e3^95?n$fFsmQ6ecW1h1y8G8X zJu_+-3ONG5`G5U;ab`Cm|G`H3XF%gDJnFYVIN>xPO^X!V0Mtxp|#b+#f&-snE#0 z=7);hWD)KRC6?jSHL82(~*gHw;Cf~)!fiuQ;gV#3S^T@yvd@tyl5GAo| zlJI@-9v_T=F)rWFCm-0=c*KE6Opn+}+U5uO6iDxZ*$;siIQu}>5_{`5pXP^wKd~df z%#Q$n-%x%Y#vkR!fWLo7{&D^W@F#cVzsXMk|G&${Bb4-MpLmQDpoXR+2`WSv=7KM( z3L#rH*3f%+mcVm43X&C5qQ2N)cK~_hTo(%0?>dzwWJ6g>$R7BL#=xgEHWSdPY#=3y zM)#&O24}>F+Wi>kI11zs5U0fQ5p8*KC(Fk`I=0;L8|}p~>_mn0IztlkzVQEtol zOP!dl7!m^!*|9{FyXgmA@tKru2`nAVDdgvLwdNi;k_Bko(`x-tcsMRckBMyYbtrwcCEAD)V)ZMWa2yDyoQ34Cz6(U_+n>{!5mo!TyIhM9Co0Ss0CGy4_QQs%mvuuY!yo3 zHzOx(G)21`Mg?v9U>f-*Dmj7VEg&0#M@JBBe01PHfR8>|h0w3T_6%Q8cW zV|diZKyW2*1>+J_q>BiJ(J28WM&H~5i4j8rfEdXdkrkLH)|bZNAojkEMB`wkaM-X- zV)V1AF}}I@2Pm8cF^vMdpL2>EIM_dK%kk}Fp+2I%jf}t|Y{G-p1T*5vSs+->AntI; z`IbE#1E&ql3&;;+iG2lCgToM$tQZsu!#q)wcOZB0@7M6?Es9;07%Eu5@4zc6NxuPm zVZALq75ZUUMGkBuA*_{kGXq3KWfp-E^kU&|DAYBW>gMA zFvj@_2NGB8fs1jB-y`w+FB55pSJ_(9pLRT!jNOmF)yXUf@)Rfn1ZLnR=Yd2fELkq! z#dapi$KqPO;pk0}d4gwbc7NJ-X`Y>WxowNe4B6)JAfj>(6(+`DI1*z*tQf#YZ~)W$5$9)}Pf~~>&h33)rK(A*1l6RH4nmsS zXV*!#4f5yG@+h-z#MLKacNQv^u0To3w>zR0yCu>x|=FP=vZ99(RcYC zkcieJOXHJWxdq<7(+b)?_v(#6tbnY-(@M$3Ra>Na%3e*s%Dprl_5x|?sp<5lGC93v zilJCP#(Dn0A`!u@*?(MLXxS8yW^$41)M*F zoWdgo)J2W(en8j+giCDo60z)bg9c^P-&RHQT%!;CJ5hZg>44;W(bh z+coGm7uxMWUv%VNv=i?R`dlL?@#_E*+}QaKR zM3&yD6v6wB#6^m? zCG|ZJmzZ>lmE4kBv=2DU(MP6djIke?$zIqNdtS72&)5VUq@H814ZTGx_q<$UIs3(+ zhhfj?>w8!U=80!F*h`!p?u5^nlY6Q)4B?DF)%I4#A_q$j+&E*m?eH%C8z;t1L#olt z!|@(uX7znQ-yPyWU~sB=yrj-1|ICpNbjjmL5RmG`t)RW&2k|9g^CFudo6_`rP0vG-#OI4^p7<;rFSpM&4~lktxXMfqnc%>S%H;SB!D7eb1X zQ-mVaqGGz=s;Q!^6}8EL8JG;3!N~@*VKQWfCc|cUve9f*l|tYR)&4sXFq;a^g%u)r z68+7VLStcNmx6M`Nu{9PYn@fhHf|50J#4lYR-vsCxXJ7&tORc6eyh>1nftBaa;INz z;qn^4yb|TGxt8U$0xRY^mea<3>wUghZd=7|kN9mJT;AZ9S95uzU+(1cCcnG} zb5X8tzPClJomI_lp3geJZawNg#dTfi_lUWb@dn_qxsCPQ$ap*BO~3(j2lH$O-eT@# z9lL=)W$xnot-z|e+aG@$%8J>;eA~HgkKeWf<*@lE%ihU&FXLUzx6kL>jk03yXBj=f z0kfCK+5`Nkd4Ta=<~hhb`+!yRkl%kl%3<>`%kGtPf2`0g4$Nxi5$=5u9EZ%K-1{(d zA7k#v7$0YR1o)`=IOAi$$ITPm_wjiWPe9@mT>At#PMRlK;wi?b7@r1y(mc&PXMk1n zNncN$%V+#@AIf1<=Mf`}`xr+VM;II2C+hd<=Qe}e1{n7<9%MYgI0k&y9AtflfK@Yw z{$Yw@j^jAjo%Ll7bKfEEn_wJgoMb%AIK?=@_#CieCRu)(>r-5x0nUotUBx^nM#Oo1 zrzaF~MqCi1ca=9)p$k)@KJjUZBH}X=^^2z@8W0yH>JpYj>%^}~6cl-h8pI`uLSjs! zMj<3xAqo;Ti^~#S5#tiIhzW^SimNVyuC3x}Sz8p75~anIL~UYPqIOY|Xq9+Iq7Gq8 zv|5xU>J$};)`%I2Hi&ByZ4}of+9aNpXtTH>QMb4$(Khi}iMER8T=anoZMTc(W&I9u zOQN0PwnV$d=OpS8FG#dUyeQFLaYv$k;;uyd#Y+7(AhGK~UC!kN=a2g}|VljWISa5>TiLr_kj8Bv+PGhoEnV6o+ z7ahfE8HvY-t-)j_YsFGd6PHrSbk^x$m`+8kK0RYtDLp&b@2uv^9LmGVs1eUZk}2cC zNhsiSyX+XbFW291q^Q+4BkC;uzvdM(iAJ;R3mY&TfoaT5kqQ|YYo)~gEBe|$G;i+w|e)F<;TVgz1NDxNvDm6jp%TOl};wJH(RNFdg}7T z6_>%aMI@Q%j}0)#hik?wl~VacZ*RGhuax(bnEm+)XPunBK44fdTs8x3(j!J%dLogu zav8r(8Z?@;63MKUP3w^%StXg0xpc&Mppw7Vx+<~;xwK)$lLIi)fD?=*`jgW6vd)Sd zBSzd=&po_4sXK**)P=?~to~dgLN=BIz)wbkZQciMU^8cfoDo>blNb)clF^4#am;GR zVvT9ztidvUHrFbtloi)U(L1H5v$1)DMp8LzXv8&tCIzjWE_b%5PY$Q7>|ol^qZz9& z27LnHb~?Eo!JHV0!KQBbNLBsXg(9~v?Bb4Txr3(CMg)-vE9_jzWh8V8&vQ8)Mlp@3 zb>4^#3}!P<)3e3A#StW}c_FTQ!eHVFMNt(wi+-eE6!}NFo=Xf_83>Zjp~OtIie8%w2= z5o_4U4kn{cC|@oYDrFgQdM0BaTG<{C8Ytr2_=ycuxb_>_$e@>ovPI8DW64jn1jlAX zUPfb9PY)msrEeOC^$XV7WFJyEF%pS*K9!!NDUlQ8Y^5jJvsCS_0&<0ti^Py&q%1ui z8!;@b5OT?g8Wi8{kzswvu=;W_tWLDbI$@+>UovAjo7g0JJf1v{6vA>xYVq2LtVuZT z>XkLHy7niLdsctyI9Ss0SYo1m7mVUA1hmS^HA${Wsdx_Qn34IPMc+Y8DA^C+rLAbp z%YT}uTo?88=j0;nYLm_(#D*!c)4G*QMKNcjx4VF6hGHozbtvWT7BURnZc)QE5JIF6 z%j*!dO~7MmYviExldFt0ty~-2d5#>wY6SD;vY4MCoj(TOFZeW~6v5xzW{`?bmGD)H&-GsKGl4>ye}m7Lhzi)%ZbFvaMSH zTwBAwWZ6K%K7%^@Ns#-ReV*_IBEGTajox^&n4i2P@~1C>V@?BUQEY*)WeuqCfLg-V zNj`O1Y!H@rg&GU>1lyO<+5=&I#9CL_@W6_vtIsI%%FI#NM6+3bGM30|6> zF0yT$Fal!2o}O|7@I+1hpFuyiPkXg#0%#+-bM$QB`eIfN*N>X|bGc3($?(n9g7LiI z3ho4;*;IHo@`8`2nO#OTMfxN7wksg_>kLc@Tyx;psBs3QYA`Rw%InekFvUtM7zbp!1rs)D zWz&*jhUIa)zRSM~)-t(#ef6NUp_1*(w1&DCzoytLE7Ak5>od6Fx|vQ9XHSFl1YN`9 zJfYYWtTl6<-9HPCGKE*O+Wv7+t(v>9hMPyclq2?L{fu9MzTTMRVyCfEm@HkMz=fn4 zc|-*IR=sY1%Zp2t4g>9yW!F)6kzUmT?2B*fRyhKQDD` zfUDeEoW5SDt-^P~!(LchTZy^W`AYbcZ$s_X-_^+2AT=Q?;GZ@u!| zOd(DxINDK4aQ5rMS=Jo&54}-46sL)=P_!%6G`Iac_-TQ>i*NfDkwx|kj-4M+<86?o zg;Dg>7xt&i7q)6iVLx0_Sb6cNx<=`QOeQZl!7Re5N3(BIK|N^GQd!d&e~Cs;q;^G9 zgC9Fv`Pj(k!suv1AI7;i7mp95_0%BFI;kvfWL0MKf1T5|uI8t;HCEpheoB>rtpiQY=!hbHZU z^`q&>#Ia~-dN%eayoOj`AX0b2KOv4qCtRf-g;ss_{)spiRj*3ex+R7E>+*%s23B9Q ze_y_^t|f&%SW?)+brp-dyPIvIjvJSh|I?-AyUQt^T-?p$;l@GxsvGE>N_AJd|0dqL z&M$8Vi@!@@gDADhk|ww$0wTzJRzt$x0$#Ia{v*zE75N!@mT9l=#OWxA4SSK4_Cx3k z;G>l8#<$a4vZuw&7`+PCkARMzC_ebrT1P&j zE{pE}0klM@wFp8R(he$&5ve8Ym()mi0^(Zm@ojPm)npTLm%WWhU5Tm+jsn=KBznfM zUzHMfk}zI4zeuQ-j$x9x=!AV8FOx(}co)g3tKO=b>V?oZLj9*m7$&@%3anmup}zdY*^(LZTDE<9K}?x7F0qo3Ok>($SZX zpW3WEbMGge_kr}tBOsrD8)izSI;YLuHD^#|liuaoWM55*-VFR3^#PJt*JX!mnxjg+ zgAmq_Odltqd>E~p>BnoTSrw~C{E63%)h9{fqOn>f@#yj-8q1e>j3mBHq^^!bH6{Aa z@dEa75_X(OT@Rfji3PA#Whm_n5&X^3I_&N~)z@`AywTGeJxkyy`2;xm+H9X7bd5-@ zr6Grb=iSWP7r&WTg~=(%bkrku-Mqd?lUj6Dtde({bo?@rx)R4~O7zTRzX2g+gu^6^ z-VW?TMCux0oWyd?*Ve#0m3&-g8(Vo#Lp z=ZL$OKDVIH9GRyRue0?IL2rVGoE#%*Y`M1xy-lQ+@Rw+^?|`Gq$=+mbnY8j}hk4Kc z@uVK5Ujl~ptkWQ7CMR#yPv=)?tiobrRk`XcxXgxw2k@yMP_NZ=RsbSsiJ1SYq}pM6 zMIzS;wM)UF&VSDv0_%tBUT=S+Cl&gos5nvkee^AiU2X(p2kybz$Bk;04ZWZ)(~(3z zr{Rb!lo`ctmYU#^=ix(}_qrIT;nn@~Uz_Yv)Vc4AwVe4iQi%MvR_Um%jDJoVCV@+5 z*f+?;i)NS~B3|Sre2Gg(-nU3>-N^gIY2@L3Cg~5usYPb58Q|ZgVHRC(eeGXCf6ANG zLa}wPNxw^`Cik}R8%UMstAwZjAHu&+I^Q9!+T0bJ)>k`9gx4@$TsX_WLUoiZHpQ<^ zVPmsf*%NFho)wMmtdXz&lZB~@buDjC(Ep_QyXs-|Xlf&zHNWrn~bMEOD1+Dur(3 z*833VwCY}!VcXNT(`Z?wf@Srn_SeBM&zDaT{|d{(^D4=)tUvbb-l#R>rz(3E3{}Pt zpfZBLatjFkhHwuDS-!cciJpKHEaLx-xT*x7F5D>38;>)K2Mt>m9&;`0hcuuDIW4OF zZG82NxNG=zU?)_$QJR>#VpF?KquS(mn`~!OaM)oYO(0GHLpp7im7kiL#$0eWEnAjN z8?<|We3sA(BBZo?(rhKvMx>p{Dk2?3RukzY!cRK<3bT%i>xuAd#lzKtHY@(Q!rnw^ zGZDI$+ucOy0Oej0cMu|PySFh~vg{rrdx$(rWG@lE3-l5?K!nc5wn^kL5e`*8)$&2~ z7}fD{kI!GHsL02fGlX;^eMHDz^CG;TiUUL*#`-eC{2cY*OuI-ZPvm_f6ejGxMSb0P z(v8~F@U>Dd!kf)XquQ;8v=+4;|61^mzCGlX+7W@B+)5lS7qxllVcpt!Tpsr1c|C}` zQ}%Xywy2GP#<13+g+i+K3vC7H&$U&c|D~+~{SR#e=)Y)NK>tzO1^V~eAF6X%|6fwK355wDFJ{d`Bw+zo|U~`kGb&eO0>w`m**s z=!?p2O$({Y3)(M%pbyY5fSMWo4^RuEIiOZXKLhGu^i!ZtM*j)4meB*CE=K=#PK~8GRpUn9=V8r5Sw>=mMkf z06oR%n?QomyFgbNy$w`i^g7TrMy~+fWb~498#qW9cKC=(O>Qapzf=mfg+ifVs38=_ YKl-Lm15gwG!qlr9qbj#sMN9bq06);a_W%F@ literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/forward_context.cpython-310.pyc b/vllm/__pycache__/forward_context.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..135a5b33518049c1589474ee3e1e62daac47d592 GIT binary patch literal 741 zcmaJ zpUI)W&=2vor~X1u9T_iOQs{t28fi3o?~SUXqY)!J{r+Py6^#A#!M=qQyr$8-Bln(Z zrg_OM&p7vMQ3@Mm0j&cSn!pTnc*gX=1lK}GdUzjZk>NV-;sZUp55MtjsDn$E9R8wu zEaB~>s2rMGw92Wu0on=9-R7cwnqOD2R-p4fp^d=c3S8kY?i;7z4UNWQMMlN>R=zbT zabaY!0x%9`UrhGJW)B>E`U8U@O`ZGwGDTab^|C%e1#^QZFYIbzPwhG_mse@M!A0fL zo3ga&BWSaF(@yEaW5~XAL#nb+zyVF|uz$Yrn0MwAe5NPXJSX>+SbXMpOtEdO`3J_? zM|Q^*Nq^bRYnF)Zr}G^b*-J-!70L!HFwOtCACDwip`5%jQX!H&%gRY>P-%tgSQ;#5 z7Z4GWM1cPu*|-LCvung(#UQ4`cG~>Qd`t(8cj8Mp2)W=fp9naljU;YIy$Yl%c=Bz8 zn|kTy5Kz#yrG1j;%DD=aht}kICtr7e@Aa;89fbJIR(e&MS6v6uv6@QnWf DBb>Gx literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/logger.cpython-310.pyc b/vllm/__pycache__/logger.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5009ecf44fd7594327eb23c1b31fd2854e1e0bbe GIT binary patch literal 4033 zcmZ`+-ESMm5x>1V9*;*-mcJB##@7}?!qldk6i5prNbAaytU{JtTTa>o3i`xdN~av} zh~1@R5fmC!0s7SDtuIXhxj-KJ(EbViC;E1;eQExL2DRPa?2)Qt$0yy~e$347&d&U1 zm^Yh&h3DPB{bT2k=Pm1B)Oqo#qVrR{>I$@N{btnc&qOn(Z>OjFv(YTG#O&_r z=rrh==nUxD=q&f6bG#Ov=K-&a^Wq$DeBmSvy(SODSw6#0ec_sZu+8`^Kh4jW`Lk%9 z`=S!f@$c|C{>Eo+^aj5mF7Wfh@45UUzl4#C;*BE-j*7ru&*7EJ`B-;+VdDcs|hceblo`t1wBN-vl3V71hi$)SXcM$8HmR-2FFIqYvlbPQ{)`v$!u> zDx7~iR$ZM;+Px$d<-|>}C7H)5`TmW09?svnIe+)&{Da8dOVj@7 zD$#aYZoj=Q_SYc1oXdW!H8$rIm2MgO7ncETdwqR_@Y+IL1|Gj=O{d7B#X-JrJp@qS;kuaPN_x7h7lRad=PJA z;jiF=R;yA3CrXq&MJ!d?g^<|a9w-b|T;Pd{w^Gp&j}xVFC!HyaCxsr$tP8o5$3cs> zw}@B$1%z9NmbO%&nG7^s{D}RQX-m#%2jfhxX!p3H*%PKNVb<2{$i}><;qu(&mESv` zpE_i^dhCzcaqWrqgdN%+L#KqoX)(hctE=GZ=B zyECM9Y#&-W9tYK+nowyi_;m*L|f&#@C?G1q%DoL8gDQ1@>9 zxL^`NNnTa9^mrh;WGITV>)|#u&Z~B7wwU{N(-s$AXTzd# zf0$7qE=ifoV(#|R@~ww!o6|d)m|IrhT!m+LtqM0Eh^%mT6&$ughs6-0h!;Vkl1%A1 z>x#l9SSW&zaHSL7u09xu!tZn_)H@yNLTAf~W+s;El;~5qga%QwJ5)OFPX_;L$riZ3 zes4kd2MdE_a6`v(Tj(1<>JNAN@Amf>(qwC4aG-bcY=JOgp$wb$;GnoX(J_>wLsK1@ z=Xw%W??I`0AH-$8Z8Mv>&t1=Dt{tFl`p-PaX5Mp~&9W=c0vCM5W4m?(U-^BA7A~Eq zOEErus7FN%^EV7dXAb1<@XyA#wu?ADGnGLj(^@ zK3(#i>C;5xG+twwcVMS667ZfSKOft>)-F4&;Ad}7OAjvL2S?eF@8 zaeuP7H?D$WEl_{#f!4-#Gg31n0ch~&nI}N8?CMLh^;cvYuakWRNq_Sql74JuH5Yh# z@P{e7W^CWmb=??SV|L`gbDHavSG#eV$~l;@uq63e!D9IXd`Go#5mV$hBbS`wYj$(w zg*}<~!-55)W_aHuka7epU~Dy!5D^|+6!mW2nWo{w?{&&7TvRbk?p*{3YpKFE&l-+dv1pPJ=!C`+@)j2FlC`D}wLQEhSni-fFv1ZINW9iD5;!2mW^fPg z3@v$4JG+$LOX@uVUzyBx4Qbjq5lM1#B{%*OzGLElsZ4xyJyZe$ieP_v@P90#DYnUVDg5k{jP^`Mo3E|3P_v6(YDcl0l|%`{REcViT6k!UfR&4 z&fx8E17q?;iD9L}0GJ*3dM3G-=N|ptr0&i8w-%Q=%MaHVH&-{-=_qBz0!^zrG~zHs zF8RyXCmxILkV=KEgYce7knPaimyuCvzK^P8KQmh-+ZlaOuW)@rO^Pw>3U|xDF z*f7u1gZAjnO>tEGm5$t1 zEJdbrp;PFpmf9^^MP64?*Q^pR7g5$NotL#vxocPkFPA!%Zq=&tvfi2N)~s5$Zq>UD ztI?ge=DSU+DNCO6f#jB5Y*8h%B|qe^*i(-j8L6Vi?;_+e-QnZ-*a2Vr1m`)bodRp)gB z^@`WnR@|yP_f)m!QPWVXxpmZ9Ujpxbr^A-0QldCV|V_(uT-2mN((i?}LX#JF#cfw>LfVbBFL0vf~8XaQ58 z{9d~=a0w3xfnd;!JdPAie&hCCf!UZ4oq#!52p0^34kRfuk_q%5JE18S5SSVF@WqI| z6~K}r=jKGEoleR*7beXy=VTm}=-K{kC390-kyIuqn^Y&UOEl(11J-NH6H0TBK3^)2u5!_qnz6`R8< z3YjR_94@V!v-_|)tgO>%=JI}X*jP{bEamLYVQrnWWmZ3uTc4L0t%NlwnWv;l$pR%r zYqmtmDkVf~k(@;-(QSa~GuvKAEj*3HkX2cIt}5ymYC(Obs_GxrqWT9_(~nLdY&N>J zK#+P)*R$=!u+V;bN(@DRSY}*amXfF7@D5=}F)9VFLOpp;Fwc@BI^$$pr z`t3|{F;Y=Uak~>Z5v!m)u_BGC%aoL;lX^c0I_wNmRg zJ(mQ)ZLY6|zq~irF8@#6!pIHoHw&mAlJ;vBVxWs z#3W92<%nd2yf5Q4DW4TUvclHz^o?v`f@W(u$W3-0qhM$ifHR}*IElGA6I`b`h))z* zm-Xkvm196RG8W!ymD#HV=_{1HM#<}x5TcC43;QZ17b!6*If=X8#D~AY8|p}iyQ-z8 z!l<>4cj_UkT*Djw6OxRZ$nGjD;^!`dlT;U07};HJ5xM4-Rwa~Sk$F1u5?Ww~xw>b# zWwcdLHjr15SCG@VD)MaHoU3dvbIYn|&3QGq;nm%FjGcp32e{js+jJM6sxXZkXj8$( zi|!J)$rNnKsbnrSKE-}GzXEMeU}I-OpN@4_zJUx;t_VS=q<{*rWe-6%m^}k=(!1PO-S)ulL>_R$2wco?OeJG%!J7?VdxznmH{8Hb{`dYvbE=jr)g%4gH4d@Bc&%{809aQ3alNJ)v~nP_6cNkimgM3Q>ilJQeg zO1Xloa!*H{u7-pL-$1vJ#C5Fjn z7h)J}wi?4+vx_l|HCu~yq@}owRF4g$Wmmc@F|q|(2D^-meT|a0C_w;2Vqd3(n-qUe zC2nkdgGwakTNOqo)BkprnkW zX76x^zYL$iZo?Dv+N8O%=M@yHcsb4f9n_2EJ!w>k3oy$iR*7U6%J*;OSUyk=rI8vJ z_6m`@SA3wb)mRNziApgP8AfS*#G6scRa|KYW@D>W?-(t{RrYDT$bJ?t#tRj!h~uyr*U;KHSU!~Eg$FA8 z)Gdr^puv9;4falrmf|{~E$zz>itOj{(!ol!8rNNgD3bn+PfCZfEXC-FH7&=>d#7Wp zApFb#w7ddbm+iU8 zJa`rB_{yP-+G{Aij{NoBtH7mra5`?FeLkP#S=`)_domyz{~)vC1iZxwIDoNV0=9F& zVVQ7v=*2bkZ(-dRMyHT|b+m%?;%F7=+UPXW*62*n03}c1UBL@}8J)&^CRQ=>O7zCT z8v^suS#Zdk@mWq6d2c;F6|cl+;yIjGdG8Y5jd(Ray={!n#R^7#j23NlK0d#9IX?II zM4R*ALy5}N3Y(;|HGGFhz{s`kI}9EOTx(dmh%j{hFr;Sk&K+~?nx%(mBav>lv@buq zzb039#&#v)a~ChS@?z;gI?RQLoGin4wP(FL#Py|nQc%%ot6va zLUxI&CGtu;b}x~8i53P7egbzxt<`h}3PjlV2iw~ovxUQ972m&c>)!RG?DYm+9z(NM z3Fzd2C;(iFGg#**KV_aYv_<+oF?{TK7&!>FvJ1c{si$xAC&hNha~Qjde!{ON|Iu0?Fc|@9)d=}u$vn(D;i8V!Nv<`xVnIm8?|RmJI0=QC z1VfIM&D)r)^-5A`cS80Kq3}&g$gSaO$wCn2VQ3i?`s#^|cpu-a3K5^S9*iL>p36U5 z4H1jVBZ;BaoZXb92wx9@7$S~U%SNQCU-sE1pJQT+tVUMJHpQC9e&)MKts(nM4lZRU zF(6DdQJB!Y$4He%qp$!@vohCu2oi->Ei+6~5M!T=tJT`{(TOm7_)8>OL6;T$)aQDk zN+rZQY6WT;|MVkWuR;Y^(YJ>9fT5u*BQAm%0U{(AYZO%U*Pm+`Un}5?Qj;5KLrg(I zPyEKvFxL@g(`yAyHWUL8HAN{H&j|+&FjZN5Zcq#NQWeah$UG8rRMj<%5kH;JPrX&S ziSIffVRl)4rWFBKR*U)f9%WW})oxH+6(KW1%j4$mBKn(W`qfn5zqi^6PR zlal&-Rabwfmi6DxT6`i}*HX9ew3;VIEO>~5qGlp?K~j*FDWTXGdxH`R%JP^QBk__{ z_{9~-up79H%tUsF5|fhgks1flNb=vr8-5pwE`ia{307BsqbSN3T0wbcsLCI;qVfk# zQ;w99@_S8Jey5d{-)e^PKSo9QjZ!uKTUjCIRd9f+&!wL>HLg+G06F!LI=Y58yoF>k zP~s{GkHGX4p%H{S5a!s{-IA+Q;LR)JG0K%P-P-Z1C4@B$9{hlGbQ|ExX0n_|gQmFC ziJ|wo%XDuxO6T;uyw_}Zl)m)gv@%8zwN{MB}0Xv%z&Tq zv(2>}@!LF}2#AF0?;N%`>LnefNR z3u_mVWDk|eWa83`1b*rWk_#wv7f|6&A)K7d$x{Vn?%#2*j^=0;C)_wb>G=DY{b>Sz zTyT{vOg~K_u?)ZLm3%j**eG|Fl9J(90Kb1qIl=@F3uol{2p_Qm74n z3*`lPZSsN2y)^1u#1ad!y03zvm9617b5y2{VC-3LLUN|a`=uyvw6w&?_atQB1}YQ8 zC-gY&K&%g`%x#hHQR#)mA|bL$#G-o%gX>plO%P3&8_Mv)aSQ|}Z%pg#8FfXHjMIt) z*;rjgba{hIeW)C|wxR19y0)R~n&M|=&_;9qIy=-Ro;%!}q>y+Tr6IxUg*c1+I)cKE zIOr#%2#yvOW*W*Qcy#)Xfk>?;9!$&x1tKVbD{dWO9^J>O$4tU{3D@%|4UX0$pI;X` z=EH&CeqaW$>>v5w<18MU?^Fi)Vu%ty^AlL5s~q2R6RKD+$)6Xln?%q{7fyia8NPOM z*GUC`aag%HR=?~nHjdq+gh%mHp0fyqgQ%bu| z=tL3Yb-ofo7BUc8i2WcF{>ro%O|&)~rpSAn2t;C0g>ZmqfGn*l!!swXV?&IDX>1RdT!h_M`#BH|s|<|9EO%;$r-V`0x#yoan4eigH?J)Ye*IWMw3k{J(j;kP)d8 zEqc<21`{R~H?Bx%vo%Txh&4}l(;H+yZZHw0;|7+s$lH&3;=;$C6Q9!mX%IG=K>#Vj zDeN$EHHFVvWe+dcuI zh@v?uXhI8?Fp4_My~KM-l&Ac2qI~V0`YM30cM&9kVn-zI{{T-Cv)0=$XKnbZc3D-* z{2)d1^7260gKftkG)br;3404E6Hl?k*MaUmAiA%-U82@7#krq^kaBIpR9}UlMP0(E z9UN9wo|Pwu(#XQj&#qNoqpqDcmQO0f>&2^K6meMl&YA4EaoUbM9c@ynY*DUI(#bIg z2unteMqX;Iv|5db&Ij%0KmKjcXxErBrGW5e;TfU7fNeKGIP#I4QbjK)u*c?)Uw1y1 zORlu8d0N#oUQM{0YQBP#XQg$rO!e@Rng9aD%=qx@&X=_HtM+t_;68G~4tD(z~PY&9J+ZY;+uo5gP3jh%|4Y%I#BI2{K=&L6 R&tiRBBxI52QwCr3^EXap18D#N literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/sampling_params.cpython-310.pyc b/vllm/__pycache__/sampling_params.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..371575bd70d6a07d3765ad30cc3774f25c97eccf GIT binary patch literal 17070 zcmc&+O^_SMb;b+^gIO&0|A)(84JAtEN=q!Me^FeDlt{^x#icA#wgp>(X3@R7*aZgA z%z)G)P$G*=m6NhucIg(CtE8%2R23(cgAcx>l2a}|rgE559651hC-HA=NtVQX@AY5? zgI&_Hs*+#}jW_-Cy8HF(*YBrinz~vz8IdVx zgeLU5R@U&Ft?P|!Ijd2gQ8ya7a;{;PO@7YR^9`$PF>Ka{8ijJ9FKX&E^ucVE@Y zdqknUR}7c;J&_SbF>);Ro_e`SS+-Y3fu2H^TfmHy{9|V^ycRE=8r~HCxgPw&wF;_O!bdT8}yPy0cQ3p>d{G4MO9o znjeJLg%%-q>S6Yx48q*AE}p25M;%|DZwgtbhtsax44gm;JPg;|T2QGs*Vd%B_Ve|3 zzV_Yset2^252yd?i63;XJyNAX_J6=4%q+j010XYCl1y0_S~)B9vLUkNoG?I!Tv+Up z;nEg{O|9PLL>Xja#owW z$w5kPrQ|jw;QvC+uepBUxK-(mAS*NiY=rUzSY6gDJPjZF8k5>bt!}3zW@v9OwHR6T znvF^`+R%3+%b!IuIc#bAz@Pbk=hmH8qm)8Xr5)J&&_*PJFfpD2K*ZR9%x`P>#g^vZ zZ1ihHzd8C{12?>^b#Y2;8s4G66j`*$zmfSi~+X&AhqHpZQsD z8pu47N**nFn8)Cn)k;N8g^qKtNcmYm@H~=R;aH{OD5(yC@1thIpv3YCmxjhlvsq7g zH7tTlgIX0Q)p}DfuX=~6#SuypmSl+dk{R|60RH1h3K>J6)5f)Fe8(|gPIWz-aH#P< z@~@k^{&69ze`Ff^hh|Ryt!e5Xn0ftu-73CUoMoI{FLcmkW=dTR#0%<-+n) z&+RzBdw>agQhXrjhHmK|=4RsM}K#0d`9Tv*4H4b zqjBDbqwny>eECj!7^69a+H>W*cx1Pu_+y{u>2k@E<9Nvto|neS-dwPoq=it9x-MGY{Yh zR=*a0hjHu@FVOFB^jnO6N21>`nG;ZTp_r<6w5|i|2qiThsW>Qiiy3@rO|hb4H`+(* zC-z2p`%#W{5m*mlbKCn_O}Gv44zw1BnepsFmcgdwyT|6irx@JHzIVm4O4zDnBkBf&n_mp^4oDfUEdm5!PC^;z}!PBFx zp*|Ll_B2$bxHua)eR^5HOLs36Y! zd-|L*2C7Y13X=47MtAz@bN1o&AZYo=7Z+P~CxF$^C_(FYomvTX7jXpmi%zR{#FMMi zlTa-d6BPbp)sr}G=Sz(2oNGHm)Cf6`&u_O{O)r2F<}_?yI$m`h`Rldnx?S^am;-iH z0}9&8maT?aQu93Rt@+0pE6v(HZZEeRE7GImxXGdz2qBm;9-ovurFkiv2jp18+3u2# z*JuT5Y=mV6a=g-9y*VyY9Oz7stR@VN^ckk(AU?wRcvJ|L;e`nQ3(3$HxLwd}*)O<# z+1!B|vJ}bq2w#Gv(e{aS_8OBIw%j`DC!URIJ9U~fP?qdRx1g7-I?zTIVrr%FfYiPW zdJW=AEf2af$XB7Yt_NGk?Z@j)%$+qS&Z~7mTUUa?noxOtdj;kk^dT2Zh;tijfn+Pj zN1ZyY7FOA=xjgNJf^3HRO?%bxN;d01zJ1a@YNP*V+jG`PUxRu^YHA7}CP}(jG6rTJ zC|#FHhw|+cJL6CcfuRHg*k~FFxmP{KL0g}K1Q>74OJR?(eah*lkTEG}8lQ6dDP*1` zQbenmOrNp?-mITu{t*kL`XP20U;(vbYzKNYt?=*Sh+`>yJ8;J*Ajz?Wt5Vov4okRW z8%YA$faAb{5hqjIq1uEt-mF8?U=ki?(`>jf9wU=87QQ9>spb`#W{ZeZ8_s4QlY`YE zVFdYU8qnr_sKPzYUazgu*6M3^gIO>NHI|ZHmR{3tNC&$V;zEk8WK3~$NTEvW#?z{{ zVbVY;zbr{^t~e{TdM()MO_4;H5(6XJqUR;3GPH-qc)p9fYj# zq%c(&@)bprK1*5HCvzY#HVOT3%< zw=*jjVC7y@8#1`ZD?KGInGS7OH3VRb_U7-4Egl-N=!(RqkTwwk+d9p0>=Pm#Ou5jd zh+D#oNP8JmN)%J%^=gcg)O1(Hc{m-*bY3c5)vK1v>D(% zg<7fDsZSu$sFs^O#~f9xksZ>IkN{*aM7E5=5!oxSO^)*>B_oA|Mru?}{fe{IqY8JO zo>qKxv1!B-5RA3}ryy1K(gbTbTcm2Y>5#Ku=&kKt$6m0d8l>OT>OjvHPQ1STrF0yM zX8hg^p_(dUB@6+*fP@Lu@6?7(vU#>@`=@vQC9mB#Fn_bws<3O)smEKUPXbW6TJl&} z^=26%j|t9xZojf3gDXs1O9TY+4gbb#ydM25*V7CzL5hQ75dl6 z2bZT1IS4j*I4=!%T;Jws$Y;}KFu53ghbQ|f~LPs+@ErKS9(rw=w_=#i^~*# zQKCh&e62#M?4)QR+OHW;X~(D;Q~U8SP@o?}XlBpzSnCE;iU$pql7?2?|yZF7G9OI(>m1LWrVSmD+Eq{co*sw_eXFj^Cui1 zbPR+Ov1!{g6r7 zYc21RCq67BS_C`bkr68bp@sDV?N=uzubw`j+cSn)VbjOg+cE^$K>P{e?>rNuw-JL?Zr#p>wZ+8aizV0m2{oP$i4|L~{ z-qPKT)b8$iUHb-Nih%zgd!N5`ITb!dyeDJedyy84Vq-OgPoE$9ivF_pvW`$HBFZ4V z{vs}gaF7XXl*JJaX?r&L2*cO?4JBJA*b-3nK5RjhQU8@OqW{tu)qi1(=|4Bd^`99N`cI8X{lAPU z{XdOq{XdKu{U^q({$pd8{;n~n|GTkU|Bbwu#VnfNKN z2FNfn*$mpv2RWnQ1qgBEkqEyEkvvDEn#7Hhskbs3 zl^DS^=KVJi#F@$rgm|Vh3lW^D>_VM6suS!+Ej5xo0-|SrWVv&9q(0yl1a#D!#VBBK zt-;gdE_Fs&v5p!pg<6S^@s8i8EZDch%<`ef!nF#e%*AsrUPK03)nRWq6m72tkz-4` zLxBi|3X9a-RkmK>;zw@D{-U(k9XKDp&%dlG+4==g<3${D7c*~bbcZgB!+Ku#zJiLOwoRlCogmZVrIvT{EcnzX zxvjr4o9QeLFnj7YMxQk8wN{dnq$i{m3o4ebcOt`kh!PT1;ZWuL=`U3-ojw0_XhH>m z4>inRaXc5!n=lI!^oi9wsL>)dGV1X4DKVRfR2H~&6#*CH5Nm`fRe^Qmha;7Xr_VqA z)Vbv+DrcX*I3TsdUA?->{aj(An>&Z6!uul7y{&l*)H6xpK1oM6KK2oTo&jLyi+st5 zFR*APuCQ2HOWT7BExKmu#z&(@0r%uAozf!xDM+8yM)}f9F}H_Gbx2FiM4fScn`}vu zJGJ^4L?T=M5xk%-Ck42p<&OurltlNlHn59&myYNO*O4H9P)bsTWRCIeEMG_>>5*gE zqIvU>6`FsKt6?`~MV?9qYMN2alU(z&QB9ILW?xN`F8N4aUF4e2M>R=;;4)ONcRXS5 zV`Q7?kK~V)ltBrD^op=t7DbHgl?V$-q@koHo(f`gGvogmLt~J4uLB>;n4}g{slZhF zivafpm`adq8^s=-xW|!bhJVdHelO|~l4xTj!G$N1gPQk!hKRP`M~_qVl&Uo*#!wcb zY#L=_sWOG(7nD!!a9!*ds5cYgn2qcA&V+I5@tc}Q>oQJw2}TP@z>V<~=8350K9o#E zHTR+B96hIMPDVA4qGXbI1UQePCdtyt9$lDnQ^E9A{hONq(36>0wO~f<5qnu;PbrCw zcArJNebMLBdXm_w@WA7%gdh4ijfcX(pi>3E1%bkO6 zEVm&#p&%~sjW$SLas%SVpv|JOv!|~fA{3JR5yzA~_g=teS;Ln;f7b&}b>;_+$G9hW zdcJ(*?gjhq=M{#|o!hL+8`fprv2*8kQ?tJXg`8sKcl4hsy+evzrLJ7#7*{k6(n34? z*+d=0bu1bv9~SDYTBM4baYaOz4oBga`33SF!}*k=Qs+Rrfuc4AxFQg<)&$ z9_sTxN?b|?^n?xMJwq$==KzX0?V$%;FX)rn9G(1H;rg7O{w?S_Gz0_rlej3V%8JMv z=REa19n~pBZD;WuU7uy`V-8nJ6^>qe>KT1spM=f<{X%Ig*C(?E>RY zq;8N>NObdUBoN%V)sN$96rPsEbW!RADc(kOS$QFm%g0m{7I&HP?#DpnUu{;F)X-8B zrEw?f*3thrh`I#;g@{_XKzzpyW;3YeO+;>VxSer#*2;mCLk>QYyGQx|}Y%*%W$+&dLyyV<~iF}1gwUDw=fkn4g% zFK46Hhr8J~;fdn5S?^KCmG5Tx8ETM!2IX|n4^d7~ptdk}h6H>){ona)XX)7%-z2Z` zYX#V$v`b0OMUFWYWP?AJE`V*D{xUAq_RV~Lm~~(hl-rn^Br-252^Ly_fb4QB&^NLd+2UjkZye9Liy5RQ zyj2dW&gYdsXcI8M3ghh}=nN&hDItq&C)p5=_`Nr5NPGEC%*U_-3YeLxZLgtY82Ee77k=K(I~JzGG1P2ZeHjj z%UYsx&1!+R0fB*+BaGMRg|J-b$7iI(Db#F5?>f-CGPs;XA+)D^F%Vn5(t^lA9D}$) z4Q73QX9BSwnET2QM~rF<)-~+%=o*LP#)uqmr51ROw?%<$+~TQSuEAh;8uVHNX_@r) zSrW0hC}FEdcTh3J_4W#1J*AhYjzEi`7jM*?R5jnnwZFxCjz0K6+*R5|+9MntI{fAe zyGUab9-iuydsMH9$bv8H_13^%^8Nrw{+fjt zYAIGCW{z12FHwSx-h}Ed^l!04?2XK~@Wv&>I>QFTS%z~A8w{Ha!*E6YJi{c zZi_B;X6Sy`;3dkX&M3m8$wk7Yj)i9mg2m7#L#+ERb@I59fIyf6P;aR-fgEZOhk2Gd z`xjuR-RSlZd}}C<<7NEVuePH!QTLZR!-y0|;j*R9B+WJzFRkBCkYK zNHq1M^vFsyX>H0GNJI>Ct8|mwyOHWXNXsI%CWoLmLm4Cih@ z;n=`A)%WfgnF(Hr+`S0zQ6cR;cVgl!U^3nK7tkXG4-rx3m@rwVDIFowL*4s zl&N!RUN4U{8#ufoo!Aczjw6C>Q3DzDia8 zl#-oz>}x3a0f`;jF2rN7YMHmjHIK^X?@;KwfWzVVoqg$hr09f(#N9lPsO!-Y9}e+V zD7a~si#Ri? z0do{jdM8SS)(O~?cwwLZUqr@#5{Z!^QqhJy1gC#5Ab>CdHV-zP$woio*LW|~!qzMm UhImuy=7*U-roV!zo(muT55ivbr~m)} literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/scalar_type.cpython-310.pyc b/vllm/__pycache__/scalar_type.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f2f2da98c8e9b064208bb56545f9570c2698fe3 GIT binary patch literal 810 zcmY*X%Wl&^6rJ%Sb{waVG`x0*T^7Y8O^6o=wGvjX(2&pG$botbN|+ieHI=zsgZ_-P>YLyEgY2XPEDdjh}^ z!!Z(=V4)EWqf^8*rhh_A|ELk28JDPMmKYLyT49CJ>VhX};atw5I7%nmgwG)K-)Mk3 zhM9c<;D}&`h{pBL2E$Cd)rbkaQFBZAy7F!1SG)<_fgFRGymjqfw{H!y1KuucFbitz zDXudc_R)@&0??o#@86U9{|l|>3;xymV# zI3&gfUc{6L0?@C7Pa5-h9pyBYn~_d%HF-rZPQufGo}XON^RuupZ=!g^Nn7Rlln=$x zd{s+&76id~G$gIR(UBMsSBd^}%6&1eGo>1d{u-|-#2kSqBm z^qaBUFZ*THXT_ENj6b95c3kbx`m?I;#I^pMKd0)Yc)q{jFZ37vMb&oWrT((Ntm@_X zOn=2+(KWuz&mQRhIaXmaUugb$R^?jn0-I&}j>T$h?hDJmc&cxnEuilbTjZD75-%%B zhb^-+=)J;LxWg}>*G_$P*;#fDUteP9`3jrmE13BLyNDi-U1FE9>Q#1yy@dBQewKNU z4e@7w?Lfmm^-cBEul}3#xluQA>t2+}+`XU5C>extUC*8SDIeTfL+!%dZYb|2{WRwE zb)9Ef*x?UC5%%%5vO4HQ1Aa$zGHQw8Ajr7b!fqHxER?+H zV1+12(BPS95Xm4;IvtGG^ObjF{&}<=)7-;8A4n{TN!qmM=?|9IEIfDdWIsm1bY^^^ z?Q2py!qd~o+EYWHwv3|1v`wvUZnO=^Pk&T5@oeJBYA7&)&h)1`nHA#3hM=`@C^Os3 zatk|?su6W^dk3oEUut3=5?{UAlKr%mMrlLhTwFF@?+?5EoBb!PINENdd$OAhTDx)F zZ)I%}r7~-#dwDI3I?#L24F@dd;vDUQwp7zcHw5p%I9vp$WwPd*ancUs>~{0?hq{?N zMNjT*uD!Rmv6Ywb-rC%HP}jvGW>p(4iUl5nK+% zpiNqt{!Uq=iQci+)1=f4W=LHzlTM7=2#0fX&u?p8EXg6D|yuaIFeoO5$VTM ze)QP)~MG1mX<5|33 zRHLmn{1U!oRTNdj&<)*s=Gu#fxPp#Tx}pV-J@-#&NR46>+R^t(M@Bj)%^q|_9{tqV zH=rwHtTXe_*oXdlC2^U`eQPi?Hhb<-`Kh*VkL_NCS?9Ih%)TY7%pO~R(f(!}z=q%jaOc5V@WJXQzP-Bs zVCxfje<@6(;4$CJ-R&^r!BE61NJg_CX3?Ni80eOlyX^uzcy6@;CG~~Ek4%y$ZRJLi zk!tUeOof0HOr}wLtC^tDS-#8()t-;SDq-P>gI= zM(i;2K;MVmdj>_#*kG2jKRkoku(J(2Yh#O~KeOx3#>jgRbJ)Vek+`66!Pz~TcvBPH z+WN}4?bQu&9t-78Ay7b@Bj_}_9Szc<%x5MefcOaZw%|Z*sJWX&Z4NU?{<*B%>QLea zhWS-BY@+SFaxyx1lA*+mFQ`&k>JCO`gh(zT#noLzjPtAt6&ha^mLw2q6E07Gl=Mkq z3h;MP>1Y%VOdIN=Ha3r-I<$bJ3V*5L}SKIG2ghPRJ~L$ZvbSKgVhb+5=l(TNuczB9R1{!4MMM9WK3@- z5ekMV2qwX?C2ryjxitiIOpW4Byo<=hEVxGGcT5wAjCIky}GAxox_5`PlKI6qwBiHKy8NvEHZZ5e{gF$@sNu+s&KC`=~u zljT7|}MBYC(x42%C5d#(Tj<^2LB8KO1tf7Zgc13n7;M2!*8*Bqd4b z8_TBmr0z`WhV!-MiuWMe#&haNn4mm{CW5yhZSLw~K^jLU(j{V_nbgPhH&vQbsm$8& zFMV|H-g@xy{f8f{J`C1BytlRX;Nktd!TRdf`}f}&wKlnw_YqyMeY>*HMbex9(JBnhl(9Vx5`i}n6DhBdl-=sq*dlCdQG>!6Td>^)L|(! zh;yjJB56-}0$d*fa;Wc9xTrv3HsDJ~rZ5pCCFZj7p^404AX8MN&1P8j5P4m-XVJzT zgyGI{Xagp}EL4^;dL`+y#Y3o1&9g-J4>gY|%is*uZ$hTRme|T6Hlw6)3t{J#lm#i3 zi^djKy?|9Tc9C6DA#AB}XD8#dqnQa`sVY{I$k{Of8hfdYZFc#<5WgGSM{{F)O2Eh3 z(fm`oyAT<_vN3XZVMy=pJ-VRGFkVn;Alp`6&W76s%)n1w!9N>Df-_JC4lgC38vS$= z%p6$}L8oS!1kZEPe-ia0?}UnaZys~P+um;{-ux+G`yI@h+e5k*3EWh*(aH%5fIyPR zC~64NxA9rYkK9HBki3KYgZA#|9}kJD;y4p#nt3}&-Cl^iO(Fn?m^aslfP`(1-cRnV z-}9ajQ&z~4#BuV3{C2XZTLgX?(2phUtVsuGhEa_Y~{X!@(#EEJAR|b~o4=4#1a^fy(Hk zjraHf42qBk6k^Ith-G+0BEi=O>QEgptYg@m68I@zKWCq_%03fZ<)_T@YuZkO3L=D~ z*N#_K3HStxv*QcG_sqXk%ugx%*yN+RMx#jnjeR=wvI#2u}-$SL=e4 zreuffMQ$KcP;gBg8;fMaG)>^0rzXpqf~b<96M;SxNQfB}d5zHZ*x|FhLhZu)#VSd? zO9dfJUIpR=lbn;OKv2l4$Z1|tW7I`lP(~MJ8dBz z*7({jSEn+D^Nnje)cb5I<9zE{)+DP?j;T4WQ31`Ym^Hm>h~HqD+zkR+Hwfx4=PsPQ z*r0;*C43iE5^|9D5ii1^9+L{9{JvAfkGib*3q_+z6d`NUVFmG2|BMPhuKr2V!~(%k zMscR@oBp-QH!1eg1iHbv9LD@M zGL@FSjRLe@{Sh~;;)Ya3Ij>6BaLwiN^6bi)+6=7v%+M>(j5E`kd(J)M-m2jK^k1== BQiT8j literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/sequence.cpython-310.pyc b/vllm/__pycache__/sequence.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de030385e1eae8834c7dc1b7b6aaab0d983767dc GIT binary patch literal 46639 zcmchA37A~hRc6(zS4&q{SGT%bt=3*zYug$#znSQ?Bm!x>c>J zuJ(P^vQ@ORAVWfkfxs|f!g!F#z{Fu6nBn_ihCqOXVHjrkSO(q<%tsQ2EyDnY$wF++ z|DXGodR4DvYv7xf-mUlUzMOmRx#ygFZgXfTpTOT;Kl<|*zxq3g#25I{`8R-%qj;Qc zsYJp`loM9cGU~~4QhtrHf!|a;)iBFuBVA4>8E)1yjZ8Ts@9BEBkt^rqJyRcOs=WR34J=T)o&BE)O?G$|H?+<#mnG@@QkMJk}U5k2ltr*GrrM#F;2>koP>^ zC(DzKjpdDvP329E&E?JFZ`8Mxw@BT#mbc<}u)eLay}Z3qDwia@P~XwmS>7q{L-k#a z-R0f#Uaa5NxV?P4ybss!XiSx-;ow`G*tMdTZjvgf(&1xSA~AWslkG?NR$~`>q9JV}k$6_rR~ir+hEcY_KMg zX3{p6?z@^Q-w&US)+YFDg3kl+d5h${+1diXEt2zF5ofEl4d2`BElUTj?f6%EDP4XW z{C8M8;lI=K-}RDNJ_O(0)@|^;?W$3pz}M~89r(JVBhF#?PFd6NokpA^_}XLb#n;{- z|LxYkmonwIBV@ldgOHi7kfR7WVBLw3JA;tBP^V+?zuUS8{`c4>TJs>j@3rp3_kEFg z$Kij!^#J@Ii1?q7*1yGiD?;8Xtv`vF2d%f^`)!e!4_SxMyGJ)Bti$-ddt+k3w2oMB zf6**I+!3--hhY>Ru_q-K#)jjA*hj5nhOv-*}IRz|AciC{wE{;k6Fg~ z#Pmb$9p~+5SMB9_yHs7aO0||#Qg*%CvaQm5z3Mo&GxKMhT(i@Ln>zO3akp5Vb6To8 z-)h*ci%n}f>87hnRj=S>(6Se*tM!&un{VNBz^b;Yk_pld&sXO!+E!&nHCJray5eS( zZLQAR(^+@u!aL8NtQ>#jMl~%2}T&<&+oZoIxF3pIAl|*PO39EmUB`$>sT`Wvdgm{G2}h z_^IWU)t2K91h2?!SIwzhuGVW-1zp1E6-&pj)a#8(O;WI>r<>=Fonyx#@1#3)w%M$s zUT3SS+Hl}8epWRbE3IQzb){viFkv#Zs*7m}@1tmZtt(+l-xwdD?1D%IuX zW(z&lTy`oIwG*SGhT*uW+HyVRU zHtuRXcc5OIJFs#E({A~|Wx(SB$D6t{D_2wrNtvI+#W9~Sja+slIi3{%9&pJb&QV5w z296Id1QyFEn*bC5NI7Gr%ULT^&RN;=fR!uf?Llk6N?3Wj0HiZ$6<##4Tnq)iMSIv9 z@=#LI8kW!zYY0F!;uifmdLDhT>a^GEUk+on6tqGmQJX&#$KYFvj!7p?Cj-aLIFbX= zM{89r&oj3F$9E+@G;@D+mB-J6*p72mls-Y}D9A^&%9^Z75s&E>ypnr6zpVJ6!^OH?mQ; z(Yc+@9dO+2Q^!tSz=BcEo_k_;7BA{840}7BqjZkZc@U0UAkft2^@`=1bHLV$h*FKw z*-M8o)6puKjf4m3QfBOIf|E<2`ubl={~E@4(nyt(5^o++B7cN~qj;nr{CQ@lu9?>p z<+S&ku>cWRpoThnV$q(R&UUrhHSOiqhHF+=TTM56_T0&bPMtpOW=@56!=zp?jn~QEJR)oC5dNSGr@AQP*YB;;*fp;*2l8)1{pk8#j?pX zr65grX0kH%%}6KKYbi5Zw3ltg#)jFj4xmS{`q-|AWHRiEq)$JXcjcBNM5N#3VVxn57j!P*|(JHDRqfYV3FBT(vcSvAYuX^Y%RGoSXKD zu7nq}w6MB7Ptsrpgix)2cd!!A)#`PWK3`w8>{6sD`$}{5=KQ4t&Xwi)QgvQ6oj1ry z$V_KnX?|sO|KiGONfOFb;10m?DAW>tbQ9e|=hShBu`g(8+Dh8$j>k<9cSQS!knRpe zfOgjP))a3>xy4|3b+)6uW9xc;on1XW*o7!%c~Db8WonuZ=`D4R&Urc)=sZCuAi4CL zp!?GZsNiuX;0PHtLcGa;Bk~U<731{Bq4y)S7l#68W=MsD;?7vaqiOtt3b%n7EmGeZ z;JmE6&R+zC;u?5avm%U+pr$CvHrD$Ugvb~oNA z)hx%Ipzr)@V-=gb+GV@q(Yx*j`eR|KkfaCrdQ>y^MGU~g$+c)8wu^_*QniPU^t1Fh z*O%CieG&m{c$@+pbgWr0a`;y;6hkgx0nE!jvQn>IvfZI(qt?RSOZJmaPiF@#p1;8g z+uPaF(V3;Jm9k81eA6!_%Q;!V^x`yN=dlIC!ek>3=vRgp>>;~o4Um3C7#F3%@~~aR zdjauAme!R=?Gbzpd8x)E)i_usMM+VN_)i4>!^ml*yaAhb9d8}tZL~&Nnq6dR)|kaj z{J6DVDCSM}#(@#*32VaM%oG+=%9|H6V@+BcUrdAY-eNszZL&6F>%Z07Vr_jfQQl^6 zx5n)Jf?;j5wo7g$@d;Xnkdn1SLU!0YmUbeCoxwNLnuxK>+AT45^`yPcx?Mtc+q;qW z4&6r7e+2cP3c~z+rmZ~^`!>l(+K=4#TKmNJcKdeZu^(RvwB>d%W{TDUJ1@CqtUIl{ z(5^dBBlM9lV$AZC_eD00S)TU3$b2!&d+;SAf!JianB~3JLF71wehylXo($dt=E>Wv zLx{c4%k8jr1Yi5Dw@bO2u`+{RB4fn_!%WwQ>=d(nr}y=c^)SBflCnGqLMa;FjQ~+R zG5G&d(V#-TXi~Q@^d}30-7;65 ztJQ00lh;#TlO-p29l2eR{_R0zDd4{2#ERUtw-T-7V&W?Ja_RwmzG5hDe%;9@!J(>L zIP=)a*~+OCm1F159eZbmJhM1F>*fyCtBpCUdgNth%MD0Af&I1z1q6~AKgKIbci2O* zl?8D0n(7Mb+&#F9$U^?yv6%H^#@*1<%1Vn2IN8m+V=-|ZJ21%UnvYCCR+Sj5BAYJ&~P02alWyk`+z~ z&LYx3U3fS4^e-43%(dhq(ytlgiH{o3a#)4pAbByJa*dfg-JGs^&2rOH{f-e8W<>rF z=L%QmBR|bTa&R(~6Yh6!aKaRK#SN4LDPi73m$hKmny{^#%r0coax$iwuah#1fcU1Y0NfXyTia_%^)n5q`H)jByqI6>}|dEqy%! zDGy*1ZC2ZmembMxgCD@$6|kJi+h#wwX+2pN_`b2Sx?*9U*)uC0pbFEm>kD12b%!dP zjVsUcsFJ^nA^!ErWiAXhRyV$6MNLazdyYicoDjGA+fCJ-`+C{T3cU%4|vo zO)1=H-yRvZzJ1lxC=MkX^(y%=93;PzNoB()t`6>KxYqcj`tLIfPKS)q-X5t+Z`{6h zQ14@jTwX8{IOuVu%S`wK27Uleduyby?x_)(p{$_O$9vjjfhs3O-pyy-2=l3`bT)A7fj>kjeI59{`5Iv5_ixrTDDNahKO5v*>IFN zA3V~7J8e|bMv5wE7LYzV3iuX?zVp;hiM-Dfm6hNP?tX_bB%mOKDTM_UT!MNNyo{;4 z!NY(Iq;%OS-K$@xK^Qelb2iu!wog8+I@oT7bQXq)Q#Njh+==eaA~LF8`D4VFRX>w$ z@4D%@>G(Z;$PpA^)f5S581bndLqk2!`fw7(RZ5WOUo!A*aJt~88+ody5T^%t0t7IL zP#=i{r5QkJDIcZj%9tG(Tx!I5>%o&`-VHuP?SF$umQf;PqNkn?2x>4PHDCvXD2QL^ zj2Br_O1*#|-5CV+xE^sQx|^-L{VRws^J65NNp=lXe2SiW`qM0&(XS-crGE7ok0YH7 z(NzRcUUJ2=vHWnXwcH^qS0h~OJo9eMeiBVpjqQ6R%@^7PqxP+weceK)WRQDMOlD9>!rlLg%z2XhmKI@7|Ik3olmwm z_mmmNyM>~qu`x@vD@A(~21QSxX!Y%=cuc`zte%3AN^o8{m4JEskq7P&o9h$q&PKnr zn8$&PP-aminwuw&O;S0Lsl-tyU2q(aa|8|$UE+Fz6z!UE$y5^>0l@_gOTWp)H-l|- zHMszKmSOazk0n0SM*F^pvEN=%_CoD>z1)GR*Nq0&jv zQM$fix|@&Dbh3S=Awn1#_DA>TAsoP|#3qseD#QUtdMqv0A+>Ul-u4VC+^9bI7@GB&2^Xi>! zo8aZ4TP2}({sqIj5cnX-*7_CQf~VMA?hA8C{wpRy<4#7f zbw}jLcp+iH7l=?aC>ZDsT&jzx7R~_-Xlvr+}^Ygx*D;x!(otD zLO{y|c|U5{7ll#PP}wW0Qki3ak0LI4Y2Y3j=IcJ6l6>E0g(iluYWr}fx2wdPB>!9|~5s>OyY2M2K0U(hGFY!}0 zq9S=e$+a^z`Jx=G-cLWGGU)r+EvV3{Z>Ep*-3R&h zt#rPPj;sbkxPAvezmv{)(~%Y8d-?Xy=x|+7AE)zubpAP=C+S?H^8<7Ua%wM~PtXY` zg1iZ9@M#3FSm(oVL_MGgE@!fMqhRKZ%nc)%2NhFHP9`VKk(6Oqe^ZT7D##H8S9jrY4gnC>|q)_Z;6RlSUfv10a%9`Q${Zm?@@;snL`HuCZ}rn~{IL zI9N3E1I2-2b|Q`NR552j%X2iPIoL_{Zq#ZXPvmbMzK-H?b^>eq!g4Bm4jA$;Wj$t@ zwrQoUjFrW24!Q}@@v;V}qkvxs%k33C3z2)p64 zKJcN~n{g&^qQJDavwrr_LPiU@%NsnONqeK{tUu*{j*e-9T z4{CUaHHGjk76sS|2-o)_-acsFoAx#l*s~8Za-P5}Z})nDX9&!4$@{v?x*K0Rysvw# zd-1gsUw$9lXWb8w9OJ}>se)+6}Zk1r_gTBoc>@iij^^5cqRTlFIF#CfnV z!L%faYC&=cid3|3^?p*+j=&cM3E2=0XKui%0nzHQR4*W{PeGGmO0BMzTAlalV97jC z!AEu|f~OIvpk-S^C5r}|YOXF`l*rF$HT-A7te~O}P3cv#mAz6Qf|sYke?V(;E3Gw! zy>M6buJtM+2uSk!L@{D8>B<*Nl$*C>k+=J+@VOUge5k7X6 zEWBNe-O*^jaejlsp7Pr0i{T@h0;qVUW@$0_sS`fG+HX#{CDCJ}!}I-2~7_`Bd1pv^)5U06OO#lbE`4B|hLg(nXh0d-Ggdts>=#Is7lGYH&A$$Li?*R*j1O(V@;2l70OadH) zBcu-`zEf*yxaL|0Zh9>XH?x*?cD1q;Ipa46qLU=1@p6i+nbyFyJc!Q0>&&en8<8QB zp_O85cxeQ`>nxJ^qa;ZyW8#jByI$N03EdzfT#%^`9G9acB@pdAeHGv(=`L^P>angZ z%Lyi+Qt0!Q+IPLJWC2vKG*_wo3i4DCk&o7*$w>t|Z+8>ISeTwvpF(G-AExsobbgf1 zkI|vLU;PA~4RGB2ljJ3eKu<^!P1y+nXL8|ihdmP7(>iqAL7%2(duEu?2|D+|DG%~+ z1117b8n|2arrD3qnFla&D$RujkcRGLG#I&gA!JqxE7FTNNCu#vDxcxJe-q6N zi9ynVB+<3V`$dxItbZn-0U_xBGdGarql_tkqiE7MXTVMA2ggYQMb8;Or=I-`s5QNx znZ%3FMOlI*z)DWydnvgI{D)V-e>i|}@E<;cA3aVn`zbfKT#?D-e1_O(c2%H3%|e|e7Ut(RRJgg%$Q1ZBK$_A(%1`3q}v<2 zDjgQDwjxPS8&dG(SU3;C;aFS)_y2lQ0h!3V!S@yvnKWm3i3f&)SSvfJ(nN(Oq(aP< zoi*~WaF}QWO2Mrf54GWR1gC2`=$7t~_7ORhxL@K5O$aojNhcqpaC!A<46KLb-g)N- z$#zd`bn**mfcn>T1Qyog&FfJe9-VILGYJ1ZcHlO=jEX7^bd0bi&S0|z2uK>QL+R*s z_`TlIsoj;2^e1u)1&Jg0=;*1e^o%jdf>KQ;m9H&eD5;FTD zE6eFKVuVt_nV8&q>co1pXz>m)btX{u<(biN(s|n##*O4FzN~ zJ-N6;bE^vrIB*Lw9)(N7A|t^2f8 zpoqz|hz#mr!Ulq7pNTj-u$m{|lE6EJYA>X*s+*S*>TYk1M|%2Dq~?oJ+E1QZ08CR2 z9}rDi2RtFtFc9vPT3B;Hm2=O+eG*x`A`1YO7*IqaN6^R5n1dF;+19N3Nwfo;p@aBQ z|Avl8T7QPV8ND|BI(^w0P~1|G#9*(Y4&$>tD9mKNx^-bmExPfSOs?UXpeSM&?m`tR)?9WZ3@{UO}A65rFWL zBL#X#RPrFS+QKndf@H>Ht5bxP8U(ve_y4rne&@~lp5y=|G;hB5DGcLr z2}ekHNdFd1P(MY7Of3wL=mBsH%6YFh0Ho!jFW%F{ME4kMBPY=Bynt*82Eqh(aENzI z2u5>89;87oRY+lm2J){+_xI+&xxZM0*a=hr`Thf<*X?Tt>!^00_&>__T~a4%Yk zogn#s1jIqTJ3;X7m0g`gRfJ&8B}3(6X?btlp6UdPe*vYoh%~U&^H7DJB`LkZKgcR^ zgCAEfA%Pz8D^cjsR_nC104*vWN)%2}WJzERf?9A(p=%*49*`Eu4eU(4+Q1}Ca|I@5 zEZbh;n{YXNMD_bfTh4l=bRjF(1j@ej7%M3&7U(5VOToGQ8HV=s4Pxt|+JYB7R4rqP zrXFApLTeJx9DpnqpMAJTEk!6yP334E5?@cn%LE$#SIkZ6Iz{K)Nz7(zVKNKr@9ZDthXH=mMXXYnZ2V6>>B+Ln5u0 zf|t?k4t=4I@iS&VxTA#2C`EPF@9AQe()CcMPv9~dw|~F;%QDq?+3!9anBDawi%f-1Dd8n5looL`39uB!(l68K0r-5uIy9xh z{BV1~7rPv85AEZ4dW@XIZDU^v)Yqq2Z_X>%q#YXd@Aw_l#Uj;w5Brp>B^0hiq?~&_ z!i&;^XK)2(XD?unc|Y?aFJk&YtSosC!-wgRNu@T?A#+0gES*o%k?OD;v}lzK93>2u zRla?O4$+2UrDX3cRHgbge*OlXF*^T>&VQ!!TXaadX}NBRZ$fX#LN80Ptdg>TiQrlm zn2$3ak*+=&{(F2oNaw%P`2#wCNaquD_C=lj{2YO^#NE?}@Ipf%#x~(RB+ajo?*Jj~ zYldljJ(D*6K9e!NmdP4l#rfv17z6n)8%HSag`tovo&kt_2O;XE*cVU6BnOLpLSu|? zMZAr!FXnTpJVLZc*%t&;ZO?-Sekj+q zQaG4w!cGU@X}=?9W$ZL0fmtgjGSvZV005bH1I>$vA%R;tPtvu0;PKTKbw))Th;45T z@(-YuYKh7OwT0SzDDhilDnST1P!7=L5p_p67;a&HRVj$<#7}eOxvu$tdur1+HQLw=9 ze1F8MShN$!uk{jd0(IKpNyTX~kRg*2!sFoEt&;d`418!kka0EzKK>#0&DL#RTTF}g z11Z;^g3mObJvL?gTcxFWYoE0rq1$i}JuCG`k3lm)%k{TgchMI;VtR+sOC3J?K>7~& zNwInPg+mG=PsmROWt9<6JK|GUuElE z___yQ)UtTD^)$Zj6^(;xX#7ZKW7SF}630q{J17dqN+>1@De%R)Eb?#ylQ>%DMG8fP z!kRdV6=8dFSa3_a#n@g*NWq{$M@N#g>YY{Sh~!yHsjP_EKOH|XpCX<%m~~D`MA5*H zew~G$(lkD-yZ1Tr@s)Fkx?C! za7-zxwJR4Pc?2A=UiR_@o97=}3KoqRSn>kHG~UkE=dTCa183hrA7Dk*nhLro2TM;l zw%U(_{m^ZIp(D@RAJYf`X<34s_dGE*c+3|E!XL-#;|}}4?2V%OG5Y31=y8kDVRi>Q zh5_b&I$JLyW2*S+TI%(l%c1|r_kddd2tInH@L*ViRmsB+;2?u%EGPC2p4i7Xc=m8J zkj8_pX@NV1f96Y+*yluU58H)!&$kAz^0y#T{2`Iz7uO)CuMDq2!MifDHUM|s+92Ff zNdfyTLYWH@6QS?1I44MIoyX&F(^*5#PiJVQ50pysyTdd8IBoJNgoB`N;Kxm&kGpA= z{2`i6<+kj|$B?n?#m5uvJ?A~#Cx|f+7Iz%(50Zh0&0$lsbk5K@OXnNlOpmKS<@=w} zkw}y?xI@^u=-I(NwL8F}uP{IL=Ztuoak2zf7$nt$KkPxL&rKBiMbxK(>HdO=zC`E8 z>HIG`e@W+m!zr(y*A`%XSknk_x}dr1UDcEF32q&khWSbM%(SZ$W!Le|?7e*m%oHhi zFY^4h0czd*mji{aWgQIF{~(VK_*l_?j*NFFMfz$1C6FTJF#zGdP)G)bpmecpa$+b_ z#eg411vM3FhSE4)tnS^ug{rS0xk%&uf@BXBR}ce!5(TNR;w98^?kpyhHuo*?Rchf# zD-)|pg_+?AB0DH>8I`{6{WQ}54BMN-OFom+V!+^Bq^^#SG~TH7Uqc=~-&Kn3+xlNb z%uloRe~*{&2uNIUVK4N`+&c%nx9>m0>csUu13t&n>2&l(*}`ZcVDJ{zJb)ehmS?&qDqq3kmEU z=t4X!m9IS@t5 zZ=B^9kUYY%L{{PUmWv0MyCo=`X(my{5b=s2m6ZB2!o$5TCjU1E9>jDS4erOAJ1lBR z8pycU{sPL8gXXki7oN!1-Pje^9b;eAo6SpAQY96q-h_Q@@eED#$p&6lj<67S*j^?- zpfZ1Pb(!?hd>VD}|G229@NhVs_rh6AIaG{;f*q`~B()&P6B9!iuS#ns9&l}8>15Iu zM8UpG7&i@pe+E8`v3D)w?3Gxs>c0Vb!tcJdv~yoaO8AGV!P~-e%<&L5gl1M6>IOaj z5DpBlO7HT#ibCxYN{-2=_{Q+^09H*;!KJJ7cXxGYMbr9p8bf z0M7~hW(lxe0J^DBy)tKm*rT;(8nxB(GZs0-&dDQvAu|e8fs;nb_EZ$h^-tWtb65e+ zbt>hcbH@Fy(7#j&VQ2+s+jZ9f{?2uEi;sK2V`SG*zf4{LD*#0tVebITU=tSwPabZU zt~!J2XabmIvBD)EJ$$ZP~+*oN%PE>Cuba>T+Ne z8KqOvPbN|!21q7}Bv5IQSa}uX9?IEC@ZeBp03Jn7q;xLTO@a1HEK-~v2w^AlaUO*8 zA#7_gk3=U>q=!kM7j(RALyQk4fNqF@(6x(rKA2d*Dv50}&n}B;J9K%RDLW1(x3ZMA zhau0H*wb$ucuHBISUo-b+%m%!v&hg$_0}W&Je#e82Pplmy+bQya5OaH-yqa0ihg0 zh&q~16oAmX2N6eukU&oqgdRb&1cW9N?fcG0cpz+8*Hn#zrdLu*07(``DJyOyoOPFh*FHCMfG6 zoyU0Bpn{F_BKUd*XE6XG2~E{P;q!$6%lX| z#d(NjA_*CE7h=&}ADI`oT48m|6~-NOu@!dLu{x2;ETj5rE8=%g=bo6t?d}gWGM$Cg zy_|%}R;)Q|G|}Llo8v^MmMPYm)a=&=d>mE!+MpaJS%95i93(N+-?G2CFZ2nuPthX^ zAW+%DB~L9>+*8@2rJ>N8ga&&?5q!|U#*ZRv;p$^eyo{9+hirt)-%)F}NBj;ERpE{o zjhI@HKrG=v4tIi(DP%;N01|9F`*9>NOl3b0lcdg!o&;H~(-SmMBeM^)c>-hvJipN~ z6W&e1gzpu$Igf}1Sm~feIi?RR+R{92Uz#444#`>X|Cu~K%DQq)2Y$CYD?ehWj-P2j0_IVP#q-Qll z?*Rn%U7p>6LzEFezoL$O8(T$++<5;)qI-)8>eHW?9ZR&I6pLRJGXWmz^;o1tvgm^> zNmDyT($2x5G~IU{W%YDmKtdLg4}nF9N36~M3g(GJ$To{Dp8+qSeDync3H82%l09Bf zPr-qla3|7fhAH8ZqlaIbe3m(f_du^M+N>#G3RSZXw@0C17nS87ZVzfrI|o~p!(U5I zqfh^ojremqG6!XailHb)f$XE~Q%*kE1OlH47hSyGy(IY-Tc&eRwd0>&1pkNy5J;x) zi`6~a%kUvOBXqcIY5lv+d?V-+q?PC+Yk=omD!N&8T0b^BFoM4q}z? zUT1)?M~M@Z$yqFBItTbhtCwn!&M=)BMv>)uoS(P_)msx~ogsmxEwOOnmNijr#c!6%_o>5#Od-T`^-N3|0gqO{2eYK`CDTs|Eh62)brBblZA?27W7}5zgkt#Fkx%O^c41dmA_Q}wTX?R zL!dtp7Zo$R8*CQ9RCf z!SNNp!i%_|h6BDLRFMoS?BO?sU#j!LR0|b_B#f8J4_w4&kE`hTBannoAmX1Fd+OqmSDy$Y5|crQB5W zviwldAcB7VNWN0e0z}17=~<>iituPO_cJ!lzS3nFR0!_T(rU?a^o|)g-l06`*67r0yzK?^oc z$Z9^Rh&RMwg*)aeP-|se@_~AZ@i1LVkpx&4s0b_j}%P|ET;E_ViK5;`a7VbSr;YuO(F6Q@cIQ^%V@u;=`v|&wm>#f0WKT%gh{WH3?@9T_B|k{N5#O+< zWKXh&;y)_>o8S)(P1IvdzBl7LEN5K&w!m+z#91%j+wiS#Zk>?t?ZNj3d{dV)Q7+j# zaQiYX0r}dLI|CnIi*i@sv)$5Kl)J?Th(+#TrWWOGUM`|T2@6EG*>@mRYfnyDdvMjy zG&J|7ML&428(b|CfzRqSP!#^e*UDmn7N6E8o1vPLs11un9oWIay)V!P#=RbXx@UZh z5wS!QuH0{^#uv=ygQce~;<{p*%aNlD!V{*v3EG5uzEhRC)AYKJD?F_}k53{e-e*eW z2b5}7Y5$S1QA8}^4YXbK#6MIwuoFq?yu_!Z(N?5FzQW?l#VZaEhiaQR0e`%NKj12u z%XPxq2AB?c<8U+X<_QQQeO&q3)fx~iH0J!v;m~yy=5Z_-B34f$WoQGYzAO*X1$Ww( zHXX%L@U*{i2ok0XHNE&o`Yqo=JFjav9lV~I@G4? zM%=phbPtG1T<6&ufC7*Dqdik{otii59Trzv87d31i+o`>yQs4>!ouH-Zf49n(mBH3 zSOIQ5#n*02#`_}OzRJq`Le%HQ%n?@Eutsw9^kC@}?q5(aBh{tAA2E09P17LzZpD>@ zF#ZFx5mfv{Mg;$`m>7){FS?E$Se?<96-oyn0K#oQFyV~tpTeb&9eS}Hylz>&L-o$+ zQ^-^o(BV0~z7Hs5dD~Yp-$+zUl4~lqd2yy*ImAf8DI)f3*JU(KiY-QQr)4(>@Fvuqx|45ldzLEV z{x{G`>Mq3SQakV$y%?fS@6Ub-iQSQoj+fcsuIm96t$j&S4wpWHhsQ%X{;iK1h4$>H zx$iiR$9V*fY~-o$SK@wJcs@JvhP$c7fDCy;klkJd^Y&WiI$}d*34|?o;ocj+Nb3F^ zG6SoJR;4v%DP3)X!O?O}qJ1|bHs~E~x+rcly@o8lgdP(ssNCn2K*-VPT^978rwwkXOCt zQ+kFprhTumrE$H?9sX-Ukq>~n38#9{K20YDGmMZGRh#Qu1&7-YLRFKzP>MIW2?7I? z@F24+u2$}PkZ-I4-rDvjTq%w6e&0X2>0yJ^hThNFAo2T-WSo5YcNTFcgvV+k&qXiU)l zx@*8P(DyTThzW&p!yHD~P3-wmV-#CNZB82ci96zl^k!D2sc9(PF^d-h84y;|$@9Qm6uJu zY~{n<{?n|K?B|6&D11X1tuHY^7=^O33E!~qmj5%1%nIw{x%YA4@2B(Q^xH&77zYo~ zj~H8X8b8OkU!x-&#J4f@5FIiCw8iD4d<)@2-ULYQLBNk-j*6u-aujkUpSiCuV%A)R z(HkQsMD*Y>$mPu0oMGfbULCC|7qbTL;DZsqykeg7s7~Y$W05$D$DzVhA9e#cY{G8H zU^AZ$*$q}&Hrp_ojP39M4qWEpGl+w716CeB1#1u-GhAcu!}J-slDSIi-~wnbq4+%0 z;9jwg zbJsq42Ua?DA|zD(gCIJZ|K%Sx>P!}iJ4-?_H&}(({!&_myU5 zV3Pno(|VKY4(UdDEp`i?HPm)x?#KA`VEI5!unnQ4P~uI(`)BV1D;#HJsHP%?9&Qgu zEy0VBz;E$k#oJ*ZoPQ8M>NuU>MVWC6n@9-m_K|>=5SWw@1omNc@t|V#X?!;g)5Nyw zSqU&dH0nEq7YNmHYbmcv#BF7`bPsKX*qsIsnzV`LtpHT^CH7xRU*cL}G`Fs${Pj4a zHRF!4?%%|^N1Ng!_Y(+r(^#^q_3|*9&vkhYI!^THn9H2TZl(hNO637Imgt^F2MUnD z8n}23hDHM3QP5z9)5YBugEFHshA!BMKa!tfDFHC3hgpWKg?%C65kxskNFXqY{0O%Y(I&rQ zNYB`_^N!+iPQ&rX{)vR0Dw_jfNg zpee5$5r}%!eiwQ()M@stRHPs5JjqT4_g7?}Ioz6h(?Z_>mN>4*@i-K8zBv@?FX4OgPgm|jG`;DrvQPyxb$|4l;dR$WTGNXabhmf_P87+Y! zVdfS!hvIZ|QSj)k0WlkQnw1vh3Up=wy%r3hGgC3FKNvrVvO_S|NURJnrUe7o0@1)b z0KUHl-aLSJ5WrgqfOp6P-YqBn!{srs+dllpln!r&_YaYSppl)B$9vR=Kkr9Xdsg-U zFh&t->1z_Pd(&DefcDgEd*)^}n+giWCi7x-#r76(LreGn0=f2GyO(?jH@I`T(Z20w z1-)4S-+~hR1Go>=rUq)>JEVbHD%ua8m;H)rcy8}{gNkZUE_Q?R9qbIkvwED4KykPj z-xAhk#jC6pu`cO)T&&F1*HjGY53Y4Y?*><5-6c(r2LKDdJSp@5a0#?DYon?3f|aud zASxSzDHIyU7_&)57rs(wK^RQaEft9X;{GIS;js3X*c* z07nkwz=1(NAvMOYgXcra1G6$Rk(n_I7{eo{4&(s+02mpxR0_RV7SVBxUT`*O>;~ey z$xCkXG9>Be2$O=$1&{v?!oCR*rg-&f`<|FV#Eb+FhCGc*r9*uxJV);l!UEXefLw6C zyutzulMRwHxK=OVS3Svga4VfoM z>9In+_r$Uuak$gXR%lnVQu!cD{dShRIldbtRy=X(_k)VEH)G&^VHp zm8pK1@x!dbZ?g)U;;TSZ7*A^jHQ-2uwARbmM{(8J5|iJQH^uZCqso0cNJ#IOf+2~^ zDOgQrr`N@zY2sC(A>6cFKkjBU>e3VkCz^Va#lM5jyXiblM-$9^`vjxNbP64rSZ5Ks zipojP(}vY(dcr7Xa&kqLV(2`QM*i4UNAWn%!0AQbpfoa?(tsT@O*Ut!$APUAkhq7k z2fZ&MZZcK!)}SEpf*^3-L`4M7&L4IQXJK{5R^E>4ZKOS#vXj?Ff}j1U42$AcL%+1aE{%+(h|sCP0`*>+hh%G05hqTUr#9)aNH0O-2Q`+q?I zY+)P%5Ky}oG;O@AX>VrMlTq0*>tUq@+-_Wv7@BqTZe^F*FBR#B0zN@lCR&DuCQve& z678|BDiI4zkHk*2A*2+dMbE4`wplo>B;=HkM}6m5IJ4wUhW#02@KIKYdYzbFCit8n znYXJ9X61zlBgXQ&sLP#r2>XUyo^=$T89I%in zFr5?$gTK5R&Z5?W4?9cxX$5)xDEo=++XDSK%2QA&k2M&3u+Y!n<{(A$)92|r(tpy= zyO>rN5kJx@5-Y)8`-HR#CtXPsiD`ATYdyGXsU-t%!?<4{iQ`G8*w2Q&`z4sq!!;@L z{p`@QITAR$#2XD?KtVX6%B*P|sHJ5sAI7JawNNt_2ToXjumb=E==AcYP!q~;JdUKU ztU%#IEh1)I_qp4ec51*)>Z^Ce;v(ofbSewwe*^k}(B0R(EqP{ozSrXVe4?4&()IhJGHST%=)WfX%od?#rZL3 z9c6kj0fsnY@XI9e8xX+W^w8V)2Em`RS2gLxv8vWk+;0=u3Bq|E^yFFoO^B)o|w@Fvl%_j+NlktXIC_Tw9P=sGTRT1$Dc5tG^+ma`VGC0@K zEIn4d?$=ErA0^|rjL&kuZ7bZ$SFc^NeF2073MyxeZhW56r!)V6LrZSxIIdb9P3B>m z7fK@rtn3a#5HF7^jVk>z%&OYP|eFweY1ikxpM}0^=qU*L&;iX zc5#6WcGo1F`~%sjj~wf|-g%-QBq@gj!v1B;ejZ;9q|i%9IDLW@`0vp@`V8^yk&awp zV>&j~!P$Nu$qgrDFuwkn%5ZXaC>4yU6w4Cbv}F5N9MW-o=WKMplHy$dI$Mg4sqoL(mb-mN?LjaKv2uBF*Js__4lk z7F`QorUE?+-Yc}mlYzT&I2mg`F52S06qmK>)!(NBo-MZ0*PwMljfUTk|~H z{FLlrW-2&mhLZ3D^{|%B>%Z)}fBob$xQisGWPZscl~ECTL=Hd6#oQ|#DUKviuc&>r zg7NzlM=e?Fo|}_ik=;Y$80dPr+ba(aHt)?djXW5GJVXEs8hFFS3M(p-M1eaY+lNqPQ4!Fs{y~2J5S{nZkxmezp6iCzZxC>OmRUJ;YIH8s2`Nl@ zlSxxYz^4hoT;{lY&gaIlgPw$ma5-|6F<*m?!W%q1l*5rCV~qC)idFDY_K1NirWNz( zp}+{VNAWlp5ZXw-;jq)&6^ck4lW&~3u(w;Iumu#}Fuf!2S#M2<0CuPNkny|0niQX1 zq#ZkZ36vl$GHPzR5&)fV<+_-d7M+^95+5rtNjw$3tB>}rx3{}k0i&wNwZ{0 zF=SGkc_R!4mat#2aBI}8ESy1`AzLakwLhjL*d<-DRGGX@n0)8&M` zIRJvYPT%tgVyxmkS6#ssZ|-Q97nmDvG4zvLFpwV7C|{*d*Gf^C>yG*cZqV>mSdYSy zs##vSYG1B3fmJoC@@PfTi4r|1cfEf`)ki80=~H(?C`B!6v^7?uWqH0r<6v{u z`AfZVMSOi#&K}BM-`722Jim53O z>9sGzLyXA3qD9o@;ZUVdw%Z*uTGpd6=osoKp$)tTFcHPd<*1n#!$0jiVxUG}WH~gL-mmHXYJyrX+q9)bQ>YYz3<_Djn{U?&*AuuGmHU+;%Xp~(>Mqqjc)ZTj zBHWMp^VQSc;N2|#lCH&_A(9}~gax>uC-w6juPb!gbiR=e=cQVs^8%eOa9rQZxA)O` zKb+~JdXDds+GY5yWZ|agS#h0{r???=rE4s{1x1|WSd>r#HSM%2r?Aw>R>QwMiEh$q zueLYFfs+(8eOv5mEsL@+t*N}H6$fSA{y{b!g6oo9CEU7BX80IZunbD;mR}o z8^JSajN;dz77BG3%+XAxK}ro}0kIvGVVr>l7>{XzAMPk>$tiAw-AwC>u#X1llcyu?FYryqf^$Z* zImsi^x3a9`n{>X66e)Cla+F zGju3}5l*q*X$eeUqMrb}V1sb_2C(Yp4xw$Ub^8dSCY)b~VLOEWP85gzxy#VuNmXWeY5Xx$SvC$-kp;py}ZfDzdj-*=AYDY@~{10 DteW<+ literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/tracing.cpython-310.pyc b/vllm/__pycache__/tracing.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0914d51351cfb170c5dde0f3f9e8267b06f5f08d GIT binary patch literal 4446 zcma(UO>Y~=b#``1t|(F8mQ72tSF&T9{-EQeL4d+F;z)F2L?#`H%EzF^V#OIrTki)w zJCrSg2HMC#ivm6M7zD77KJ+KF$Nq@D_S91kx%E)@y;)LUmhF<**_rp=ym|A!H!hWm z1b(;v`uFX>jT7=uTpa${u=os~x2_-1cc$621ZC34NYbmxHbbT%(0xFTR}c7 zuma3;JikSmEplRL&tOGfV8h&CBf=D=G7+QPzD>Fx!S118EF5R!dUr9H2q)R3o(~68 z;WV4p^O4|GIKyW2ycC=c&#EWRXc=IuD3>76& z+N5Pt#{H%k2TF@y;1_}NGEnT}0lPm3iVyfDptu5ZT;^9mjw_G=NVk2!a9*SPf;B-nPzA7V zdt080WwV7UI{X9o*Mac!2uuWF1}BVaby8UXQ2iAg?z)i|3fE1Gt{cX@6X1Nrbsu-U zAj7;O@)S_(;XfbWRAKvO+i%}cp4<}Z#;0LtJG>P>xf%EyH`}{vJC1Jd1VMNcwF!2r zws-Ym4zT#yg25)1p&@UTS3*=1Ff}MJ5l7WkmlJ?HkSCkXAnXhb7jSZVuzKGC67}*9 zj)=ar&k_@7Fn+Lxf zOi(Z_;XRlrvPY7lGPt4WkK`x1XZ&C&bB{cu%IX<=#xt`=U*uH2XYLs<4TGpc4`>Vm zP`gK78qchr1#wR6-Cma0h2A4kOwM!oL^epfi2gZ?ID z)u}~EM+)aCmo`MTwG68IZ91sM8SDl9Rw|vMUj?MtTELMgxTdNV3nMlNBB8}k80R{Y_rl) zrul5kA!LjJ-C48c+85@0g}(*NhV=HYrz9nqh`^tUKdS` zPYc_^gDR37F(7Ibjktun(T_0bWT{8md7nHZ%Ge_>zz{FN3>*@MJPRZs6Gpong8@|a z6tApa8S)AO_tU!6{&lL4&OFuytz!lJ8YmM~Yo6N1h+$EA4HhQ~hQjG<{ef(PH*xyh zR%Ds%p<;U=o?aEqE87^vG|9(GCAdc8H|-?ipqB7y6jOUG_zNMN?Y+Fg3R2 zRoIomwe-c?+;6E?9JcTL8kyn==Sa75^iqAjR&NJ!%L|e_)q`E&TP0oB-7jl*mustQ zhfdIEous5bKQ=gVl1dEC7Cc9OXuHqgNj`+(pcXTeQ)Y3K<+#Q2JjV(=&unNphqw)` z=Mc1>MNr8*QB*Au%VFMi*KQT zh69?g%~(G1B!3T;cq&@ZG1V(05+g^V=g4Uo*yzg219!Fd^?Gfs>E2&vrNiqwJB`cr zwYy)`-1_a(7mxTB3AA;VfDS+y#Ju)-mklWT(7b7Z=jkD_ul(wDX6^_AKxTQ~{*ps@^{<-Nx0w|7^U-$hdUuC}z^tQ|zbf!}M- za6t_T>H?Z8(=rbIscmYCBf*Exh?PLcD|)peha%R%h<#hk;+Bu$(Y5FI7_)^z%zgu( zDUa!X=E+374z!2jNQcnSZEy$3y7JCrE0Lz*joN~T4~ZlMRCm`_`&wGqcbR*x6(EC&ay zrn+WmkJR=&(JSoR+2`3EjNYUPLy?RdBXk7X2b)@S9NNGU`~zgfn1;tjTn-uxI%Ja=}2TI0t50P^2j00000 literal 0 HcmV?d00001 diff --git a/vllm/__pycache__/utils.cpython-310.pyc b/vllm/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02651decfdd0e2396072332bd786a79bee63eda1 GIT binary patch literal 41025 zcmchA34j#Wd1hDj-E%SwBajekp#w&S0U-&=SjUV8kVYDY5ez!Y#!b_&hH26oZc_#7E*Iv9mcH=T0_poMU%ulieitZlX=r*^L~>mMz%t`(Jf+ z_sk$W-eaI&RlR!m```cm>-|;j>dHj$ck`!yW#(1C5Q%(^53PS|@i2^^cYQ}BawwuA z$|^-1bF~~xo^8jLYt)I#HRifDOwWa+l@PG_g2O_gph-{IU*ezo)J@-AmrkpDI1JDoe_K3%%2e7AFV z`5xyUdCruE%OlQ+6>;xz?>%EVqpHK%tva1Os>|7{x}Ez}k8{83b;i^x=Ket}TxW276|TkmB5I4;`cy=1J!3n^-Q#B>&I!Z~scnec=Ejbf&P1IuVz#TB z5py$ARq(V!-GZlE+^jpZE`k(Q#NDcHL)>jhF{^gEC#P+7ySn44n4{e_%2Ka>#tLG0 zsnN~PpNgZF$K}~p zd(>XE<+QqwDU^j2_p33))baiywDke?AfDb3mStkc)jp|Xp(UO@n^5~Deo;-j8{BDk zX4+mCK}o#7nq$Ie)bWryfKuP64!Y~z8Pv*DQ|^Xi5u`e4QXyCPZaSe3sfUr{O&GC9 zK4hs!o9~eC`u3bs<9c_KJ0H|9<)e(pl=D==d7`b>o$f|yS4_QDy$&@z8T9+GdOe=r zta9!{h&hQiA5nS4oKZ&+Q;_j|2xTheBKD~u*R(o@r?=o~9-}d%igy8=SPNz`q2|;n#Quny2TcE{ zdfYv%PUFY@_?^M;Dg54o-w#qAA?dBCNOD3Dox!r~~k+KFJoj8`UQfZc?8@I5Qnv7jZre zy!Zd($bfpJz(vocg&IXlP=RpI2YN+wWmi5;wTdW1gOu zH!1ad>i03nKZTb40rw!m7XL0x%bns6)gQ@DaRIF$Hv5YDW3=L@@kC7jRrM!$`h@z^ zmKFiet3N}%G4j7nj`=g{ zFHy>WcCF(-i&Xzb{S{JuCjI{OZ>n#rzm~e5livRg`ujDkk*~`-If*s*U)6uZ8vkti z{pvgFZ_yJ%o{SXW>F<_2x0Yl|2veh^7u0`8NuQIFT2uc0l2VrDOsRiR{{uNcAB@pA z)IXy2KNmdxPxZg>^z-QPKdBed;}?NdzM!mwk-?WY%`stFM%N6p48P`wF=5?)@FZuC#ouBmvEk8b2$(P;1Rep4& za@y}6@lIC?_qi2U=WA8%r$^@V#ajMo$@NpC`BF*lV!MllnxB+Z#e$!@Kkv<$7vn__ zFUE^CWH0%t{j;@VwSv2j{aU%&Rgy7!(5?CLskvDc8k;&ji&imwIIsQqp$anisp=_L zm#TRs?VBsATXBuH`B&vSL)ObSrF9uBP(^ z4n?KFreG9GXic$FtmR78W5ffzW-pUsEUf@8e zS+`WIxLE)*<`ef``4xsAvxsh9uC#mi&6R3J%wfLtLzgj5P@Dtw6i>NX0Siv@j|xY=q2&=kx;%gFzL!*&0pu9jzO*}TfnVqJRKJSO)tkow9Ed(G(N zp&zzagB||bgHs1`6Z@xfW586oJrkq3-FrrJ2Ok;nuU+xYn@Y#z=(gvimxUoFY{vGKL6?! za~zyR$Iv-{{fe*e9T|Oa&%|#3x)t9}9snkv%8l$EnVi~l;4-6znQ#Uk9v|PA8=0Eg zGch%`etXIy#NbqIUC6KQ1?Zbb9sq!?z}V>p|a|@~uM!?w_rH79NK2^Ewf@5lWpPapGQU`{~?4ZU7axG=c3WmCJF@ z%;o${E(gpphkYjJcjj^@=kg`<=0ZfXED2t^d0VYKyKS~OyS0|r$K2Z1+skt^<(td% z+e*cw+h$MKW~!BKr%I*rwz*od87gFoH8Qg=wkGYk(vLDOO)v8YPK0sC!;MHuuxf@~f zdl7sDKQDnG6-im0mc%V$LF(S7<`TeY}>-eG>0wpc7WN z6>DpTNzm2|mfYyS$NsBk|LqAGKdJbFB^58NHLhLHt$hah^gaaju123F9+_DF1zX>T z%L*VAEmmrJoVh0%9Aq%u9*ppD4}$0M^HK;>k$$^7+8vkJWgx^9!%PanWfO!f0T)|E z0VFXMbK@%RCIC(el|)PuPXwowO5-UNyienOMs*-2BQc$B8sOIj?yXz(AhyG#LrhHd zs#S>TY>DYps}a+c?oyj2w%_e`)8G%{=896*r~!a-&yupQP-~H|x22?Y>Pp0{YKgf@ ztw&5>kWr*4I>fDcg;rdzZa^yr)QxJ>QxWG1)dh%q zm7f|vaA;IW{}hP3e4$u7ec5H>U5VkYMa}?%b5_lkYgCpIB<%+ydJL}z70E+D}QtpG5Y zR{S>J3-;}8Z$X%#ZJ63HaCcZYho-&e>a0Gza?Rl2OwGU7sJXkn<}kr!Yktx6x;ZUj z-8#=oSE@N)tIic>{vBV>AgFhTqb)D|F6B<=!Jp)0-r5G7gUvyg0pEffE>x}r-4{Yh zv<*t42LUOI(FI%uT%}oqu_k!v3W|}rT{<^WO#N5zQfQ-0Okg7#y9D^?3|ovOZ4G4O z!Uk?+0|jUotUCpaL*R{^v(HCbXDVtI&{1UZ*W|P-z+5c>zf6O@ca^JFf*(DN^#2LJ z5PCaXTL-9HHgh4o-o)0K;j?sC4HDP|>x0nwJr_M=y_xa$Ip9HoCk)?8?@=g>l29R5CPy*^>pId6?^ZkJeSKU(0ce=9O9z;-V<*ufkD^_gYwC%{j!& zWuw%XLtR3)0MD69K!WSV%gG-;11<=Gqy?(@OIV z+8gp5(}I6sHevw)F_p*;3MX9)qy%@mYQMw^%Ow zWk0b#QXdS5p}BxuuQqgNsR|jGclS`3ufA&pk|f!?MKpj}!)Cq(ASS?PvS+dWklg?> zWfBGe2NMSVVkAiY)g$BMxyiAK32@{G$&^m)8TUI6fPgQS4RHD~)T4C~tsFG_lHV1y z&s6IqDXZkxKa4bH6P128l9_h=BVG!voxmOyvu(T2vaQ&~R5WGTm%3xX-2*H4zimbc zahw%Gowpr9V?XDLKM~Ud*u$d4P3`d>>cBW;IJAb9g&F4ih!G*~9dwjQ_QOtjm|*cnWd zkP|JaraHT9G8(AH+licwL01;0qMuWuK=wa_)z$jOOX7`Xris&nRVS><)S#u`#2&QH z&J&0LuV>^D2D=!z43054&Y;X-6@w}RZe-eH5YB@EDT0rgB{tNokm2xdgm;f1_&NMM za`&l7x7`=-j`l?d;(ZA1cb@?;V6WGap;*n4nf5is)P#UkL$GMJnWziaaJ{-ej;LT!}%L;dxK zJnVtBs=+5&BMZq^B?W|^)olJ$wWxxKEQGzlUKGT8vPxCy! zz+fEQ1R9$=I#&dK)P4fe1LZ!>y$P{`BmKC}SB|-UB0oC|ay&qLPDg7tE7vlaGLK}E zt|TOO;wE9?pWSKoS+Ps?fwln*`K7kC#B~(*bPFCDD`pWp*#h>eu!mSV#0fzUVJOwa z34+wlBC~!QgSRtyD}qK3QV<%ko1Xp=z8Ve#cEIn*9n;l$GXxvi5iY4rqZ8|vbfTHI ztsjjoQ)cV|6hJIj6N2oFbt2wG&~5P%Z{UH5A=?-aZV5S-3uEe!9}qDT$v04ITLa(% zILf>=fNm%|Hch$?Li0eN?^ddF$7Vt@4MPBM_O>ulD+TBk>n`N!*jrU~4yvdoVW%h1 z6(FXkaWQ*`4D*jNHV_L%QR>nm&49w@fTOm}4bbR~rL2Cd(}sY^uCHw$)n@XxK}FW) zu9v|>ASeP8IT)?)+m}D#Vo1FP>b zA|*w)u{WISOtikby$>RNnixzi!LmL$X;m%cat70K1saPtN zN_2KWt@iLA^$xF(bU1nd}J_TFx3Uy@1KMMFMrIPsMhvE2&xRD@GcbWM+>ua zevHEBy%(Z>U$IP;CB)K0Ww!>65#+5uj%1o?_4^nMhrtN$eD(}Qz*sQyB8REhwjr(T z1je#2>E{q}*^cQS!`nt9rEDiv1&TaXtW@;}@NTcB1boF>J5XyQ?fs}){{&Nh7(rWg zcoUAJY*_U)Mlw>0EMn9aK*Dh{W56ho%{-XUzq>_>8$9B;={R_5t4FRE^k}gLVqbji&L+Y#0;n?YWt;k{=fYeiq zpTkbQ5I>$2Nllz-077{C#&`ux5wG*sJjg|FT+*h5!9mHjR4olg$gUo$rH^OMb)4t+ zm<~kEgcU#f`0SjS%BlJ~Ggw&}EkPk;lY1WBmA%2M_mRn+CSYcv*?l|<@Eh#ZA3_OD z;B&5mgxM|SfO25ofW?->Y~(;Ma0Tk~tSw>Ncbe=_?+xCv(;%YMN+2eRZmSj`2S zWU;sU0azlF6|F@VK)7K1WA!^5+Zs_cuuHK#SI)vf1LhamG8lG!Iy(zesWzkCyviO0 zMRkHgh@+=#t~X@F{1&$F!kkyDVuPRc>Kjl>&XkZt3D_7aB8NEYX6vr5VkhHr)r3`N zJ9nM-^r$h18go)rPKg#O$0;cuoddl_f*CZ$Pvhc8QJvG>o_*<(x@zSbr2Qv33tT*% z(Ufgp;_j$FjmH9hZGRzZXW|+-r}LopFnm^hN9*v_sx_DeQ0*OzTvccR^j?7+Xbc)u zf{9s=x;v-)R8iNU1>`uP-`pWkLQ5BhQJ>ipy4eq)&LHU`o6crsZ2cKrsHAdL8*Cs& z6E=3^fej}-fF-hp{WJHX|I=$~@#BeeV2|W|>P!T49zULL#84|}G=ud|3N3Tct%>}K z)n?I87X*41Xiq_OWt5yl8A&)^twMiBiv-f{)yl5yOszKS?b@~th9om{M~4d4^0x8f zv6&iI;ntBcsyMk!x0V`_+W-qkx9u3Zb!huGPZyvE4N`BL&L7pqf+#|ZwW8~7V{@Pp zy>5KRcBy@0FargkG)sS$<78K6b(S|>Sc!!>mCxl%#S^ZqcR$7mr>C5+a8g(n-j-zT z2`&KIW+5#TG{T~KBqdFWpqxb~-ptAA#+yu{(@KGgP1wC4#`PBvwG6hx<;x5;>@E~= z9>+3m(AF`iO={YY&B3xsuVa1D8Tav)HVh-q07-}GX^u(XjYjAbeD#Y6mQ@>qbRXW5 zTA2mrrxxdfk>?27cKz>V01Y_{dl-OfwNi$VGJ7hoi?o)?ibcWL#KEbNiP1f|vE3wj zA#>jXqabQr!Cx8LZa)d)`J(5BZ}JULIdo`jw@EZsfx5O*z(P^T4-(r$T_7z{Oyfz)b zVwM=~+uBeC8%RQeu;^Wn0H6SnP&Ps^>b2XdHLyp>w*qL+@q<`6072x@*ezbLq(lP? z0tVeVDLO|JPHe0{LvY|^r>p^x8TE~2%%Mrr6!*4gfn4tn))rQ&e~a}d_8OB>+tm2G zOft;icNj?BnP%b7vtEu`HWiEO-^Y!gG6EPFj*`X#oltOLpQqZyFw#;a9tDhgEci(P zC4>;zg3nmTK-;9bn_-Ma+-HwPRP6r9>!7%zSo2K8JnMUFv2*cS;=J`NxH_bbuZx^Z zo(Bbtl2Ye`R2Im56^mhN&qS?AC3Q3GQRcB*8akP~=DD;&84wmkb)nvG$}u%8#+$8T zJhWDEO6Lu|UCW$4+9-#nD)W((>A__EsnKc$n+A;S2)}_i3d;kuYZa@8keP=*r&@x& zC~+6i-KioJTH$Ijz+cNkL%;EI-Q0W?ax~v>z>$ip1T)AdA#1SR+Qh z0N#>v_v&2DkmN7*w&!rwV237tfOS(YZ!Ag-7{qAv>u0D12e}Pu+)vTg5D47!yY9uV zzvuCSE2h1E!c-6GR-pu{!0#Sy%(>9{Fzu>UXGb^~V-3+92<|^Xt(sa-aB zh~_pgCXhd|fSAQ(eeFWRq+3iaB(Vy`(DqKgefUJ(wHwr*^5jrt5;FeOUE zm_2Cv7_dpn1NsPz=|OoL1VXXp67&H}^_5YF;FTW?eJrCRxL;AOCQ3_ef|0`=kRxLm zD9Ba~+Fr2UW1qxUh`nk8DIvuqV$okQ_os33W3%~UD^)YX$jqGzT5#|Svv>bBCR&8V ziHL3YUP@f54?@^-EZE3sTO=nEyylkH%sUa(>U0=7HHRs!=s#o={)kQJ_0u$QWC^aI zIQ_@cB@YVZ-%*V465*!qZ>nt3P4zrwsm#!Ji`->=G=Z|1%^0BLk_` z=N*j^d`s!S;=^As*v}XnxYe3oQE0wq#U6i^BQuVy_7jPogq^t98-ubF_zb!ntM_7m z6!Ii_;xv`9xq3b{*nw^kJYMH;rGKI~4 z4fYF`=oP^J%a-tURBc9Tn0FEgoUAXH@^dt#2uPW*$cr4GPGqp3h(#%SLEmlt0wN0d zg@2sq6n>-lc@+7OAYKGGYw*aBX4&UJ?H!Ne2leP8tRm3UxH1M-*15!a*n_OY2#CNi zdHVVWeyv#Zk$QiSJJrZ7d7~U9+%3%u*&F{o7GYk>&Zsj}iDzsGAd?u+6zIu%@{S?- z_rfHaDqk&)3-3Kse&*o*(Fb$8?>jKE&xg4# z>=1Cyr~}<$LK;^4=_cus3UclAQ-Qz_WCZU(j3;NRUJVjE-}07H5eYJN#n~kH^(|wi=;?B)s)VF3dK6X zzHFcFYs|d@y2K6_otWAS{x4me4N8SndGgfG5mSXeiIT8LUczN*N7jJC#>5q27d3lo zXD-OTnf;Q*fc_*%@=y@}GV%lQ%mNETyx&HSbJj@2%d*&v8>)j~Inb#<-oOY7dIks( zK;kOsLh&r7QG6bZPrPL9swZoOS%`jKTelDIswd2YG!>h`MHHtmF&GX5SV73Fu~ZbZKMRi4Oc^XlcdVz&KPq=-uf^{rYyS^K2g(abO(`{j}iN1z;Q-()b1i zM*IUV=Ehb^2^`4)U@JkHKL3%zyd8>+TJjt$p!u{Gap%!IgKYJ9-+aJ^$4=`35 z*e*(Jw?f^jy>6e>LvAo%@+NM)Q;V4+{IDKgI;HVg*FtKQY5M5ujiJ>2EV)I1B_#xh>#!tZM0o)iN z1qk{L9OUCVZ~rC zynnT*9H-Oo2y3bD~NnpfvtCSR~*J_<2JJo&?xH7zl)U219Ws ziec*@CkyLb48j$Ww}8qKE5tN+C|Cq+d5nw>&Uh9$b3EbLtjgR5LxE-({Sn9)V>6ON zedu;Sr;*BcA%K^5J!m-z!fAGtG#V&ZyPpE~0d~;J`_|EAn-+U1Ai&)Y?YBB2R|4Il z%{Wgm>K1WA^jKCEPZg;x9FkhmE+~2DVC1&(Ork0TpyN4xM4YU_Y)Nlh*=#383=Xp{$u6-hPdDKzj1 zFlT`_Tsdhd&m6!;-aNF;6~~J|nX7ZSp9*o}79*Njz!z%nPiv(nF9SpYO~y1tFUP6oF#@!Ce>6Q_*jOqf3DX8?nN20?!j ziOr$T-BXWE?#WH<$A6x>lt-hb-oUvc>4($#5Nq6lDmOCfIvI7*rI~?VI)7`q|C~5w z%&u?%1WGrP@GYD}Qg<0_J>(N(J)Qp)vtnJ{|4j9CES>r`2;pS+cS-WbqTQD?O^o!< zvke&1EggdhcJ#8VggVr z!Oct8?XE)GMGJ`FJhz-Bkd-BpA|QdR5HUu^krth!nVq+xI-}-{3sTTo&KZ}c0FPm< zd*R{$5jQb-6$AJYi|DNkLYOi-P_|TWLg1`{V8k3b2_Ol|f4FIgdoUfxyz{Lz5N;QQ zbqt*$c+z*E;cbAn8u>#&qmdARfY_#o05uTNJ=g~UVvWfM_!%&?LokFb zD*!J8wgO-V15unBe8xT(KaYK8u>;r}xfeRF1nyes0H7M+gydGtD~AL)lK>I|XCeU3 zB!F`qz?oXe05~%taI#&G*3tk@i4TA?qEaou*>x_n&?9n}ok893ytUYiC9w)%-Ck>N zSZg18&03#c=v%SY_n_9kCAFqoYF&j|`=r*B3tbDV1mH6hg7RMAQm*g(0pJ&#i192C zqK2&AMrga5!FC2a7~F!Oe#5c?8{!FUHsO+hM1cshWi`B$kKFJ4PPgPzuPsQ(P$IXo z_~!C>JJJrW{@1SzO;bhRdfWiHtN|LvKt)6L&cJ^{2S0u~YT{O@n1)UTJ}J+MQfJlmx7>d^)j+G<^R>^iQJPHbNVb zP3op$RY2sdY>`!w)`B@Sk)Qqt1R$@-Fd1sslJb`Ju5VUjL(2sM8yqAkxZh@<@Qd_C z^Wt+MXz1X!?O#hL0I+9 z8xsZ0K4~v1zJ~VJubYHwJg^o*?iR!@#xmJQCl3wPZ~Q)qCd9&UXbQTrFQNA`F2-o( zBYdP?rC2+*4DT@GMiBT{KrqzoP}9=wA_tl>lSDKXjRBhifP@VtaA4QB^xZ2BKk3_H z{JeV+gcko054lnJG=z;2y+$b<>c{D0p1;Nx8JxyW2e$tiIeiVVrxXl_`umM1f_-8l z1>O;^Y+OB|x&O3Lj%Y8qS2Owge%zSjDwd=Kusz5q*|mmwCjm!5jDq{4xY>uN`qe8{ z7aGnRV*peg{;^7AHZaG?+k0X{D}Y_bC9OiJn5y=Ps2C^TC`%~Q3#xcyuczpiXa#f zCyFF3fWZ;*`x`!vg6wl1%z>{!FNLp6VUvXgc`Wg=T&?=xibJ+i8OsnxW5UE-%5dg9 zUqKPO>A9_kG1`7NMSk!^6?AB&^EgyY1C7IUFn2ARN=St9q+Q>zV)Mep1tx6#@jh(R z^>}D)({Z%xxJ~g`E?NV7c0OX{BBrGYX{pl*$wjlBBWS$S11`&hLP-U!_w8c^I+ze8 z2uwK$nY0KgnJlI^s zKsCi`;N66x8h~&ozL-Fo1SA*H6P7n9$2=0C!a&)X__x^8HUv3W^Kjc>y#BQ4J$a-d z3vDT+Sir$J`vnv(!kZm1aI4^CFf_Z8kh&12qU3-A#@SieScpo+XhWKIkvt=*VWO+j9j305y}S+Y@$fVVAh-i`8?OoHDz=z}JUF_L zg!LQbA3&6#X!Vr~DK0hKr!Y%){p<>}WMmZopQlNWp_`bMD_b~nGUyYB_Dwz(Fz6ho zKoTR$NHD&$r+pi|Hpgf%z0!=CO^WZGRAn|`-6vU)$Y+Kzd}alX;LZn{hy7L{q!C*e zOxt~&HcGluJ(x1kb7YG2!zcr`GdQNIjF4 zD8Db+ZD|L&mTgdcJBptOojDRb`bSGX+Gh$pZo02w3e6*2fsHB$IZJ#Bl+iGjC;QUY z_+4zPFg3%dp=D0R22cb@!jzm~Gr24E+4{BYT70^wy&4ciTnxPAT&*8Pb^0*tu=VSh zDvDh{q>vCHWD5I(;v$}dK%`rct$9h(T9{EAoaI`BS5@?cJ$nUHP`7gyxK~n8TDGYc z6=P63WlAl`k$0La!+r^TrP&e(lM{QT6-F45L-lP#%igdlx3DQ!B6t!N9Znfb5mqKb zWt&v3rgo37*4B0BdXONI#ph&d3qM7z1_bc-bjrItJc z55QreFrcJvk>Y2~hU%+vhlx^*kSaK;rk*_SM(~1Cx<=6_IX^^_nFLmveI-`g00a

KVNJft!2TqTbgi>i@V=AHHl! zoO8U4cpNN%tq~FTHoY@IRf;7`{|o?CzQuW@$) z#S0FAw}$JMOnnPzhXzt@O(R$ktxMAea6=?^ILXx}y|LdV_E-iAifqA=<;{Tk71xQF zUwxv{nt@krO_+XJYdBjgtbNm(C2L<_u0^uoKZT~~qiAk@5C*^1{liTX{mX&`-R ziq`_azk~Lf!RK0fp52uNj)k3p_#}gC%7&^F1LC!H4yl_A1ha6@digR zNe^d=iR8qvY#2qDG$Xt_jy8XuJtUZ12{QwnXJZ3%JCtC!RwFtVA`l??VQW%G=4O2c zHTEto;Ioa=yKP-Th5{-I|M1)#F#;z?%p6~(b;21kS;mjfE}{I!^L1+Rg&i!`#iF;d z>)t=>!>z3}22oI(oBFl8G8FVZ8XBQw*1BSsI(N9WqsgnRKy1ljCHHhqi7)e2PS=`7%!J!o*7s$53Ez(H?%~1{78?;`ygg=r~9$2^< zc4^{ck_8nPR0d&Z$#VI_20&*cb<_6&h`9ddvxzO5239b<|l)>$W#q2a6igmar*g z2qUVqk@Elm-S*sqq#-ouAyC0gwl+gK0r8OjRTS4~d{n`H_SEv0f0ZpKXF$W*e#?Ft zlHN-iYb;Xlgf+gLttC9vdzL_etf0XZ4EE-Z26h>Kr;wjH92x*yO#K}X+aLu;k%Q+M zJF1@X-MR6xdk>5pc+414)z~gER*T8#IvDDs9>QwDb5_HuAqW3m!HBjPSV?zA&~~3f zmN3Pyu}l4k?&}zcCLo}O#u9d~F}SOOu=gqUbfo}QwH*Q}zEX$Z=)=BzR?<`L2U9@!$x5vRjSLtb@g#}SszRj-4Xn~wx` zpVAz22peraQV`M6ku06UmVuJ#IQ?|6USBOv1hOpDFz74w!ki^+?eSeVcoX$b)E%}) zp2C*YZ(?d3dI7r`=t8~h=%L*sf_UI%7%tR|BXV!3e&y0O1i*#!lyon|A`g7Pi)EPDIkfjFeymcm65?Q8vB-$$vpOU5|H(q&ezMCseVH z&hHq*OCA)3RQ*k;uMJ+2t7*VQ{($hRUrl(!_JG~eh$6+H5hr-h6V*p+DTtL}mxue< zbIEfVS`5=p8SslU9N_7awJvz<+Qh)a6Gg1Rley}NiNRH7 z(pRt06hDOxZR;A{rXq3fMRDQGq1E3_1 zv6euI=y`$=k+U2D^;4ACph%c~A>{6Gxq2p--Bj_)k49?*C#iaYylHWvpJ z^6()2bmVF4X?t4yN#g{o__c&hvTsjUR@Ui+{1Qixe9{?eGc9>&VnAnF0&qTP~6kX05-W% zni|4a15}l?`RA%U3n_TxWGP5EC?yA`Kr%szJP&`6kX9dqo`L74>$i)0zUC+kS`-IE zM`0o&t{7n#OD@EG`2!1^;5#Ji=7lI)ZM3v8{WaeIGlOp-@cVWLGlvO##Q4r=J&p>G z#o^56f)lhqiF`=Ine@BcUVHkzoMV~jVOBB1D&itAUyCs`8c4y68MtUj!OWhzNZNID z$Lrg}vVxkLoN#d3wHYBr0&K~mgGiEUe;jpRh)TQk`x@;)8hI%#Ne1P>WZI}&5+Isl ztLoQ&-?~dnWrDRBAQzq>XfOyX)i#S9EIkV5tzv&a?#C$bG! zdwzezsfT!2&*jU?D+3v#4}|z2XMNG6|QVCJU@t;3)yv0YoZM0~kNJe4#*F=tj=TQ0pLU z9+bM~DjWxC;<75?O(!(GKk7o}!Q(;!n+Fe}_;8HzjVp~Gg0zCeWVYu1mHN;!HNzM( zvqr8Gpl@wib9%-&JsjptMt}4br+}soh6e-ei#*)$Vu6(Z>QOmL4QfW_0^=2mAYkmV z7tr)hHt6RWoI^!UHvqLI$%-vsEiG?P_%K^cBt>9?@Ze)8%uum;jz`~SsW>;l!1AK$z@cuGoh?2#f>W<@@_ZhK1)#3M&f~}{ z>{X1i5(lth?}9T{MH&_LoX^D}3^pljtnMP=J9kD}@Gu(i=l2yKeiw$@UXJ z$-cQ6H`sI$v1seQv)C(_dR8!+hEPQyBMcDwGBO5^5eUN<;AR~?@RM^D8ALM?BS_?D z*uk8{s&Le;E3shfSDRPhshT@TGqrK%38xK=LgD(SI7!gU^GgxHcD(`*lT(bQVC70$jU0i8#ZLfe{E-SK>c{=r zXH0d?ql8k|2sAkilRnG4x8n|Czd2at!GeLaBiUg}A?9KPYgnsu*zq+CIIIXHUvvyb z7co(U#?iJy$_o~4kP5I##O-s^Vst-91L2B;zyHY zi}1&Lcv|6Xh9b3qDq?CfS9ljNRVcQlNQk^qJ?JHDGdM#-@ts?n z2bB}n#cCS4Mak1rvio?x44sqFX|SpzM~5b+OlSCU1{c+{MKbjm-T>S)a(VP zIPB6SJpjpf`rPactc?wPQ$1q3%8&SkP?{InFOxu&-&h7lSj#Hl!p_Ui!|07EOQhO@VK)ypYtf%0`kerls$aj4 z`yBSb3J6E8OKG<#p~4k?ZT-ju+!?L-ijBVfUobSHsmPwL&cPMZl7(*;w>gaH#05s{ zBdEw+(?wXyZoSObv+};Ge-qWV?LeeM$jZtNM5%gn2O9yS#An@NM%MbIJj_1uY|NU@*of)OX}ra=`fM&SwplEhf#Of+!FbyMq+ zZyaUs$6@mL>D`ubPn)Vi01o%GMhO#tGYH_&AuXs8>P`>$fu2b$!X?r<7@RL8t}v&z z@Kl4Nq%-2c{D9%f3Z4_7Bt>RC#~8O<5WZpSz!z0u4GKX94+G@G zB@n2>u~4WFG+I)u5m_0Cf~hG~c+kZ(H;w;S`#{j-=7z&H(@xLql19E3b^?d9OOZEI z1N#bk+qE)!3zDpe;*jOmCNgHq!%g(%TSGMOS zCr|@)TI(;xp1{I42;m}V(I@Qh^(N@}3}|5U<ohfo`Q6H8>~w08W^LCI$9ch|*1?i4_+X zmcckChTafv4)S;#Td@ztqcBI_atH?9(0H_H#Q<0Ed}&-6v2A4CVC;kTO~_OM#eap} z`(p-QM#Ds0v1Atg)nu)6%x|o*ng3;u1F`VOK;G4`I zgyJ|eRXPpBSMrv05fI493mluqAJ5zJc@#hI(+G_C=scv64n6p2r(=~!AblOV^YDZ!geXk)~{}fwY|Gwr*aR{-ow*Y zuxB|0t`W}W7Ql_1;rHuc`W=+fdBUzv;i+rEIsu~~nS*Hk9iebezO5yy5Kt+3qGh;~ z6)l7)Df1%rr1RR>WFg91I#7bO;bA49-xMegkICPw_)y;9`~Ef$#W+>0@MJq3I@|=s#ib zrwsm#aFZYxQTCj)TzT-sSv3n5mZL-n64kzVK}SbGiy*O+0gsC>p!0qy*mg9O@yHv4 z5=~s1bf$S?42KUvdElS8HRmxR#?r|N=@t6nip6^IIVO7afVukLuyy9^>t-hWUielq zqR5rtn351`?F5du2prQ%g&Zy5(GaY5tauoB;Gjqx>puQ*eB0>bgxdsHa^8g$Y(#bn z7&jG5m=Efj2Tsf z_(7EQfaaLR#Sgn_D1Wi&xIqS<1{?mK5ex6X^h6P?F!+AMB9bC7CI!6pfJKFwHsxWw zKxT<}8BOItg+m16L@Grp0WUoUf?x@DyK73%!6GeK#l|=Rz30#bYZ%M7tp;h9nMaau zr7Fe>SJZo&9ja`CG6q(Ghz-X?@JSP+(o$x&3etzC)TFq};|FNK)8T;0B7`uNe%R!k z-aLd$(iFknxiBGQQeWeu{W=3#$Xve8D(n(e>Er;DrUy_xh<~i`dG!k5x^&rI6&U2P zn1Cn@L33Q)pYex!?h7^JkUGRrV1j&^ENhXu>sA$7X`}j-X{yEiO{j! ziiH`t>!kXDj4SvzfMP>4E_Ol&(}_qmFRMdJf(<>+E@5#Hoi(t1)sXb?g{O3NHh5?Q zATDdbk&L;N6G9Xqma|5WKxoMF#%z_wkW&s$31cA#I=6sxn0Dv0Vml&uB5*-!zOm3C z+0nreGz1pK<0*Jne1bqY*I?zlL&t`;NCHeN&>DLxJbO6Hfv-?6T(%7}G_*+vBZOrz zJ&oN=mSR&cSuO*UoPg6C@jFa)i=1mVl>G=g*yJ`E>vvanw1Mo*jMZQb<7Kz6JY!3C z)^;)RSdzM7IW^5lKMrBpqTo;ppwkPBs>9Gm;n*vn$gg!v)k|Z8#kd zkx{d>r8sC89n@2vA68?4hv?A&3isCr5;0KJK(hzTm!_Iyw;PC@<}t9%1rtEQ3ulp7 zF~C`-vGk&-VLU^j4A%k=p9g&id;R@j6YvU^UpVXrR9gEUmeiO~GX`yP04{n`sbb`M zBW|$UMh^FY^QPr8luM8VK&^zGI7V~ah2T)cpv@CsA!Pz(4@rQA4SU8QMb4-n#WoC0 z6A_?6g~O=CunT{UyGjFLB_fG{$pQQGGQ5NVY1ME2ewenYjcPiLtv@`4;hx5hSe@tK zL;eNt(-`XJ{BmlE6atjfT;&GgKx)ts7!#$;x{9B(0Wa`%w*6}iKE~kV2w*%S7@lZV zdg=5TiUfN@SRuYUZpz^oFayt(FEGOhi}`Q3dx+ye#dd5hPVL9U8*GL&7m+xmJ~2sE zK!WfOxddmA;wr|9FiQvw6ZlaIitXtUj2~1Z(KJzy%Xc}@jVqBul;HRrOpZh-AQrmN zPFg86p5ZyMPoBl3pb@)tFX{3={I+nS6Q%_h^d*)$l9q!h=- z=M%!O(*Ty9M*?H?kqnGLTdE)COAepk7q%hH{3B?FelJt}kIOZI`9%oVrdI6YNlAQ} zJ%%sH^E-zYzIWIy-VQ7Wibdn^fJ#Nxi3a!hT_dofEf&N~1=_+5BVZpThJ+qP7F#J) z04l)F53-9>*2ef!o6DcfA@^-8F?_nPCI>Aig7vPZweZ;@jX)p>jVq4%XCnBG2}bkH`1=fKzTqujxDBxYf$X8xG!Ey8`b%Y^nzC z=7|yg;BEY?LJ&GJly3M24h4-P` zFR}xM^SS+35;XD3MIt*+5AELwhMRxa_42IDa6O7=_kD|cmxJ3R1FPC-= z@7i_pMF7nK3{YzLnyjaZIgRll2O7Nv2Spb&<^wVb@b}ozGyzjn6r?G^ z*BuOqOf|o4pxJFB!0YARDh7QFxW#MY7`=u8&u{4~7;xJ$pDT)EJ5W!CoqJ2Zbjw0KPv&t0E{Jxu~uVQ_)u9xGxPMR9d zAsjUb&Y*P2v-#$mm|N>>*dF=r&roATJe_3$0y44}U(3gh46b8vJ%bwD_nw`XD$Hh81CJqHft@ZzARY{Gc<>*2RFKr_fGHX03V z<*OkE+Zb$Ta5IBj5ICv(?#;=U%bfIm#o%_So46YZ2RAUd27zOoKHjn;xY>ED14UE~ z&%ErsRZlUcSlYar)lh+?U&G)|1`7!M*d8LZd-=SF!6OWwVsM7Rdl)>&;KK|)!r%gf zhZ%gD!DkqJfx#~__+g!!zgoalB^jT7B+dRGoK$`L;~Y9!}3#GJlq>MI7i#cz<{Ik?tE)aI>CDwfqL~{UUsm8VU}+ z$@E$`#_e8w=Kx>fhGsLGdD*@?agon`c<$r#7Ct4c%%$#h-?am&?nHNd?b?CNKzbm7 zuU+FEb)t6Ee%X#?zGn}_F2ZZzC48b0y3x!<`x>E|{{>-7I{SMVk?-~rTVSzt1|2QR`tg)CKM8(Opbj5ErAC5<g`k0>icYGE9K Union[int, float]: + """ + Min representable value for this scalar type. + (accounting for bias if there is one) + """ + raise NotImplementedError + + def max(self) -> Union[int, float]: + """ + Max representable value for this scalar type. + (accounting for bias if there is one) + """ + raise NotImplementedError + + def is_signed(self) -> bool: + """ + If the type is signed (i.e. has a sign bit), same as `signed` + added for consistency with: + https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html + """ + ... + + def is_floating_point(self) -> bool: + "If the type is a floating point type" + return self.exponent != 0 + + def is_integer(self) -> bool: + "If the type is an integer type" + return self.exponent == 0 + + def has_bias(self) -> bool: + "If the type has a non-zero bias" + return self.bias != 0 + + def has_infs(self) -> bool: + "If the type is floating point and supports infinity" + return not self._finite_values_only + + def has_nans(self) -> bool: + return self.nan_repr != NanRepr.NONE.value + + def is_ieee_754(self) -> bool: + """ + If the type is a floating point type that follows IEEE 754 + conventions + """ + return self.nan_repr == NanRepr.IEEE_754.value and \ + not self._finite_values_only + + def __str__(self) -> str: + raise NotImplementedError + + def __repr__(self) -> str: + raise NotImplementedError + + # __len__ needs to be defined (and has to throw TypeError) for pytorch's + # opcheck to work. + def __len__(self) -> int: + raise TypeError + + # + # Convenience Constructors + # + + @classmethod + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType': + "Create a signed integer scalar type (size_bits includes sign-bit)." + return cls(size_bits - 1, size_bits, bias if bias else 0, True) + + @classmethod + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType': + """Create a unsigned integer scalar type.""" + return cls(size_bits, size_bits, bias if bias else 0, False) + + @classmethod + def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType': + """ + Create a standard floating point type + (i.e. follows IEEE 754 conventions). + """ + return cls(exponent, mantissa, 0, True) + + @classmethod + def float_(cls, exponent: int, mantissa: int, finite_values_only: bool, + nan_repr: int) -> 'ScalarType': + """ + Create a non-standard floating point type + (i.e. does not follow IEEE 754 conventions). + """ + return cls(exponent, mantissa, 0, True, finite_values_only, + nan_repr) + +elif core_C_available: + try: + import vllm._core_C # noqa: F401 + except ImportError as e: + logger.warning("Failed to import from vllm._core_C with %r", e) + + ScalarType = torch.classes._core_C.ScalarType + + if (hasattr(torch, "_library") + and hasattr(torch._library, "register_fake_class")): + # Needed for dynamo support of ScalarType. + @torch._library.register_fake_class("_core_C::ScalarType") + class FakeScalarType: + + def __init__(self, scalar_type): + self.ScalarType = scalar_type + + def bias_getter(self) -> int: + return self.ScalarType.bias + + def exponent_getter(self) -> int: + return self.ScalarType.exponent + + def mantissa_getter(self) -> int: + return self.ScalarType.mantissa + + def signed_getter(self) -> bool: + return self.ScalarType.signed + + def size_bits_getter(self) -> int: + return self.ScalarType.size_bits + + @property + def size_bits(self) -> int: + return self.ScalarType.size_bits + + def min(self) -> Union[int, float]: + return self.ScalarType.min() + + def max(self) -> Union[int, float]: + return self.ScalarType.max() + + def is_signed(self) -> bool: + return self.ScalarType.is_signed() + + def is_floating_point(self) -> bool: + return self.ScalarType.is_floating_point() + + def is_integer(self) -> bool: + return self.ScalarType.is_integer() + + def has_bias(self) -> bool: + return self.ScalarType.has_bias() + + def has_infs(self) -> bool: + return self.ScalarType.has_infs() + + def has_nans(self) -> bool: + return self.ScalarType.has_nans() + + def is_ieee_754(self) -> bool: + return self.ScalarType.is_ieee_754() + + def __str__(self) -> str: + return self.ScalarType.__str__() + + def __repr__(self) -> str: + return self.ScalarType.__repr__() + + def __len__(self) -> int: + return self.ScalarType.__len__() + + def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]: + return torch.classes._core_C.ScalarType.__obj_flatten__( + self.ScalarType) + + @classmethod + def __obj_unflatten__( + cls, flat_type: Tuple[Tuple[str, Any], + ...]) -> 'ScalarType': + return cls( + torch.classes._core_C.ScalarType.__obj_unflatten__( + flat_type)) + + @classmethod + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType': + return ScalarType.int_(size_bits, bias) + + @classmethod + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType': + return ScalarType.uint(size_bits, bias) + + @classmethod + def float_IEEE754(cls, exponent: int, + mantissa: int) -> 'ScalarType': + return ScalarType.float_IEEE754(exponent, mantissa) + + @classmethod + def float_(cls, exponent: int, mantissa: int, + finite_values_only: bool, + nan_repr: int) -> 'ScalarType': + return ScalarType.float_(exponent, mantissa, + finite_values_only, nan_repr) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py new file mode 100644 index 00000000..64a5534c --- /dev/null +++ b/vllm/_custom_ops.py @@ -0,0 +1,1105 @@ +import contextlib +import functools +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, Dict, Any + +import torch +import torch.library + +import vllm.envs as envs +from vllm._core_ext import ScalarType +from vllm.logger import init_logger +from vllm.platforms import current_platform +# import ixformer.inference.functions as ops +import ixformer.functions as ixf_F +from ixformer.distributed import _distributed as cdist +import torch.nn.functional as F + +logger = init_logger(__name__) + +supports_moe_ops = True + +if TYPE_CHECKING: + + def register_fake(fn): + return lambda name: fn +else: + try: + from torch.library import register_fake + except ImportError: + try: + from torch.library import impl_abstract as register_fake + except: + def register_fake(fn): + return lambda name: fn + + +def hint_on_error(fn): + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + + except NotImplementedError as e: + msg = ( + "Error in calling custom op %s: %s\n" + "Not implemented or built, mostly likely because the current current device " + "does not support this kernel (less likely TORCH_CUDA_ARCH_LIST was set " + "incorrectly while building)") + logger.error(msg, fn.__name__, e) + raise NotImplementedError(msg % (fn.__name__, e)) from e + except AttributeError as e: + msg = ( + "Error in calling custom op %s: %s\n" + "Possibly you have built or installed an obsolete version of vllm.\n" + "Please try a clean build and install of vllm," + "or remove old built files such as vllm/*cpython*.so and build/ ." + ) + logger.error(msg, fn.__name__, e) + raise e + + return wrapper + + +# activation ops +def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ixf_F.silu_and_mul(x, out) + + +def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ixf_F.gelu_and_mul(x, out) + + +def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ixf_F.gelu_tanh_and_mul(x, out) + + +def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: + out.copy_(F.gelu(x,approximate="tanh")) + return out + + +def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: + out.copy_(F.gelu(x,approximate="tanh")) + return out + + +def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: + out.copy_(F.gelu(x,approximate="tanh")) + return out + + + +def paged_attention_v1( + output, + query, + key_cache, + value_cache, + head_mapping, + scale, + block_tables, + context_lens, + block_size, + max_context_len, + alibi_slopes=None, + kv_cache_dtype=None, +): + return ixf_F.vllm_single_query_cached_kv_attention( + output, + query, + key_cache, + value_cache, + head_mapping, + scale, + block_tables, + context_lens, + block_size, + max_context_len, + alibi_slopes, + ) + + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + raise NotImplementedError() + + +def paged_attention_rocm( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + raise NotImplementedError() + + +# pos encoding ops +def rotary_embedding( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + head_size: int, + cos_sin_cache: torch.Tensor, + is_neox: bool, +) -> None: + ixf_F.vllm_rotary_embedding_neox(positions, query, key, head_size, + cos_sin_cache, is_neox) + + +def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor, + key: torch.Tensor, head_size: int, + cos_sin_cache: torch.Tensor, is_neox: bool, + rot_dim: int, + cos_sin_cache_offsets: torch.Tensor) -> None: + ixf_F.vllm_batched_rotary_embedding(positions, query, key, head_size, + cos_sin_cache, is_neox, rot_dim, + cos_sin_cache_offsets) + + +# layer norm ops +def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor, + epsilon: float) -> None: + ixf_F.rms_norm(input, weight, out, epsilon) + + +def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor, + weight: torch.Tensor, epsilon: float, + residual_alpha: Optional[float] = 1) -> None: + ixf_F.fused_add_rms_norm(input, residual, weight, epsilon) + + +def advance_step_flashattn(num_seqs: int, num_queries: int, block_size: int, + input_tokens: torch.Tensor, + sampled_token_ids: torch.Tensor, + input_positions: torch.Tensor, + seq_lens: torch.Tensor, slot_mapping: torch.Tensor, + block_tables: torch.Tensor) -> None: + """Advance a step on GPU for existing inputs for a multi-step runner""" + return ixf_F.advance_step_flashattn(num_seqs, num_queries, block_size, + input_tokens, + sampled_token_ids, + input_positions, + seq_lens, slot_mapping, + block_tables) + + +def advance_step_flashinfer(num_seqs: int, num_queries: int, block_size: int, + input_tokens: torch.Tensor, + sampled_token_ids: torch.Tensor, + input_positions: torch.Tensor, + seq_lens: torch.Tensor, slot_mapping: torch.Tensor, + block_tables: torch.Tensor, + paged_kv_indices: torch.Tensor, + paged_kv_indptr: torch.Tensor, + paged_kv_last_page_len: torch.Tensor, + block_table_bound: torch.Tensor) -> None: + raise NotImplementedError("FIX SOON") + + +# quantization ops +# awq +def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor, + zeros: torch.Tensor, split_k_iters: int, thx: int, + thy: int) -> torch.Tensor: + raise NotImplementedError() + + +def awq_gemm(input: torch.Tensor, qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, + pack_factor, group_size: int = 128) -> torch.Tensor: + return ixf_F.quantized_linear(input, qweight, scales,"awq",32 // pack_factor,qzeros=qzeros,group_size=group_size) + + +# gptq +def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor, + b_gptq_qzeros: torch.Tensor, b_gptq_scales: torch.Tensor, + b_g_idx: torch.Tensor, use_exllama: bool, + bit: int) -> torch.Tensor: + batch = a.shape[0] + if batch <= 8: + return ixf_F.quantized_linear(a,b_q_weight,b_gptq_scales,"gptq",4,b_gptq_qzeros,None,group_size=128) + o_dtype_str = "fp16" if a.dtype == torch.half else "bf16" + deq_w = ixf_F.quantized_weight_dequant(b_q_weight,b_gptq_scales,"gptq",o_dtype_str,4,b_gptq_qzeros,group_size=128) + return torch.matmul(a,deq_w) + + +if hasattr(torch.ops._C, "gptq_gemm"): + + @register_fake("_C::gptq_gemm") + def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor, + b_gptq_qzeros: torch.Tensor, + b_gptq_scales: torch.Tensor, b_g_idx: torch.Tensor, + use_exllama: bool, bit: int) -> torch.Tensor: + return torch.empty((a.size(0), b_q_weight.size(1)), + dtype=a.dtype, + device=a.device) + + +def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor, + bit: int) -> None: + return ixf_F.vllm_gptq_shuffle(q_weight,q_perm) + + +# marlin +def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor, + b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int, + size_n: int, size_k: int) -> torch.Tensor: + raise NotImplementedError() + + +# marlin_24 +def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor, + b_meta: torch.Tensor, b_scales: torch.Tensor, + workspace: torch.Tensor, b_q_type: ScalarType, + size_m: int, size_n: int, size_k: int) -> torch.Tensor: + raise NotImplementedError() + + +if hasattr(torch.ops._C, "gptq_marlin_24_gemm"): + + @register_fake("_C::gptq_marlin_24_gemm") + def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor, + b_meta: torch.Tensor, b_scales: torch.Tensor, + workspace: torch.Tensor, + b_q_type: ScalarType, size_m: int, + size_n: int, size_k: int) -> torch.Tensor: + return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype) + + @register_fake("_C::gptq_marlin_gemm") + def _gptq_marlin_gemm_fake(a: torch.Tensor, + b_q_weight: torch.Tensor, + b_scales: torch.Tensor, + b_zeros: torch.Tensor, + g_idx: torch.Tensor, + perm: torch.Tensor, + workspace: torch.Tensor, + b_q_type: ScalarType, + size_m: int, + size_n: int, + size_k: int, + is_k_full: bool, + has_zp: bool = False, + use_fp32_reduce: bool = False) -> torch.Tensor: + return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype) + + @register_fake("_C::ggml_dequantize") + def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int, + n: int) -> torch.Tensor: + return torch.empty((m, n), dtype=torch.float16, device=W.device) + + @register_fake("_C::ggml_mul_mat_vec_a8") + def _ggml_mul_mat_vec_a8_fake( + W: torch.Tensor, + X: torch.Tensor, + quant_type: int, + row: int, + ) -> torch.Tensor: + return torch.empty((1, row), dtype=torch.float16, device=W.device) + + @register_fake("_C::ggml_mul_mat_a8") + def _ggml_mul_mat_a8_fake( + W: torch.Tensor, + X: torch.Tensor, + quant_type: int, + row: int, + ) -> torch.Tensor: + batch = X.size(0) + return torch.empty((batch, row), dtype=torch.float16, device=W.device) + + @register_fake("_C::marlin_qqq_gemm") + def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor, + s_tok: torch.Tensor, s_ch: torch.Tensor, + s_group: torch.Tensor, workspace: torch.Tensor, + size_m: int, size_n: int, + size_k: int) -> torch.Tensor: + return torch.empty((size_m, size_n), + dtype=torch.float16, + device=a.device) + + @register_fake("_C::marlin_gemm") + def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor, + b_scales: torch.Tensor, workspace: torch.Tensor, + size_m: int, size_n: int, + size_k: int) -> torch.Tensor: + return torch.empty((size_m, size_n), + dtype=torch.float16, + device=a.device) + + @register_fake("_C::awq_dequantize") + def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor, + zeros: torch.Tensor, split_k_iters: int, thx: int, + thy: int) -> torch.Tensor: + in_c = qweight.size(0) + qout_c = qweight.size(1) + out_c = qout_c * 8 + return torch.empty((in_c, out_c), + dtype=scales.dtype, + device=scales.device) + + @register_fake("_C::awq_gemm") + def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor, + qzeros: torch.Tensor, scales: torch.Tensor, + split_k_iters: int) -> torch.Tensor: + num_in_feats = input.size(0) + return torch.empty((split_k_iters, num_in_feats, qweight.size(1) * 8), + dtype=input.dtype, + device=input.device).sum(0) + + @register_fake("_C::aqlm_gemm") + def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor, + codebooks: torch.Tensor, scales: torch.Tensor, + codebook_partition_sizes: List[int], + bias: Optional[torch.Tensor]) -> torch.Tensor: + out_features = codes.size(0) * codebooks.size(2) + flat_input = input.reshape((-1, input.size(-1))) + flat_output = torch.empty((flat_input.size(0), out_features), + dtype=input.dtype, + device=input.device) + + output_sizes = list(input.shape) + output_sizes.pop() + output_sizes.append(-1) + return flat_output.reshape(tuple(output_sizes)) + + @register_fake("_C::aqlm_dequant") + def _aqlm_dequant_fake( + codes: torch.Tensor, codebooks: torch.Tensor, + codebook_partition_sizes: List[int]) -> torch.Tensor: + in_features = codes.size(1) * 8 + out_features = codes.size(0) + return torch.empty((out_features, in_features), + dtype=codebooks.dtype, + device=codebooks.device) + + @register_fake("_C::fp8_marlin_gemm") + def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor, + b_scales: torch.Tensor, workspace: torch.Tensor, + num_bits: int, size_m: int, size_n: int, + size_k: int) -> torch.Tensor: + return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device) + + @register_fake("_C::machete_gemm") + def machete_gemm_fake( + a: torch.Tensor, + # Should be the tensor returned by machete_prepack_B + b_q: torch.Tensor, + b_type: ScalarType, + b_scales: Optional[torch.Tensor] = None, + b_zeros: Optional[torch.Tensor] = None, + b_group_size: Optional[int] = None, + c: Optional[torch.Tensor] = None, + alpha: Optional[float] = None, + beta: Optional[float] = None, + schedule: Optional[str] = None, + ) -> torch.Tensor: + m = a.size(0) + n = b_q.size(1) + return torch.empty((m, n), device=a.device, dtype=a.dtype) + + @register_fake("_C::machete_prepack_B") + def machete_prepack_B_fake(b_q_weight: torch.Tensor, + b_type: ScalarType) -> torch.Tensor: + return torch.empty_like(b_q_weight, + memory_format=torch.contiguous_format) + + @register_fake("_C::causal_conv1d_fwd") + def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor, + bias_: Optional[torch.Tensor], + conv_states: Optional[torch.Tensor], + cu_seq_len: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], + silu_activation: bool) -> torch.Tensor: + return torch.empty_like(x) + + @register_fake("_C::causal_conv1d_update") + def causal_conv1d_update_fake( + x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor, + bias_: Optional[torch.Tensor], silu_activation: bool, + cache_seqlens: Optional[torch.Tensor], + conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor: + return torch.empty_like(x) + + @register_fake("_C::selective_scan_fwd") + def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor, + A: torch.Tensor, B: torch.Tensor, + C: torch.Tensor, D_: Optional[torch.Tensor], + z_: Optional[torch.Tensor], + delta_bias_: Optional[torch.Tensor], + delta_softplus: bool, + cu_seq_len: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], + ssm_states: Optional[torch.Tensor]) -> None: + return None + + +# cutlass +def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool: + return True + + +def cutlass_scaled_mm(a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: torch.dtype, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + m = a.shape[0] + n = b.shape[1] + out = torch.empty((m, n), dtype=out_dtype, device=a.device) + ixf_F.w8a8(a, b.transpose(0,1), scale_a, scale_b, bias, output=out, out_dtype=out_dtype) + + return out + + +def cutlass_scaled_mm_azp(a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: torch.dtype, + azp_adj: torch.Tensor, + azp: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + raise NotImplementedError() + + +# aqlm +def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor, + codebooks: torch.Tensor, scales: torch.Tensor, + codebook_partition_sizes: List[int], + bias: Optional[torch.Tensor]) -> torch.Tensor: + raise NotImplementedError() + + +def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor, + codebook_partition_sizes: List[int]) -> torch.Tensor: + raise NotImplementedError() + + +# gptq_marlin +def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor, + size_k: int, size_n: int, + num_bits: int) -> torch.Tensor: + raise NotImplementedError() + + +# gptq_marlin +def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int, + num_bits: int) -> torch.Tensor: + raise NotImplementedError() + + +def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor, + size_k: int, size_n: int, + num_bits: int) -> torch.Tensor: + raise NotImplementedError() + + +def awq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor, + size_k: int, size_n: int, + num_bits: int) -> torch.Tensor: + num_experts = b_q_weight.shape[0] + assert size_k % 16 == 0 + output = torch.empty((num_experts, size_k // 16, size_n * (num_bits // 2)), + device=b_q_weight.device, + dtype=b_q_weight.dtype) + for e in range(num_experts): + output[e] = torch.ops._C.awq_marlin_repack(b_q_weight[e], size_k, + size_n, num_bits) + return output + + +def gptq_marlin_gemm(a: torch.Tensor, + b_q_weight: torch.Tensor, + b_scales: torch.Tensor, + b_zeros: torch.Tensor, + g_idx: torch.Tensor, + perm: torch.Tensor, + workspace: torch.Tensor, + b_q_type: ScalarType, + size_m: int, + size_n: int, + size_k: int, + is_k_full: bool, + has_zp: bool = False, + use_fp32_reduce: bool = False) -> torch.Tensor: + raise NotImplementedError() + + +# fp8 marlin +def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor, + b_scales: torch.Tensor, workspace: torch.Tensor, + num_bits: int, size_m: int, size_n: int, + size_k: int) -> torch.Tensor: + raise NotImplementedError() + + +# machete +def machete_supported_schedules(b_type: ScalarType) -> List[str]: + raise NotImplementedError() + + +def machete_gemm( + a: torch.Tensor, + b_q: torch.Tensor, # Should be the tensor returned by machete_prepack_B + b_type: ScalarType, + b_scales: Optional[torch.Tensor] = None, + b_zeros: Optional[torch.Tensor] = None, + b_group_size: Optional[int] = None, + c: Optional[torch.Tensor] = None, + alpha: Optional[float] = None, + beta: Optional[float] = None, + schedule: Optional[str] = None, +) -> torch.Tensor: + raise NotImplementedError() + + +def machete_prepack_B(b_q_weight: torch.Tensor, + b_type: ScalarType) -> torch.Tensor: + raise NotImplementedError() + + +if hasattr(torch.ops._C, "permute_cols"): + + @register_fake("_C::permute_cols") + def _permute_cols_fake(a: torch.Tensor, + perm: torch.Tensor) -> torch.Tensor: + return torch.empty_like(a) + + +def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor: + raise NotImplementedError() + + +# fp8 +def scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, + num_token_padding: Optional[int] = None, + scale_ub: Optional[torch.Tensor] = None, + use_per_token_if_dynamic: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP8 and return quantized tensor and scale. + + This function supports both static and dynamic quantization: If you + provide the scale, it will use static scaling and if you omit it, + the scale will be determined dynamically. The function also allows + optional padding of the output tensors for downstream kernels that + will benefit from padding. + + Args: + input: The input tensor to be quantized to FP8 + scale: Optional scaling factor for the FP8 quantization + scale_ub: Optional upper bound for scaling factor in dynamic + per token case + num_token_padding: If specified, pad the first dimension + of the output to at least this value. + use_per_token_if_dynamic: Whether to do per_tensor or per_token + in the dynamic quantization case. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and + scaling factor. + """ + raise NotImplementedError() + + +# int8 +def scaled_int8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, + azp: Optional[torch.Tensor] = None, + symmetric: bool = True +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp. + + Args: + input: The input tensor to be quantized to int8. + scale: Optional scaling factor for the int8 quantization. + When not provided, we invoke dynamic-per-token quantization. + azp: Optional zero-point for the int8 quantization. + Must be provided for asymmetric quantization if `scale` is provided. + symmetric: Whether to use symmetric quantization (scale only, azp ignored). + + Returns: + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] : Output int8 tensor, scales, and optionally azp. + """ + output = torch.empty_like(input, dtype=torch.int8) + if scale is not None: + # static-per-tensor quantization. + assert symmetric == ( + azp is + None), "azp must only be provided for asymmetric quantization." + ixf_F.static_scaled_int8_quant(output, input, scale) + return output, scale, None + + # dynamic-per-token quantization. + input_scales = torch.empty((input.numel() // input.shape[-1], 1), + device=input.device, + dtype=torch.float32) + input_azp = None if symmetric else torch.empty_like(input_scales, + dtype=torch.int32) + ixf_F.dynamic_scaled_int8_quant(output, input, input_scales) + return output, input_scales, input_azp + + +# qqq ops +def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor, + s_tok: torch.Tensor, s_ch: torch.Tensor, + s_group: torch.Tensor, workspace: torch.Tensor, + size_m: int, size_n: int, size_k: int) -> torch.Tensor: + raise NotImplementedError() + + +# gguf +def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, + n: int) -> torch.Tensor: + raise NotImplementedError() + + +def ggml_mul_mat_vec_a8( + W: torch.Tensor, + X: torch.Tensor, + quant_type: int, + row: int, +) -> torch.Tensor: + raise NotImplementedError() + + +def ggml_mul_mat_a8( + W: torch.Tensor, + X: torch.Tensor, + quant_type: int, + row: int, +) -> torch.Tensor: + raise NotImplementedError() + + +# mamba +def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor, + bias_: Optional[torch.Tensor], + conv_states: Optional[torch.Tensor], + query_start_loc: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], + silu_activation: bool) -> torch.Tensor: + raise NotImplementedError() + + +def causal_conv1d_update( + x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor, + bias_: Optional[torch.Tensor], silu_activation: bool, + cache_seqlens: Optional[torch.Tensor], + conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor: + raise NotImplementedError() + + +def selective_scan_fwd( + u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor, B: torch.Tensor, + C: torch.Tensor, D_: Optional[torch.Tensor], + z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor], + delta_softplus: bool, query_start_loc: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], ssm_states: torch.Tensor): + raise NotImplementedError() + + +# moe +def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int, + block_size: int, sorted_token_ids: torch.Tensor, + experts_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor) -> None: + ixf_F.vllm_moe_align_block_size(topk_ids, num_experts, block_size, + sorted_token_ids, experts_ids, + num_tokens_post_pad) + + +def invoke_fused_moe_kernel( + A: torch.Tensor, + B: torch.Tensor, + C: torch.Tensor, + A_scale: Optional[torch.Tensor], + B_scale: Optional[torch.Tensor], + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + mul_routed_weight: bool, + top_k: int, + config: Dict[str, Any], + compute_type, + use_fp8_w8a8: bool, + use_int8_w8a16: bool, +) -> None: + ixf_F.vllm_invoke_fused_moe_kernel( + A, + B, + C, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + mul_routed_weight, + top_k, + config['BLOCK_SIZE_M'] + ) + + +def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor, + token_expert_indicies: torch.Tensor, + gating_output: float) -> None: + ixf_F.vllm_moe_topk_softmax(topk_weights, topk_ids, + token_expert_indicies, gating_output) + + +if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"): + + @register_fake("_moe_C::marlin_gemm_moe") + def marlin_gemm_moe_fake(a: torch.Tensor, b_q_weights: torch.Tensor, + sorted_ids: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, b_scales: torch.Tensor, + b_zero_points: torch.Tensor, g_idx: torch.Tensor, + perm: torch.Tensor, workspace: torch.Tensor, + b_q_type: ScalarType, size_m: int, size_n: int, + size_k: int, is_k_full: bool, num_experts: int, + topk: int, moe_block_size: int, + replicate_input: bool, + apply_weights: bool) -> torch.Tensor: + return torch.empty((size_m, topk, size_n), + dtype=a.dtype, + device=a.device) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + slot_mapping = slot_mapping.to(torch.int32) + ixf_F.vllm_cache_ops_reshape_and_cache(key, value, key_cache, + value_cache, slot_mapping) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ixf_F.reshape_and_cache_flash(key, value, key_cache, + value_cache, slot_mapping, + kv_cache_dtype, k_scale, + v_scale) + +def reshape_and_cache_flashinfer( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, # for fp8 + v_scale: float, # for fp8 + kv_cache_format: str = "NHD", + key_cache_scales: torch.Tensor = None, # for int8 + value_cache_scales: torch.Tensor = None, # for int8 +) -> None: + ixf_F.paged_attention_cache_appended( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_format, + key_cache_scales, + value_cache_scales, + ) + +def copy_blocks(key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor) -> None: + ixf_F.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks(src: torch.Tensor, dst: torch.Tensor, + block_mapping: torch.Tensor) -> None: + ixf_F.swap_blocks(src, dst, block_mapping) + + +def convert_fp8(output: torch.Tensor, + input: torch.Tensor, + scale: float = 1.0, + kv_dtype: str = "fp8") -> None: + raise NotImplementedError() + + +def get_device_attribute(attribute: int, device: int) -> int: + raise NotImplementedError() + + +def get_max_shared_memory_per_block_device_attribute(device: int) -> int: + return 32 * 1024 + + +# custom ar +def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor, + handles: List[str], offsets: List[int], rank: int, + full_nvlink: bool) -> int: + raise NotImplementedError() + + +def should_custom_ar(inp: torch.Tensor, max_size: int, world_size: int, + full_nvlink: bool) -> bool: + raise NotImplementedError() + + +def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + raise NotImplementedError() + + +def all_reduce_unreg(fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, + out: torch.Tensor) -> None: + raise NotImplementedError() + + +def dispose(fa: int) -> None: + raise NotImplementedError() + + +def meta_size() -> int: + raise NotImplementedError() + + +def register_buffer(fa: int, t: torch.Tensor, handles: List[str], + offsets: List[int]) -> None: + raise NotImplementedError() + + +def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[str], List[int]]: + raise NotImplementedError() + + +def register_graph_buffers(fa: int, handles: List[str], + offsets: List[List[int]]) -> None: + raise NotImplementedError() + + +# Add our new features here.. + +# broadcast +class Async_helper(): + # For now, the comm and the other kernels are in the same stream, so we can remove the stream wait.. + def wait(self,): + return True + + +def broadcast(tensor, src=0, group=None, async_op=False): + cdist.broadcast(tensor,src,group,async_op=True) + if async_op: + return Async_helper() + else: + pass + +# w8a16 +def linear_w8a16(x: torch.Tensor, qweight: torch.Tensor, scales:torch.Tensor, + group_size: int = -1, format: str = "TN")-> torch.Tensor: + return ixf_F.w8a16(x, qweight, scales, format="TN", group_size=group_size) + + +## lora sgmv / bgmv +def sbgmv_expand(x: torch.Tensor, + w_t_all: torch.Tensor, + y: torch.Tensor, + b_seq_start_loc: torch.Tensor = None, + seq_len_tensor: torch.Tensor = None, + lora_indices_tensor: torch.Tensor = None, + batches: int = -1, + max_seq_length: int = -1, + token_nums: int = -1, + add_input=True, + ): + ''' + x: inputs + w_t_all: lora weight + y: output + + y += x@wt_t_all + ''' + assert x.dtype in [torch.float16, torch.bfloat16, torch.float32] + assert w_t_all.dtype in [ + torch.float16, + torch.bfloat16, + ] + + assert x.is_contiguous() + # assert y.is_contiguous() + if x.dtype == torch.float: + x = x.to(w_t_all.dtype) + + if w_t_all.ndim == 4: # shape:(lora_num,1,size,rank) + assert w_t_all.size(1) == 1 + w_t_all = w_t_all.squeeze(dim=1) + else: + assert w_t_all.ndim == 3 # shape:(lora_num,size,rank) + assert w_t_all.is_contiguous() + + assert add_input == True + + lora_indices = lora_indices_tensor.cpu().tolist() + lora_num = w_t_all.shape[0] + + ## 单一lora model, 且所有request均使用lora + if lora_num == 1 and all(x == lora_indices[0] for x in lora_indices): + if lora_indices[0] != -1: + w_t = w_t_all[0] + y += torch.matmul(x, w_t.t()) + ## 多个lora model + else: + ## prefill + if batches != -1: + for i, lora_id, start, seq_len in zip(range(batches), lora_indices, b_seq_start_loc, seq_len_tensor): + if lora_id != -1: + xi = x[start: start+seq_len] + w_t = w_t_all[lora_id] + y[start:start+seq_len] += (xi @ w_t.t()) + ## decode + else: + batches = x.shape[0] + for i, lora_id in zip(range(batches), lora_indices): + if lora_id != -1: + xi = x[i].unsqueeze(0) + w_t = w_t_all[lora_id] + y[i] += (xi @ w_t.t()).squeeze(0) + + return y + + +def sbgmv_shrink(x: torch.Tensor, + w_t_all: torch.Tensor, + y: torch.Tensor, + b_seq_start_loc: torch.Tensor = None, + seq_len_tensor: torch.Tensor = None, + lora_indices_tensor: torch.Tensor = None, + batches: int = -1, + max_seq_length: int = -1, + token_nums: int = -1, + scale: float = 1.0,): + """ + xx: inputs + w_t_all: lora weight + y: output + scale: float + + y = x@w_t_all * scale + """ + assert x.dtype == w_t_all.dtype + assert x.dtype in [torch.float16, torch.bfloat16] + assert x.is_contiguous() + assert y.is_contiguous() + + if w_t_all.ndim == 4: # shape:(lora_num,1,size,rank) + assert w_t_all.size(1) == 1 + w_t_all = w_t_all.squeeze(dim=1) + else: + assert w_t_all.ndim == 3 # shape:(lora_num,size,rank) + assert w_t_all.is_contiguous() + + lora_num = w_t_all.shape[0] + lora_indices = lora_indices_tensor.cpu().tolist() + + ## 单一lora model, 且所有request均使用lora + if lora_num == 1 and all(x == lora_indices[0] for x in lora_indices): + if lora_indices[0] != -1: + w_t = w_t_all[0] + y = torch.matmul(x, w_t.t()) * scale + ## 多个lora model + else: + ## prefill + if batches != -1: + for i, lora_id, start, seq_len in zip(range(batches), lora_indices, b_seq_start_loc, seq_len_tensor): + if lora_id != -1: + xi = x[start: start+seq_len] + w_t = w_t_all[lora_id] + y[start:start+seq_len] = (xi @ w_t.t())* scale + ## decode + else: + batches = x.shape[0] + for i, lora_id in zip(range(batches), lora_indices): + if lora_id != -1: + xi = x[i].unsqueeze(0) + w_t = w_t_all[lora_id] + y[i] = (xi @ w_t.t()).squeeze(0) * scale + + return y + +# temporary fix for https://github.com/vllm-project/vllm/issues/5456 +# TODO: remove this in v0.6.0 +names_and_values = globals() +names_and_values_to_update = {} +# prepare variables to avoid dict size change during iteration +k, v, arg = None, None, None +fn_type = type(lambda x: x) +for k, v in names_and_values.items(): + # find functions that are defined in this file and have torch.Tensor + # in their annotations. `arg == "torch.Tensor"` is used to handle + # the case when users use `import __annotations__` to turn type + # hints into strings. + if isinstance(v, fn_type) \ + and v.__code__.co_filename == __file__ \ + and any(arg is torch.Tensor or arg == "torch.Tensor" + for arg in v.__annotations__.values()): + names_and_values_to_update[k] = hint_on_error(v) + +names_and_values.update(names_and_values_to_update) +del names_and_values_to_update, names_and_values, v, k, fn_type \ No newline at end of file diff --git a/vllm/_ipex_ops.py b/vllm/_ipex_ops.py new file mode 100644 index 00000000..31fcc4c3 --- /dev/null +++ b/vllm/_ipex_ops.py @@ -0,0 +1,243 @@ +from typing import List, Optional, Tuple + +import torch + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +try: + import intel_extension_for_pytorch as ipex +except ImportError as e: + logger.warning("Import error msg: %s", e.msg) + + +class ipex_ops: + + @staticmethod + def _reshape_activation_tensor( + x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + num = x.size(0) + d = x.size(1) // 2 + x = x.reshape(num, 2, d) + x1, x2 = torch.chunk(x, chunks=2, dim=1) + x1 = x1.reshape(num, d) + x2 = x2.reshape(num, d) + return x1, x2 + + @staticmethod + def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ipex.llm.functional.silu_and_mul(x, out) + + @staticmethod + def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ipex.llm.functional.gelu_and_mul(x, out) + + @staticmethod + def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: + ipex.llm.functional.gelu_and_mul(x, out) + + @staticmethod + def gelu_fast(x: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.gelu(x) + + @staticmethod + def gelu_new(x: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.gelu(x) + + @staticmethod + def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: + ipex.llm.functional.gelu_quick(x, out) + + @staticmethod + def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + context_lens: torch.Tensor, + block_size: int, + max_context_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, + ) -> None: + assert kv_cache_dtype == "auto" + num_heads = out.size(1) + num_queries_per_tokens = num_heads // num_kv_heads + head_mapping = torch.arange( + 0, + num_kv_heads, + device=query.device, + dtype=torch.int32, + ).view(num_kv_heads, + 1).repeat_interleave(num_queries_per_tokens).flatten() + # todo: ipex will refactor namespace + torch.xpu.paged_attention_v1( # type: ignore + out, + query.contiguous(), + key_cache.view_as(value_cache), + value_cache, + head_mapping, + scale, + block_tables, + context_lens, + block_size, + max_context_len, + alibi_slopes, + ) + + @staticmethod + def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + context_lens: torch.Tensor, + block_size: int, + max_context_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, + ) -> None: + assert kv_cache_dtype == "auto" + num_heads = out.size(1) + num_queries_per_tokens = num_heads // num_kv_heads + head_mapping = torch.arange( + 0, + num_kv_heads, + dtype=torch.int32, + device=query.device, + ).view(num_kv_heads, + 1).repeat_interleave(num_queries_per_tokens).flatten() + # todo: ipex will refactor namespace + torch.xpu.paged_attention_v2( # type: ignore + out, + exp_sum, + max_logits, + tmp_out, + query.contiguous(), + key_cache.view_as(value_cache), + value_cache, + head_mapping, + block_tables, + context_lens, + scale, + block_size, + max_context_len, + alibi_slopes, + ) + + @staticmethod + def rotary_embedding( + positions: torch.Tensor, # [batch_size, seq_len] + query: torch.Tensor, # [batch_size, seq_len, num_heads*head_size] + key: torch.Tensor, # [batch_size, seq_len, num_kv_heads*head_size] + head_size: int, + cos_sin_cache: torch.Tensor, # [cos_sin_dim, rot_dim] + is_neox: bool, + ) -> None: + rot_dim = cos_sin_cache.size(1) + ipex.llm.functional.rotary_embedding_batched(positions, query, key, + head_size, cos_sin_cache, + is_neox, rot_dim) + + @staticmethod + def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor, + key: torch.Tensor, head_size: int, + cos_sin_cache: torch.Tensor, is_neox: bool, + rot_dim: int, + cos_sin_cache_offsets: torch.Tensor) -> None: + ipex.llm.functional.rotary_embedding_batched(positions, query, key, + head_size, cos_sin_cache, + is_neox, rot_dim, + cos_sin_cache_offsets) + + @staticmethod + def rms_norm(input: torch.Tensor, weight: torch.Tensor, + epsilon: float) -> torch.Tensor: + return ipex.llm.functional.rms_norm(input, weight, epsilon) + + @staticmethod + def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor, + weight: torch.Tensor, epsilon: float) -> None: + tmp = ipex.llm.functional.add_rms_norm(residual, input, weight, None, + epsilon, True) + input.copy_(tmp) + + @staticmethod + def varlen_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + seqlen_q: torch.Tensor, + seqlen_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + pdropout: float, + softmax_scale: float, + zero_tensors: bool, + is_causal: bool, + return_softmax: bool, + gen_: torch.Generator, + ) -> None: + ipex.llm.functional.varlen_attention(query.contiguous(), + key.contiguous(), + value.contiguous(), out, + seqlen_q.int(), seqlen_k.int(), + max_seqlen_q, max_seqlen_k, + pdropout, softmax_scale, + zero_tensors, is_causal, + return_softmax, gen_) + + @staticmethod + def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + ) -> None: + assert kv_cache_dtype == "auto" + ipex.llm.modules.PagedAttention.reshape_and_cache( + key, value, key_cache, value_cache, slot_mapping) + + @staticmethod + def copy_blocks(key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor) -> None: + torch.xpu.copy_blocks( # type: ignore + key_caches, + value_caches, + block_mapping, + ) + + @staticmethod + def swap_blocks(src: torch.Tensor, dst: torch.Tensor, + block_mapping: torch.Tensor) -> None: + torch.xpu.swap_blocks(src, dst, block_mapping) # type: ignore diff --git a/vllm/adapter_commons/__init__.py b/vllm/adapter_commons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/adapter_commons/__pycache__/__init__.cpython-310.pyc b/vllm/adapter_commons/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10fe16804ae0776a1e07c775f9b6f98a27253765 GIT binary patch literal 161 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HEeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_equ^uK}l*+d~$wnZhl^|etdjp eUS>&ryk0@&Ee@O9{FKt1R6CIA#Y{kgg#iHkTqfKA literal 0 HcmV?d00001 diff --git a/vllm/adapter_commons/__pycache__/layers.cpython-310.pyc b/vllm/adapter_commons/__pycache__/layers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e94d5a815ecd50e3fcf59e28b5386d9cab3823d1 GIT binary patch literal 680 zcmZ8fy>b&V5MJqiaGY~+LBmX5z(stM1{x0&ph$%qpt)v8SW0}1bSESkVkVvQGAVck z9z-n_uRz7>JY*bpW_R|x-P_%7SN&u%29on1Ki542_?5t+EHXv41)~BOXY|6f${zvyw>XP;`LhOW5h;fW)db!cbuMod%gx*J7+I-B~ zF2tvZqU-t~LQ69sx?>6rOb$$XFD^}xSo;x~F-)^5ulVqFIV42ONUJVrEHLcx(n( zy12#4!DN;HL=N_mhx~>7kh$iRvu{2m=^iY=T@tEFicn2+rl)86>zOM~z6 z-~VyA)7G?q;$ZzVVek}M_BS9*VdBV{)sQfMA9Sy+76R2_oDRRfFslX!^=PI z`@ta(P~rM#z~Cvg>|G#Sa|zR23T9EJpAlE*2J{VPo)Iw2WCTDrWak<4=Q$TIQpTf6 zKw)M1VISK*v<#2wgq&y|c>R@jP9_L1WMHq6@n|OP7&q`jcCLvnkp1TQULMc)=HdKN z?u!GSKl&hE9LA61&-S8lx;H-tw5S=h@$A#Q*IX}$go%*>SS1sNiHdnVEt z&MxpH50C8iJPnf^S7Zp#HkJ=I%&8R@Xg-7xxFhjn?s>B1d7yt0VZY;fe_8lZF(XDG zD$p0=4iY?yHG-H#7f1)1@{FyuDAflEuKiC^|RAXQKit~1IFZXOV(&**J! zW_+Q|hzhTcYcY$PaFZI`u4Zi3JnOj4FEm4QTM&9J*+#tX2?PgAn>p9iKuegJLbL#R)o zWp~MDG=6H#s7iDOq&9H8W;0>!EJ+Nn?sW1oteQp ztc!fNyTR?TzT)}~drnt7{QgLv;TM{#el5d_f#Ib5CP3(U+f{KSX) zAx)LH-kj$#Ac{O6LV$WiQ3@fed2r)_8Vo~JU;cGpaJZh1@6@Pq{Df7JJ4*#f9fU_b zag_Gmmb3bvLCSnmjgYAXoEnpWb-Ns7oWNeB@09uY9?hnEYY8hu< zAulp_;HgByFF2jmoR>qLb-Nyvg4~P!`5bQBm=?(aH@uuSF}%M6O~t8YTg0LkT$<(G z6_9l|Ut|VL@O&YX0K&EUuuq}wLsPru;Ob8yEX%9~kBEpTVi)GmNrC7hk>jP24bKA! z&-)1s#5R!SPCX1IbwxWOK}*OiOQA&v#EHD>9Z_K^Xx-hi3S+@@_wD# z4e2*37vD!sSliyHSU52I3?+UDq~6t4(P))wLv|&;OM@@zJYSvnk5I)Wn+H_|TJ8HQ z5Gu^3%Xe$dE=$&G!T*Xvd{Co)Ypn#X_QMJ+@Whu7>(pOeARGTDk~02(S_7!r z*5;iJ+3Q&f)u7ZmtCZGSx+V#sy9dIP@$E}bC4oPmqy;4c-t5Ig2}_Ci9@JDNHD#xr8Je?uyly; z0{J$y4EMA(N)6ijcdt)%cz2PKj-YqBb%1xPYHFX!_@>nTX&|r~DEQKXvI>>70IME{ z3X^&vDx6Iab7(v{0i6M%l>h($ literal 0 HcmV?d00001 diff --git a/vllm/adapter_commons/__pycache__/request.cpython-310.pyc b/vllm/adapter_commons/__pycache__/request.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5beddc2851796a64104cf25c96fffd281854118 GIT binary patch literal 1203 zcmZuw%T60H6tz80NG2pj0s4@tY8Kt3Dx|EssSpoURX0&pq%N|tCS#`=m>0G~MZ}8m z7gaxiMSr0m;%!&_LO1l@Nm8J2CExfyJU-{#IBzz60=D(@*JzIs@*9KO#v#~&WnO}l zMAC$$v` z1Us1JO1)tI>?JvoutjeVe-ml4;szco$a#@!?WE%7C zhFpedSyli`%e8i8SXiY$tK+dUcJj&(!1h9;f+z`%35K~2!dXwCXH90aY;skt>{w?J z1!xPyJ+6IN2F0=DoLr3jmQOWR=7e%Gc&w}^`k z8|tshZoEy$qjW3%)=lC;w>SljvhGQeq}@3t5#?!`XQn&HwppAye^HBN9bv%S2=^w> z?o!>n>FV91;K7lS!VB1(kwX~l5z$RcCBx`D`N5|29EJfRdc7-TJXwonkdBQF1}fMN z-naJ~>}F2@P(mxVYUhsnIMB!tW( zNY@IT7fRbxjgmSq&g=})BMhtg;7~=@@j;#^dKFT79o>ym7+sAY|C$~dEK o>U(Y8=4G|l;m!*Okwz}%eP#NguesX08S-uX`<0Jvg{*mh0cWWa#sB~S literal 0 HcmV?d00001 diff --git a/vllm/adapter_commons/__pycache__/utils.cpython-310.pyc b/vllm/adapter_commons/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a9082d920faa1d4b8695746efd0b25df49ce5da GIT binary patch literal 2979 zcmZuz&2HO95auq)6-`N&EXR)RIE~``G(;M!1`4zXBWMsexv53m9!j7PXzj{YOp5Z7 zN@|1dA+eapG$MaY;a38tASb7_JeRu1Pdb1>?HVff_x!u5nXP&P1ta@e;S* z8m`3`1mQL!1W|}GU@JMw=2gB3s0CgFR2{x0wL{>1gD=C5tt3pJpGdgIR{&9-xt&cU z^$fnsuLGiT~A_eGrA9`}Z^kexnHYhE|*zY?8%DQc%RF4USAi%xGe=%!4Hcq9j19Yy2= zdHkNj7i|Gaw1m9UWG$hIc0mZt-oe^?;5gZbID(FP`^RSvN%dhkbaYvQL()RB0OW%v zAyKsdp0wg%*c$eSk77?Aiulpj!RRR130}AS{y}Sa8Xtv&)+^r+T7?bWFbKjy)EdQo zKiVFirj056bC3yFN0@XLwTMj&_>+q;O-nf-U6$goAAwHW$r6$V67*w6s=%dXR6V~F zZH|!Gv_dTS%YCtmz*681hyB4}TJgOo?i90nB*v4>1!!_Hkf-peJXiw;O*_`5Jg2-P7m_)2xM>41HGRn*KEtEi9gt>z8Q_;HtlPd{{VV)?GCt?iZ-+@wG zdrvPw5xD^7GrW3B{z|zLj^fB zD>Y@^oU}6R7s4LG4N8_xC0dwTQk;xL6h~Kber(+}lv)(Lju{W-4gps_&cbcTEdZ&Q zyUUPZ{%IjFQrYCLeInj!XFduW3OL)XI|KM6a#zD z*obYIWQBX7K?k$2Uu8y}n^3am1vzJlk(e))c?pAZrR`^hTbJRqQ<8V#CvD6GFEek+ z54~~ysim|&RgoEJW!!KJt+DaCi+Fa@kX)9{=N;3SzAtkKHlqlGUD1tkp-W6Bl%6&J zBD`s}SWz8k#!*-v3_g`(zX1a=*3L~%IDHFF+k?D5XOQ1a^*Te|j0xAX`5ziL5(?D( z6I}R)#1v8IrKFUYYudS$m`MqK7m|fN?H6`#gR5=KpOc4)1uGV;7zs=4b@iN_e1&=C zu=Y%;VGJ?s`|w5BQTm`19EIj9X+p3W|4vWf?a6YHs7ip^r`unuh|PP_-o?^{7iIYP zIbMlX#WaJ`kBqR(GiNS;Ghc7-`eE1eqsQBm(`TFQQE(t+Gwd~UKMURrLN5HM`QU8F zY4)S$p%n1O%4U4z4S=l3`)Y*nUg#{;9fTTK9gHx6daN!;j2Z#&L z*mS4L>gYc1EM>wwAS~{PU!lvmcD@9i-n9qfbtjv<)dCtqsdBeInP#_V{ba6{tbEcp zDsQkhn9c~L>|@2Cq@pXmH0lXr$O~?)lgMmQZ=XyRo74Hr2MH~Y-<-hKJvw5G(UO)WK(N^-{>VJ_GM!s g`Gq(UW#pczp#qKZX~qRld{yvS3nsHObKcNovwQTrPrJkh!)q9y4Sj{?C@N$CFx zr;w0Y{`{*Kc96^~qmzYE(p(fG6MM3k=y`Z7`mJRKJ3fB%*>u zOi;;x^3IrGf?s=4OZu`cJQ)De68^P!MXzYo0jw>%^5BGsK=$0PCpu@o=!ygMiTdCH zxQp~eUkn^N0CXsaTXZB29XbN^o;VU?XB~p&!P&zT0(VZtu|tnU2hzT8kDqT-{jtzL zOZ|P4C+AXIU&zaJAr}lXsTb~1UVv2iIIrnz5<{&|$qk)_R0n`T3B5{YXU@1G`T(%M zzMC3VPRq1>Y!ZDg&EwD2@ zRc2Dj+(_}Y)`h;I8nbpOZB!3T+fA!j=E*#h0_`_Q$>Zxs&6IAPZJ_1LrL0V4+bSu` zG(Vp$I0V-V_ccL11jCWjBiMy}L**FJd#JF^d!)vdG%+-AdMV>=O{+t6nSfe{&D>v@MW`*Q8C1dN4nrZEM+ip!K75|mV`wdJ6TB_o< zMDL*ss`aPMRNi&Jvz`9&TKcHr>T!KmmesYW9`5eHd-d+Dm zbNcUihfEJ3U>1&XCi6teIJVt5R)tt*xbMaB%Vm;n|7n>*FD~3?LH8lf2Bj`asm)4b zqg!uY6qz0)1s@eoq;8?QkLm=J int: + return len(self._registered_adapters) + + @property + @abstractmethod + def adapter_slots(self) -> int: + raise NotImplementedError + + @property + @abstractmethod + def capacity(self) -> int: + raise NotImplementedError + + @abstractmethod + def activate_adapter(self, adapter_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def deactivate_adapter(self, adapter_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def add_adapter(self, adapter: Any) -> bool: + raise NotImplementedError + + @abstractmethod + def set_adapter_mapping(self, mapping: Any) -> None: + raise NotImplementedError + + @abstractmethod + def remove_adapter(self, adapter_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_all_adapters(self) -> None: + raise NotImplementedError + + @abstractmethod + def get_adapter(self, adapter_id: int) -> Optional[Any]: + raise NotImplementedError + + @abstractmethod + def list_adapters(self) -> Dict[int, Any]: + raise NotImplementedError + + @abstractmethod + def pin_adapter(self, adapter_id: int) -> bool: + raise NotImplementedError diff --git a/vllm/adapter_commons/request.py b/vllm/adapter_commons/request.py new file mode 100644 index 00000000..2bb17fdc --- /dev/null +++ b/vllm/adapter_commons/request.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod + + +class AdapterRequest(ABC): + """ + Base class for adapter requests. + """ + + @property + @abstractmethod + def adapter_id(self) -> int: + raise NotImplementedError + + def __post_init__(self) -> None: + if self.adapter_id < 1: + raise ValueError(f"id must be > 0, got {self.adapter_id}") + + def __eq__(self, value: object) -> bool: + return isinstance( + value, self.__class__) and self.adapter_id == value.adapter_id + + def __hash__(self) -> int: + return hash(self.adapter_id) diff --git a/vllm/adapter_commons/utils.py b/vllm/adapter_commons/utils.py new file mode 100644 index 00000000..1e9adca5 --- /dev/null +++ b/vllm/adapter_commons/utils.py @@ -0,0 +1,90 @@ +from typing import Any, Callable, Dict, Optional, Set + + +## model functions +def deactivate_adapter(adapter_id: int, active_adapters: Dict[int, None], + deactivate_func: Callable) -> bool: + if adapter_id in active_adapters: + deactivate_func(adapter_id) + active_adapters.pop(adapter_id) + return True + return False + + +def add_adapter(adapter: Any, registered_adapters: Dict[int, Any], + capacity: int, add_func: Callable) -> bool: + if adapter.id not in registered_adapters: + if len(registered_adapters) >= capacity: + raise RuntimeError('No free adapter slots.') + add_func(adapter) + registered_adapters[adapter.id] = adapter + return True + return False + + +def set_adapter_mapping(mapping: Any, last_mapping: Any, + set_mapping_func: Callable) -> Any: + if last_mapping != mapping: + set_mapping_func(mapping) + return mapping + return last_mapping + + +def remove_adapter(adapter_id: int, registered_adapters: Dict[int, Any], + deactivate_func: Callable) -> bool: + deactivate_func(adapter_id) + return bool(registered_adapters.pop(adapter_id, None)) + + +def list_adapters(registered_adapters: Dict[int, Any]) -> Dict[int, Any]: + return dict(registered_adapters) + + +def get_adapter(adapter_id: int, + registered_adapters: Dict[int, Any]) -> Optional[Any]: + return registered_adapters.get(adapter_id) + + +## worker functions +def set_active_adapters_worker(requests: Set[Any], mapping: Optional[Any], + apply_adapters_func, + set_adapter_mapping_func) -> None: + apply_adapters_func(requests) + set_adapter_mapping_func(mapping) + + +def add_adapter_worker(adapter_request: Any, list_adapters_func, + load_adapter_func, add_adapter_func, + activate_adapter_func) -> bool: + if adapter_request.adapter_id in list_adapters_func(): + return False + loaded_adapter = load_adapter_func(adapter_request) + loaded = add_adapter_func(loaded_adapter) + activate_adapter_func(loaded_adapter.id) + return loaded + + +def apply_adapters_worker(adapter_requests: Set[Any], list_adapters_func, + adapter_slots: int, remove_adapter_func, + add_adapter_func) -> None: + models_that_exist = list_adapters_func() + models_map = { + adapter_request.adapter_id: adapter_request + for adapter_request in adapter_requests if adapter_request + } + if len(models_map) > adapter_slots: + raise RuntimeError( + f"Number of requested models ({len(models_map)}) is greater " + f"than the number of GPU model slots " + f"({adapter_slots}).") + new_models = set(models_map) + models_to_add = new_models - models_that_exist + models_to_remove = models_that_exist - new_models + for adapter_id in models_to_remove: + remove_adapter_func(adapter_id) + for adapter_id in models_to_add: + add_adapter_func(models_map[adapter_id]) + + +def list_adapters_worker(adapter_manager_list_adapters_func) -> Set[int]: + return set(adapter_manager_list_adapters_func()) diff --git a/vllm/adapter_commons/worker_manager.py b/vllm/adapter_commons/worker_manager.py new file mode 100644 index 00000000..83929e82 --- /dev/null +++ b/vllm/adapter_commons/worker_manager.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from typing import Any, Optional, Set + +import torch + + +class AbstractWorkerManager(ABC): + + def __init__(self, device: torch.device): + self.device = device + + @property + @abstractmethod + def is_enabled(self) -> bool: + raise NotImplementedError + + @abstractmethod + def set_active_adapters(self, requests: Set[Any], + mapping: Optional[Any]) -> None: + raise NotImplementedError + + @abstractmethod + def add_adapter(self, adapter_request: Any) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_adapter(self, adapter_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_all_adapters(self) -> None: + raise NotImplementedError + + @abstractmethod + def list_adapters(self) -> Set[int]: + raise NotImplementedError diff --git a/vllm/assets/__init__.py b/vllm/assets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/assets/__pycache__/__init__.cpython-310.pyc b/vllm/assets/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d8da66521d1babab7d2b2e4f99fd5b69d9686f0 GIT binary patch literal 152 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hueo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_eqwQPYDuwve0*kJW=VX!UP0w8 V4x8Nkl+v73JCLEpOhAH#0RVzbBPsv@ literal 0 HcmV?d00001 diff --git a/vllm/assets/__pycache__/audio.cpython-310.pyc b/vllm/assets/__pycache__/audio.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0638ff696d3c3ee124fb33cd694cf8940d43183 GIT binary patch literal 1260 zcmZuwy>1*g5a!R`?%v*>bWs$?R^@gJ-@$O<6hdIdHk=;2Vt8kub?Iw?jf-Pn7|B&9PZ3F-^@1SafV<${{4?-LJ|5i9PX|N z4$naB5|l&|YgA!|aj=Od6=k%FSk#}X9#k=ldpptzOW;kUQ#OQsptEYkMpe#o@W*Vd zuuNongI^&zl-WBZvukpV*_MosQ8D@pvY`U|p%hNcwXn7zJ~=F%GD7>oN!Mxx?xZvN zqABYm@ZLLDj=#~m;%#@P%Q=USl(T;4&EesX{P+p~=JfdwFHiXC>q9oFI_=7;kwR~t zoj|a7VVX--&qM0FYXFC5Aog2O3Nb7ZBd17Fi60;pNwSaBKvET}kFR=(B@Pmd!yxSrC@LE$h0h&-q+v&Bm25E4~zxYf+u~s1}v- zu~Duw^&A6^e^=;Fkmn$FACyCDbb;SPr#=uzB)K71*wKq<4X`D>AwQ8Taz))>O$HzeG zi|=0hM3-l#vBF2X5z=R+EoQD=wH;I-tH5SA*rC*o`VuTpVTkv@umse7a=CQ}9=MK*u}?VD~eoWt05 zP#^a>=RbEsZ#;&ESlgV7x^A3sWm8*tleVU5m2oQ*ro@xFWl1fCF=AythZ_?r^3*zG z0){)-dXxd68G;&v^h{V JA}QU){{WkcHf8_- literal 0 HcmV?d00001 diff --git a/vllm/assets/__pycache__/base.cpython-310.pyc b/vllm/assets/__pycache__/base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2976baa6676200bf84e141b70577a58786888db5 GIT binary patch literal 1281 zcmZuwUvJws5GScWmL1zkgQ4lLq0p-_*g_i&9fo08=6P+gBpqgF4|-7)la7^AqDE3l z6F9(t^tGR$7_dF|Vc0j=2hr=E_7w^Y*^z3n038J$@5mGH?sq?mUax~-Jox?3Mr;5R@}0>$Ai2bx2^8OekbmL z-ip^~z?=o4ZQ6P1#$ATnE?uWo%8F zGEe7A28R26s+X_^&q|$%BIWyF+&$*vDCNmg>oS>(qF{4#2q*V`+}+(vp6=~Do+RH+ zX5%Nx?CIX**}?4MQ{${YkD^n~^I=&X@oYX!m10_r)b^-S!!xGz@Zl)U(+dc6rbcs- z*S=)BlEvKQb~7+2fF~NmR3C%TNaGW7g)h+xoj4T3M_v&O7S!cjeD;{>P%l|nLM@>d zR<{L)3n4?Lh0KcMkczW{i649Yy82 zUW&qk6E)GJqg1g`d0zLxx#>p&*LF?OK8OzKW0%MclS;72I4jbe)orz%l#(s77jpt9 zH^ac?!L*RYplEajg!>3xk`*}txZravWMqY4*I7BglHZW}WQD0)VR;``Zwmn3LYGK8 zC$9EhBkHZtYxE2FYM=VAfUGMA0B>FV)CpmH^A682EgC>WlV!M&A`cG_)%O2#;qcJ@ zn-(-|I{#kdZneqWH?scmrlD2R&?;#FoJsS*_TSa&W2S4me13Ps84lHuh{IhN&nYO?S5b`|kG#jSxw(E^p z2VZzJ(8l+(`~c3>JrJE%9|t7B0q$ZKCV|iY$MxhE91igQV7+cHssiE&uB2(M;ig^! zdm29a`n&`lOVjJPZ6b{fL*<&WH>~~NTWyNA70J8c(D2AnTH1KGk~id)wu0b`T+oWM fFU=lNqTpWNN?3Ve2gR(W@h8_x0QaK-rVno50 z8=aj$EOb#*{M%Dn2gW^a2$?5an8I*I(@I)7|~Mj~+gy4|bm@m&f|G9|&@;o5*bm2)L)&2@TIm}x3Rewe2_{p8xmV_-Z>;K!woLI4VIRd!11z9mfI`5ta={w z)@($nYcHiXrf$mT!s!X6ujjdFZxsAsWY1EXS5<9t1OGvTPb3wzS(!!CbG7MkT>z1V z&v>P4r5unIq^|MG!Brg)u5|50j1oe~IZVPW+}r@DwbTE~wQ|#IL$i=TYy+ju;yvEo wYcgjWZczH|p!A2i=IwYo>Y9N6!eFCYZLzoAN70Vd1SL2k2~LPlB5xD_2F&LzegFUf literal 0 HcmV?d00001 diff --git a/vllm/assets/__pycache__/video.cpython-310.pyc b/vllm/assets/__pycache__/video.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..098a5a1a8f74b354a56cc461aa6b6b37adb1820e GIT binary patch literal 2864 zcmaJ@OK%*<5$^7J?CfxNNs3E~q>*6bz=78hNl9@M#~=jJq%9a$NSTs+7}yz%wtFAs zywuYxn~Nb3gaVQrbjUd{Kpu5Seu95UBe$G%_Du-FRP~S|8N}(u)K*v3^rPyls>zy7 zpTM~Fx9^6p>xBFhCzpo@le_Q~{{X@Xr!h%rKq>MpW{DM8hPPrnaRMiC1J}&$xE6R| zW5@Nx4tyhX;;Ez&Gz{;?&7>8y3}1_Nq6}b+>mssuJcGmy*MfgoWyb*_M+ZUbeXpAj|!!0_eV;|DBcG3 z^spZe$4^3@eVxWx#6jeInnVM%UM!*{k44y*Q6dWX%aZWfh>J`wDR~f%l01{Dl7xBG z+lRM`SF}1qb0Ac;ZOBJbd;L)?Qdma2QmC;^duVq)Y+zh?HsL9@fGBbVk$Q_9(vt2` z&fsUgV3d@!Bp_vqm2mGvt7OO42`M_t=5|TAQ&R4}ut9EtKWdZT`5hQrXIJY)Deq*t zNP+2$@Qxk4QfD|G3`XgoAN7Q>mm<$T?8sEvVtUg-~5PsNweEh|31}A*h>~I>?2tn;@b$ zyN;X!rFN2i^sQAWsa>;IA<>OpyLx%r91q_RJ(bA=?N@?4Qp2Ei!H2NGOCsmt)z=@a zsU%;^NBN41B*bv#*U5O8+)BP)i$_n^@&h%@(zR!CoU8%7geoehtmX&0eGZ{BL>T=H ztXV7q@!b~nm`6Rz-u&Eu*{-|}%2%f_w2kM|emYLh5Z7a%J0FnS1asM7BMh*c7 z$AF0w#DaCq01I=<;vo4e`4$lJ2dZq4J0;t5PbeV99*{#v)k?_2E%Md$p$l5?gnY>k zYlmK0lQ&D}xL$fEP#>QD1a$rhp+rp`H%bTga8$Ddj2$uHTBUOY=-emrG3>~2M{boA zs9ouRbH1(N8A=Dx=x$GzH?wifJ87mW7@7!lFf%%nl?$B|5$}jJ8xMv6P)xlFr_%q*OTVVw?zBXA1&uE-TQ{Q|GEM~EsMrDY3$FbyqIfJmmNEnM zj8dg7h;-22+;|W^*xh*$ezLpq+mFM|oyXgcv?HT*AheCA(r%Q)3S0w7V*!BGEX}o3 zK(*PI7#wXumUruN33$0|1ifdswbhIAprJAq#b=?_Zgn$FN7{mlpdB+e7_7(wzFj3) zsIoBSk(ALv@eYWK_kg(62ROCp46|tiem3dteoWLxBjLj0`lDs=KS?8Gf*);>RK0plkdLSaTPfej>R8Zc^wadeb@!>gYTL9Z+R1@kD#4Eq_c@g zdoaB+u;|}~r`QD|NI*FW*kf#&bcqO?vvo+Qz!5I+_Bs(Y?g&rR_v+9P8FzWjG*jQm zr}i2kgT`xQP3}X$rU2#${|&5Dli9Og#zm5aw^ozH#l@g9~D3JnOrvS&dlfD-f&LbMIB#abqRfdnsaR()w{yAW|-wo!t>{cnu>7g72Q z)Zb$T_Zb3uGJkP_)eEa1JfnOcm;WV@D`!FYS2z3>a#-dsL1`+@-z3-PsQL|%$-BOO>nPlsWYLa~d~C`*eltj_QwRC&3C zMl98VL=>2ukRPDJN^|%=+1}c~WS1YJ^5VRPLmf0%gtuHM%Y} zaqYgPr`~)Ei9%x1>Uw{iLOaajLSlfm3*8kOe036Ya0gmYejtAe+Mr?h)iayBD4M?X z!GPaRrsoFMgHP{Q$vuzlaWxsoY6R|y;?=Pl!Dg7FCfBt61WKoA;qn>^4Q$Xaas!Ep zmjMu7^MBo6d+U|s@8DNdfvMUc4LSoI7E2*;tYI;$O_$qj(L05XdkTf|)brc)KaJ+n A(f|Me literal 0 HcmV?d00001 diff --git a/vllm/assets/audio.py b/vllm/assets/audio.py new file mode 100644 index 00000000..49bb6aee --- /dev/null +++ b/vllm/assets/audio.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import Literal, Tuple +from urllib.parse import urljoin + +import librosa +import numpy as np + +from vllm.assets.base import get_vllm_public_assets, vLLM_S3_BUCKET_URL + +ASSET_DIR = "multimodal_asset" + + +@dataclass(frozen=True) +class AudioAsset: + name: Literal["winning_call", "mary_had_lamb"] + + @property + def audio_and_sample_rate(self) -> Tuple[np.ndarray, int]: + + audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg", + s3_prefix=ASSET_DIR) + y, sr = librosa.load(audio_path, sr=None) + assert isinstance(sr, int) + return y, sr + + @property + def url(self) -> str: + return urljoin(vLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg") diff --git a/vllm/assets/base.py b/vllm/assets/base.py new file mode 100644 index 00000000..f97e8c21 --- /dev/null +++ b/vllm/assets/base.py @@ -0,0 +1,39 @@ +from functools import lru_cache +from pathlib import Path +from typing import Optional + +import vllm.envs as envs +from vllm.connections import global_http_connection +from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT + +vLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com" + + +def get_cache_dir() -> Path: + """Get the path to the cache for storing downloaded assets.""" + path = Path(envs.VLLM_ASSETS_CACHE) + path.mkdir(parents=True, exist_ok=True) + + return path + + +@lru_cache +def get_vllm_public_assets(filename: str, + s3_prefix: Optional[str] = None) -> Path: + """ + Download an asset file from ``s3://vllm-public-assets`` + and return the path to the downloaded file. + """ + asset_directory = get_cache_dir() / "vllm_public_assets" + asset_directory.mkdir(parents=True, exist_ok=True) + + asset_path = asset_directory / filename + if not asset_path.exists(): + if s3_prefix is not None: + filename = s3_prefix + "/" + filename + global_http_connection.download_file( + f"{vLLM_S3_BUCKET_URL}/{filename}", + asset_path, + timeout=VLLM_IMAGE_FETCH_TIMEOUT) + + return asset_path diff --git a/vllm/assets/image.py b/vllm/assets/image.py new file mode 100644 index 00000000..8e303811 --- /dev/null +++ b/vllm/assets/image.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import Literal + +import torch +from PIL import Image + +from vllm.assets.base import get_vllm_public_assets + +VLM_IMAGES_DIR = "vision_model_images" + + +@dataclass(frozen=True) +class ImageAsset: + name: Literal["stop_sign", "cherry_blossom"] + + @property + def pil_image(self) -> Image.Image: + + image_path = get_vllm_public_assets(filename=f"{self.name}.jpg", + s3_prefix=VLM_IMAGES_DIR) + return Image.open(image_path) + + @property + def image_embeds(self) -> torch.Tensor: + """ + Image embeddings, only used for testing purposes with llava 1.5. + """ + image_path = get_vllm_public_assets(filename=f"{self.name}.pt", + s3_prefix=VLM_IMAGES_DIR) + return torch.load(image_path, weights_only=True) diff --git a/vllm/assets/video.py b/vllm/assets/video.py new file mode 100644 index 00000000..05e031af --- /dev/null +++ b/vllm/assets/video.py @@ -0,0 +1,85 @@ +from dataclasses import dataclass +from functools import lru_cache +from typing import List, Literal + +import numpy as np +import numpy.typing as npt +from huggingface_hub import hf_hub_download +from PIL import Image + +from vllm.multimodal.utils import (sample_frames_from_video, + try_import_video_packages) + +from .base import get_cache_dir + + +@lru_cache +def download_video_asset(filename: str) -> str: + """ + Download and open an image from huggingface + repo: raushan-testing-hf/videos-test + """ + video_directory = get_cache_dir() / "video-eample-data" + video_directory.mkdir(parents=True, exist_ok=True) + + video_path = video_directory / filename + video_path_str = str(video_path) + if not video_path.exists(): + video_path_str = hf_hub_download( + repo_id="raushan-testing-hf/videos-test", + filename=filename, + repo_type="dataset", + cache_dir=video_directory, + ) + return video_path_str + + +def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray: + cv2 = try_import_video_packages() + + cap = cv2.VideoCapture(path) + if not cap.isOpened(): + raise ValueError(f"Could not open video file {path}") + + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + frames = [] + for i in range(total_frames): + ret, frame = cap.read() + if ret: + frames.append(frame) + cap.release() + + frames = np.stack(frames) + frames = sample_frames_from_video(frames, num_frames) + if len(frames) < num_frames: + raise ValueError(f"Could not read enough frames from video file {path}" + f" (expected {num_frames} frames, got {len(frames)})") + return frames + + +def video_to_pil_images_list(path: str, + num_frames: int = -1) -> List[Image.Image]: + cv2 = try_import_video_packages() + frames = video_to_ndarrays(path, num_frames) + return [ + Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + for frame in frames + ] + + +@dataclass(frozen=True) +class VideoAsset: + name: Literal["sample_demo_1.mp4"] + num_frames: int = -1 + + @property + def pil_images(self) -> List[Image.Image]: + video_path = download_video_asset(self.name) + ret = video_to_pil_images_list(video_path, self.num_frames) + return ret + + @property + def np_ndarrays(self) -> npt.NDArray: + video_path = download_video_asset(self.name) + ret = video_to_ndarrays(video_path, self.num_frames) + return ret diff --git a/vllm/attention/__init__.py b/vllm/attention/__init__.py new file mode 100644 index 00000000..2cd4ad3e --- /dev/null +++ b/vllm/attention/__init__.py @@ -0,0 +1,17 @@ +from vllm.attention.backends.abstract import (AttentionBackend, + AttentionMetadata, + AttentionMetadataBuilder, + AttentionState, AttentionType) +from vllm.attention.layer import Attention +from vllm.attention.selector import get_attn_backend + +__all__ = [ + "Attention", + "AttentionBackend", + "AttentionMetadata", + "AttentionType", + "AttentionMetadataBuilder", + "Attention", + "AttentionState", + "get_attn_backend", +] diff --git a/vllm/attention/__pycache__/__init__.cpython-310.pyc b/vllm/attention/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99ccc3b926324ba258fd097fb0ce333fc112aae4 GIT binary patch literal 519 zcmZWmJx{|h5ViBsq@_hgLV|&THA541VgOap9hNeAvE15fBkUw{uAuxKHirHMeu!5l z{sI$rilVBVy~s8#PsLnar12)xf4KGH080%Vw13MGlmv ZRb%2iRr0MAGi)tucit5m217jbegUwYjz|Ci literal 0 HcmV?d00001 diff --git a/vllm/attention/__pycache__/layer.cpython-310.pyc b/vllm/attention/__pycache__/layer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38235bbc2d9527658056d59aff8bc439524f035b GIT binary patch literal 3236 zcma(TO_SV4wbjySG@9?(_1Y#Rh>wB{?9RkC!KuIkW^Jd6BpXPK4}vO{R^79sU46N` zXV*KWxv(yB$}ga(vX43N7x)DnIrWuOZmA+b9KhyvYi7Jq9FR({-+TRDcfZe8-fp`H zo@>AV!}N~>g#N6Ei;o4wkKyCL0YH$Ch(T~-q}a!)>6@wLTWQU&8H#Tvwr}gPo;tpx zc~;U$UEfWcepA!6q?NY)w#MzGlXm^C#_LHh?fZR=JINs3@HY(f79tJeK0?H$&e7(% z>0ct|2WZ%w_ihP6GZE*RmxO1OjXr_Xp`j(sTiKbkZpV?3);lp5(s{2?VqvmB#InCu zdzXrkgd&vfHS6A4L5GIy9a0g5LS(@~7#-7$0I#_hMpL?%XOs94$idHNA#6Sm)yivb z|0f}*Z{OFVu>3S9kvi1H=jxK+e*%Oe-vEbVV)$kQUwF-iMP4Q~Y7ra0I;|0h0(KRg z0VB{?5#TQ14vbA08!)zDbZL{eCRoq6fd)y^p-s{SybXAd;K`K_(ZnEqGI-SXJG6V$ z18Rfzj|OB@!%G@&fd=qE=Q7_MYW^nB%?TzikuN{8{7WAq3;A0R znpb3F?a(kX5L#2qK)J35&%J5Py(kGe_e6M1x%Xg3+1a*tOaX)$@lL{IMm>lz&l&eZ z?!{R#6C+Th6}plhEZs zNDk+KHM~ilB>8Ea9llu|_S(q%K;#ULv)&xe`9F>c+bWT_?|YHTmuiFSBkyOFO>&lc zucos^#5+?Ol5vGRKAEpTJxy-&rARg32{0ghN&UkU6>bXPXA zSsJL7xooH*;PITw8jr$+%9ffuK3UGU!X!S31D@ms<+7#E8i3y*xn=8tO6?UjI7ss7 zm=_`AGyoW;Ty~TEFcv)E`2^&`LfQqRllYZo<00yY6x{l0N#y+_EQ{=a}#P6#d#L(UDE; zW6a(WEmA)>fYXDO+Q4r=?v&_>VIa|c++?5On0I`#qhsVj&1AXo_^c>$CMfYvA?H1i^B~)z<06kU!AHYQ2s`8xlTHxCSuBFU zZ&pDj0(zWubyTYa>F^{bkm>>CJIPOFKjy0WRrwc87^SiUiE~li366s*t(@Sb!hGE{ ztJFz#>yvKG3#hcxMY1F_|9tcj}6%f;1tltg5YEH#rMY|EyhJ$?1+%T zDR*wBvuS!QJsm?qjEgh46SDD1lBDCci^tXVRh+SxVZWE)8L`dOILAQ)opPHFJ}@|kq$L&3s;(Sl{5!d0K{7z6!oLf*_b zUu4Y-QD|WLVQmn8-M5nKPeZjGti>{|j6ApLPxZ!?p1jb{#*p z9sGHxf&Xs1_;0w0|7y4JXSm(@i}jZE)B+s~bS%(uKnH^k1|7?BKi}G5YS-ZwdlLp} zLmffK)n;m|D^F;jXyY8+|3+CRahCZV)i6d-fK^YfOx7;B{GT%;=zuf=-KtGC^=x&m zYd8BhGza2DlYl;?5!^y-r27va>E6Bkqa7WanK*z>^?f=zWHIS<1+wba}(AHlF hhOS)|4C(x!%9lKz7}Mt?K-_!v+8J6V$MNsi>mzGP-^*_83E z_)1VROEQg>d^ISWW$vqfC8(O!Kr^*q%pBt}+1E{-X^aOov&Lg(eooH-ZFoAbd5^F(mcJSj_V!(BL#%~Q7|XUx&RkR08W9-TfY znTyW2yX4f|#VysDaOz*E<{5lx?&5;9r8tvL10!c0**3B<_nb5Jf#ghQUrx=Dfj8sK z0_yw--kdX!kvH73BReO4BafYQ-gXw;H=R?!dvif@PP_V}3(g|GmhiQNnKSNN&RIM) zcg(G0?S^AG=e{U|=hwj`{_DZ0AqE1JK2gv&@87j>ln!mODJ(j=q{5_t(Yx#SE1*N2#m2a-ytmDH9?ZCCHRJW`ka(X`XYnJtQJ=@P` zR5$DescQFL8+X^Pe`c+Jy1EPr zuRfFuXMfc z<-`^b-Q@CnL2o;_8th*2z0E7#y<|HIuk84KaK$dvb|rRww*%$2x_e?0l%`3TvuHF) zlaKz@WQIsg-t~m;N6@o_M5{VN?e@9d^%gqv-*?BBK4D2;yRj|Tv0Sxhb;Bynkp_}=!g}Q z+)8*@Rt7pGpQlPvdM59a+~l4l;9%CJL>)*6baa*eSGs}UEjKZe?csinXvY=8b6g|t zx*c!Ja~)%Ik0Ew$-@bDwI7TjnN9RjKVSp2A?AXF130^Y1Eh7qD!;24TwRFZ?V{OOR zvaffH;%peSX;6R&3EK`2*%_t6G~O}VL_GrYV%LjZ9&E!eC{dhNbB@vyELc#uVVN~$ zR>q66Ujh*1y<-);TdJ~dnO&x z!#dK?8AvuX!@BrS=KdzpGi={K6te;61-}X37#m4s^kU2(inFlNT&$~h9DCtIHg7|xvGqo9=?Z|_;BiLEt$cp`3VoZpSH$#>PapCNfmxFZ3q=)2S zvm{cd#W>;;)*rDi_N#or&ybOgZt$q#tN|~k9jax~!V5958rV-_ukYSW$F^PDv8bQc zWB4y9+1m9&C)!QRPJ$?!)*tU=!Pd&Y^-VwOJkC(25Cex>w6=r`a11d-+_gpQcF4~b zzbSNvem8_*$%78$fifr!)Is^6vR@rk6Zw(ySP}C`N)*2osQa2M4YYNMXUdqlm{gut z9rzz4Q?jG_NXYh)kPXIi7#%Rz7^a-VRD2|8`#NCs9A*MA_mcX76o2*;DVcPt2Qqtf zqO9duV}2u;+8+njc#bsl+>bAb6}G7ri!k4$+ohiUkyg46p*BnR~zW)U!dV3?^KX4+p0&g@SDW-^C43z&a{RAzi5 zIJb|~bI^ciKJU!Z?wWJ`57*h!VqukSloc-ud5kA1`gIGX@5t1$T%3~sU*EMU)Nu*frlqr^tB7G9n%@PD`g zkG5@4R#ITl*@;4M3|Hn1UDWl}dxqog;5@ea3)!MCqY{e^9wvgt8aa=5k>pX%V+=Q3 ztVn1li|OHPf&Lz)L$46=E7^Q7!sb%O=Bv?Jnk!xR3DzsJ+G1lYNiW=T#VChUJiJ1~ z1#5HJ3abbR_7OeJXG1UU1^Ya`>f0T6JMtY@yg1!Ib!_(2YQB+9v9V=d7wHGo!|9Ef zgEgh+DdB6`g)OxQdTV!L5B;E(0m?!8;mba%Wb{Uz*akx+5h{*)SUlZmDH0ImK z!HTF8SR;#FBb1ZWjH5|wYdu^bf-LbekL4CAYG>Pu9XP$eA>0Kt$MaX*8^i|$yGRY? zV&=KynDn6VOcoy!@F@a5mKWX(n}ML@R-7j2GsltcvJr?M6XX&>UOX1^&;l}EJ|>(u z&@`vS_o;uCnjcW}Lu&qtnjcZ~E}FEO>o(ITtA>Tm0%J+==9p(@=VL6nNayfkd$k*|9992i5&bIG(7F%KO?t{Yp|jkVI=x z0iO1_B)*r7J=Gm$Py&Yle^!MFOlo6Uu|ieuc+oF!l@3nRP193M@+A~@FkzqKNqIb%vk3iQ4sxvt+ISuUbD~97(nXOiE^jlfz{UxX2O#7#U#VmJt zy7Wxi*O86r;Nrg-b7mmrS^ytE#eUYMtW~At%x~pApMsw%6!-K2ByEJ^VDx1 zIz$Sq#8BMpQX+uR3SQdF_g9QttM}qo-vE~{fq0>@eCN{);_FqS_|XXCH;<$bmw!-1 zD-PQCBR{cmL350ao}fENw#?677oW4#?DB>?3NCu=nh{PSJGuI=1Va+=OF{(tclcE+NGqX=lKbnkr&1ZVjK+jA}kkzcHAwy=O?JMoIOlLIQH@V zC`Em2CpPC_Epfi>b83(!2^0gwr_@YPLvf#Dc^1{j@M%)zFzNO<4vQKAa7Hq>ZO2xY zn&N`hwKqNAOZKunr_45LR&yIQwZd9c9h*TqPKK3McWn{EWW*;#l(HoAB!$P}QMHD* zyEq+R;Y6MbNX_x@j@M=*yOE*MzYxpC zGC2Mnp7;-FD!Qs7fNRQgO{vJ05*7C<2;?YGz4X?GB@JuR2*&=bmDDliztoBPH#v40Sl_4`^V}@8 z+wCJ27%m`S6I4BjDHZw!;v#YYIYIUC08z59JMw)*i=Rpu!CjN?u85j>e-~9V`%T=g zvbqA-DsYkEa)MzSN9$ul=Q1p)kukvI+Kio-%ZE!om<4J+TmEVK~QAyxjNGNY9g*u;EE z{4EBW^3CQug8qdh-l8S-923QBLHbX}wt69I)sY{I5`9-tKcUk}g(Fd>hLT8e8;v=} z-i{oN_?*6vy~@~|aU$$aBB*d;jx$Xx5P(OOo$USI!gI9{LRXmQbYPqe~ai(m!@mewZ{B>X?jxA6ixk> v&Ur;q{;Zai|Es9V|0-qWf0T;yC#9-g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H!eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_equ>UYF^}$5IK=0ZQiRQ**!fy zb`rBuuZvZ$s@|)5^}ZN4o2CNKtAF>i9qp{5{2RW^J_fv8fs%AoMIj1N10_^#HPmcP z#dj?zg}SYadMPM}hHZouyCUB8pc>ZfT4>s)crOR_uwggCrriu%b}MY#?XY8a!Xh=X_ z?UH3^U7l&Rpml|;LhI^G>odf-tMu0X9rvoI@zT|6TfF6NCnjiF-aAWAxfF#hU--yN`*WGa%_e|cr_ljIVq zG>8hjr19p}G^G&|i)0OY6<&3m$PKCEaMN+Zn2ZCgHyr2T*bQV4tHRX0a{b2EJJ)aA zu`4$QveM2fPS*28VP^nSfRAr&rr~IFyHlD8cf-Nk{}gT#ZV2^R7LBty=Vc(6-2!QyOQq(QSB1d zX_;u$7;2ySQKCMyEBEpS(9hpa(41!WzIMwb9?3^f>UlGEf&Hohyc z&(h@wE99Jj=LOWrg<+Lg)*F2g!SULY@$&(rk`3n}P9 zc?l_i85`tf^2%e~-XIr1el~fhke_SvtR^p9&F-2dE|WEknN^&qm7i#z8;qu779Be? zl;15VAL;;7YRzH+56`TWv=ZDM@e|B;)f1 z8Rv^NX2~cJJ>)ijN>D*Tcn8XdT>y`izf%5Md8GQvBkgb0AA%MWhtg9SF7TwpORyGb zY<3^?`0hUb&zzNzkY?!a(qfA)!mtFzq^nJiQ+5QEuNRc(M|DkV;@@hY+PF{Kaw6QR zSAiILU{`u*~p14=+Wt_wU06X&qtiOU_Dz#!KR*$fk)9^ZA${i`eZO)Z(PQk(< z+yp!6YzMKon{ZS7JBdG`yn*ezd(IAZNy2ORCp!8bo@)MJ{RA2B>Fjr)m%WAnDPvzk zFvmz2w-A=`TyKXu$&NdsUja11g3OW}fJJvhR+vO}hd8Qdw0dXOB@kgUD++n~9_tsRExP@lb~I2kR%1 zrLi}lmwgq%+&0#|cyy3aG=P3dk+FcH8AM(OFM$S*kNyf~7pg+K_zb*$mu&2!Rl$(|c|APXrS zy@v?T16ZJnXYqO(6I%O4K+WrT8GEPMDV$;*0h$$FaU2qR=og{d@;%U@;8!w~23`VJ zlRby1O9);-Ae7%r*jpD?F~h%ved2b05&KGhlrnT}*ze+NIgOdO!;O11O2F7%#*QX} zD+tiHVYd)W4^hB@(BuI*N1h72U4fGP z5J2vteI3G?+)2Bv2*;(A{p;{=Jg9*CR)$)Y)`&q(q6nv|LaLb?Q=jXZI&6(vowP}V znzTt<8!!&Ww`q%ZgkzGvylXvb2m&$^bXI!p==5)tRUra1UAC7${h_u)^vNCc^D|_5rg2WK5+S*9CX~gqr`F8ZR|o5$*Ut4k0?tIg!Y$e`46%8E`m0Kw-MA4AZ6^E2o@=A z)Hc|epsg|$9rP{yN@f|AH>mBhfQJ6m1odiapi*%JSMNbWqr4rDBXUf=;(}M8B>w~; zSFcjJl5<2SWmK4U8C0wRim(FmI8~xmszVN^g4vs)E;2TV3$bvmMb3+!HmF{c)`y13 z*_5HTk@a>ds9DhqYL>Qzay3MRi&&iuYZaBS>?OqrvIcSMDYE_;F_(ab3*-zrE9mT! zHMk!Z>j5*9Z@w8rl4eHHZOaJsVltwhKlD8--hM#66gA#B0ewft3~mW(N1+}9e@Ih5 zq*go>8dG>Ns1YF9dYTak$;ljwh_8n87@ed^r zM^GUiK&vR25VmYGjd%t6AKm-bLgu!Rip&_iwiU$DtW`t5i=uu^+&yaf(F`|*m7e2j zxPYsAMk>z!W9-MPH#t3bCNoGk`MRGtG(vOFGPBJvPey~21A~l-%eN@#?cBT;WC~|u zdKlRW3Avo^T+T~Lo{->WX8}XcAK7~cP#N2WWjAvN>5W1HUh zBa}ZJ^^b@l?%6cbm1lAU4U~~kqNss;>P<)lh^diep+*M1<%I?t-7U6_0QHlw-;84I zJ^-$7$8o?MeD@Iuv-~#J&{VQr1R;V5!ITeCLq#Zz&!0mjL03|QRwWriL3pLFK+$yl z(;{>#H9my^X{N=LEoJ=^b0)mf^u{MzwWEsJk6Cdf@(M1A9?6^)Z{>MvwBn>n!HPpr zQ_KEMu-|nkI;}$1ILl7kHBtsU+GZHsPBwpS!ZuePAkgBQL(-q#5YfO^VG?;DCri@N z5emt~aYoER!i zuM8l6u%6p;kz4JllS^WSMQVEAPj{?1x=84r57!;On?P8e^hG|nSC+TZ3PYrkS7toz(0CasO?U+XyIhO(zBwy^*bfbIbZ;*VECP2?sN_vX5DH&5)MgjOK zlmt^7GEIQ843dliyeOTgLQIayBxV<`;Lv{nkjo@3)l~7*5W-gq_|gKh2Yx05>;}lB z4zgK6*`)A|1?niT*jA%e@FWci|I|cp5!&kT4p<TpE{=HL<5X$b=OHf=m;4ojp% zTMydstq@^Xz-0kf;Csa?thol|6nu$kVC%3;*2#wSD}|$UhJ6l*AIR@nHH^bWo>HV3 zc{zb-go>OaZ@BQ0%6A|vAJK%jlE5e6K{@+=MB;tk%2RR{r)=y%sCRuv~DD2w^{sh6FB6xt{I|#mu-~$9lx#E?~ru-}PNzNLR z)uWtc7(HV!kS9GD^p1sk!f1$4uRIK5H`zP~Nmd ztW9YYHL?9DX#Wf)LGPuifPFA9Z6<&EM}@tBRGU%Oeq|ck$3{i_ZwLVYt5wr}X_(qa zTD|d~S__kKx~l&&Pr>OhLWdDB-^XBS#0U*W=w|05A>O@bdCA@O7%Cf?p)|9P-yqY% zM)LYhl7XR86ahEaDO?Fw!S^ZvZot8U*fTs%X)eXKVEiEylP#>Z5p)nNA?PB&U9!*N z*WLbp%yuEbk~w1;WPwwfP!VZ~^s%%kVv8}2DS}+NyFGkGoh2Asw9mYH8D@rW)^DJ^ jB^H2=(oj3_yQEf2y0)Tr${j{K{85{7EjexAv@p0A%Uu$? zSY2RAlU+?`BKOgqymmU7k!CWT@k8JG+_(M{cx`7|_obb7Y9~${_IJ+3E+bPeB2FdH8Y3j!r%X6?+4d&xql`k`xJm&!5jTMfXE3g$PF|{8|aR%5v~XM zf#Dc~f>Rh2o#LS6lm=y|Jg7Jo&d&#?V}j2Jssqch1~sQPs5|vR!)b7SA!rWfoVh{E zX)#_5<_8PT!eG%^WV{rd7@TxYa=IKW4VInd!71mImh-jObD|>5k8{Gjuiw|4)4;33 z0&e-%{k(IAIclN~O5LyD&x7YIcp9P!O4D!oXP0x}dkVBU(E_dI*ML7Q^0#v<^Pi!= zxfMOB2ru?Jffq$dq1*QZu~JO(SBJZaalPM(6XWfE6es0dqqrXqy&x&v8IJ-#G4AY+ z{IwM$XYAYalO!Y3UCOD797aFCCaF`O5m>S0GCuMYfzt#|5?CUzKwzH0 z83IoeI7?uTfDM41X+(a|O-k`d%?TxFsXEm-j4?Sh#F0~1$aEIDs!wzB8>r&TTj%4! z==`WZT8%x4^}YJN!MHcLFxWXC^w-aicH>?+Jii?TgY({WW1nAFJ3BhB=AKuPi|k>0 zw42micN9jkOLOeHm(fP_0zh4(--1@ui<*gFQ7>yn&7j}Dnb$!r?w1WqGk?4H^dpP2 zeW=d0m0D7E-Jv(|T{kgZ+N zCM_s6xHhHA1inRph9?8QN$4T~4Sz%vX6DLzIgi(%pRQS!m2cr^>CD#w%b}+R>@Hx?d3u zyiL4wcs1VyY;~zEQ_PD6=B$Eq(YH2h;snE!KFawO;u4xz>XyZ_IK`X|aGv&?#D%;w z%!j(E-ITW|&Wfj)cTOx}sXv|AnYp|PrP7<7w^Q{{lzu#x!{qF-oTN4$8BE$ebPoKt z0T5$feX6akX!1p1nMvb0q?t=BI=lO*$?XJ@Lj`pJL*H9Hw%t_1cDAInQ0iD=UE`&N$B;N&@Ub6f@{$(Gh%>q4 zb$Y%V^*oq9hP;HVgQnyU2t62ZK7{B@-NYMFTjdg+pnESpQO6FqmROPO%p^!Gfm3_R zk(iVhhc1X~oje+vK9nxtcTgAO|zq4DJ+LOvudYCO!@nNk-9ja~m@U{o?$Rs^v1OKS>op7`}m0p)uQBYnZ za71vbZGx8tZx)_F@NC5``8?IRLV%Pd#3zN-!B*~tLRN_W5`mWq{2_tw6Zkd()>yPA zl0qEHPA@6mVG~x;EXgYbXrV|FF(Wp2fY(zS8Gnj5S_3HObbUe7^VwI|@Y~mQ{kM8v z|4KLX-V_SW|e(4oOWw|=9Sbgc|poq3yINvgYp_s~Q8QJnl55-OX6e;7zA?j4cHiZRcxG2Ai?s`9l z=DKRr?Au}2o}R*XHDkvpID#U=puz=$_-n~Rrkw+kmXt0K zGqI+sJUKbdNoAX6$n#iIhkPWmi%9kG@rDzP{15hdWQmSuVDLZ3nru*tah>q!#MDb- z>}-l*eIxI=6e}?H?G!6Ab}_|JJ2<~jSX`zy@xtT;L^Oobe+45EU$h38OR@YRtco0q z%b;E2BL6AnMEyr7kS0?Tq?ux5Mt@R=zeTO1k1GY`ami;eE;m1|E1x!<`Y36Uj-M`s zI7G}MO6G``j&(}1q9BA}=8ZZ^pXTyJ_rIop( z_Cv{aMc9G2TOvVO{j;rbj?fzfZW34}@W%vrS<(@jfkv}a;}``3LlYT5xH0JMt=t| zm4Ymk3hW+nN>KF9z+3POf?{$MjVt+Z*s(av6q&=fg2hU#|39j=op*R z?EZQm=vZ@PG(cm=(5(I8XdI{dHG<_aj6LXHO!Hs_G9P&7m=TIT1Sf0*DB*4x1mR9} ziEEl%UOoKS>d>Sr;3d$RX&+{IWil(MT5?}pU44b>Nzn_0w%hFtX>FB>%8I!K(dAA* z@_%pHAP9@M;loJaxX)C?=iga5A|`kM#Z@8uLfXihicykh zniL`!cs|D~6U#%yu#YG^jt`hT(`frne<;G8q(1e{ToEJ4pS0LOQW0nlvm*rE2*Qni z9Jx{0jbZSNlyzgGq3J`>7xISKx*3vngCieyN0F@%vQKbqyag9hX!r3$Oqi?`gzK1w zTN9>Lmi9L%KxA(qtcrnlk(;vczfImZiPqu8P)OU|8hbJqRiJ+4v*)!iju*RmF z8GG+ls)GEURB|{j>$~>hd!V*`TXl;zwPx1ize&j;!A^?-Q44}J#WG>w#JW3qq zSK(@ky*Nu8=$Z5Vgk-9$k18QiVz;gshqskKmW_?IX>k&JTxQcj|a%K`MQAtY4 z@_tAu?i6>AG^R7J2GSRqmYs&WZXgZ8_YWC0OqK)1?$zbM8B9?06Oa%&hXqvJxAX%D zLjs`$OKZYhP?rWpn5hdeWli1WD+EN|QHtK;zJbewGIGe00{`r1{c7)<*EoL1w-kpx zYzi#nYC#FJ0Xgve=c`G9J+dUfi-^z=$0w#R z0k0dmdo_7?R*xQaCNi|5!3Qm5-V zb;`2P5P&-KyXB#Nsc6!y-W`NXD2M}_#)t8@JB{1$;Hz=_;KjqELbcE_56c9>7K&cU z@laVhlts^>O?ByTRO~<_%*OgXzA`#8cmJ13+Z){&Qm{wbbdS0^Yl_#6ud>c7Z%f)8 z8Aef-2XQn~CUmRh4`5zFV^BZ8_PqIQ9}iyuukF2XJwy;KqmM8|h(~?2>JKSyBr+-# zSH0TZMB7TVZO7`2(~J7Zy?Y4Ta>~&_okMVt=*V}L)by)6+U$+Jb26KgxxIHzWHUI; ztHwE-rK=r~{YS-9k}f!t()Am!-MW6`y<~nZjOm0;hlbT73`W{oiLu@HcQUGmbzuRE z(hvA@#HmN40Qbk!W#=sH$UZJ}a3L|Ge{HC6n2L|EN@8D?z4so3PyrZLwdB7#bxaFI z3e%8|3f!<3V`a$djTPi6Dtwv%sRP!tD@~;r>BdEF5eNtj2n-2?1V#kz0yunSq|OYd z+UpD957qEha5^Ztohoni6tS|wl%xm`4DK3yLKTIipw3YRT1vY^q2!2UFqfeF3N4)i zITl45;BXrV#jBqR_#o2oOS5W~OlVZ3_l^8PuAvpbG%E!>XQ#mXW zvcBro-7`H)fRbfv-P&t(QCTKbh4f77&W7ls--$+&FoCnCOYX_x|6A8I#$h+Iz~HN z%c5?oo$KUl1;Lr^Vy9FqbtY?*opP<*nW{~7Dz%D~O|+*wGqst{Y;CqPSDWk1*XBD5 zwFN0lw|8}R*LF)f)85nBTie^Yt#+H1a18fD=)-DPJGa+v@9eAXlRDY<{?38g9g@x= zeP``1Es-%Yd0Yit{L9;??4pxhyW1`aoD^6VI3=*+EZWn!W}LKh&wRq3b?&w2ke+ZB z=M%`!JNMWN$fFJ9b`4Xzhp9bC-HWoltg}+JZ?mf}rD_Kv3h+&8^7fPV?b72uf%^p> z5O@drzjN5$sE;^JBN|9iy!U&|JoJMBej^&Wxu3Op$AkaN&p!rg~)?X``S zl6{|jkW=M_oKKq3IIG*@^HP-g&zk(iBPkZer?33u}0oyo}SbS>xz~jE}bp2MZ zTWvSC9JhL@<*!yxw;SH-uw>~~?zTlO$k+|P(L|ZII1%K}{?M~e)=&PxlP7=h>E)+_ z-lz&N|L;Vh$>Y`wYZ`MplPx8W_Og3_>~ zCmPLlr)vlKA?E4MMmv}sX5a7l4bl+Iy$$umW~*&GZtPxmhvu`-K7Oix=6%nctv`J# zsGRI|I=$}j?lXSFcY;c@*V)+gox0cV`Sng?W24nQAMCvuw(j{2*RQwi7lP@QS3lS8 zHP`EY<6PVE>P~0F-@@}vop=2DO3%I2aP4}t*Y%wj{6#$|HN35EvyS2RdTxF7s2x-~ zjddqNaNz434I4FQ-`}&H_S0R^bh_t0_3ZP}!yVXJ@Il;krh!2M!O`G=f|(UgV7=k{ z-TK9b+jhG3mCbGwC9|;->2dvHv(a32PJfMa%uk?Q2(nx)au&3OLecc?F197(`;-6xkkHnuEmkx zaJ(QJ8_*!x+w_CNriXdCfH9Yff$2K_rrT{66}19OfRngR;_}V{^fPv1GvQ9y+Mw7i z`N&^}NLbgs;(HUH(omY5XB}fcv7!%(zSd9lwZ7g@_Kkk3Z}unp>3*?q^|Sq4Ki@Cd zsVhZn7IXO({nZ=zxJ&5bRUN?9m?Bs#x_gM&OK=-OmEd-QeFPLz?g4^32<{}fi{Ng8 zMS^<(YS!pku@aIP{9v{oo3!|Jmi}`3bnj)yCD&=R>#;d{CE?zWf^R>4*zar{-e_$c z!qUgmKJ-v$bG37%bLlWu8TkZnd+l|ZP zqo$`!t)!PU{k!~`ugQHu&UTvE7xfF9j=L3P)}5`oEZrb`vC-ai!b}A#wk}Jyxv?4U zEiWi?Ifh%2r9ob9jh1&a4|<{x(_CG$nmBh@KWcqMYa@fTVPLgjec0NSJl2N3ylA<{ zP*=_5Xr5(uWu6~FX>D?>+d4>;S(h2W?wpIybbNa*=G_O``nw1oB6yg9ZGM+-InvU+ zOr`=45Vs59DO?_xtX{nJEST4&Zji)?2PqlrAiaJu9Piv1&n!gvy6uDg1{3RbPnf{O zMU`kypttD5G*vqp=_(R_;%c}TZ4EnAw{^g9S6)kFoRiz%e7b9IHvOJkZ8o~yo?o@G zY?ah%bvN3Lrc(tcaT}C9UKL{PV#~SYEm>83&NxnW)%Q2NV}}o)2j|>8w*(%~CERnH ztHbY>=WROP;Uo7SIr7L+xv|l;|6DiTcjTc5POj-5OwExGjXEgHw1ge>qXs#TI2Ub zvil+SI$Bebk^w7{zH>^uCV4+GB=3&#ZsX*|>bum;zBd|J^QJW9Ndq)*sOX^JvqK8@ z-wPRPC)) zkGyNiJ&gKTIMxTK$~@`1J$KP?Kg>oxLhw<7j}eTfNM@(Zkr@hI5eYwyYMu#@n@DQ~ zZTs$957PLJLC$lV!^WU*d84KSi)DVV4&-t(iPobJh4^rQP9W0}8zR&x z#Pw@QjiunpgA_{nZAvK>D=D?Hfftj@ae3!*}lg9kEtqc?_{*P+bRY{vs|f2aq;7ifKKqm9_2LZ$6eW;-Hv@s*$J0lVb*Z z*lUvMMW3-%MRS5KtCui1rxKU70Yqplmy{u*b)sI=zkW1V5(O+kCY`c=!e(>f`-t28` zh10Th91FeC_8vtdYT=etx~pgw6y1j}HTp6|L8^KORVd4dlT{18EnfC_@!00>f|mImfo@fgUe zbV+*baYjS+Bnq~1c^?2sCv?4{=}8z@Nq)yKQS@)#wBmZVPTK--&7I1{#h9R5!y zy#_rGxuoP!hO`k&r;s**GsEu*DU0q(kL5CBIcqGJMb1Jyx#I~tU&}fM_W|++$>)ao zqU7^ql$Hcnh*3K!m?AKW;!Qo`(l~&*i`0fHTvt0hfRSjo#Hd+*=IoQls_^$!mGNKgom+F7eic4F*MXtrxDIU| z8fTjuaMuBMY1QdgTRzazLUYq@96IkdHdd>y1EX!rv6re(uT-~so7L6EMVLhtgB~oQ zhF|SEXaF`7T25)81-tvu(3hNJpz%Rlc^R>RDJ?DvjpR zEE|(XBKpMc`uNG`Pd)w=sd{d++jZQf(1-W&>f_Hotqids9k~~RVw4J<3_+eg260Y^ zhXKyL7wU1BgQ4RdH6H6LNlPuue?=sxWE zb2yX2cLv{vtpTQxvux6lc^O&}FecKV0Wc9_CXfalF-#iiEb3m%9^;rf^)E?kHTrJNWSF?;SMf)@xV?c6Pb zPY}?~RqDimsUIfzB*BXSr`^j;zC`dh2wo=mG=Z2~l%tDN?vE4kBLt(qq}@?#(l~jh zD~6DGS4QN#V=vI}4bN$>#N@Z=@TKVaqg{vN$Am|f$I=bW(oLKxsejv?(Ep#A*55EP`nL>A|IFa>pXVduRKAXY!Jiceg_ZzT(#XT(U z3U_o8ZfqD7DdqAtZRcXE>8QOVd*ljti)@t9u9AH_+Bc&ee3Q+qeBGn1^!Frqkr9;)n7H|^glA^&99le^w;(4 zWD(z6?d=`TsUf@A@mQxZ2t3WK!#ls}yYn2DNsmXC&_jf~<1p0|FGr9jqV{Y;`u|dhEl8bTiE?azSm!rFY*Fg`R+M8q z(#G$En39zLew2gW!=oOKen6Z**-u^8P9;8?9gt5?^e0k`UeO;)BooII7yf%Rq8RbC zUtXKy+|o6&R-prKjeL^24|#pbHm+pgFiArDNNt}Mg`dYj&x=t~jihb0)jgyRj$juw z?2C;q&yczfPL`#kziu2+}Rj6MMX004~s4JDFH6-takcun(bPI-k=l z&Cn~Ul4gc~WxW8Ggi>V6_{Igdi>NgE48Cy@hdMa}mj(p~=fdD%xYVEZ#`X?>?~v!j zsaVubFD|%UK0}XSgOpFy>vpeMue;P$#3|#t#24#0K!c&GjtfyvVc~AF$T-*QwUsg` z0%atBgzNuxRvamiqEwC;e|p2^DWt#U{!P~VS%SYqAP$=m5HApK?qbuvP47)X3U1*p72@-Vo%~<_J#<+&K_3<|YLX z2MZ24`Z*-%*Q_h3Nm>_}6bODszAVrb2(J_Q1RV2;dIsD9`dr<@H+W|~i*Im>dJfoq5IS-a5^aYg&+i4;Pus( z@4mv;O;=nELBVUcYy|Oy0T{CVY9)`>I-HCyV;5Fh=YtBPVJ;pG-Asd=G==K(T0qD!?(>pQ%~Qxl~@nYl9r<80t;dobTJxh`{2&7 zq^S1OtaA!g5XN-Ix$qRuzc+%}=IUm5J?4dQx{RN)!;y{+(bz!@PgI7%0!y?sv^AVA z0BVDjPu>q%DsFRa&7d0{7@RUd5@?Vj+8S2DTITWuB+CmQnHZRZi9vdh8CZktfbt@J z72!VDG@MTu?i2k4G=Qv~6#0*}f?TdYAvxB~qi&jYyRZdNR_I$&MvI_`vJA`4CkFX` z()|zvig3=6?=JLp?<0M~b0Jp^*fB1GM^f-Ww1zb@D%H-rdkl-X`RZLw@A%v;YXYbQEM@b_rg|^2Go>#k^))E(XORS%{{rSgjyV|H?9dE*W z;#ALbI_L~zAVMS!m#SwroMvl9dDcP)-eH8qAuw+!L60m|!#z-i@8!I|I+Bk^m#X5# zsY+1X2!H=lm60OVFuHD3FlJ72v(uzJgUhP`aAWfz12H(-WlbGq$QsAl0p%ao{u;gl z&EK)|`S(QZ2iDL%=(+!Zl@=7=7~A{NS*584J;qG_K5kK(!Y?z$ z@o}dJeudyy3CaYbGf*K}EGf?L4~gYzj{6mYe@yU6fSRdDu4R-aj>%L{!34tpMm(*S z8IoSh#`s@t{+3T5p-{Dg8Y%cYojQ13Eu&nR2%y|CVL@Sh5`LQu>wO^7OvB*;NkrT=iC zluI#*RP@Op!NZ-CxV)bNz?Q>iUDH3J50YR@P)XNQUOrTFQ37|GCYw6tC;PgnxcgBT zjHr*89pC7~Oc6ecQ&)xE}oj05|NK~#=XGJQ5- z(MeuM!Nyjg!`07r(UOQB(Sv106=?FFYxvDo2|uY>lIIb)=l)aN;r?xcMS>3#+&=^l zAQkMb^FUkCr9|TxdEerqsG?#am6PB>LzYEZvrPSl1@%XNkqg1Rv3>lzO!PZ;qm*>& zWWm(PfzIe$6}@VW9`n01cMl4Q5axm2oMMlcDXbtl`zS)93=-E2X@NUB)B#) zuw;0^dqMoMYg`yfJGq`5r2Pplm_Y^>4yRYi1`C#ug|ODty5T;88b}YaUEMYZS>)(X zPT3Pz^g&K?*)T_ml7kF^HiY`ce%jptCV#cy7tbSnp^^|82;1pu0jxL5BfMd4axO7| zjJaAC=~B9;4@zCjpF&B=PWctc9z=9qF$N_B)`BMQ<@YA+%vJcSuAwIc-Qt`Nw(F0J zjYt30oVo|8)Y|-IV=(E@_O%_jGH^O@3)l`(y(!>8@eXVS`Ca{~wcR$N&}?%yf&3oy zX~B0mEoJ4Kmd&7SZ@&x={LI>I{TZmdGP-v5iUEnsvMH2JqilLdSrt^I`}EIELiTF? zaz7IW3{UrG`jh=qKW*nEa5&%iES9Z^n(8n_b|`RF>|(EbNcqjeb1FnE4yEnm?s@On zFjvKgT$_g;$K+(I)7ZkP7Z#_DgEAB;>8=DbLLgVdfI_%mhNWRZB-`R=$lJpfB_E3R z@2!QK>=H#B$pXt!@h?1CWKF0ZIAt5@0E&v6z(Yr>t(DjCd2#Q=)6Fnl7?I=+I^Q0s6-u8wms(K{Bf^(C0n#6i4q1MKLb zcVNI?JhG@~K zEWEpcaJo(pG2%wM*F9fLZ96z`6v5_x9`$NwbF<@ZhC}iJU>0YU6uZb)$o1XdB>3k9 zoNhJGpF^rvrU8cc7T}VE1tZH?PNQm*q2F^f1?p^SG=}nkLqVwys4lV1$mp1$rxI(8AN9_P~vR_@x4zDs;-w-Oh}3`u2pk+PtOch_ z*3R4d%8a;LQneh;cS;UC7b{6PTZ)jvWr0%yD*~sTNxTo3!n=SeypNWLvj+86%J#gy zfC!MXO(k)+#L71Q5bRGFD#9>m4~l0d5=LU}H4sptY#!K*Jjg#N6%oCG_jJgZ5m~X) zYqxusLTS}MerWttrB^>XPE>7F$7>+>Xg?!#@b0MYeMHCdN0?t)T8iVA4*?6blAF)q z8Qe=P&v^$TNB!&|Lg)&dbnr4&yN9=D5Ws_1_$Uk2>qQ4qy~*RZ=#xWYXd(Y+WS|i906$Hm zNK68{W%=G6j@MulBa#{^1I{fTu1Q#pyU#aa)usRu2moUSc$D(}D>`x|*ao(;**8~o z9J}N29&TNj3UKPGHvo^@2FxZL!qvrviGZ62fqeapV#&Dsiun-r^=0_7&@7e8IW@CTdYuRjnL8VC;pK zm@pSPPDS8l>OLosNF!zydI;iUOll?2V$=~7;8A|6@{$|}%<5J`ixb40kJ7Ic~$ z2L85pjZY-sIT<>=zlIL*`Nm^GN>sKTuUZ9p9;yxBx@<>!xA2@@O^(6B>p;4n^TR!a8Jd3fC*$01T){h!@trn8rxciNG{{NDP)q zUMX?yV|^Npdoe!n!Sjt|hyc&K>1*EaWO2 z&2N-Z`tjr}|-XkVp#{a+B&kve3jw z@=;+;kx7Th?02|1dI{s!_e(tJU#NvBDxbt3Jt!hUv9^P1YYNc{ACYtLR!_sHQbDl8 zR%QRf3Ww;H*TAE2Q@+F*j@pOs8y^$4LW39r7~WCxbvV36_9=_U*3+?SblxeqiOl1X z^5~Z5V%$EASvwrQxA}H{1JOIaRhFQy|9jjg8+XwPW;eSyMu3$B z`K(?>S1$*TIF8~nCwPxA9bqZL56Du&b`^m@_i!#n+ zr0|nR1QQr&F8+T+K0!%Uo+mdz1iaWL>VTQ{i*OG*jU{bp<~NL~?S=6N6`l|__)2nl zc^Q#|T~aVSZ+8DKvtp9`E>jZ(l!dNGK$oG*V-97%h$T}*Hkdf|z91}vhJ0{Sza0-j^-*OZNTELB?n-AqBb1G>%hEcztqyCMoMac zHO6v!7w;0GX2N?|Y8DtTh$x_5-@IyG14D(s6lr<;sElv?6ur^?`EUH33%q zvHm2^1~X6uD%}*)mP-5Ct2j3XOD^}P-~;3j24t?`tWECYsoqPdn-_XNAygnYm?my8 zW9x(2?lV%efSSJzY*EUJ1NaTsN(1;o5n0HnqUrwZEaumH1%iMlos&>J;Efa|WcCY* zFPfr8={ss8_69MIz@lDSYG?Yx+THT7KcUb4T}4$lDfo~AzkC&*-D}za5r*)5N;KvG zQHD^BrsWVLnw2-g5qpCp*q@OyjLR@hQ58PRy8c#hqH(>x8)=9Smr^^GI0-}{%ltVoQuX7*vFU`>})hgz`hsQ zdBHBk>OK$bf?(&NXtHI@Q|Q~Bi$w zU-SZe$)ssltj!!S)A)&rc&k~qiqW_Yqp|z5y1Tc(d#xI_e@#Y;`Q*9*`u_!bn3pxU z8$AGzYYx}#eG3ubImD{&M*DNo4BCY&c~JZje_wz8>V7+Ujb7`i=n2u@S5f}}o_zP( z9Wm^GLw>Jd=VRDEV%vhfGlsnm>|KJrdq~5u|FF-ZUA&UNy2$?Vr0`0tC*?WwSatV| zsO{%>VFgwa5#~5j5nsSL1$`H#W%(_RQB2mh37J(rR^6Tg*dG}}p=r5YCUV@k+)j&X z!CNMG+~QTYOXD|q)y2a1@sk@O(e45hjQ!Tb&D+_>O1Bg2CpbWG2Z1!P$kcc{XWiXE z;@2U_XLtVi$UCfrzWX z;BPLtlxVeyCznN{)l$$Bop!Al=IfE9`!rjCc_HeII=QB-kY68gCCKUznEeR?@t|H~ z$_EH6(X`-dUUyWO6%Qcgw?b5m|A5(-3B(C2-rb*JR=nBb=Kfh`w+O_|y$h*YX0z)- z-F4tLmLd5u7XK&#y|ohfi$ii%BQWsTtBZSFk%%MxWQS}Nb>#%_9;qubw09CGlRFfj z%I+!gSRMM($eIvs=UuG-5J0UGzPdE_8jTtz88sQJ|G?@GvU)CZRWQ!O{aZw3LuZ9x zM#_cQR9{7fKcrrPUrPM?vOp;F31xSl1xV5Lb#(0Iej7Q_QB zb{H%!*jxzmGj3E6?q}XGaVH~=P@2r4J+z6x3d8?W3;aL=*V6?nxew>eF2|!; z0Ej+JCvk~hLpx6WbjdZm>7srFKvz0kfFdxBzntYUK1qRA{}M=$-*&J{?tf)Z>E+YQ z!V;jBo<3m*csZX9mb`@mH^0K9tf;MN1W>b%MAwL56^v zuV(NUe_TGf3NfuRzceyR&V|1KwA462(!yxgav(p8LsL>@|Pdac@e z0WBi<0Y43cw{~0SmUz?1hUqE>bCiveUAXTi*d%ZXMBo1sQy#%7f+q<+K=2`g=Lr6Q z;Hw0jxSJ{WLHf9Q#rRR|!G!lIKm&W^0Q?p?eYdvT$QdR0GO%;7gP_aHI*y?68dm!7 zZQh+4J>L?dMsI~yKpJle!E=K3uD@>L6#%1TeM3JIwWYp|>pNSdU6F>3kPhEFVvFfm zi+F$dx>2xR*Q+5-9>8A&E)D9nl9m&PvK+}t6h-zzE?0$)l|wnpNfgVjN>Vu;JNc(mNt7r)C@GRCMY2qj zKz_gP^;`=;W{%2C^*8T*{rdGgzxz$MP;e6X-2KHbU-51NUD zm+*{MqHVZ_!L->j+etSm>14}lr`%NAcI|fBO}8^{rk!=OZO3)mIXBnNyLltAxff4a ztwOu#7Uii_Yoc9pOOm!*lkKuwmUOx`)vmY|q%-dH2NIo0E8)*ve%wg#>t)>{t`e@Y zmvE;)l;{}GuszwC3ne4 z_$l=nytC9g-oDGdt9`e7w~=@>;Z1tw7ZP6C-@Bc3??Habs~}(TtH|H$CxMyvW`LRT z_irb+4fj4^X1zIJ=KKTxee(&Fdkb*$-U4t7{yya2nw5{Y& z0In*ogMvBazui0RKj0m~brjbve%8U@;RF&VK00 zleI@4JNd|aPcJ_jB_Hlwj;s^SMi^P|YX)JId15_mc02V}lsw~yQR?i*ddrWjvzOO> zRCVax^{{c_gx~0T{s)@jg(vX}@4fGTq~5sTe?ZmO*X4zGJo5C3haW}u!ka37SV}#q z{FP>_b@N(hR?vkDwR#wKYM1J&<#%c;8=XcI-7YPoZMj-wBM7_gT6aBIw4>6)Vd!_* zmv`42YktRz3IojP_IfLt9AqE&!#c+(n!675?u};4^Ho$F+&L50Lw}?$85t|eJ^Ap7 z+L`w~akh5)L{w>X+v^*lUkh5@u-3-7G&|>`{Wrqag0QZ_TGM+rnr;TQbFFS;trphL zwfvywx7WkVoKhL)K?{JHI`4#Kj%&2D&v~MW%zl(%8H4Sj?fUcpn#II zesOgI)Przdpy4g7WV}7zUa75wGW+~V-obv~KPX{)2fV73C`&8wHfhB{?~q`pfIaM6 zcyn>JvSN8hyrY7j_7<>nZ;2*Pab=(GtoW+GR&C{n8>(~NaAFV>k8n6G0MEeV=5pl7 z&S>)amlNtJ?hL_er;ms2_2cW!^L8grBTUlvyyAP{W(mvOT)3|BOZ{R|ad?kg?xwx@9(xle z&Lmzou#?r|SPyhxZUHXJv*sphsx?}{oh-$cZ0;YvCLTX&LcgGzLZO?!Tiwoj?f}5P zDi3Swujt`Wjx?mihO}d{!N%OhN~AG|hE1X z+jUOUEH>-DL9-4HUpd;Wk+RXmu!s=3H`kWCZ`PIq+XCtaGKiMkKWNE;;cG@)GAJ22 zosG68ghAwJf*3S6{U}d7XxRn7?gdenKQ-Q1YH<1VKTM}^J%TGJ0`#y?dPdLeC0A0i zUtZzLmEl{o)V)m0FujGTxAt$1Om>+a8PL$+LVewT5XAykQ%r9jxp9BSABggSY78og z@?J2kGIDq??ACx>ZX{7_|HJg4FJ)xH#E#KRus2skXgm|ITvP4A-6$74Q(upVZZWCe z4xD;`;KvAt3x;b&u@5J4Nk7W==DzFuF;ZQW?RO~9ePMecFa!+_L;oW*0T~(m5)%Ce zUok2B?QU+PyMDRfTvNRR<<&b0cD3|fz&(yjT3SlosHGz{HBfTw!$wJiGDw`qYWRYt zn~TX;9s=f-=LjGbkG^-j1)7Zp8|&*`75bj6Kv{ahSv)Jv%9xwCUq67d1#a+IRP2wY z=Ee=u_5O!x&b+jl)QKzjQbMJ}#41{joVjMIB68-LM40rFy~O7dpEtLS=Z!^c87w@x zvVOmt0jT-+weO8nZ)xANIrnr&L3TUe!LISDr=GmODzd->)kP~xiE!W+R$AS9sGnTv zxVe57k4Mh?>#Ys{q*7gomFfgqwri%Mz3ebwEcH@CsjZN=32*u!9tfx_WlY=17`9O~ zH=1tUEHX*$efrXF=;kHb1!tgC>exG-OASH z2!wAeF(pL%9;BjVvlA*dO$nVkMVx(BWFy%pD$RRbhl;ZjrKErAF(QUbMN+c24&eq9 zfeVmHm}bc^la*bUY2bU!G|j7K(tN|T%-5Y%{*TR!X;{Epz*~aPziy^Y44B55$^09p z&AJv0#rHIDjeSzcJ%lUxBtT!}@(j;>A>m@Rcou#w5wa3)%1e70g5^5d32%WIic=6_sGOzTykLnjMLkQog%J$ZvlJ9fQOaD1RG_|l6#2z< zG-I$u*&&d(M2$o{QG>JyXbMaUv`9rD^qg8uMYh`LbRe8h)gU>ar@jF>cO7yQrR~Yu zGhMY7tk)YpIN}OqqiCY$iRMy!Myo1OskW}Qjy?vP#|r$`iu!T1V`OGImt1+xPBX04 zJmh2{+gQZ>I>zSQa3KrxT@jTiK-A)PrdKqGmo8KK6Ub`+6SI^>b+;4eE5-* zwRb;#iik6(-IH!=11tT0-MtUt@s|zte%3Nv#M0I28aMLp0bD~#VW|#RC@yT{H*9P3 z=yiR+(bLxr#9-2AQ14(MGBV9^w_~|XJ;#0*(nc^)$QoVcLtR&Vx6D?Q2d(oV_;x!h&GXSLdq8<;AhgF~(ZqUP z)myD$NfA{?B`1|!_`}K9g4M;Yu0e?$RA-r8i(*Q~dVCB0B+3PQ0UV*|)-}?0 z>ssD4G4v#GvOo^saDj~16cyX`XKUwZUl7)M9gX0GlGX*^0{FP557;}T5}~=8>=`RY zXd%UUuvgQb`4Vxap3AW2!`wB&bQ;T!zDf~v*tU~+7$5RsqYh|O=QnM^(wYr`2!<`SRsUI7)QMgvO z;|Ea+MB?6i+=+e(|6&d17D(eO3icH&CMbht{q(58A~M77pxV(Jvf85kyV|*nJS*}_ zH{TFTgD~prbN4zg=*l!J@qZd60uo;HIa}nGA>k7O2quk{el25Ut~sV<N^J zWqJ@BHDS%_1(dBpLxXv+R@2La^iwmJRi=an{RC4a!_X^)V?2v=l%k;J?;gNh+z~7| z|5s2fAjNSoG#DiCO3T>Xvzs7{JaHr%ygWvq;mqT)yZvhD-)Jir?Bs(Wk^>% z`=#VowvQcP9Hgd6(d5~g0Y29+$({|cq+`j|+0{93Uo!!LF7;`{V~gkeJw`q&Sf^(o zos%@5T0pMQ%K{dA4&X#D2UzN{hRL1{SeCl2ojq3)$W2LZ2C0h2{B$o52wH{r??r01 zmjRs9&#X=&4eLAL9$;8AyceG~K51+fdWD^RS3m=M1yHyB%ZgT9RU`N)!ASxVImjm> z^oZ^sb8IT0qjuFYiA0~9hh5lugJ_btm>|ZeDe{+ou|SiwC_hT3A}~+bv#F+QN)nYu z5u`ZL%m6b+R@4exOx77?#4zDC+fi1Cj@T{=d?7XqOuex1MiUn|e06z<5UQ>X^)<21 z<}TJ~k=B_a1%t+hSJwpXY*ZX4^=S9F7%LlcalhkE}-71AVV?e{4Tlb$Qa!hA*)|h zG-%-iUqa)9zY5@(Wus_lF>4@f#f-^(!@5>Xk@48q%w!RQm<{HH{8c+CZ0l-fkk|Mk z*pzv#80(rAbWkg0{;_SFuUTpHkF1ROLn~|kq2-uAuyW@2^Lg`sW(wx_tfKi{Yr=dr zQ!>9}O`6|!%I3GMDf55W74w_cwDpb53~2m)H=*&Oq4_uI<`{)1mm4Sv)Xr(hTTL{z z`9^&V7zjl26Emex10_pjH%k(ES&ws*u8nw!1mDI3VoSFz>)Ph6*EJbZ_6?r8&frK= zC47kT{J_qox1koY-NM*=)lFNv2YVSZJO(HhX+~Q61GbhUVp*Ga>~8Cicy3IVW)laa zN}UR{IiX_t0dj*uZAR_EFL|x5Py4x=y73KAK#KbuzW>uk3ZY+Ok^DY40cD}%$vm< zh-+g+YWZd3)M81UX7%qOcrU>R2&if)X{j`lW3QG8o*|$H6zde=4Ck8pF6*jkI#*4{dc#aQZ{%%l ze!*j7AJTt*f^T9W_boLRH^`SIKR?J%Nqzyn*@LGmlG_{SrZGMcX1tjzhPw~AeJC-D5_9f;8LN4u z7rZ^Nu^oW;vk$-3Y|1+h_hH6A=pT|`f&>0x|B!z~T$0u&5-WyR^$xyJaF6=8AS3}6 z5pgFT@(zn-?$$AE-D`X&4qSKo_DbG6<}JY{xA=*K zm2hu|=ykU`hIv=d14LyCNU>xQ%SUGw_v*5eEhf9`&#?HPCD`UO6LG;H_9RD>BLY?J zTz8}6sTWxKMS`Cq7)VO$6UgKaAwy9G`VIQXEWL?c%-K-8t{^POl5oX4bhp!tU zDqb=|zkzB==V_Bt0aEBUtd4p1oa4k13#POVDff(EAihBp+9PF>i-4^Fkk~<#f zY{|Vf&Y7^mfO~l69A^a@&sr)t5#P%M7xC8QP8QI)0xhRT-O3uti{3htr-o*avAj?MU8y8R!IbGrRM9p}bsz>|Msi&(SzB1)!4-YspDJHLiIyIYx))_f&? zV{RuedJpuW)dJs$i&`H0ODhyLNa~G~bnC$Olz_fyz2s*GVI=>C=d73VnscNHK4bjY+KolvJrogH-3TOvjkTNE)vkhB)rtkA#SEGhHHNG zBX@7#s&X?Xdy0#jB8MYpfFhQ7|69T$x2+OHSBR_EN^qD|4C}RGiT6wRYP^#y;YqML zYzksVza~5=mg#)&hTCL}SHn-b)?BZxH2szrsKaQ_n1Dpml8i55u~@v^EbzZ=%TnIK zV!na^zMg>;j4cLCf7=O|gnk)F{a>|aD7wC&^gQGc3%Gq;<1mHMh@UuF3e+nPa1hYE;+2$2!%@1Ui- zpSpQdkD#i!sg~l-U{lSY_#RLex;!-n)5Fd_zV?`%^`deM9^5n8apqb%%Y&>`ZB^qST+ZAqgY4HOU;tx&L^y;J~&OJWvNo2Q&= z{CRpEHW0tsAMt9lb8Ni@e_l20R_orSdIw3E7^`{=szK9>g40 zVEj5p*@SnSI@TBw?-O*8@sT*80ofR>T};^ObEtmtj@^3KGO%#>*6JPXe1rmESYh1e zNU6U#@-^p67NBr+Bh zS(Ci9oGcso66VTgISHJ6i)PWRa7xUKV?JbEbD-`zX7Q?-Lx4dB*M#*NmC4LVkb(&V zk!hD-leabSqOlJL^&wo+guW_?C`*PNSO`BrY^Rq-?5E{PJSPGW9H>dKD|k6S4Q$>k zpj^f)#@LDYw}b$NNtDQVWp4^4vR(z$Vn#yYqJeEif^}6D@4t?tldC_%hV=l>gQe6o zG$d^bObUd~i@Pa-aKj^=mXL%DNErmq8r+E_Uod+ZBw=s^7^`^-bG3pzU-XRgd}=~+ zC8=Q&?%zZW!S+K&b!u@+HaLvTvGumU@1uMi(kos!t!Nb*8%mKUXq@n>=g=1D}C?v-^xT#g+m@|1wfW_5H^cnv^fyqy1`#P?lKnT~sE@huA!=*!L## zVP8~{_YGH|2r9I7{Ch4w=5$=d2sGjQ!DXAUNfsc+z+#E0i5sxXI1q9WKM^_Lb94m3 z9KLzy<8%X^zAiX<=`_onKoP_&pYbm~s=6EN(QHFY0)3VVA(?^I-k6cGgw{H^WNrKe z+K<_!!w6=qEep`xN`=f}Rd|S?K`M13u}wpy7{u_a*9F$ZcKSr(BQJe8y=8Buw=!GV zEoUpYMFyAM$@Ov~F(%c?Ucv&~^OA3aI99`5z$vea)uOAd9Z~jNk;y^GmTosdRs|sK6t$^yc3CjMWX&~ z-Vqan2dY20b!UGZNF6jnJ0jxj^5WhotF0+GQ=lozRD2x4i-`av%^*r{HrL%WYo;(t zA;m3pRrCCi^89V-)u8r3{Ua7y(Bx^1_Vk_hww^UDUq6S^>L0QW8)lVuJ=6s3AF~oo z2%n7)D_m%XYJsKa3BFA5PY7tF)sm42Qsk@ATum!U)ZEvx7dnxe@y8&fm1_QEtQ-CiK(IY9Anud+c5) z#oE}s{}Cyvix%1$Qc)AEo8EJHYhEwIzMF@aN2|_`Nq-Mw1FsdKh*&mDvt(J=I;_5w zudX^!9P#v3D=qh7&$fPR^R_p;2-Uykc>NoIG15t)g_n>{ib_J35+R7=wjrb`fv{^> zmu9GFCe^IY(W^e2=ve)nc;}NlAhz&VCrOlqRMN&mts&*XXxmH9YHVB|NHYjBl-GB# ze&nECptQv&g~?hYmXcSl;e%kRUq=(wZxYDLc@e3|#!)Gd6m6vZm%R5=0E;QTYzF;S z6#d-7nB#BP461*PB5&g!=c+B+kgY^+s)!-azUCwu^I9@C@A(nKev@Uaxa-uv0EniB zj$oeHkyt75YBV?%{m&jGs0PZ^JeSle^r`{=x&zWAcK0)_J8V?EZEV{P#r zr1hO}a;FTgXsk~4Y>Jj>FW*rS#}zqaWyk52#@ai*<9Ox}gFp=iFyuWmoLrqnEt!`r z)M0xT6yRT=iW|-$fK2Y8oV7Z?ZEZ~g$MOr{hJ9QaIE>FO>=g2QdS#3NTJNP0wlTT7 z7qw>*_)(F2lQ+CKjeGlglpCj4_xGkDILl~y&Px{fH1CyhuY!A(UH7Ed+mKe)tWO#0 z?P$}1xW|drssyDR*f!=8dj3$;L6-1xNX>eAv}z{CRYq{LF>ZPUHwWAvsjq^!rRE^Q z59*ljWIStY2G=aEIb8EHPKS05dyq8Ht{)GN>>SlNgv%^o7H-iwfw!*iN4U&Fc$;?= zAuW_U77;!&(VGHf|8B?cqy(;%CFEoZD;7it*h+5#aJn}OIMZXO%4}~AaIQBGINxJX z%7T8!&K>f5=M_H35EX0n)}3P`IJyvsTaw_HeaIb0Nz4Y|fnEi$DnT*_dkk_pq)V^v zNBVFt4R{2YyBP2Sx#yNTEn~vutD>q?D;?T4#3t449Mi$Q)tG%PIRjbeVRb%uV34cg z6Z8Mzf$CWbPW8*@d>9@)52i2N(y|rU7{sEW+FcRpN=yxdyTMTUVSNMs7S|~GSg?9? zHQazS>Bg~HP)e$dg%NiOOG5BN$o6;}*pBNy$L_8+SE|ED`W8LtFy8p%!;rZf42s;8 zCJY}^{|-b_d#H^l?fENp*mra|(%D7B&3C#8Vd0rfv>pd6ljw+!d=VS@2k|hm2=>uo zO3Cg9Y`u;34Jb-dLW}?>`3}+8#NhHAvmYmTp5XHY+XTBT3u0{$yTc0aGPr2OWj1ug z$%_UaG;QU$!oo)Z;NhdP=EBO<*P?0^B-L-Rgh)ix8>O{T5*|Pi&UEvnwZF}~vjoz{ zb*4TG5M_NxjJ{(0-)|Cmil9qykzi=7I>0O~GHM@y>)auDPk)k0yknkbb zZa@m%(4(o<(O*ZA$kzKopE>yyFuS9X&JlNvV5k{4nDq#Jf}bN8h>SRiAcEn5v8vzU z{bvcjLcmdV%My8|BUU^ek)bDz)9048VqdB|H5%p?BehA2lkd(B;*p^0k3=iE!x%N(U_h)MJ$K*Wfk88%!47aWX&{PCN{$Y`hw={2^(XQwMgpN zurEPaHJ@{m^z+b{fk^Rd6`X?6M+%U8J)aiu4lGfq2M~fQJVLy0UoGJ|{_21xl*0i8 znh=>PrEDCsVGKGkHiDh>uZ6eR5|hs3HfYNm7Q|NzGCBabUH>?@+>&~qCiZy5cfjw{i{*4TqeceyS6i2vZ z&#Ed$e9_r$+oYXETQOw|Q_9o3i}I)XPXxoMk`(7l3}wHKi4y5FgD^7lsDz0iJY?pb z^d=a@fa6AllLE^Er~C>03xwrO z7w2M)Tluj4fHbsT_h@`(PrA87rO z>pUv!-3t>wOk`rn52A@y_k1%9YC(4;sNii8>C?C-h*p zK`IHuNCGk0;x|_dA)~5Nfbf(M_X@I0j3A)y* z1-;6Gf$09&BO|y4z*<3cAb5+n(2~uiGg>dNYMWr6eGY;X;8+<6Eubq*xS{~8m;^ix z{~15=5Xy-_^ls<#vbMP;{)35u-;0Eur)iPuF1^O2J3^U^3MkU%>`?dO ziS&V$@%Ql{KHqs)loC1|CD;7RQR)&8JVu%RVNG$kMCrAfETbrWDNbZ1B#nW)4chhl zAEqa8$(r~u5|~-)teGC>8}kNttDfOrhDST0A>%ZOCk8&69IXX? zb$+z%BGx=K-v5qwY~F#EACFu1Cr$Nc{iKewhfLU7IX09_>mB?LioSJRyVq2;0WKYk zmO+pV9jiQN@3-+^8xTAhDlvuMm8l;2Q)( z>d3_trO{^okH)K9K(_-^7H<{Vj%m|B)%Q^GcbEhF`0nP+cwd*|s&GDWd2w<#;T93- zYixmdo`i5e&a6$4A`sfn;ZoluI7RRTfrvKWW@^}aNwEt;?0$#M=OV*#Q+D77qX$dr zvjOjY%|=i!4w(M1h@f5@LA_?q`M!BrI{1hE1E+nyht9u-gQrs7lpx^Z5AYB26mfrOKik={5tO^CGZ;;XDkcJ*PT8byHqha*wIB^&5VAVm^^ z0Xu<&)MQ0BH+n@xUzkBjh+t8)|0HY+r7aw#+MzE8ME6Q^3j|qma7`nJ%{7uUdlN7- zL0?u6AY~(E1`l9!YFoR;Sn9F3lo>oWxM%C5eR%TOFoTdFZd#CU564bCSSVnx=;!cv zC&*2#Iua_Ih7CP?g_iU5PTou5_{g(iVW-$jLl-XhY#77JNWr$SGl8HsK3B35TT|-q zd-hhPR|$RwW#mvH$jL8;=&8JU3IRjY^R{mZVRtQ&bNRRHiXSyJC;8veup@f5=P1v^-nI6_C30UuM)?7%b&cZA+*qTyJV zn*KBDITR3m$W2k0601Sw>O-O%5XGfEZh}Q5lg9DcP!&MnmzBb4`u;r;<279-F7y3-v8dTT z?BS(B9lE^zT0X7Vo-ryzii7rPD#MKa(~2tw16%q><&yfWiz5fEp+T?t>W3%_(dhrl z$f#{_$V|MVSW>U?P03|2m3QbB1fw;CSH;^=j-FA841gHRen4Bk%s-q`7BZTz z*h~q-5!OS&ME~7Ej;SLAc>s5ExQ3Tvwom%&mPhdMLXgM_x7vJ`ZX5)FR0SQ2PmJ@o?wl7SGUQbo4m{eWXG3Itc*NR*bv(h%or$Of3>TL9k5lIKkHl zxNv1Dj4YO;#BrNNnFsL>xcJ{mi9PsN0Qp4TxXm!J!{iHjhKE!6#m;&?W9QAPF=yrB zHGj{*@BPzakRn5#qA0X>8FAIj8GOr+~BI;uCmU&C} zSCCibHRRPqO{}bF;5@2ZS}X-w1FG^yiX1&Q+r@jrnG&DXK5m@v35 z%uU7_H*S+CEtO+)6Cz^V;x^<;m0RLvm0Lt^MYz0*P(#2rKp!R0 zxlIO6mpAyl;w*!+DJnZvzM$x$qB>u~=2+${il+vi6WD0r!uB~i;jsBBep+$N@g;1F z$FkZbNC8XM8!x;qMKYAV2NnV+N_(9a!UW+uDzo+dBFXSfi0XGWxrl-z@|(BTlI~z_ z&>5UbLb)Z9GvDkEx4SQPZ>&X~jkUpEvfb~kU5}#f8tUg8b8RD+e!Qmk2DQ67*vlN$ z4til%G||JykdkJy{pFc~ua23r`I!>hK#!cx$26TpyQ1CI*IT+gF*S#LHcQCM%G7)Z zEd}i;UM8*;ko{9LeW_85*|Pe$biBtNIShG{nmh%9K|DtB>D82K5Oo?vcXR+~AXFhA zh&S$PBu6!nyrfyVrwl5%M@I(vK=wZIejGAH{71we@HUiFT-`TB@fkHSR#Q<%`DT9Lll=4H0d& z!}hia;_YxCTBdvgMe<1^PZ4>V$kFoB%F3^SP<~8(IR#TU@9#hPuz5{4o|R+So^+U% zd3@A;=2BVG50JUuHqh?)qd8d!YEC3SsSUD>XTX9 z$7bE8`Hw%P0FbzX@G+@`Ej0SReW0gM&MVqquxkv9rsr!$a=yzlw)#TmjAt;j5}A80 z!c5xz?%rP7ep zk33D}aUvc_W^{UqEKrh;TxKc!k=a*8FYZf8ds5N{k~G+3E2LIUom3Kd10h}qaWuy2 zIy0Kb0@Lw#pE33YGuS<5vd>+s{241U-9)~Ld{gC@KW7fpolzb$>h~nI-A0)D=(L_k zh<^?;MgTv+&h`yq3X8LIny_~Yh#81Eq9|~)uV_e{DvdIgvQ!FXD6!9Jyuh8?nqLxS zUK9>(Tp$Mmjf5vEQ@I2jDYwe2*w{67KkPp_zEix-zI;IYE6|n|J2Bz8?jV7TB7nPY zI2d$#TbUD!Ye9rHS209_Ynj^(ZwAOmYHBCbuB8*9v=!|~j^h?6%{02OvLfBo!A2*H z0mIB#M4QJ%r*5uF5DfZIj-6g72?7#bOl`XM{$t})HlEMw6sueS+oQ+hp)}QYAdOun ztBC1{nQ{5M+Rt%ITv4OE%W_QuC^$!V4j1h{7tqWAnowqxHIQPn6GgGSfc9p`ULTcC zZr=C6^f7I2*QjDs0gOEHz$9kte5EOvlPR!#3Z(vN8uCr_)?)G!4eu=?4;yH0G_%RI zPmJFK*`Mr>$s*6Nu47yM4zZJhklzBy3IXhaHjcNs(eFoDEeOJ1uTLkv-;0Alo+D1u zmy+(x{M7y)#ohqP9PGP+kjb9>F6Dinh&seaL>*_k{=S2tFqI2n=a{ZuhiWS84r2}u zwzHfgq6ZEp?TqtO^&dgT2eYUbb$*f)L^lvjVR8cJwyegIMf3XXVp0BaGdMlMyJR7H?r#ZrVDri+bC<~@M{ptSOlP9H_mENf2)(kLEPU= zaA6GcgGT3j{FM+P(W-<0D@0PLsi7{N1jp@`jssRy={nL(rE%l}0%lsk5+gNkQQE|% zt_4VFjrUlRv9byD)QRC(=i@%*8UE~N6x`d`o3**UEBSa(NE<^{jlEEcvtFjJW@b1{`umr^d`xd#{+9Q>_dJCW zR=vxS2xH+55pKTGmt7&_$trYuae`h}TXUKF9s!GsQsR>SeQbr%j@*p;VItq9jHif@ zPRMLnbGXd2f-3#SOOk+^yh`K;RNkO{eQhXYN8oxA68PCK_5}P7-7wx&K;BJ%vBAkOC3`w3Zeq6?c*a5bb>NnWJpCnGyHr0LEj>2CQ4AbxWsTl71zJ!UZz!Jquf z+C`Q4Jcv^!fMMQun4{bHa}gY9EQ;HDlU0%9P`LqULGVJi?^7zT1+!w-WTeU9&da-z zFQSV!L5}f9_YC1BbqYz1a{T}>nXbGQ4d(}@1Agt)hFOIDP9XXOpmu8Bv5<@F5T%8* zFfQdVnvk~B!X}|)b6x&6+Em-|qx<-9Nm|No%FOE>aiitrD@(&VvYs;irZTEv6pXQF zRyqbfk7;^!`&Z*ZgnNIim41oFV#08yjkUsxk-6mn$T|80MweQh>FWQ6n)v?X)y$$@ zw3iteMP{k~vx;_6}W`*mcNRCxVEw;&?8h>QX&m*X1{uf}A)K-|nNelEF zyGji~_3h|TAM*)(Ke@`=F@DSricmZHp^0UO;zF?R7%)fJCVwH&jas)bhd)W#P6O$AZu0_EF-A9)Hc+#PHZ}Qo;98*8sl(}6 z1Wz?_cTVRvep*GFkB!3P_p$>H|BS zOB+p%&+VEA3+cj#x_k<<_!`EerzDG9P98=yhww~VPix3qK*(i7qq>kho!0Y<>_a_% zo@Arujh549I=^YARjg!{T)mj};!)YAf%vnNk|i~tCCq0TWB5A8u#_&Ri(hdha9F3+ zNIX@mb)3uA3XgMTR> za}J}P&(VMyGzLJE`QOJwOS1mX)r)7nNxL^-ad;a73+oA(XnS6MZ{{+{*X>6yOs@*H zLgpU8HS852K2HvNt5XX%rT@-)W5BPfO@w(=5&mg zj^WXZ1HB)>uB4*1-1LxMTlVAlIFA$xbCLUWbh|&ywxt_Ew$YiowDS9_rDq23bmb3` zeV55U#7|~(alO+PTZpUgjjJu&cLp4GNq?C56#~A>dik|?uU@)(>7Di9$|e7z-x$yv)sfN)uD|Hl#=ee`bY+0v9>V!jYbD1z1odP@ zq)S9WpB}~fL4D~;SN@PMCNVa(AQ;Z}K8f|n z*Fo-WNl0DN>6RNrj(j2n+4BQR2V34CLcR|9QzA!B5EWCN5PEu|6CKlDgJZLV8GLfw zjma$bnQOD(*ai0Aa9#eoT4bNvF8hR)-2Yi53`I zV51h`6#dk6tDof0UdmD;)hW*tQI0p7+W4EYME(J}TiStv#s*0T+5S0Ua#`nN`1s1ksBn@W3+w|IY_HtQ`Xo+DZ#c+F_YMA=STTrUoF8wJu}K zB!4R)OX#ldJG>Ka(UWznq6F&O@R;JhigEmY0Fs7#0I4eOv~kG723Eqf?^ z2d>%JLxfI2L-(*5PU*`y4f&Zc?^O-^-Xa7qi90Lh-XW!RGsN@j>K7FIQzAbBAv?FTiT_hjUWb|G-3P-CE$=BrawS9R4+;<}r-=n#gbHfF z^(%?9<;l0H#nR+oA@KSFXidJ5J#x6`Dncm>3)#6AQ7VGgK35wpimBV&QxAL-@U%Me zO2w<;2HsY}b|T3&C(A_W{}U8;@J~Og)-dV7vrTWO5AfhwaCLHJ_LS>Il3^|nh%|{P zrA4|a_w|xoa&<_1O3nUZ+AMR<<&Q*$ui4z)48gE@mb=x#;pct;5G%%XVKB+t3*2YzTZ@9bK)kjKV9^cHo_vXEs_c6?8 zXI%r&jlX}jb){n%|D?|3L$kNg)L#I&!Og@-P2V&LX9-I!-%4%YP95LTV^&i0>zK2W zhTp)~PMWFfyXlNSqvxHZmCpLJz-yxZsLksdHpConqPg7KbomT#J+uAJrv{(p?PmsW z3-`eC=Y@%pIo`oY=YSoU{sOn|8{PSr(9GzXWs`?R7$u=nU8}S|i&arN5Bj}Cl=g$& zo>=d)vh&NL5LprD+3hge78x(+$0NTHMMx43&2sTho~C&=p1NOzg(zJmqX6tGm!->9 zF~(^;-`rH9sK$qQO^Tu~v&bg>hEE;e8rmH+wFbb=!gyqIRxoZ6wprsgcb?h%_P`z( z19rgn9bo3b0#+M1&#gi2h2DOcURy6~Lxox!NgiznSwGznQnn#EVgRFuba2q_qp2?d zXsze$A=dw^`NG_1RexYSw_X_NTUe2Cwq^`WZe2147U2an0-bZfqj+#&r2)(ad0 zdbQ(T*Iw`1(!?rfftNJVt(Pp{@$0gW(@2Hy-pO;xs>0D#9!h zW$l69dcXC#Nb^EeJ*+cF>%eslzbW%L*boon41$5;d=CwI5mW#EbhSu(tG&3lT!a$x zmp@4RTj`DT$!ZdBtoC+`tvp-ZNs@Fm9C^saqJ9fuxu$I|kZL`hxu19%qH;BS$O9c9GR@QJtVWMuXoRR@E79c^s22fgYR`_+z zUF6-mq!5%d1X=`U30wy7+vIr1iui4E%HzJayaJa%UUp6@MB)@lVwP>PBgWW&ZOeVt z=&+-P#g5uczK+QVZR#NfZlS4v0SLo4x#2TmaZ}iv@ELABvk{yQM#z&5;5FcH^Ez+n znYyNJ@aB>MdW5?kZDQ2Z0Z-g6dUIlichKh17SK%50=%%vz~}Ns{+i~S1?5H2el*7~ zX?R(5h)&E9-4gtpFKssX>--H(JrC+PH3c|`en@o|SB+ocZ)vUtzJzGIT3)^fPx#&4 z^?T#YFiay=z+pS49&&UY=oEe2gb21?Hb&kwaXjRr69?6x0La2ryod2`D2#2kclErr zS4PyQ4r;XiH?)3X{J}gl*OA>Wtmnnu6nQa4Dir(^DRX&J8SOAFk{p9Fh!S-jGL+WE z-h1~ubs2F;N_JC95j!b$4^6&Jz$0)K;Ez6aI5io1PD|)_Bu?LE=a@lGExb zO@^MlayD=2qOw$JOqFg`YgD`^$`*}p?*v;SDJ`W@X=PupAhy_0Kd5{hQ-gH z*$IGb6_0zV+IM1Y)H{)oW1 zc1!XYsT0EogdGRwIElzNF!3&$`W&EPFa|>{*JLo7S&L}&WhP!-k%7NrHv86fTK{1U zW>TfLQK{Q{y!9=OEnuvTF=lmMY55UEPd%j9Ej0Bpz^G>b7GdaH!hYn48r|=Pm^H@i zG3Jc1+M2=Z2-$|tKmC6opO2E5x0y@tTo-j+56|sT#gSJCnJTZZ1ot*Xl0W%S&nr7fnNf3EM;1oYS#P6Q~mVX!~)%8hBSN{L^+D;Q@u85 zF>-BUM4cc$PO6~g@lMe1T{|gc)w})Ky*s}S?%(x4nP8t_B@0jPPZd&LoOxBUjx!!( zCr7&a&0dJhB5jbrp!yr5?X0fP%AcL$@)xJNBcizzs;=X*`g8b~q)H<{1Sso4!1D+p z<`lV%wyF#6To8m=h6AM^F%<-7h6#nQkERQtiyM&Jc58U+ZAW_^LVITX!zYxa`hmZp z-CmqMtin*orw-gVF;@lYO_J)-MSo>{(O=mZUi4SO4TOFe6_RqLPA2}9^Jx1;oG5** z&_-1$f$qyX0ITABM);SFwJHhSq>GU909wW({u>%*hvCnGBLApr?onrny}xMw4+~Q7 A6#xJL literal 0 HcmV?d00001 diff --git a/vllm/attention/backends/__pycache__/pallas.cpython-310.pyc b/vllm/attention/backends/__pycache__/pallas.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d92265b6c3370d01d3222713bb5cd99e7641378 GIT binary patch literal 7378 zcmb_h&6C{5b;subgZUl~KP^&-#fmdFxhuLHNulM46uFWtOG_5H@&_aZg`Mu%ompTo zY+$(B&1^X(%14#&Nu|O)swz38%4eT^_8)*lE;+d9Q>s#yV_G)zdkxI&?2@av%E1=i zbU!>a-s^tf8`o-%g6qn^{e08=nxgzCO%|^zCf9H$zW`tgQ$r5<_X zk?EOH$ty*cXGOMW%jbGn_R65sL&vM&*9fao&8tOquO2nLM%470lFkfUQQK>)%9{!+ zG3yhBS-kc@_fBvX7@L)WDf7ky{ekLrfN@v_m7{loO5_^uFr}x2Slbyu5T*yw{4niEqd{1WETM|5- zh*5U#+0SXupV~PU8+HTte;WX)a##6GUG1u(h1q9{=-@|!$1BTeG+rJL#+TDTZ1D8* z%h6;rx)NJPw(-wTsEeOCi8 zwA~k;&GXVCeOAwjX4-PBFrrO-p8$X;XJ|YZBp;mwE^E@-QK5Onk9_Mi9_@bZOSoBl zLy!o60vm!9^qEZ5h>3)hnm9N`WRio6U&6thJaa~q$X^fR-d2)1^4CuWyF9BBe(S!! z$pe;TW%`j1mU?u|vx~-b)A=Xlz^2+%nW|I0Z`{qdPqN%K#8>c8JV)R>feQoSnuy+frL4-5gLN~9o~5x5Bfr(qnD=?UDj&Xxv9V{S%1|9@5A~_R)O}4=@QZl# z6K$%I$C=OczV<5itDrRtTJsR?Y$j>%l^8q)c6JSJ-b`cB+syQMoMbvg$VL`sdyDS$-#&XdOShY!y(Tm)>f`Wo>gemQybok$2-Mw zegkVsM`D+%YEyj<_XXAX;@Hh!c(vQgY~Kf~-1jrb_oJ9iLK;_n|K21Bizihww?PkP zzZo+@swJeEo~03~c%~0Vsjz7JJb}L=TxpGul30j~G_iVdG#-Rp(20pl1P*jaj>v>W z1Axphc?ZB&G_9j*`s~(J{5{b$?T?zS{XsLdFC4S_d(GBV19$^?L*lDnI2O^ifvrpG z_8;WRJuG_YBGbQyJNY|+S+Q=uwl>4sOcYVM@cU_Ine?_NAg`A4ZAVzWdOZ1gA|%`O_xZTeMArpge^ zY-z$G{liS9SyF#n`B=qFW6DHHP8Ks=&YsI>h-=!apar3^(mRd??+?N-$y($q8Sll6 z!?PeyNyN86C;l1$35b#Ku)lCzdFUwKo`;X?AeJuF(G0b&)zvzVW%v2q1I*&p($Tk$ zm-cO}_kT)DvM)-jCWW*^9L+>cUYN-)Ucz!+Q@ny7@iKtdCZT8S9F0ndBmV|Ajguu~FYao?uvJSp_689-yK`Kg5+UzttgSA@HuFlC!Z8a;?dCF)n zgL7sky#W4(qIFgckfIIzB*f+=Sv?5{Y%tpJx8ZN&?W{hFHa<&{a%b&4ofroq;XYsx zC0QekHwJ0qCviW8Zyx8SDhncffG%=d^Hkyu*cs9Y#g_?%ucq1*B?;yfh+v|kHC0FI z)BpzekQCiGB@H{dP#CYoodfjEsm3(iD4xeJHpXK6)$6s)EILFB+OsK+>~>R zm9}*8Cgzr$+w%D>%xyVGxdr+kU|yzqY7Om&<$aV)TZ&i*wnA7&-*@D*%D%GaOr3{S zoTRd^jsISbYYXE#jrS^36mT%TkEI-yhc&!cVU20!zAFBOHRbO<$;(=~#Nr=Wo4!I0 zjh&c}XhtEnlRK?P|A777z4#XLIF}MUH<~1=yUyJJqc9zy?sDV4d*z$2EbU&nPRH%$ z$34q-lF4`+iM zJmc~5yhw3#9db#BE<=aL(7~r)ctlFHEAz4YTqLmmxv1ZAk*sYFHa59f@|xFg+*o;M z<<9lBTX+1mAKhN@3<*2Pxdx?@hQE@WhArL1vnrSdCMvrg((_K;Rr>0jmjfmQg%|NU0m}EpO#&+bo>eU5*)$*n zUhs(0ZzIJSj0UOi{|NNSTL7hMNp+APbP%Shs{M>V${&m;b=_7=ngbdKxrBpUq6$nC zZ#k-C*pH8^*=T2$W?%gkK8WkTUdv3{#GOpv;yamnKL{s0vuD01tD$Nf&7*0C6i_-a zY+^AeO9B$GSI(D|kV-c4L$obP38xEBKna39wPb4GNISGCxrIMCk;0cq`&Wnh2l`%l z&w<6N4~=Pg>cA>Z+@*bEPn~LWoK@i9!=A#CKZE~&Xv1^pTZUMjn#1y5b!s5}uI<%R z2L+WrtxfCG>H`z$Hzg^Rj)L_~v_oI3xc{BzEuGNf;t;QOaj zZCJy&PHmr0k>)S(H{%&!Iz*!{@H?Z?)5fqlY&~p4<`Za%pk>6IeFLj4&h*6@t6)DG z)5aohuQ7V-u;j_lX`h*~*94b{b~u4wPRq=elIh}?thU!0p)?tG_FB^xtHJmFI_G70 za;l;&Rfb%u+$nQ-8oP65+5}zwQw!f}7hE4pxnGhTo|UIq*{9}JBlveH!yB>K4g_<@ z=v~3xrJHV^oVfEI#ggN=#dTe5BvdIQy{f_&-^cJ#?zT_tS9%ZeU+<3z7$ zk0cU_KSdug87&=pdx|Z`PzcY(86@xk?po4O7mxFi&%pn+l^L6RE-EIS=hfk z|FBED!1G7Td0H*po`{oVkw$RY-k=^^5uPOb&+bK#tCm&KBPzaj-!07J4RjYbuh05; z?iLp#9=d|xn+ya`X1edD@iS78m)!M9>TaOJ^5~CGo;*QgLuZ3Xd>89KqWbHR4zMPu zz@bLtZTxo63CbTc>*mUhTQ^tkh<9mb%AL%N`vbnc5ELmWdnWXihiH$4F`7a9Hk!21L~0Pvgx_u^TF zd+{op1IGABIvv45hVBsG$@?Q&Mf!tcQi>-?0aEWYv6yEn)O7&sd8b4Qa5OovU=ZXX zKgG()6rgj~K|WA5I%)@T-#`}7)JwoLRR@_soq7?otrx7HPCAXy#{*2 zsedkuxdmO>AdWT((9!2r7YR4JXiT@ve#RKgn|NB@#GB9YuaJFe04fjh4-uxLQb9H{ zAIp5Fz{@wLAcaIl@%i}O8+NTiPWY4aV*XK z!_w@3981AE6fWqcg3?n#IjN`+Af5BhANSTI9fXPeFM{mR<^QZ8ean@TM1Ph_X7;Oj sxAJevr6w-|R25gXHCsKSRkfzBU2dz*^U9C4=NG$!qi4WcAh^B27_OM6h-RCJKlA&Ml6B0yonQA)=Ji&Wy_SZqMc-FH?ttRNrD_; zpq_zgVu4E4(mARs$tg$ak}4fimE)!=m21wq&N0cwxuiCGOKdaW>zTn1MQRv+O{0ALDJeWk0qs!CXzNItk3zCZq==-3fILIv@`u$cfnmy zmAeYdGwVx*S=>I<-8xr6DKHxpo0kstL)BdbrN~O4lz5pht}1A^1X`I@K&$W~@H0$* zq%^8ufor9q#s$`jT5Z1-hK*cY-uSbJx4oMm-oE+qz4g1XalaErvGt%Ib%I{Yk8>M? zzRzQ0DHMXiW~$FrXVjQjU`{Xujm5O*Jbns#Z^8fJfGAlLDP{|UeqS7nN;G1b+% z!8D%R(wNSSFAYe_l$1Qnttz0JEH9~6O0^`FWJ_%eDYYo6C03SHJEc}6waRLeS_E~0 zmslOoBAz8Y8up^Z&TMIDX|rW^R<^959YZ1+;mE*QsVz zVc(myFqy+%l`R(8dFh{-pH1f3Zj{Hh*wT(MJoI9)kexA<+-}vP{s1x+LBc+f)nBY(%U0GU{=rEtv*UA`;%9pz&eDu40O0W)mFG7>NFJ-i<7RT~h|P=C zP_p^PNt4gAInyh?a8l!p(^CaG$Kr+4Sga9?-PC9hkFQR6ymZpEvpmjr6J!#;123X< z8YgAqBqU2jZ2ZoIjmB?dW6~$KdxNgG=?Cq-Ft+8}3p+y|mx#XixwpexEQ|~Ek`>K1 z9j)v~<;ngo0Ks`{1DWAa z6Te40WX)MK-CrzB^#kS{l12*tE$|cWxJZSl3{D9>> zj|FWalg@S8H~>2Vi}f%PVu{$w6OhK}JeA~me+8vj@AM*J674lYb5S7LJF&UJdto31 zDT5&O6PE}~7bi=kLZS*lShxXTDTbjy;-+|BCb&)MeYL%7YW6ub(mBaTYKF!I@ob=> zz%k$9q8VEuf9(0#+qjyQJOpsq>bTr(eF2r?;(!{&ON1ul%upAJn$F~WD_hJVg0R#E5uf zo(x01vRlMck|oSZHB393wZ%-wDBy%Bax59~KEZlC8i#_l+`1r86Hp zom^73#ZiEuK8zQLmdZ9Il|ny&e{S{ro!++i0~+rg0)Ir{4+)UD6YmkYO5lA0*9iQW zz;%E|S=^%X4Fc04WOLb4_W2QN;~ekyqXT&Y#BFLmnc}$~8zJ|%#9boGv8QHAI=Nc5 znoKQ1vW4?*Jk4@C5iOa-KlfW7Q`UT7$Hx{W&S=@ z8Jj`iCwfUzh^@W|`dmN*Kc+G66PPYmmZUQz@lc15q7@t2NKLP3CB39uhNV{w{Pl`u zIvWiX2szBJq%ZUErGkRuk; zcL{E&%=5@4E3AqZ7OUax*JVx+mphb7XBmK~V3HT29SuZjCh25uL#@07sWX3#mr0(YNmgW?3sx&nXE1n#}%p#tpjd5;lj`L${To~Ip-ub76QQ;Y`qDH#- z@klYq&FJqan&P1rSaFn-m~<;>Yf{_Ct0Rl)ds=w^BV0~BEsxAmVPrkSwGz2L^s0<3 z*{gyXs_2zZda)c7%N(*Bj5#d_`#a#hHg~p342V4(gdUSnrRhBC^LA&e({B0xfivhs zvV09*1b0Walm6anI;n0r2oJWSo#v!!&XAIZQXKdQPq+j?n)x#mZWdBIRR3CgtRa)2 z(}^o29YZy=SZlr+tI=`2v9J1w_}f9Zf9+psFdBJD8QyuBk)P;Fiy>)4bR!qsyozY2 z<%ic$$cB`jcKAlB`R_Wv7+=a3NbyfXn1o)0t~Y9Np%cOngQY0ehn>Egr~Vxlx~1gs zCXU!G2cok*Jvek&V|y}KD1HKhdoDR$euu$Zd;6Up3-+^o)J&v5bL9S*#xQX`)NMtH zD= z=Oa^6^1Bv86nChPEjplxPG64-4EK_VEbWnYsE!NLrx%${9)D~{rICH8vfQ2_uAv3W z<6;kYoZ})eWw#9Pk$<9%OA@Q3SQ$0dab=VjpN>&(duxGtVl(SJJw=tgD?^NAs+XYAiQe z+&wp1gwK;JWcCw7r7>1V3#hB3u714k>3Nnz#Jv_1#n&*EdgE}Rp6{@FmJ_C zX;dE7M~kEC=*%e3ichLAW~J6&VYB1`pSc;$!bfMn6YV%b?=s_`cVOs}1S)LW6DxH? zY=`eoFtTdt*$>}!Hpm*b4mP<1^}#jk)L`Poz}AIMuq92VJmr&G44;`tdQXm0I-)~4=MFw+>~wltT+H2e-f8mYCFd^hc3YRPG@8x>N}2aNA-^QIBN#*onVfECdna;4;=nQKM;}g+5P*UcsDjSZm)0Ld$8_(aO393x7TmIf9L*zjOO1 z&1p6UH$oR^wPU9bs{F!LcaMJp3P-&ZH&*ORlyCU!niv;Y!xNmi~-NMdR zT8!<_TK<6F76KOyI<5;6PXClfAZH`Uza*aK=TxFV+%=PPB^{V}8; zoV=}cb71i8E3Vldbi+X+jhRe^#L!qy{FW31*>JH*fI@B|eHpF6t;^enRv#Zc5VJ{b zcdN&AL?)f?LYiz$_a{kur$Ut>t4v@idvX`1Ez_HhnT%NO)=61wPF}SC8wLpf2B1`w zaqN+$TS!@vk{(%_OplH%v!a$%TTL9EiMOFH<6@0cDDzuWpMTd*TWO`hoq*IX3hCsAi-1Zp zyPASUqMZZyM?B#QfTh^FRs@5%UbMfhpO+=IawJ>PHqZTZVAt`;6m61OvIt(4F*r4Jiv48R`|It)h;&hsVj`ly?uJmEJm(_R{lN) z_zeY|hB|!pq}YVMN=-77Q>PC}Y!Pmo(vLs!;(8EKSKh@JTF&I*6MbTIUYKznksi2X z*TIMg(O~GD%0}80`zeLd$e*~ElSEn!=I4_$%av1T^d2S*O%$+$+LA7X+*po_lx@im z(mWI-nAn8jqiZG-cap$Ul$Bgt=cyzOhC`nYn=nMl7wP6^@|`nmwl>LdwIe~PmM9Rg z2~hUsUOcVWAnN#`{CecB$Ts_d*uy1iV&jrrRA$cdf;zbwA-z2P%b;}?eQ|w%jZ`Ll g1)!u{P%Ui*R|E!f)0(NhSy%1PafXkSzgW`$5A9zM<^TWy literal 0 HcmV?d00001 diff --git a/vllm/attention/backends/__pycache__/rocm_flash_attn.cpython-310.pyc b/vllm/attention/backends/__pycache__/rocm_flash_attn.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bac8ec275d1dc866616793cf2f35c68d3836028e GIT binary patch literal 15127 zcmbt**^?yKS#NH&uf6y5Y&p_s=~;SaWP`D;WY2gsBaJP!gr+5<#3q$pC#$=vYnzi< z)9Sue*mz{_gYk02w@&Dg^rKM?qO@+y44uz|DgTK@@O=ylLbYZmgw&55$ z<;|Aawj8USa#HQIlWu36Ogrmj+c_uK&O3Q-W3>uS0d1{Tv0ZXX3`@1j?TS<7e7ZH& zo_40&GtNwV)|qY3IdcrpwC39j&O&?9S>$rIb*#PQEVYk2$GMzqooJtQPI5lqI@Lby zoYpnpk`BfywCwg7=M2gx{U`3}&Xc0(oE0U%(0pp|X;I$So6p?UoM&IwL`78J*F@Ey zzGphiz5&RTmh5JuV8dxXquMX9xZujNP9+D^}31)73e>jz=C z?RI;?vK3WF1AoD*ZTTG$l}3j9T6fz0i;5 z6g27fg?^(Ye2M1eH@ppBj9N$Y2g^wZnrx%f2;Ek9V*?%as9NhwiEVOwEibHhWgD=w zuXTmry4C6R!<$`+9s6=4+`R2!>W$7uf~uhzn&wy=_4lqsR;@3*8g1(2Q$pb?-r#SM z_?n~R!!mqR=)P4qgdxoLiR@AgNec@kWQdeVGc*%JGYrj&97D4)G|$k2C^9q`LrV-T ziwZ;YfL8s2n8G`acLuNS7m?1^4YbUQIWf;IOK7>^mzxz3H>Jlo)x}ah%Op*+tfI|H ze~M~Al&3~5(&DtR8OyX-0@>SxB@Eq+rZIp#X=s;tv z?p9>C*1lLLL>IjaE4l)K(hPv(c$79<|!o| zB+=hv=O9lJzSBrR@JmNACS?A|iB)N0?fBcJ$vs8Lcyi~CqMDrCWT)uN(YDF3@t6HW zO_C^(38`g=p!rW^ux`AT0U|fC7MS-9E@ghaWso( ze()?tJ)(T4-*(qq-P%?V<@whQ8oPc}B>2`{chmPo5asBHD^_YmV#!1KG~VlYgDR3x z+tUZyKpz+bb05+|(SaeSP!~15!`%x%c!dXtiHd=&jfRejA{b93%2Qp~bpc$hnHV8?D4)X1tD8nl;%w{57 zXiy_G$-^85g(JtZPfFL3Y1g{Fon+dEyo8SOG9`y*Oeeq?34Jy7gJwL5aVE;T%=}z8 z%DZm6EBY-T`J(II>3c0=i&P?#Y}D`-Z+1mw2BDP4X%0C`Y)YP>*Bo?8@3^lwR{YL03xJT}jj;Y1R`f;WaT)Eueyi2;M}J)eNJm8|Lhx*U<6z&@hY# zhG~3aSjPABsp7vHIYYMqw*a>oUi`k1G4yNWlWB3> z&qJX+Ax=W+oDv0yGthS z@2+p(^;&&Du)|F+v^zc!gk5{xw`-eT2MQtM5IdmWM$K!rb}kU=V6)qA32GJ3qtUUU zAAlfN?6pll&I3Kr5;8`T%&#uHZTsEpZ@qNwWt!ES{Z7Z1E7*u_Pl|W#Yj50I;r3BB z@b9=Spp7a?&Q+sCrMCATzA9htP=jcOiaqJq8?BZbw~eL=DEt~U(Gjeo`rsRsq`}sr z615%|%l!ZX20P~Rjzu%IZYT8LLt9KE9(*;LoxnVv#G>N&VldwZc z6qnA&r8JkGi%Y*~ z{}2STndpnzBqd=A>wbBlximUyvZ+-`+wvhLH$NyVi5(wy5KWW(PL4k8!dMETDT2j& z9fOKNs|$t8>-AuO$mfAdl2DQ_Q1T)rpQGgSlw76c3rM1(QWQdOz2yh;8bHe>`3hCP zM9CNp_oFWIRZ6Z>a)y#ylzfSj(MTWYkrnu@`UJBkJ{?t)RV3u`4M2k*MUpoxJ#VD- zvQgHPKLaqZ=D4i#)A}@*!Jz*!UEMu@RMIAVXXWr%kEg$t~yQ7x5!+P{P~KC(GNSY08U zZqCr&56pqyoP|-LH|MY}V#OBTsz}|>L8CRn4%53gSv-<<@D_`FJ7E}hqjM1wz6Jr@ zwLNjy>mb|h__m@42y4HmWGi*v{rT&g{mvEye#{IKMIz|;dR@t?1iUu(Av@pR@xztJ z(;Ol3?AQ9OuyOG=#lvM=emSn7?-70L$~ttQD)~&!0I2)ki6|$GAgeLaaYZ z4n!s-{2^h$tR+fG(uyJzemZA9-eTwveouKq$Ppsn#xU~BG>xoWrTmXmLi1MU-*|a% z0T$_v!;GjniTF>^CZI*73*a>mv)Q7ak0k-Py+Isbas648=25~+P-9jvT6pzz%&~}5 zslWX9b{}Ckk9gGN=*#-eToVab3}^xvnJsqO6b45WM2LUI(fu3>=&Z z>VJ&tKTB2Cdbit>#Cw$JBxW6Dd$QZ}Ww;|>r;0Zy`6)`qjCpLv5I?5_46Y){!?9G& zm5pK{Yi7-|RkpIJvQ;&!WZYoohK=`(^)VSW*$Jb@NE!cXq{|<(aIOK-fd@=cyNZ`B z{KRPSO$RcJJZkJESzS&Y+78C$ln!`v9=U=if%+k#CvQSl%bJXEEh>8Owl!R6C_V7= zlnCL!RrFbIds{wx*PyxtTS{M~-pNB&BOL;5Z16WoHngENFl29FfblejsSr98*a~Fu zePfs&K>E_(Hw|)Ns?s;1(@|+rLp^DD$b(@WXt`$&GSJJ+y1B3K5kpGt>x69&Jrq7l z>*g_SU)QyPDK{`KdSK*K-5jKwSsH&Y2WY;4Hq8QSd1GAJfwfm0qzBmr4fyqa{OQ`A zw1OUlutxZSE(KQ0Y(sGk-N}lEw?W=@=x9WEP8#e5AFKQc6e4&|d%hP&M%azg!oS<7 z`H05!TW*x{+dYsV;d4@KL6FT|cT%Bj^xY^8oe>^*uqC1wH7qNBK6NP27w@h&yZ}d+ ziA@j3B0!WQG0lD@Ckuq?2*;$Yls-V|h?xSYR|-TJ&{t+zP$%#?+vz-rnk;U57g_Dg z(ue$@Y*-IS4TRnPVBVUB(Js5_c!aD+p{YMZZ9>*$iRWP7!-j=LE-bt$AHH+^X2|aL zQyielA`id1$oW~3M;^bhYe6D_6;MViQ4|KkDflX$c2cr}W28RoANOr*mZU_sw+b8SThILx+wp!io;CaS0c;Vu~$EE{R zC}QbjA8hcs!Kh>0?DEBnpCdeR@d_1JR#qlJs}~ul%=_0db-CRL{QvOWFcF^Kh7aqR z8E!JgXJ1=BG)=UfnCV0ho-L*qKQzmU+Sk-fX*He@u$veSulVKBsPW#~HLo+?b=6#X zA}0I~*mKy|La89vRn&+)3I|Tf7TwM>;Ru&Ut`CJTc8QRKLG=jic4K2RRBjB>sMp~Q z>4Y|f`bM~k)_!$Ba?0qBR(O`IAXtgReNI9NDm zd2iUGb!h7@KtuI9ARsym=|4mMD#jKcEIdTvwhymWvaLt4EVZn#c_DNVC5QTjU$M$f0c|tTw0cI}MD}@|%d5=3BT> zFf2fHVlr+U4kb@Xj=4pij-3r~%DOihloJiWU6{848H zI0gKcY|;*oviS@qb> zzk+y?O@SnP_wz^Xr}Z8CU=)4DzCALoNHgHBkVMjd-j4K@n-orm+7Wg;r1hzY>+Y$y z0{V8=7;fLZMht{j6o=b(|C|aF)uU~R1M{t&(FVoeh^@Yni9I4Q+UH?5y;hw*#9AZ3 z=7H6tCYDtgCo1CIvGHE5e!sKMdoq-S#_o?Yo||-+lYlS6_4A zx_!f4d-K-X>#Od~SFhcE#l5z+w)*1bcRzHnrE6II{5YWOq8qrfTkm&jyGv>h)R2eP ze^%9Xyav44qis91OUtLD{M+Q|z9A*zCKHUL-s*ax?9rJ%ONr7QV@Gp_Yd`gzQvU8x zy+oqij@e|bIoeAZ zP%)$#%#K_S6C1~i;E^Y+DpsBmZ{w>d?&nBprY|&GWbYi=eB`Bjm^Ra}meWw5tCSkq zhYU4l;YmdaZRkxuL`y3R4=Y*AdGvrU?_oZNIs*`t$mTW}8)_cxo<7P@=(@s6GM`9) zFG_V>m>dVqJlcyBc*eIn^9m+Q%&2X`Sv8VQ}J;a+S zKdYfc?jU3LGDieglzl_~KOdhB0z1#XcL7ql{o<7i6!?Md6THY%T3$uWR@p@Udp(F+ zCnJ3oq)w|d`=6Ma{5d31S}7JzzB7uEJ6VpA1M}hI$i-9Gy8Awz$2<@eX`jh!fn7Ep zVh`C^*=7E5^S})k-oFuo4fMtH_n(hatQ$qU^O$&Xor?9c3H zAv;se^00)oGAs`agmqwA4@!gbf(GgT4oiRY5LB?wvAQ=!=JT+Es|h7xk!yWe1!-A> z66Z1ZQ66KD^5l1nN5Kd`&m&Z0NEwh{VMwYuZE3@)Z|m~s22;%$VZpDv$2G}njo^Qw zW;R~CftH)X8j_|lC;tZNJoS-(5iU?3V=iKiGjwO`KDXLCrmj?AUQ^Ab_&tu_s3a%WGzT+Ha|=5hC|~%f?gg!L2DJ60+DY^+ zF@@n&Fv3~9PvLzU?=yrK?-NX2M0u42{|iKc&*H4-zHP|k15g6k)ckN^xHu?+DsX#l zcx*s(&n#)frEnQ776*%iV}qr^T=U$$)bRM=IC^SS8lD)O2+uFTU$Cf!=h6OTcz$pU znyi}rpDDU+UKq>+_Zl?^M^*|xzoe?VvO(SN5uGu;0YhJ;ts zR$BT6rLD7agay#T{{8vTy40135#DXY{OKkF{15ENOD-Sq=Bs`C$;$_Kg|N`Y z3)sQ%_mQS5`<-!MgsKk+mgnunH_qM0n%8Kr-{fnVhtBRlb7;Gb8y&i_AfZIJ>U3>W zX>v?AzN)&UjDBg?WH$&V+u$+?nL~7c(Ql(8ybH9>M{6r%GiDWbha`9A+J_)vZMhKX zcOrc&((gLOTAyN$(8B}w&VkaFQ>1GUs*>s^N?T5etS5$$neL=(UO(_!PC*$SeCVuF ze`sPr<-s)(7Kb6FJgg0x$8)lbpD6Rvjq9(!bmPrv1}Z1%3Gv#LCeNCTb+_SfkCl2> zpT$CPm7yQ;zSz zs95A$C0mVon+a<$*<8tw}akYpMweV$zXx4q>+M8>))^5Stx_!&J;erlc zlTE@!+bai@dI8BDzUx2P-ad6b5S7V&VW7`wNa3p&NvJ;afgD6c+Ighc{` zpr9rswm>v{CI{C=Qi$mF1t&eyt?W`x9$;ySCmZGeDMDjD6r=jmxNS! z(aGbUv>Q(<3$57=`t88UQ%mJ7IIIh2;1NS-f>_aO} zRto*8`y2TEUb>J*1aBVqJ&N=P%Y*trk2X%|mPOW#YE5@P(%-jIlNb~hgbj=$?40~} zvgIP;fMmT)_L)u1>a#ko^Z`oniMo$0wx1G6g3r>^|GZRz>mh>0`F z-=Kty3HbpfBe91E;3mS%X$LK`tIaSOA1Qn)5gc?jPWl<*7#mIXMBX9 z&v%VN{-2j_8WrA#kCHo>7Vc)^*b&AwKE_>4?twVYM^?W4U3C%u&>4uSIy-pwI}>-c5}{E?(7LwAPmXFw;*hZcD@5n}8qTLxNCaX}lF&?D7^ zcay?<_)Fb4zxB3`>%>jm@NDLjU?Ez=8i@muWZB?S8wDkgz~eTD;qYvd7Zz5V{E0z! zP+%)SLx?hSKLeXe+bb$vUFgu(%Y*VDKZzk37y4UinENtDw(xfk7o9uVD_Zc=+q&{k zz5^fSotZ%%x?N6PH$n(l&}I_%T)Vk!;%+Gi1Q5duH=SZ!V0Vc#n^gI2*RSKYLj&?S zHftf;EGH)r{02*LdrqzqfCaLER0b;td-6yCq57I-pHS&1o{SFakC0V{^-*Wsuma}fBe}8 zS#wdRKQI6|VelRNOBEEqluhd=zGHmdDl-na2smO+3|{^LC4Y#dMxSr;P#z;Gd;d*j zLanHEpB-qS-lUsM(DIRkJpvfqFIz@$7Qd)L8UJO3@(Hq=xopA%pX#24?QCx8sJBFV zi>?P`RzJK7%pZQ8k}4%O5>{;*+SlpGaX%cAc=g)bx0H|3B3G-EU1t~Ll^?~$g8xlD zmMJ;7n8RDQhbF&9TQ`l&l7+2#SjH`fGOlQ+@k**+*4Naf{U4FU=c2;sKh5HISM%vih>S-j-GTYM5S7@9@=j{-*d!M_1aU_S@| literal 0 HcmV?d00001 diff --git a/vllm/attention/backends/__pycache__/torch_sdpa.cpython-310.pyc b/vllm/attention/backends/__pycache__/torch_sdpa.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2647e0a560cbe33a81071dbf7e924133ca4e9925 GIT binary patch literal 13852 zcmeHOON<=HdG6Qr^z_Wm&dv_^K`yBl<;Y%zTuO2jJB(zJ5-D4@wPHj{w%d=^>~s%z zcIUE{>5C3+%_llzYls+bZ1$!BR-9^q@?ONxUdyMmi_EKlrT~?J~`L-gyx^JO9yS>s`byrp8 zOG@{cp#;bGTveezF00A1uPEL8ttrT76~E}0?kIjKDBsrI6M+WEoL>f{98`j1w~gDX zy9U_2UjeKV%m-_$3NTOl3xU0J%C91Q$X^W1z*$xN8s31eDox8j<}ckb+=sKWE zvc@FlioXiX)4p+4SwFsi#km+oK{smkx=y>k7YOH8E82FVo@i`4VWZv-e9!MiUSIV5 zK_l|&DYot4SMU1HmHPF-&l=fyht_XhjSGG~syEv8FkCld{bF}7HZHXqQEYs%6-Kf3 za-X`ax8vNkLBAcu#MI-OoO!@L^RqX3;RO;09m03~KC^cwvEa^-292F-W!H;0c4c*1`|5-6?; zzSRQVR|BJ|`I@iap;gW?#PkiUp62I#lcD(pnrEox7Z_?KP@AE|LDWJ4(7C_{=h*%{ zo(di{C?Z{GYQVJps=vsXC1BQqxt+3qjMF7fReu?L!`y(@=7Dn@JP0gs)`<+m^w<28 z+{%K#430S!SFRCnT)lMVV!8mP2%>@L9-svFtnxfj!a(AysI6`8+ss;LL%(0YuZUwP z87J?(crNPn&-GjVvr%1K52CZ5>I}9!PjqgbYqz$}_4lIfUiaM1cDr+~p7Pq_X>#IYNFKuzQeRfyP>l!s zj~qI&>DJ>?C>mLhamf$I{l_+yMLiF|&4!MK(;ww?cz9AY@bRSX>Ciu5$*E}-!&~)! zLc?`k`~hkRUVx8NZe}W<0PK(OgtT3%=Kk7&X-u>h7t;BJaw;@j$o8;7wbLKvX(4#L zlZ$YVH28thHhTTNbhMgSM@2!jogL^O0`@c>9%vs>YVbqQLWF+w(fmWbFX? zlIPtR)Y~K^=!^KS)d1&j_xxB7BO#WlYm1UKN_hW1gj}q*x{=5ej8;F+MI;hq^IFgi zdqNNy1uc@GaZZe#$<0i&hyvPv;U*GG(X@)H>9vVRQ}Ope)3gsYUHd>YwD;{?=~r4o zQw_ikzzv3%-q-S)Y9%mDuY4e=?FOEyA2H_zJmGVY%Bfa*6;j#N10%@!+S5v4?&Oiv zk+XtA0By6X;M;gw@pEo*3T85_1Q_ZtR-PeqfS?{~S^Qnjz6<<4KT*SGNM!<21f=B8 z-BH|ypz4?X`8x_U(<1T}e}VHg@aP3Vl&s z%#6p377YKme*(79s=t;(C6=FSn*K@u6v~b#^-ci$A>RRH&CfxTJuFUwn){EWV%KT* zgmYlKu#w~zU^~Em>hvQpSsTVwr{3?ky4PbX3~qRBFwJIM3A&A*9|$i6#mlo04|W>Z zD_-o>-|%J%<0^^76d3hj7=>}QA$noxO?8>Tq%e)@TkRlhke*I|l&3w*`;}PLSB8ow zQ&=*{YvWz3#|7x&uD8{yhw(z%`xrDY2ao~otFaLV?dHUmkov*%`aOuhR<{*--Z#)I z3sciHRa@PEC?ji1?2Re>qsLDhcBt%t%Li(&jGUU~G|G`hvi~wK5?Qks+ifSaE1XmX zIwH6+Xo&MPf_w|N}i=;TGqx6n4;cpXKThpO8W#C zMBAtol6z$5XnR}O^7~I7B*m|5=4k4W9YNITl`_guZcLf)WOMorDRa&b8V;}d*)bhc z`ecn=9rOc%2|B%|bC8@FwNqM5Q}#I`<#|f}l#;_JdnzHx1oNeB5nXSf9@Ev#XLNo1 zNV*=;h^V2Jqe<(alt}BVyP8NCZ`TM*pI5#G3*;&*zWdsypxw5v#S64~K|2TlD1F|$ zY~u7q&}_Ba;i1#4B&C^MH$-DUBW-2tWoQYQyQdFMYqrvFGzTM_gC^%dB7o*0F7X7! z3wVjklpMAo^U2iGRd|aKKZj(0Y<#|y7^fRExQT>`;I~cL6=YLnc10T>Lp8$w2D#^n zhesNxI+=P+4TGgp9i^uLH@MUoO_%0SdN!*&Q5u=P8s&%TZT0)g5479rx72m5{x4X% zjH55V5s7*uav;`4uP<6KbRiRN48Yxi1NnYE+76v2G{A(|NQsd(Qg1YR!iQRb^F4t_ zGvBZ?tc&9MpaXwBJU@m#M(Ev)56(}x^l2{+%o(`v0}(jz$a@VMqAy+jY^P>@B<)ep z=MV1CGmcy$fho3|APO!E0n6}lV8N@nfS33@)v!}_@7;)tS$_BCthQi3GA;i$YKA1e z?L|{HRjXkCg5Re7D}L+M2m52!D%)A?ge3;Y)Lxph(MiB&dxFtK|ow0n|o+Ak~k zHm@pC9y`RE*&%m--44kZ&msHa|M$o5v_EEu$xQ8BZ`};KPPC0ppN2~u+SM7ii!Wfc z?-!V}#TStmr!o8-i#P@@Gsy$*irQ?UMmBhNq%-EC>J3Npu?edFLAV!4E<!vw=LM{FWcylFoD!3I{YUzQAwf&iuM31B2JJeO<#w+H z3j%DGS`%lbHF1_B@9e}OQVJ*TtO(kq(zBh7g_VjYbZ%|8D3-t78-QiE0w<8x3OO{1 zNiwz_;FW+k`UtDT#>g7O*#HR<5*^b#--9KOph>SgVWC_~G(ASpb+>-w^Eix1il(``%RBBfGhb3N*04W$fr*gJDsy3Pbmp^^?A3tL-L zGdrSpSsf-uw0JkT+RIt(V4H z)16OEvm{32Rw+g!ZQeAZ6PG4~E>g3OLj>&2vfj-Ut+;lG%v+nV;$)!IW2W+OUK0M4 zL?9c6YnD!dXj{wP8X|`es`~!ML7(uWnp{6A&d@N3*`p8^r>3HI|T2qivC$%~Yb z{Vi}ztcX`Bc?}6duXKDUzD$5|goSfU1fwV&IjP|lBsR`~EW=hSYR#ys6}_TY@kV6D zwpGo-V(F4$jnnU5&l5T&XouKH848+>+YsY0+w)_FN-ax}sKkEDfp70-#q@mV= zT2TEHJ{<(=c#OdC=?KsaayZ1rAs`Oj5PS`EysaSb7m&w0D9}+JU^c#ih2vwjS>qsS z4hP$19R3w|N=+RHfONda$9Fg?rhJvt#b6#se+CZz41Xyo`bB~^tNyaTawq3jxVCyo zZK?-+93rmz$AP)P$9HRd(6t#CC|<#bFX9o*G0yRsEgzA@B{)x8EebmJgD@_KaHZhW zdbe6#zjrGxr)LJ9Pue7|@}3Bx^TNPGTJMDMe7kqO6(MffYeqPZ>C0FPMZ(fw5gB4G z;*aoOkqj+g8K}aF5OE;~eOTb{3clH4XXB5~ePAj^`cS`3Mx!w_kmiOt#(gY8Sa_I2 zYWe!x+6Zuz7KYI5o;`$a_lnTwsJD|_QhZ}q3!jnE90@y)c!)>gGw!+n5^^@R3+P-_ zzB_+U8QJLHzNv~Wj4C(OzOVS^&__ip6K;IN&S+_U$Kgk=`xq9W& zMaS<2A+$;4goA#+C!$1MN&1a#(T)8xA5}@x*CF~kpzj94vN>K3jz{g=pTl{YLkDfn z{Q%pbn)p!1KlnJ<&nbFss_?h7tPOSYL?k*T{pH4mO!=2~RfLy?P`( z0-UCV*7*f^jFP2ls8){D}L%|Rnu~z&>QV{GL9-{`Yf`5}zCD^bVimR<#7H089&52pG;l^sdP@a9B4n4qd!8H168> zu#0yM@m5s4TOxU~leF_SZB(Eg&po~+dqjJ8B?Eu{y3?>WGJJu_;<9s%%rrd+;zFw$54fcRAxK$TET%{21&G^0&T}nCZPSQBa2t#`!#7MpXbMkq( zM>z0zo^ifPs!hh=&p4Tya~@;K#@qbsnITBku>(G$GWG^l$eOOAz9mXn$KpVgb) zkB_YY_Tt`bwA1Noh+DfQTDZr9%N~8Y_$TknOeoLvfJyElc__rDK7|)yDbhZZHvLloh%XUz z!2^5n%`FdHLt8s8y)pm?bP~1B=0Dl^ZaG0noGYnJUq{J$P3l=X-V=h7kdla!0VOvn zxkU-ZlH3Azt{l8uZJ zB@CBKvht)Y8Ug-{|E^UI-nU^pAsKdqO!M?d&JdaItlIwr6M~4RM}?`o1Qo?0HQbuj zU~)s5OnvK6u*NWVPaT;s2$X>$zKojET;GT&6=OkpK;4*yGU(AQRhd%y{<;3Vg9ySQ_!1ncMfVOcMo@z$ao*B7z66wt@t|9 z1t@)E1qz8w3WjMDFft$rPoIE6%}3Rp#a#_1!MNWdRCW&i&yC7a4QjtMG;iydl&^mS zs&al*87&OwhV#qHs0tNZ!Fw?}HiTP1e{X3htGdzJEGVk3CXl8#&O>$g<#hbq0>I<@LMTZh{KHPi=6C>CU){h-3 zpjbAsM6#ty1Q9}cxTt#KFQ|p0Bm#%J#T9D!&t${6BpZ1*TnjhXklDTI7E(EL!%a^X z$%8SNvN=R`fo;+YxcP9f<$=&M(nruWf}ddZLKm5mRe}G2FcLfm@)!)&{E)5}r1`Qz z(!|zkkVG|-E$n${YMFfyO+C(kBsWNc8NXc0Yri~hm5|?mGz&x}7f+?HF{=r1XLxhH zIunEvPY@M$C?R1ZgG`T8j!ub$PDziFY-uLKQWO#rZ&E^2otQq!D!#DFVGjIPfD)3@ z!1cT$DM2mh=E@(eDZS7sHX#bD)1WU}Hr_^9>()h(9i4!GiC*I)pELswRX2 zMMWTz6G1~QG!e0au)qbvdn$w_PC_915Ls!GX9Mr;ZS5|3Fohzp5D`J->uq(Ehj4`O z9p;x6cuf$lRQHB4#2LwYZc_x9hm;fFMDD)oT7Ij3o&H;3eNOI06Op3s^+5bN%41wV zCsBZZ0~xeE*B*3}4d=x<-m*+TH%B*VJi58rbaODc23~C9fRFxxK#O`f-G3czjY%CB zc{h^WKo~2Z>?_w`LBbNqLQ66#zf^g%a434p(eqeQpzj<}vK!#Tr@yxG3C0Ey{^(9(stU~n>u5XS^&|3p_;r)X8 zD8stp`U&!(nE*??a>WP zDJfWHYb<;~hl%9}7)k_*cwE-@n_kC-~_(Fdr(*P1P( H{Qmy}F+3H0 literal 0 HcmV?d00001 diff --git a/vllm/attention/backends/__pycache__/utils.cpython-310.pyc b/vllm/attention/backends/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0669a5f8502ba412d501e147255bce1b4affca97 GIT binary patch literal 12346 zcmb7K%X1vZd7szL?hbYriv4P`phNxzpL5H7s&Yu>q+@bQsYD{buX|?pK|r>5Yrg5} zclY=Hy1V7FrQr9pI%YfXri{qSwwlkSz<-!FteN^D0>fOGp(cTW6m*@m3)gCzPYX)QPwq9 zW|aqqbDYikCs@@#$>#9X@XY%s)-ATc4x#iETlDk($s-C|O5TT2;|QLk{z=q2#*U+w z{VQsnU?)-QG&_Zn(*A7paT*Ve`kd?!^E3UEavU+oDtn&2fR@ksMfNTBB0KX?bDn=k zVP{$Wfx_y3)qnnova35UAn)7kCFH&2&*A%9%($i0&kdF?hoRpIo86AR<~6qc4zv4V zvlV#*3rLM9!&a*eV%x`_z?{s|rmKXSgOCNr4rG4Su>o@IY zV0XHq-RN_URkwERpx^6tdFV5nIbyw>i3N@xP z;{m=O>5rAG%C5Sr0!z8U4a>&w%BJ7gwwvqr_&Btrm2StkyWI9{20UQ9J-#E?~pTD!m5ex-fyLaVuUp|=xmb~_h7X|>uHyp-ZD zBor515G1$U+lh-8H~fx&zsKMF7R`qCO#AVoGC11|-0@Yre!Ca$EMLTid#&Kj<*ZrV zh^=PO>;$3LY51{08xrf-hS-!_P**ugq0!7aEjd24?*J!h5+u?JhM`*Of~xIt%8QGr z-s4?NA}-$T`+Uc3`JH$sVHWXG30h6o>}zwhNu;(k$za&)lfSJdNBsCN_xTk z`k`-!o4$?x;GvC~`1kGZx*fE@1#Pd_!vrN!aH7$b!4Gb7f4zC%Zg`DNlo54}n1xtp zieZvPg4$@2c7|IFWq?bbD4+z27 zl9htyz@NZTk%qt{IQdN^z`a_e?`l_-kC(9jYLttNqskFwXi{oX8RjE1G8U95zoZNc z;A}N2JXXO{IbtckfG)sNIWcl<0wWl`UgT$x&uLM-PT<=FUIK{Krjy8-PQSedNtt|g z*Vh9-bPQVR^xhqJ_uV|nl;1_KK?$Ium(+qS1z1i|z z-tWXFjo4&yp571(oJ3ip0FeI5ND#A&jvpWaCND+mBdCE5u=0XJ1i~Qtp%648Bf;E- ztVTiB2t^s#z8L94E3%^EVP!ZIe~u`Jv2a+rKpH!ny-q;L1KKb_?Lt&LRj73#N!wnqEp{~H2`2{(`7OyQ znKF_hePKVMzlp8`8^Fj{As;G*s%mM5#HdkFk=iTh1o`ixQe0cj?2y!G;iL~>2ISVd z{SIs3W8z16zJw?ELjYfKphF$)w&Lr)aYXTROueYE8Z%f9Mw`h@mS+V&zizN1vyf9@ zGk(#xU`&>hcNqq0#kbZAY?f6~V}{K^#Lvfvu5{b&ZYP~pYJ-&Bc4Az)jaCqsn`EZA zYtpLXHO#3&`ab^Fwo(9~Oqip*5U=j8V3Vk7lA7l2?(= zuSL*}TP3Ey2BLCNim!46d5TL9t(p;w5mpy zXjMh4IkYO$2ph_HqIH>C zhx3uKs}3vSLZm|$fPH@EVP(M)#0 zEj$4!abl2<%^~cu2v+;gw7anE1@%Bw=+FXnjW42FtUjZit&1)AO(d+|{734I_;EoQ zynX=fKo$yi*dRI>vw*Wsz9dU-@KVB5Z`hv>&)H*qB{MRkl|Ji7rA0=DaXe2mf!W9s zOBL)C#QLDwbA&=`vcM@xsUjK9DR+5uW5l6_jGL`&kbu91uAQTjmnZCEIZD9KqXNH5 z;2MF;0I|8=l#}?AdP4ag3|g2UhDo)_{HVcDE9e^B5<@kfTIdBr=c)0Z zgBN}s!r>p%iimoF@ILe04es@X1MQN{3;Wj#8=E3MVFnTnlUNZV<^M>eC|h7M2nUON z6&Qkm$bm5k)-jMawsJ5Fb$BiMFdv$c1|}@vt!(8P{N0BzU&L1-`GNx3Dv}w)w~(?P z&4i_qQG}fGqZ#m8VkAWdYSI`LW<-Wy`cg>jTi&HPRez`r%fm`k;eP}MFGXc$gw?3B ztA4JnYZsNfE8*OuT2y+hZp}v~%P|wPneA8*5$J(w;T%K+RP;k*2oc4!Z6o**1XkhE zVx&C=mw%?B)aa|h=kF=-#Fo(NuxumdbY#~UR*|bt<;@|F`aTMALvyEkHPPCd%ALpe zv8c9noI#wi;*x^z6ByHas1FxJ+1$QmhfsDhnuGCpXzNsT2*!oj8#eRMfFPu@8p;+> zwlGkGQoMh(Nf`_C#EoX25JpT$s zaYjwjuoNN$Y_j=?3-{R+Q7EP`YQ%{-rJ1hAdFHQs{Z_cL`i1J0I^B*Gw(#o_j?=M* zILAVY)|nDZgUa}peLCbHptCPD{sZchtV3$Xnr)F0>)n1>uSl6K+*KjCg{&sw63?di zOhR;V8P30(jS$b%=adN;*C;cw-jeq5tZXBFeMEfXGL?)zp|M1al7x4V3?_}o3dLMT zNUT54FJWfR?8xkLNp14m$d#($9ZJzsg-0XQkkC9*2Z?2Vhf2w7mSfbU5oJxC{*Wc9$cz+Uj;T zoZP_Y-5|~dn;!6$L!0ls>X;4a=DtJ+|9xtmDbV_&RC<}1=O0n|9}p0Vkk;tb#Qu1_ zu+O3I#X`78Rn`bJ2oOG`WhhRO;mnT*U{VY*YT*-@ z(v&&juKW@u!7l(xGs41#Uq+TSDMeE?w3=Fl;;R}pK*Z&KOlq*Or@?na><5qe^`cqQ zEGR`|&(hE$gOpQ+QbnDoRZ-u3YC@4xgwK3p8fnUa3YL9ocu)V${~a9LU(t%$ufX2B zL~A3Lt6qxXICQ+W@4B(&x`-;n>p;5Xf*-t=Ea7>oe-R*_bzQH6Xq!jD+`x7Db;>o? zy4{wfRgA}7+iB;k2A}#}e7GAyZXWs^z zsdYT>A_^oWK|}Bp30Y0@py+;tB7q%3=o0j&2yvUByupe6rZ#dFDfo7~5=9Ui zgAm{+a+ZiEZE2CZu7)~NWQ^suaOCh1DL0QCcz1yC+>?AA`Bq3C6XlRuL>?5i(I3*P zD@>dQz|=yFAX4t+@xWZWgM92pZe^8!jKL9MC*ksTeU{(Hn{NTc#(KZil2WrnnPN|< z8g@=@+qj5u44h~Lyhr7;X}OH11$>*@ZxU#Yw;vb4>CW#^t;wB_%i~z`aptE4-$pg< z$MDz%zr&D*6d$)0ultYZ|UlUFrQl8Ofn594gY*fUlv@2f&ERh zrs{9Uhe5;130R_oIms%&H4#-Hw|5SYSi#>5G@q5?5bWVB;uIRcF6wBH3Xc_*L%iTEnBQbHV-(8!CxzwzC5ak1#ASmn%D>KWjbP#Tv- z%hZytTat|k^KogsQwIhW4nWh2{8L&-MAa5)9f|pbAQ;?bq2l?F3JW-$6Ur!l$?f+L zdxD9SngD5EE|x>2Kx80Epl(QECW!7c8tG36Op)IU6I|fZNE2NO0rm>fk?&IK0fEl} zfOy)>QM7h_9i&t*$S~<_Vh*PpDXc8njgX>$O5OZ9wO2?~AlToNIJ@J1&1nHST29rjT-$e4L)g{a+Uy0tq9WOrhgg+S@)Tp$KJQ zVH+8L{+w@0i3?=!7P((W>jxq)DB>Ksj5xPSz zdzz}sYyJj&9Eh2F9%p@;IQ42Aq+KVfPcX5LX1R1SWblSi4;uubVjgOAlOobw1K)J$ zrEHmx@{j3mkCxQXlNtqCV;!OJsgmM>B^0YNanzx?2zo>17~ZE?Nfy+&Eu?E8A{7wa z$vvBtj}hD8&Fa{KCK8A8D6yO7s(i0*Z=?h@6EkjmKQl(C4)>LyO>(cxy7 z(Z!YKdeaX^p_%jCZ$Z|xG!%oLQ7t(vvN-}QO*nzsU=-BYe@6RGBUrX?LGNKqT*zqc zoU@x@)|FH;avK`(VuSOe&7esn9cE#iY=&4q1$nZLE~k5VBb|14Z41hF6#jtiDS{Og zxLCmvBiWvgDT7bKUy2P12gEfokBM_(PP8NJD@TYtVQ&kEw}&Gdp}5j!rZk${|l7fCnk9r38M^CbPvWTc$<_YQcs{7 zpA>}aYCJVGJUSd=QKzB}F73MdJtb!8uw%)|IRv=+u_?(azcTHk*x28;Z#GgMefaHk-Ko zS8prpbGi#OCJvy{THelx@AyA}1o+<(Adg@L)}qjgli6P+WG*OtA6tX}HQN51$h?Gv zSvL1@ID*I88(i6+gy~4-*S2?_A>hoqcpJkJ=?XhS*ngQFp(WM@<&LZijpeesMp~Dd zAn&~QU*qI`v)k*F*bS!WTHH3Ib6-27*=e#9+Vf11;dCT4QyM#frs+&wO4|vFow(tc zF;L22*)VCO-oB1|lHDHZkjZ&Y(>Za@w!d?)%i;6z4Fo-45O8fd3=eJdyO@z$w}Mz%g=O2n*5vCET)~rc3w9_5%B!kE@V)%Pq-zyM9%& zey$~jLTaTx&pF;aAV4uA5o6?Jgvr1n-EiPP zA@B)-za&6*R;(jn&WUyS-w^mYfxjg{N=mRWr*)03Rjitp$nOChq6NQ&L;((nSM zoR%|i=84dxYVB#y8Ba7_dt&L<6K%=(S~D~}*4I|8qH9MjsxdDDvwH|$Qh)UqZN&WT&AAG#}aaq7m+_uXqN zSKO=DuDH0Qe8a^hWf>)VgE~8WP-h~nEYH3Lc0?%nMVcBJ2;zd3JO#f-U&%FIS@cD9(o=i0yhciWGCE0g(e%#43# zkhy_x@O2}T@iMiHXLx4Y=$JLr;B&T}?N~LdldI)AcFpeOYx#~-b2^1uLCURmu~tOA zTzjTds?7?_ZkIcAwYg5ER*`(ZJ>RL;s-1<}g5;g{V&_EdMCWAfWM`?i)LE`A3%t-i z)j3@|-8oY`Bl%+cZ0B6pqUCXmJ z9q**Kgc?sujb;C-xcw!RobpbiWCcB)@y?>Bv;G<9UhnPt>TT6(bhifWhGP9Mf$+EeZqQRixMD?>Q4fvg9lz_1_4!(--;U*2Gr`Jtuz7;k?KRThINwb#*t+wYYR4-3XR&;VQpESTU%(S|# zu-@)%ZTYInDINdJB69=Z;5$hCOwGVvG5xG(_|~TBnO^n-@>)(Hwr62oO)ux!0?o(J zyg(hVAW$cU76m%vl>}Odp|b)dFB5MO(273;E}Q}1%y{dFKf_wsrKiEa&S1N9t+TtkT1sjp zUBuhxXChEN&YmV$R>{Q7k){p36G_sODM^nd9gdSUR#R0`^=IpyHIM9lmc0i&9N72e zwC{6A^_|v>irqn{zQLm}h>G%C4_f?iWQxvffZf1X=&a-p(>ac3F>>m19M$Vl zv0m@=yg}Q?bE#f`Z_sG-pkj_p*J@%XZuh(>8-z-oVb2Z|m&rvYPcZoek|^8ihAPi2 z*FVaIJ=NTfY%O9HSx<2~6&Fj9;ITo-aD;c3P_T?|@C77J#xyI2nO&Uvng;$JnWp(; zGi(0Hw9Jo-xzZ2Kf@xTQTYwh@Ui#R~n}!p^%xvXHiujxOPJBq*4Sa)pkP(Uadj?jHV3A##!&22<>wd(DUA70zL29 zkjAdMfHi19o6aOBy$j)-3WyCYTUU!k7oYVv-@bj_g&OH5>4 zKXAkCM(B2ZAP9TzhVM4Fp+)+h;PCdaU0TgXyS;aXsDtg^pzX0%Oh>EhLSKfGx$3^X z?Z?kR541!kND}%sRj=c|`|>-tZoW#YzA)%^eYJ`y*lj59UH9haUt5*>(OlraS8oGt zUE_`xI*ohvzVbI)?RGsbjH(2BezWJ{^wtfc@*se~!NBV>@b$<^P(`zR!lqZ$GMmvt zv)2v%d&rFSGfR~34ScmnaI`qq!w5R3A4jN`wFvQ23KQ`jEhQx}nQ1}A?~-nn5FSKJ z(~ya=PZdU0%EJU0_MpuNQMIXhK~N9cJ?J8he!tb-N?=-&hK-H3A2g{aCLccY$d=>! zBp$rXFjLnllx#ekqeD9z6`%=r>l<=GNqQfF<}?8*;9Evy1%7*T>>w1)WVT-K_aOX4 zd6V;C=pK9AMvx8%8VD!p(GSVlLPgIEa-xLnx2D{mA zyQyO9CM$!j^4_4u9mcI0k8!LXwp`$cnonh$LdL#=j@7G7ewoReOeO_bggsfuCQ6q~ z<|yeCOb|XqqkzVOVvgszgEM~r+M${FX;_Y*98NqZr&mfRhEJo3`GcQ#V%EUZuB`ag z(KII0lOlU-(8s;m58dA85wn?2b}|){6H6JXeuc^Bm>e~+=i@0EW4#0yr}fX#PNvn) zr_;K2+_WBpLs4E&%q+ElR44mm>>6Q$b4GaVc_XX%0_CDSbaKMJ_G1!x`^nE90?;%pV#a zaXI+Sc?ME@Xd=xIvq+txg_I*J;kSt2S^VNsjkGjmIy=lEEe~y^b6y4MJZiCR6~8Q7 zz;7A9i};>YN{L_oa2k1Mb^Q^c5nOAvi2~fF=?pK0xOQi%;qpR2n}` z)eFF*t~0sG6ZXk(D<43LfEI_YJpARF3<{_~%uin9ry2V5~kmE#pkuybb z{~Alb$m9l-UuSZG$%qg+ds5shdG&dgk6Mq2zrg}QF?C8+k_kz+&|jiv@S8}ArezdO zJ6kl%IHSu(Ia|dkRYvak)Uj-o+BnH9XsZ^qSSx4#(6Y^s?7aB{+cE#qDwzLZ70vHk zGv@cKlKJ=6todDg&ip&OV*af?Z~l#4HUHXPu>YsMxPRf$rk$pBVuw=Wt>7zi?LXlG z9Px|qP}~KQy#hSgsp&4%`|Mg=%AJS3n(jiqM{mTX+^0!tHgsSRBA?3bRGtb$?i}uh z8GQN76M7ckvS)q>ftK=hR`c~rS+kDt#g{NhC2S(BGvZBIXfoole7G{Fr1fo98Zq98 z=Y&B=JSQwa!Z^ivmS(&WDMxYI+>}B@BWiF1Ny=!#WYg@0s}Qb6xCF@`e{uN+YwAA- zjI*Dr({IG)nANGSRvuH&aG=jJ`6VP#zFzlw&3aw&KCeE_QjVT8@*mz?^LDeE6KIj87snG(sG=5Pl9=)r7cXK8t(>c^F-g z=5QDrjROOOQfaz{ zT|cbSTyZtKDX59<&5>qwCDn*H0rH8JOX^X)3QX%#n({A*trz29lgsYS<7Z56bM98_ zE-rx~u3{EuGqV@NO&^)eU0*2(hpCN(1;m%5$XU@`%xCq4 z`^W@Q`DPfZ*2W<8Z!6VP>R$keJJq}R5gs~*38CAMnD7dju#6VP`68H5|5Z)34Nz3T z4FShYryowRBUv$J$1XA{JDwOCiW1m7$aX1(vjiAL<}oE40M^j+7{CzJYHp9&YGCHzdCN;JO)#iAItyt;^87yBfjBP(5nP`< zKV(5Cb_fNO$in!Py%ir^#OGdpfhQguWpV3KU8BPx%tgL?@kob_JUUy+LI%EvZenH@W!Mp`nJD19f|2;k3X!Cjv7r)V_|o@JStB*WgtiT^E6)c+Sc?hHctZ zW**;FH< zlWvS~k@=!!J`U9HlSZDEPR20>;UOwcp!5d5!GA@PP(lAf0bUD)Wx&tiS@`CBxaIMi zm#~i*$~ey&y#EJlijxWpyzE&B6g^-JQm&(*3_ zJyy<(GrIBLv+?l_lRD4KR<}P0V=fL{T9+Fzj_chCMU^JnD^d}b_#SHF+@+q|Yqxv5 zv3@sv{_5n%J#3|LCu<=0!muFx&DP1K_-TU2^4FMOU0ofAtzQKcV6AArnVb_pfyhxm zQ$z@z@jXiq^)Q(ebjEdsOpq0SNL^g=28q){Xn551N0h$m!)+jPD4tfg+31cYTvyl5 zkTDksrW?KiPa7Ct9w-GdMW@EZ0=ZtV`&5|XYS8zVZHj79CIYw9+S(4aC&g=R zZo(nc4HKDkg)+DeA&UB2)AY(}HQKw4y+AkDg3P@ZcLIiLZV$S5Ag|-~B(wNkihPe0 ziytHzJVTQ7q$LneN9=mpXoPTX)3XDu9fAN{>Nea>Xbm`S0l#+36>mn|GZz4#e32iTEtvGHEUOJxodRg zo=A7uQ}mVzY4Mt3lEPgd*QI1e4^5@!mi4H>eJHU95)csOD1Us35Qs`pgg07ryY&4a zDhE)D(Pe$N)%AM2Q8{s%*F7p!QB?#~0Cf+JDx{505X~X#sTIO=)7uQ;FzV|&cM&IP z@?o4HjT0~98~kS^LkDJ9LpdRgTD<)XX|y#|1$ns*$xVr254cx_*< zKg=8y(NpoRp}vIvb3^k_;V>Q+k>>v_N9scR!5sRbJ%Ihqvp+nS9+JK)P39}k`pDgZ zT?n@%J)6Ox-|wj~*4Xr{jqV()`=3s%f`^+3hwC^#nO&E&i<3h9VykdXm0jeV{ba>&xK!ul%bGpAx&|fE0_c#V|8UADt?j9Rehzog9VHE zX0zREgsM%zB_=dB)oiW$)bdjL>@%vvBHl?WRp-L z3@TAEL2Vu$U%x>YTj;B{4{vW|gW2HMt>plF_L-<8F*LECaAigt?NUm|NSq`_^-0!y zBx^yhj_-|iJf=g4wE&SA+ROK^s6)_j=#<2SM-Laxcd!p@yMbd=Xg$E+BCN`e33m-6 z8w~rA^~i!13-;$9@1IE4o1tUy3u@d{%<%c!7o(gUW>NNzzZd1?avV9y2Gly+1yqdk zcj~f%qx@a{AX;3rAZT>5gr9kQWyjrss2^gliIuA~%^#D;JJd4ZxnLRa zcn3bP#T6nK#D`fpZA{O?9(`e$;mHR}L*t=wFo%<_J5@o1N=%)$IO4b@9$ z-TbBzEDYy&PQ|N$yt;Q-8>1U*Le&=kQzsWpS>0FGPE5x-z?`(zdd3+zk_be!I@x34`IEUWHdL|{(duzIP z%<)BxZ~xgW6w=)PJG<+Q6-K)F+#_COiXxj1`;}YF~buOn)1UYBo2$f4KZl|$Fw5{%4 z=<_;6U;`IyZR?f(wV5Hs20}_=E*P$DBiB8(QAEgGzoePao@V| zP}h3WxsF#LXkibV9G~l6y0$V&PI?VPR6GKjG$kVbaO_6=+nU!fP{4sdFo~Q_0 zs<#`DYzh^}rH7Liqz^aQ7M7W=yN%|&B8Wv!3~rZ3dW+{k zE=ILSOuFGCx~tVwdX~~&SskmaE&ta+g6cHdlY1&@`P@V!Hk1{N&3%&`J8t=!Xm;6+ z?Y->F{oEI28NgxB>AfHC!PK%d(Ux6eXo)@ZQd~JPu>@<=fH+uc?GP3kW~M5y;5LM1 zRb9W0m^QrU(eekhw(8~5MkNxEd=g951VynuCs<1!5(>sFLE7w;ph@Q=v43Ab#A`o! z{--wi_{8hm57?YL_;X*RMOVQQG|k_xw*Mwmzuod+^hG2eY{eVXEx1&82+VZnzZWiN^I6alX1?aNP3&I+;c?1!}kQJdsTHRw*B z`&j#tAL6>3IP9fadaxbQga_kw_bu&hNGpoN0t1fk^a(MA9C2bAU;Mi z<(HUPeT_rXw5wJ3mcQ8;v_px4jn5UWB&2nYB}Y4Na0jJexcmC+Z(uVEve=M>P~4P& z3GZ-1$uGjk`!C+sE|@Wi5(no=z+=b-Or|vPl#U&UtNZW72&U#L9d8m?IYKvmo*jku z+mG;xqg@QSrMuYm4=(q8% zS;E~q6oc_9wZ&c4!W%1iv!TzI{B-oLT*8$EZHQYl4Yz?Ah)0H8;-jK$jC#8T$x6?l2DF8Iymi~j>ow{grkeN#Dvfr zFsdCh**#XId0}Nq-{|)Vc%R9T$pMphnfyy8Ut`ihQY+vT)5ERJ$bl4R7USUUiLhcF zi6%3`z@f;7JsZaFT1iu(BdBYo#MsA$#cPhJFcViGq*h8rC)69fusU*~&@)o2$PuPp zB?z8XThq3Md^}VCfu@E32s(-{@bw7CWDF61tCnMwjHGjC(Fl0yNpmZ{vm(?<_Oz3nO!z)3-u*mMEeq6)F=tIDstyd3#Im>(=b&q#0Eia zFjXXxNje%SNd^9Qr{~eyM6VymHM9CR&czW-%$5Frv9iB7Vd#hl+N23E`Ow`>gM}C< z{s*RDXdKvK21bC~ee)ncM1TZhJOfPlN@&HEp@j%g1VjgC>Z+(Y2$y3)?U#r{RYq~@o`YUXeW&v$22oUce(L1SSVD8MVqFnoMzn14Xy zLQ$akEw-zs?KY&HL?@%pEUx8cmfg1w^5Gmt1ykkytdR*TJM%l$2MZ5req`*Y9^Jng z8V^o*CNSgj9Y4{OqG4IIIvHP`@tzC(N%ZJ1;k)QrALb97q4QvQXg_4EZZbE2F7%vw zaGLOgf@dBS!!tsWaVzuH=V4%(fsvy)EG=aYW;K39@fB%SpzjLihVdMXMPp_8;Y>n< zm;2C0|GdnLapJ*QO-qu02qOo~FzDw~Ftfh*SHl9X(stu>ut?oZW0=4nsU@G9YzLyJ zg_n#ed?BubRrifaNGd>LXAKOOV@?6S)6sL^6}9MH?bjN+K)>bt?sgdVgX?Q+n~kQA zaJW0l5BvsRhgyXldwa04+Ulh*h^{q30ztw9fc!Nw2md_D826&LIfH6|MjRu`$>O)9z9BH z7_l80!qd``aYkQcX=L{L2uWf@oBAClB4E$yI5&~Xv=l4RKt>VV^x4!(47lO76;~;| z;mK+xCN%EX@Osf5pD!ad+Kgq=wwtJemv;cxUL{6(X-`4GW)je3-NQbrS(}~hMlAwe2Kv99l2fAAQ zC?_viBH;M*Bx6dcc$YBIBFa70+G^n+7HAbfPuXv=_!~%|FY)!O$Y@3RSoTL*1RhNZ zcJ%@3{%aFLo&8v+i&aNLx)614NQ?xcBlu)N0GG28C&73LickpsG6cB8=O34>at@(j zCd0v!kYPlmv&^u6P%M;f!VwdTl(@?f{4BLUatkcke|AC$|7@BZ-nn02S(uKt_zI_h z(ZlL46UuS@o+!gf^t+=t&@*a}i7=CBnz9t1A?&xATwub<8Jniw zM=S^8^^4!ne9dZDAF*Z<*-y-@Y30j5(C_NYz7QXma4z>nlQAU8hq_Cuzx)?uwlW8Z zv4d>&4svw0z#VCN1{{<25K;5|izTb%zF~U!Cl9CaE-lJ)d^5(&p2l}EBN_*c{diA? z?v5-Rk`8cD53zvJ{xEG%j2bWAV#xtJ+C0dKyhoo8;0R4h(O33VMkKK_H11?o1Lb*O zHh|kW4N9OZ*>DXwAsndpMP|PT9SX8KzpfNU_w76r2Z?wb)!TT&|HrUB|1N_46z6EA zsQ2D3D}5bFl#@G!`Xd59M1ubWGicYjy<-nPoO*qR?x|UnBKGo#Nw3YwslbE857ajW z3p2GMz2LaxFl0$COTtPce)?ThJprx^R`Foj4)%xZRIrdBl9)$HC$ul?R8wUeq^3uu86Zjzsp|ySVKnV eQt{)(Us}v%zQ#Xu89XIzd9v6zgX~=ShyMh8zj}uN literal 0 HcmV?d00001 diff --git a/vllm/attention/backends/abstract.py b/vllm/attention/backends/abstract.py new file mode 100644 index 00000000..2bc36ff1 --- /dev/null +++ b/vllm/attention/backends/abstract.py @@ -0,0 +1,232 @@ +from abc import ABC, abstractmethod +from contextlib import contextmanager +from dataclasses import dataclass, fields +from enum import Enum, auto +from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set, + Tuple, Type, TypeVar) + +import torch + +if TYPE_CHECKING: + from vllm.worker.model_runner_base import (ModelRunnerBase, + ModelRunnerInputBase, + ModelRunnerInputBuilderBase) + + +class AttentionType(Enum): + DECODER = auto() # Decoder attention between previous layer Q/K/V + ENCODER = auto() # Encoder attention between previous layer Q/K/V + ENCODER_DECODER = auto() # Attention between dec. Q and enc. K/V + + +class AttentionBackend(ABC): + """Abstract class for attention backends.""" + + @staticmethod + @abstractmethod + def get_name() -> str: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_impl_cls() -> Type["AttentionImpl"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_state_cls() -> Type["AttentionState"]: + raise NotImplementedError + + @classmethod + def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata": + return cls.get_metadata_cls()(*args, **kwargs) + + @staticmethod + @abstractmethod + def get_builder_cls() -> Type["AttentionMetadataBuilder"]: + raise NotImplementedError + + @classmethod + def make_metadata_builder(cls, *args, + **kwargs) -> "AttentionMetadataBuilder": + return cls.get_builder_cls()(*args, **kwargs) + + @staticmethod + @abstractmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + raise NotImplementedError + + @staticmethod + @abstractmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + raise NotImplementedError + + def advance_step(self, model_input: "ModelRunnerInputBase", + sampled_token_ids: Optional[torch.Tensor], + block_size: int, num_seqs: int, num_queries: int) -> None: + raise NotImplementedError + + +@dataclass +class AttentionMetadata: + """Attention metadata for prefill and decode batched together.""" + # Total number of prefill requests. + num_prefills: int + # Number of prefill tokens. + num_prefill_tokens: int + # Number of decode tokens. Note that it is equivalent to the number of + # decode requests. + num_decode_tokens: int + # (num_tokens,). The indices of the token slots that input tokens will be + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot + # in block 0, and 1st slot in block 1, respectively. + slot_mapping: torch.Tensor + + @property + @abstractmethod + def prefill_metadata(self) -> Optional["AttentionMetadata"]: + """Return the attention metadata that's required to run prefill + attention.""" + pass + + @property + @abstractmethod + def decode_metadata(self) -> Optional["AttentionMetadata"]: + """Return the attention metadata that's required to run decode + attention.""" + pass + + def asdict_zerocopy(self, + skip_fields: Optional[Set[str]] = None + ) -> Dict[str, Any]: + """Similar to dataclasses.asdict, but avoids deepcopying.""" + if skip_fields is None: + skip_fields = set() + # Note that if we add dataclasses as fields, they will need + # similar handling. + return { + field.name: getattr(self, field.name) + for field in fields(self) if field.name not in skip_fields + } + + +T = TypeVar("T", bound=AttentionMetadata) + + +class AttentionState(ABC, Generic[T]): + """Holds attention backend-specific objects reused during the + lifetime of the model runner.""" + + @abstractmethod + def __init__(self, runner: "ModelRunnerBase"): + ... + + @abstractmethod + @contextmanager + def graph_capture(self, max_batch_size: int): + """Context manager used when capturing CUDA graphs.""" + yield + + @abstractmethod + def graph_clone(self, batch_size: int) -> "AttentionState[T]": + """Clone attention state to save in CUDA graph metadata.""" + ... + + @abstractmethod + def graph_capture_get_metadata_for_batch( + self, + batch_size: int, + is_encoder_decoder_model: bool = False) -> T: + """Get attention metadata for CUDA graph capture of batch_size.""" + ... + + @abstractmethod + def get_graph_input_buffers( + self, + attn_metadata: T, + is_encoder_decoder_model: bool = False) -> Dict[str, Any]: + """Get attention-specific input buffers for CUDA graph capture.""" + ... + + @abstractmethod + def prepare_graph_input_buffers( + self, + input_buffers: Dict[str, Any], + attn_metadata: T, + is_encoder_decoder_model: bool = False) -> None: + """In-place modify input buffers dict for CUDA graph replay.""" + ... + + @abstractmethod + def begin_forward(self, model_input: "ModelRunnerInputBase") -> None: + """Prepare state for forward pass.""" + ... + + +class AttentionMetadataBuilder(ABC, Generic[T]): + """Abstract class for attention metadata builders.""" + + @abstractmethod + def __init__(self, input_builder: "ModelRunnerInputBuilderBase") -> None: + raise NotImplementedError + + @abstractmethod + def build(self, seq_lens: List[int], query_lens: List[int], + cuda_graph_pad_size: int, batch_size: int) -> T: + """Build attention metadata with on-device tensors.""" + raise NotImplementedError + + +class AttentionImpl(ABC, Generic[T]): + + @abstractmethod + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: Optional[int] = None, + alibi_slopes: Optional[List[float]] = None, + sliding_window: Optional[int] = None, + kv_cache_dtype: str = "auto", + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + raise NotImplementedError + + @abstractmethod + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: T, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + raise NotImplementedError diff --git a/vllm/attention/backends/blocksparse_attn.py b/vllm/attention/backends/blocksparse_attn.py new file mode 100644 index 00000000..c216d195 --- /dev/null +++ b/vllm/attention/backends/blocksparse_attn.py @@ -0,0 +1,444 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, Type + +import torch + +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import (CommonAttentionState, + CommonMetadataBuilder) +from vllm.attention.ops.blocksparse_attention.interface import ( + LocalStridedBlockSparseAttn, get_head_sliding_step) +from vllm.attention.ops.paged_attn import PagedAttention +from vllm.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) + + +@dataclass +class BlocksparseParams: + max_seqlen: int + + # Num q heads per tensor-parallel rank/partition + num_heads: int # per TP partition + # Num kv heads per tensor-parallel rank/partition + num_kv_heads: int + + # block size used for blocksparse attention. + # This is the block_size used in `local_blocks`, `vert_stride`. + block_size: int + + # Number of blocks for local attention, i.e., number of + # local attended tokens / `sparse_block_size` + local_blocks: int + + # Attend to one block per every `vert_stride` blocks. + # Controlling the sparsity + vert_stride: int + """ + If to use the same vertical stride offset for all heads, + i.e., attend to the same block of tokens on all heads. + By default, it is False, i.e., attention on the non-local + blocks depends on the `head_idx`, that is on + blocks satisfying + `(block_idx + head_idx * head_sliding_step + 1) % vert_stride == 0` + where `head_sliding_step=max(1, int(vert_stride / num_total_heads))`, + `block_idx = position_id // sparse_block_size`. + See `..ops.blocksparse_attention.utils:get_sparse_attn_mask` + for more detail. + """ + homo_head: bool = False + + # If within a group, the kv offsets that each q attends is the same or no. + homo_head_group: bool = False + + # Decided by homo_head and homo_head group + head_sliding_step: int = field(init=False) + + # range of q heads to for a TP rank + active_head_range: Tuple = field(init=False) + + def __post_init__(self): + assert self.block_size > 0 + assert self.local_blocks >= 0 + assert self.vert_stride >= 1 + assert self.num_heads % self.num_kv_heads == 0 + + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + total_heads = tp_size * self.num_heads + total_kv_heads = tp_size * self.num_kv_heads + + if self.homo_head: + self.head_sliding_step = 0 + elif self.homo_head_group: + head_sliding_step = get_head_sliding_step(total_kv_heads, + self.vert_stride) + # negative indicates sliding along kv heads, i.e., homo q group + self.head_sliding_step = -head_sliding_step + else: + self.head_sliding_step = get_head_sliding_step( + total_heads, self.vert_stride) + + self.active_head_range = ( + tp_rank * self.num_heads, + (tp_rank + 1) * self.num_heads, + ) + + +class BlocksparseFlashAttentionBackend(AttentionBackend): + + @staticmethod + def get_impl_cls() -> Type["BlocksparseFlashAttentionImpl"]: + return BlocksparseFlashAttentionImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return BlocksparseFlashAttentionMetadata + + @staticmethod + def get_builder_cls() -> Type["BlocksparseFlashAttentionMetadataBuilder"]: + return BlocksparseFlashAttentionMetadataBuilder + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return PagedAttention.get_kv_cache_shape(num_blocks, block_size, + num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: Dict[int, int], + ) -> None: + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: Dict[int, List[int]], + ) -> None: + PagedAttention.copy_blocks(kv_caches, src_to_dists) + + +@dataclass +class BlocksparseFlashAttentionMetadata(AttentionMetadata): + """A copy of Metadata for FlashAttentionBackend, + to avoid having to install flash_attn. + + NOTE: Any python object stored here is not updated when it is + cuda-graph replayed. If you have values that need to be changed + dynamically, it should be stored in tensor. The tensor has to be + updated from `CUDAGraphRunner.forward` API. + """ + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] + + # NOTE(sang): Definition of context_len, query_len, and seq_len. + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ----------------------| + # |-- query_len ---| + + # Maximum query length in the batch. None for decoding. + max_query_len: Optional[int] + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + # Maximum sequence length among decode batch. 0 if there are prefill + # requests only. + max_decode_seq_len: int + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] + # (batch_size,) A tensor of context lengths (tokens that are computed + # so far). + context_lens_tensor: Optional[torch.Tensor] + + # (batch_size, max_blocks_per_seq). + # Block addresses per sequence. (Seq id -> list of physical block) + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks + # in the kv cache. Each block can contain up to block_size tokens. + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph + # captured. + block_tables: Optional[torch.Tensor] + + # Whether or not if cuda graph is enabled. + # Cuda-graph is currently enabled for decoding only. + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention. + use_cuda_graph: bool + + # Max number of query tokens for among request in the batch. + max_decode_query_len: Optional[int] = None + + _cached_prefill_metadata: Optional[ + "BlocksparseFlashAttentionMetadata"] = None + _cached_decode_metadata: Optional[ + "BlocksparseFlashAttentionMetadata"] = None + + @property + def prefill_metadata( + self) -> Optional["BlocksparseFlashAttentionMetadata"]: + if self.num_prefills == 0: + return None + + if self._cached_prefill_metadata is not None: + return self._cached_prefill_metadata + + assert self.seq_lens is not None + assert self.seq_lens_tensor is not None + assert self.query_start_loc is not None + assert self.context_lens_tensor is not None + assert self.block_tables is not None + assert self.seq_start_loc is not None + + self._cached_prefill_metadata = BlocksparseFlashAttentionMetadata( + num_prefills=self.num_prefills, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=0, + slot_mapping=self.slot_mapping[:self.num_prefill_tokens], + seq_lens=self.seq_lens[:self.num_prefills], + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills], + max_query_len=self.max_query_len, + max_prefill_seq_len=self.max_prefill_seq_len, + max_decode_seq_len=0, + query_start_loc=self.query_start_loc[:self.num_prefills + 1], + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1], + context_lens_tensor=self.context_lens_tensor[:self.num_prefills], + block_tables=self.block_tables[:self.num_prefills], + use_cuda_graph=False, + ) + return self._cached_prefill_metadata + + @property + def decode_metadata(self) -> Optional["BlocksparseFlashAttentionMetadata"]: + if self.num_decode_tokens == 0: + return None + + if self._cached_decode_metadata is not None: + return self._cached_decode_metadata + assert self.block_tables is not None + assert self.seq_lens_tensor is not None + + self._cached_decode_metadata = BlocksparseFlashAttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=self.num_decode_tokens, + slot_mapping=self.slot_mapping[self.num_prefill_tokens:], + seq_lens=None, + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:], + max_query_len=None, + max_prefill_seq_len=0, + max_decode_seq_len=self.max_decode_seq_len, + query_start_loc=None, + seq_start_loc=None, + context_lens_tensor=None, + block_tables=self.block_tables[self.num_prefills:], + use_cuda_graph=self.use_cuda_graph, + ) + return self._cached_decode_metadata + + +class BlocksparseFlashAttentionMetadataBuilder( + CommonMetadataBuilder[BlocksparseFlashAttentionMetadata]): + + _metadata_cls = BlocksparseFlashAttentionMetadata + + +class BlocksparseFlashAttentionImpl(AttentionImpl): + """ + If the input tensors contain prompt tokens, the layout is as follows: + |<--------------- num_prompt_tokens -------------->| + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->| + + Otherwise, the layout is as follows: + |<------------------ num_generation_tokens (M) ----------------->| + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->| + + Generation tokens can contain padding when cuda-graph is used. + Currently, prompt tokens don't contain any padding. + + The prompts might have different lengths, while the generation tokens + always have length 1. + + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + assert blocksparse_params is not None + assert alibi_slopes is None, ValueError( + "Alibi not support for blocksparse flash attention.") + assert sliding_window is None, ValueError( + "sliding_window is invalid for blocksparse attention.") + assert logits_soft_cap is None, ValueError( + "logits_soft_cap is invalid for blocksparse attention.") + + if "num_heads" not in blocksparse_params: + blocksparse_params["num_heads"] = num_heads + if "num_kv_heads" not in blocksparse_params: + blocksparse_params["num_kv_heads"] = num_kv_heads or num_heads + self.blocksparse_params = BlocksparseParams(**blocksparse_params) + self.kv_cache_dtype = kv_cache_dtype + + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.alibi_slopes = alibi_slopes + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + self.local_blocks = self.blocksparse_params.local_blocks + self.vert_stride = self.blocksparse_params.vert_stride + self.sparse_block_size = self.blocksparse_params.block_size + self.head_sliding_step = self.blocksparse_params.head_sliding_step + + suppored_head_sizes = PagedAttention.get_supported_head_sizes() + if head_size not in suppored_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by PagedAttention. " + f"Supported head sizes are: {suppored_head_sizes}.") + + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + + total_num_heads = num_heads * self.tp_size + self.bs_attn = LocalStridedBlockSparseAttn( + total_num_heads, + self.blocksparse_params.max_seqlen, + self.blocksparse_params.local_blocks, + self.blocksparse_params.vert_stride, + self.blocksparse_params.block_size, + homo_head=self.blocksparse_params.homo_head, + active_head_range=self.blocksparse_params.active_head_range, + ) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: BlocksparseFlashAttentionMetadata, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with FlashAttention and PagedAttention. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [num_tokens, num_heads * head_size] + """ + if attn_type != AttentionType.DECODER: + raise NotImplementedError("Encoder self-attention and " + "encoder/decoder cross-attention " + "are not implemented for " + "BlocksparseFlashAttentionImpl") + + num_tokens, hidden_size = query.shape + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + + if kv_cache.numel() > 0: + key_cache, value_cache = PagedAttention.split_kv_cache( + kv_cache, self.num_kv_heads, self.head_size) + + # Reshape the input keys and values and store them in the cache. + # If kv_cache is not provided, the new key and value tensors are + # not cached. This happens during the initial memory profiling run. + + PagedAttention.write_to_paged_cache( + key, + value, + key_cache, + value_cache, + attn_metadata.slot_mapping, + self.kv_cache_dtype, + k_scale, + v_scale, + ) + + if prefill_meta := attn_metadata.prefill_metadata: + + # Prompt run. + # normal attention + # When block_tables are not filled, it means q and k are the + # prompt, and they have the same length. + + assert kv_cache.numel() == 0 \ + or prefill_meta.block_tables is None \ + or prefill_meta.block_tables.numel() == 0, \ + "Does not support prefix-enabled attention." + + output = self.bs_attn( + q=query, + k=key, + v=value, + cu_seqlens_q=prefill_meta.seq_start_loc, + cu_seqlens_k=prefill_meta.seq_start_loc, + sm_scale=self.scale, + ) + + if decode_meta := attn_metadata.decode_metadata: + # Decoding run. + output = PagedAttention.forward_decode( + query, + key_cache, + value_cache, + decode_meta.block_tables, + decode_meta.seq_lens_tensor, + self.blocksparse_params.max_seqlen, + self.kv_cache_dtype, + self.num_kv_heads, + self.scale, + self.alibi_slopes, + k_scale, + v_scale, + tp_rank=self.tp_rank, + blocksparse_local_blocks=self.local_blocks, + blocksparse_vert_stride=self.vert_stride, + blocksparse_block_size=self.sparse_block_size, + blocksparse_head_sliding_step=self.head_sliding_step, + ) + + # Reshape the output tensor. + return output.view(num_tokens, hidden_size) diff --git a/vllm/attention/backends/flash_attn.py b/vllm/attention/backends/flash_attn.py new file mode 100644 index 00000000..c5a90a2d --- /dev/null +++ b/vllm/attention/backends/flash_attn.py @@ -0,0 +1,951 @@ + +"""Attention layer with FlashAttention.""" +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type + +import torch + +from vllm import _custom_ops as ops +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, + AttentionType) +from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState, + compute_slot_mapping, + compute_slot_mapping_start_idx, + is_block_tables_empty) +from vllm.forward_context import get_forward_context +from vllm.utils import async_tensor_h2d, make_tensor_with_pad + +if TYPE_CHECKING: + from vllm.worker.model_runner import (ModelInputForGPUBuilder, + ModelInputForGPUWithSamplingMetadata) + +from ixformer.contrib.vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func +from ixformer.contrib.vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache + + +def flash_attn_varlen_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Optional[List[int]] = None, + softcap: float = 0.0, + alibi_slopes: Optional[torch.Tensor] = None, + block_table: Optional[torch.Tensor] = None, + out: Optional[torch.Tensor] = None, + use_sqrt_alibi: Optional[bool] = False +) -> torch.Tensor: + # custom op does not support tuple input + real_window_size: Tuple[int, int] + if window_size is None: + real_window_size = (-1, -1) + else: + assert len(window_size) == 2 + real_window_size = (window_size[0], window_size[1]) + return _flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=softmax_scale, + causal=causal, + window_size=real_window_size, + softcap=softcap, + alibi_slopes=alibi_slopes, + block_table=block_table, + out=out, + sqrt_alibi=use_sqrt_alibi, + ) + + +def flash_attn_with_kvcache( + decode_query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + seq_lens_cpu_tensors: torch.Tensor, + max_context_len: int, + cache_seqlens: Optional[torch.Tensor] = None, + block_table: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + alibi_slopes: Optional[torch.Tensor] = None, + softcap: float = 0.0, + out: Optional[torch.Tensor] = None, + use_sqrt_alibi: bool = False +) -> torch.Tensor: + return _flash_attn_with_kvcache( + decode_query, + key_cache, + value_cache, + cache_seqlens=cache_seqlens, + block_table=block_table, + softmax_scale=softmax_scale, + causal=causal, + alibi_slopes=alibi_slopes, + softcap=softcap, + max_context_len=max_context_len, + cache_seqlens_cpu=seq_lens_cpu_tensors, + out=out, + use_sqrt_alibi=use_sqrt_alibi + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + """Inductor cannot deal with inplace operations on views. + See https://github.com/pytorch/pytorch/issues/131192 + and https://github.com/pytorch/pytorch/issues/130174 + This is a workaround to hide the view operation from the inductor. + """ + return ops.reshape_and_cache_flash( + key, value, kv_cache[0], kv_cache[1], slot_mapping, kv_cache_dtype, + k_scale, v_scale) + + +class FlashAttentionBackend(AttentionBackend): + + @staticmethod + def get_supported_head_sizes() -> List[int]: + return [32, 64, 80, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "flash-attn" + + @staticmethod + def get_impl_cls() -> Type["FlashAttentionImpl"]: + return FlashAttentionImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return FlashAttentionMetadata + + @staticmethod + def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]: + return FlashAttentionMetadataBuilder + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + if block_size % 16 != 0: + raise ValueError("Block size must be a multiple of 16.") + return (2, num_blocks, num_kv_heads, block_size, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + src_key_cache = src_kv_cache[0] + dst_key_cache = dst_kv_cache[0] + ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst) + + src_value_cache = src_kv_cache[1] + dst_value_cache = dst_kv_cache[1] + ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + key_caches = [kv_cache[0] for kv_cache in kv_caches] + value_caches = [kv_cache[1] for kv_cache in kv_caches] + ops.copy_blocks(key_caches, value_caches, src_to_dists) + + +@dataclass +class FlashAttentionMetadata(AttentionMetadata): + """Metadata for FlashAttentionBackend. + + NOTE: Any python object stored here is not updated when it is + cuda-graph replayed. If you have values that need to be changed + dynamically, it should be stored in tensor. The tensor has to be + updated from `CUDAGraphRunner.forward` API. + """ + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] + + # NOTE(sang): Definition of context_len, query_len, and seq_len. + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ---------------------| + # |-- query_len ---| + + # Maximum query length in the batch. + max_query_len: Optional[int] + + # Max number of query tokens among request in the batch. + max_decode_query_len: Optional[int] + + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + # Maximum sequence length among decode batch. 0 if there are prefill + # requests only. + max_decode_seq_len: int + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] + # (batch_size,) A tensor of context lengths (tokens that are computed + # so far). + context_lens_tensor: Optional[torch.Tensor] + + # (batch_size, max_blocks_per_seq). + # Block addresses per sequence. (Seq id -> list of physical block) + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks + # in the kv cache. Each block can contain up to block_size tokens. + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph + # captured. + block_tables: Optional[torch.Tensor] + + # Whether or not if cuda graph is enabled. + # Cuda-graph is currently enabled for decoding only. + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention. + use_cuda_graph: bool + + _cached_prefill_metadata: Optional["FlashAttentionMetadata"] = None + _cached_decode_metadata: Optional["FlashAttentionMetadata"] = None + + # Begin encoder attn & enc/dec cross-attn fields... + + # Encoder sequence lengths representation + encoder_seq_lens: Optional[List[int]] = None + encoder_seq_lens_tensor: Optional[torch.Tensor] = None + + # Maximum sequence length among encoder sequences + max_encoder_seq_len: Optional[int] = None + + # Number of tokens input to encoder + num_encoder_tokens: Optional[int] = None + + # Cross-attention memory-mapping data structures: slot mapping + # and block tables + cross_slot_mapping: Optional[torch.Tensor] = None + cross_block_tables: Optional[torch.Tensor] = None + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence on encoder-decoder + encoder_seq_start_loc: Optional[torch.Tensor] = None + + # Our impl need info fields... + seq_lens_cpu_tensors: Optional[torch.Tensor] = None + encoder_seq_lens_cpu_tensor: Optional[torch.Tensor] = None + + @property + def prefill_metadata(self) -> Optional["FlashAttentionMetadata"]: + if self.num_prefills == 0: + return None + + if self._cached_prefill_metadata is not None: + return self._cached_prefill_metadata + + assert self.seq_lens is not None + assert self.seq_lens_tensor is not None + assert self.query_start_loc is not None + assert self.context_lens_tensor is not None + assert self.block_tables is not None + assert self.seq_start_loc is not None + + self._cached_prefill_metadata = FlashAttentionMetadata( + num_prefills=self.num_prefills, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=0, + slot_mapping=self.slot_mapping[:self.num_prefill_tokens], + seq_lens=self.seq_lens[:self.num_prefills], + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills], + max_query_len=self.max_query_len, + max_prefill_seq_len=self.max_prefill_seq_len, + max_decode_query_len=0, + max_decode_seq_len=0, + query_start_loc=self.query_start_loc[:self.num_prefills + 1], + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1], + context_lens_tensor=self.context_lens_tensor[:self.num_prefills], + block_tables=self.block_tables[:self.num_prefills], + use_cuda_graph=False, + # Begin encoder & cross attn fields below... + encoder_seq_lens=self.encoder_seq_lens, + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor, + encoder_seq_start_loc=self.encoder_seq_start_loc, + max_encoder_seq_len=self.max_encoder_seq_len, + cross_slot_mapping=self.cross_slot_mapping, + cross_block_tables=self.cross_block_tables + ) + return self._cached_prefill_metadata + + @property + def decode_metadata(self) -> Optional["FlashAttentionMetadata"]: + if self.num_decode_tokens == 0: + return None + + if self._cached_decode_metadata is not None: + return self._cached_decode_metadata + assert self.block_tables is not None + assert self.seq_lens_tensor is not None + + seq_lens_cpu_tensors = torch.tensor(self.seq_lens[self.num_prefills:],dtype=torch.int32,device="cpu") + encoder_seq_lens_cpu_tensor = torch.tensor(self.encoder_seq_lens,dtype=torch.int32,device="cpu") if self.encoder_seq_lens is not None else None + max_seq_len = self.seq_lens_tensor.max().item() + + self._cached_decode_metadata = FlashAttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=self.num_decode_tokens, + slot_mapping=self.slot_mapping[self.num_prefill_tokens:], + seq_lens=None, + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:], + max_decode_query_len=self.max_decode_query_len, + max_query_len=max_seq_len, + max_prefill_seq_len=0, + max_decode_seq_len=self.max_decode_seq_len, + query_start_loc=self.query_start_loc[self.num_prefills:] + if self.query_start_loc is not None else None, + seq_start_loc=self.seq_start_loc[self.num_prefills:] + if self.seq_start_loc is not None else None, + context_lens_tensor=None, + block_tables=self.block_tables[self.num_prefills:], + use_cuda_graph=self.use_cuda_graph, + seq_lens_cpu_tensors=seq_lens_cpu_tensors, + # Begin encoder & cross attn fields below... + encoder_seq_lens=self.encoder_seq_lens, + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor, + encoder_seq_lens_cpu_tensor=encoder_seq_lens_cpu_tensor, + max_encoder_seq_len=self.max_encoder_seq_len, + cross_slot_mapping=self.cross_slot_mapping, + cross_block_tables=self.cross_block_tables + ) + return self._cached_decode_metadata + + def advance_step(self, + model_input: "ModelInputForGPUWithSamplingMetadata", + sampled_token_ids: Optional[torch.Tensor], + block_size: int, + num_seqs: int, + num_queries: int, + turn_prefills_into_decodes: bool = False): + """ + Update metadata in-place to advance one decode step. + """ + # When using cudagraph, the num_seqs is padded to the next captured + # batch sized, but num_queries tracks the actual number of requests in + # the batch. For --enforce-eager mode, num_seqs == num_queries + if num_seqs != num_queries: + assert num_seqs > num_queries + assert self.use_cuda_graph + + if turn_prefills_into_decodes: + # When Mutli-Step is enabled with Chunked-Prefill, prefills and + # decodes are scheduled together. In the first step, all the + # prefills turn into decodes. This update reflects that + # conversion. + assert self.num_decode_tokens + self.num_prefills == num_seqs + self.num_decode_tokens += self.num_prefills + self.num_prefills = 0 + self.num_prefill_tokens = 0 + self.max_prefill_seq_len = 0 + self.max_query_len = 1 + + self.slot_mapping = self.slot_mapping[:num_seqs] + else: + assert self.seq_lens is not None + assert self.max_decode_seq_len == max(self.seq_lens) + + assert self.num_prefills == 0 + assert self.num_prefill_tokens == 0 + assert self.num_decode_tokens == num_seqs + assert self.slot_mapping.shape == (num_seqs, ) + + assert self.seq_lens is not None + assert len(self.seq_lens) == num_seqs + assert self.seq_lens_tensor is not None + assert self.seq_lens_tensor.shape == (num_seqs, ) + assert self.max_query_len == 1 + assert self.max_prefill_seq_len == 0 + + assert self.query_start_loc is not None + assert self.query_start_loc.shape == (num_queries + 1, ) + assert self.seq_start_loc is not None + assert self.seq_start_loc.shape == (num_seqs + 1, ) + + assert self.context_lens_tensor is not None + assert self.context_lens_tensor.shape == (num_queries, ) + + assert self.block_tables is not None + assert self.block_tables.shape[0] == num_seqs + + # Update query lengths. Note that we update only queries and not seqs, + # since tensors may be padded due to captured cuda graph batch size + for i in range(num_queries): + self.seq_lens[i] += 1 + self.max_decode_seq_len = max(self.seq_lens) + + ops.advance_step_flashattn(num_seqs=num_seqs, + num_queries=num_queries, + block_size=block_size, + input_tokens=model_input.input_tokens, + sampled_token_ids=sampled_token_ids, + input_positions=model_input.input_positions, + seq_lens=self.seq_lens_tensor, + slot_mapping=self.slot_mapping, + block_tables=self.block_tables) + + +class FlashAttentionMetadataBuilder( + AttentionMetadataBuilder[FlashAttentionMetadata]): + + def __init__(self, input_builder: "ModelInputForGPUBuilder"): + self.slot_mapping: List[int] = [] + self.prefill_seq_lens: List[int] = [] + self.context_lens: List[int] = [] + self.block_tables: List[List[int]] = [] + self.curr_seq_lens: List[int] = [] + self.num_prefills = 0 + self.num_prefill_tokens = 0 + self.num_decode_tokens = 0 + self.has_prefix_cache_hit = False + + self.input_builder = input_builder + self.runner = input_builder.runner + self.sliding_window = input_builder.sliding_window + self.block_size = input_builder.block_size + self.use_v2_block_manager = ( + input_builder.scheduler_config.use_v2_block_manager) + + def _add_seq_group( + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup", + chunked_prefill_enabled: bool, prefix_cache_hit: bool): + """Add a sequence group to the metadata. Specifically update/append + 1. context length. + 2. block table. + 3. slot mapping. + """ + is_prompt = inter_data.is_prompt + block_tables = inter_data.block_tables + + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len, + curr_sliding_window_block) in zip( + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens], + inter_data.orig_seq_lens, inter_data.seq_lens, + inter_data.query_lens, inter_data.context_lens, + inter_data.curr_sliding_window_blocks): + self.context_lens.append(context_len) + + if is_prompt: + self.num_prefills += 1 + self.num_prefill_tokens += token_len + self.prefill_seq_lens.append(seq_len) + else: + self.num_decode_tokens += query_len + self.curr_seq_lens.append(curr_seq_len) + + # Compute block table. + # TODO(sang): Combine chunked prefill and prefix caching by + # only allowing multiple of block_size chunk size. + # NOTE: This only works for oooooooxxx style attention. + block_table = [] + if prefix_cache_hit: + # NOTE(woosuk): For flash-attn, the block table should + # include the entries for the incoming prefill tokens. + block_table = block_tables[seq_id] + elif ((chunked_prefill_enabled or not is_prompt) + and block_tables is not None): + if curr_sliding_window_block == 0: + block_table = block_tables[seq_id] + else: + block_table = block_tables[seq_id][ + -curr_sliding_window_block:] + self.block_tables.append(block_table) + + # Compute slot mapping. + is_profile_run = is_block_tables_empty(block_tables) + start_idx = compute_slot_mapping_start_idx( + is_prompt, query_len, context_len, self.sliding_window, + self.use_v2_block_manager) + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id, + seq_len, context_len, start_idx, + self.block_size, inter_data.block_tables) + + def _get_graph_runner_block_tables( + self, num_seqs: int, + block_tables: List[List[int]]) -> torch.Tensor: + # The shape of graph_block_tables is + # [max batch size, max context len // block size]. + max_batch_size, max_blocks = self.runner.graph_block_tables.shape + assert max_batch_size >= num_seqs + + graph_block_tables = self.runner.graph_block_tables[:num_seqs] + for i, block_table in enumerate(block_tables): + if block_table: + num_blocks = len(block_table) + if num_blocks <= max_blocks: + graph_block_tables[i, :num_blocks] = block_table + else: + # It may be possible to have more blocks allocated due + # to lookahead slots of multi-step, however, they are + # not used anyway, so can be safely ignored. + graph_block_tables[ + i, :max_blocks] = block_table[:max_blocks] + + return torch.from_numpy(graph_block_tables).to( + device=self.runner.device, non_blocking=True) + + def build(self, seq_lens: List[int], query_lens: List[int], + cuda_graph_pad_size: int, batch_size: int): + """Build attention metadata with on-device tensors. + + Args: + seq_lens: The maybe padded sequence lengths of the input sequences. + query_lens: The query lengths of the input sequences. + cuda_graph_pad_size: The padding size for cuda graph. + -1 if cuda graph is not used. + batch_size: The maybe padded batch size. + """ + prefix_cache_hit = any([ + inter_data.prefix_cache_hit + for inter_data in self.input_builder.inter_data_list + ]) + for inter_data in self.input_builder.inter_data_list: + self._add_seq_group(inter_data, + self.input_builder.chunked_prefill_enabled, + prefix_cache_hit) + + device = self.runner.device + use_captured_graph = cuda_graph_pad_size != -1 + + max_query_len = max(query_lens) + decode_query_lens = query_lens[self.num_prefills:] + if len(decode_query_lens) > 0: + max_decode_query_len = max(decode_query_lens) + else: + max_decode_query_len = 1 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0) + max_decode_seq_len = max(self.curr_seq_lens, default=0) + num_decode_tokens = self.num_decode_tokens + + num_seqs = len(seq_lens) + if use_captured_graph: + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size) + self.block_tables.extend([] * cuda_graph_pad_size) + num_decode_tokens = batch_size - self.num_prefill_tokens + block_tables = self._get_graph_runner_block_tables( + num_seqs, self.block_tables) + else: + block_tables = make_tensor_with_pad( + self.block_tables, + pad=0, + dtype=torch.int, + device=device, + ) + assert max_query_len > 0, ("query_lens: {}".format(query_lens)) + + assert device is not None + context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int, + device, self.runner.pin_memory) + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device, + self.runner.pin_memory) + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device, + self.runner.pin_memory) + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long, + device, self.runner.pin_memory) + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + torch.cumsum(seq_lens_tensor, + dim=0, + dtype=seq_start_loc.dtype, + out=seq_start_loc[1:]) + torch.cumsum(query_lens_tensor, + dim=0, + dtype=query_start_loc.dtype, + out=query_start_loc[1:]) + + return FlashAttentionMetadata( + num_prefills=self.num_prefills, + slot_mapping=slot_mapping_tensor, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=num_decode_tokens, + seq_lens=seq_lens, + seq_lens_tensor=seq_lens_tensor, + max_query_len=max_query_len, + max_decode_query_len=max_decode_query_len, + max_prefill_seq_len=max_prefill_seq_len, + max_decode_seq_len=max_decode_seq_len, + query_start_loc=query_start_loc, + seq_start_loc=seq_start_loc, + context_lens_tensor=context_lens_tensor, + block_tables=block_tables, + use_cuda_graph=use_captured_graph, + ) + + +class FlashAttentionImpl(AttentionImpl): + """ + If the input tensors contain prompt tokens, the layout is as follows: + |<--------------- num_prefill_tokens ----------------->| + |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->| + + Otherwise, the layout is as follows: + |<----------------- num_decode_tokens ------------------>| + |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->| + + Generation tokens can contain padding when cuda-graph is used. + Currently, prompt tokens don't contain any padding. + + The prompts might have different lengths, while the generation tokens + always have length 1. + + If chunked prefill is enabled, prefill tokens and decode tokens can be + batched together in a flattened 1D query. + + |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->| + |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->| + + Currently, cuda graph is disabled for chunked prefill, meaning there's no + padding between prefill and decode tokens. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + use_sqrt_alibi: bool = None + ) -> None: + if blocksparse_params is not None: + raise ValueError( + "FlashAttention does not support block-sparse attention yet, we will support soon") + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + self.use_sqrt_alibi = use_sqrt_alibi + self.sliding_window = ((sliding_window, sliding_window) + if sliding_window is not None else (-1, -1)) + self.kv_cache_dtype = kv_cache_dtype + if logits_soft_cap is None: + # In flash-attn, setting logits_soft_cap as 0 means no soft cap. + logits_soft_cap = 0 + self.logits_soft_cap = logits_soft_cap + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + if sliding_window is not None: + # NOTE(woosuk): flash-attn's sliding window does not work with + # paged KV cache. + # TODO will support on next week. + self.sliding_window = None + + support_head_sizes = FlashAttentionBackend.get_supported_head_sizes() + if head_size not in support_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by FlashAttention. " + f"Supported head sizes are: {support_head_sizes}.") + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: FlashAttentionMetadata, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with FlashAttention. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [num_tokens, num_heads * head_size] + """ + # NOTE(woosuk): FlashAttention does not support FP8 KV cache. + assert k_scale == 1.0 and v_scale == 1.0, ( + "key/v_scale is not supported in FlashAttention.") + + output = unified_flash_attention( + query, + key, + value, + self.num_heads, + self.head_size, + self.num_kv_heads, + kv_cache, + self.kv_cache_dtype, + k_scale, + v_scale, + self.scale, + self.sliding_window, + self.alibi_slopes, + self.logits_soft_cap, + attn_type=attn_type, + use_sqrt_alibi=self.use_sqrt_alibi, + ) + + return output + + +def unified_flash_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + num_heads: int, + head_size: int, + num_kv_heads: int, + kv_cache: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + softmax_scale: float, + window_size: Optional[List[int]] = None, + alibi_slopes: Optional[torch.Tensor] = None, + logits_soft_cap: Optional[float] = None, + attn_type: AttentionType = AttentionType.DECODER, + use_sqrt_alibi: bool = False +) -> torch.Tensor: + + current_metadata = get_forward_context() + assert current_metadata is not None + assert isinstance(current_metadata, FlashAttentionMetadata) + attn_metadata: FlashAttentionMetadata = current_metadata + + # Reshape the query, key, and value tensors. + query = query.view(-1, num_heads, head_size) + if key is not None: + assert value is not None + key = key.view(-1, num_kv_heads, head_size) + value = value.view(-1, num_kv_heads, head_size) + else: + assert value is None + + if attn_type != AttentionType.ENCODER and kv_cache.numel() > 0: + key_cache = kv_cache[0] + value_cache = kv_cache[1] + + # Reshape the input keys and values and store them in the cache. + # If kv_cache is not provided, the new key and value tensors are + # not cached. This happens during the initial memory profiling run. + if (key is not None) and (value is not None): + if attn_type == AttentionType.ENCODER_DECODER: + # Update cross-attention KV cache (prefill-only) + # During cross-attention decode, key & value will be None, + # preventing this IF-statement branch from running + updated_slot_mapping = attn_metadata.cross_slot_mapping.flatten() + else: + # Update self-attention KV cache (prefill/decode) + updated_slot_mapping = attn_metadata.slot_mapping.flatten() + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + updated_slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + if attn_type == AttentionType.ENCODER: + # Encoder attention - chunked prefill is not applicable; + # derive token-count from query shape & and treat them + # as 100% prefill tokens + assert attn_metadata.num_encoder_tokens is not None + num_prefill_tokens = attn_metadata.num_encoder_tokens + num_encoder_tokens = attn_metadata.num_encoder_tokens + num_decode_tokens = 0 + elif attn_type == AttentionType.DECODER: + # Decoder self-attention supports chunked prefill. + num_prefill_tokens = attn_metadata.num_prefill_tokens + num_encoder_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \ + f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa + assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \ + f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa + else: # attn_type == AttentionType.ENCODER_DECODER + # Encoder/decoder cross-attention requires no chunked + # prefill (100% prefill or 100% decode tokens, no mix) + num_prefill_tokens = attn_metadata.num_prefill_tokens + if attn_metadata.num_encoder_tokens is not None: + num_encoder_tokens = attn_metadata.num_encoder_tokens + else: + num_encoder_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + + # Query for decode. KV is not needed because it is already cached. + output = torch.empty_like(query) + decode_query = query[num_prefill_tokens:] + # QKV for prefill. + query = query[:num_prefill_tokens] + if key is not None and value is not None: + key = key[:num_encoder_tokens] + value = value[:num_encoder_tokens] + + assert query.shape[0] == num_prefill_tokens + assert decode_query.shape[0] == num_decode_tokens + + # prefill_output: Optional[torch.Tensor] = None + # decode_output: Optional[torch.Tensor] = None + + if prefill_meta := attn_metadata.prefill_metadata: + # Prompt run. + if (kv_cache.numel() == 0 or prefill_meta.block_tables is None + or prefill_meta.block_tables.numel() == 0): + # normal attention + # When block_tables are not filled, it means q and k are the + # prompt, and they have the same length. + flash_attn_varlen_func( + q=query, + k=key, + v=value, + cu_seqlens_q=prefill_meta.encoder_seq_start_loc if attn_type == AttentionType.ENCODER else prefill_meta.seq_start_loc, + cu_seqlens_k=prefill_meta.seq_start_loc if attn_type == AttentionType.DECODER else prefill_meta.encoder_seq_start_loc, + max_seqlen_q=prefill_meta.max_encoder_seq_len if attn_type == AttentionType.ENCODER else prefill_meta.max_prefill_seq_len, + max_seqlen_k=prefill_meta.max_prefill_seq_len if attn_type == AttentionType.DECODER else prefill_meta.max_encoder_seq_len, + softmax_scale=softmax_scale, + causal=attn_type == AttentionType.DECODER, + window_size=window_size, + alibi_slopes=alibi_slopes, + softcap=logits_soft_cap, + out=output[:num_prefill_tokens], + use_sqrt_alibi=use_sqrt_alibi + ) + else: + # prefix-enabled attention + assert prefill_meta.seq_lens is not None + max_seq_len = max(prefill_meta.seq_lens) + flash_attn_varlen_func( # noqa + q=query, + k=key_cache, + v=value_cache, + cu_seqlens_q=prefill_meta.query_start_loc, + max_seqlen_q=prefill_meta.max_query_len, + cu_seqlens_k=prefill_meta.seq_start_loc, + max_seqlen_k=max_seq_len, + softmax_scale=softmax_scale, + causal=attn_type == AttentionType.DECODER, + alibi_slopes=alibi_slopes, + block_table=prefill_meta.block_tables, + softcap=logits_soft_cap, + out=output[:num_prefill_tokens], + use_sqrt_alibi=use_sqrt_alibi + ) + + if decode_meta := attn_metadata.decode_metadata: + # Decoding run. + # Use flash_attn_varlen_func kernel for speculative decoding + # because different queries might have different lengths. + assert decode_meta.max_decode_query_len is not None + if decode_meta.max_decode_query_len > 1: + flash_attn_varlen_func( + q=decode_query, + k=key_cache, + v=value_cache, + cu_seqlens_q=decode_meta.query_start_loc, + max_seqlen_q=decode_meta.max_decode_query_len, + cu_seqlens_k=decode_meta.seq_start_loc, + max_seqlen_k=decode_meta.max_decode_seq_len, + softmax_scale=softmax_scale, + causal=True, + alibi_slopes=alibi_slopes, + softcap=0.0, + block_table=decode_meta.block_tables, + out=output[num_prefill_tokens:], + ) + else: + # Use flash_attn_with_kvcache for normal decoding. + flash_attn_with_kvcache( + decode_query.unsqueeze(1), + key_cache, + value_cache, + seq_lens_cpu_tensors=decode_meta.seq_lens_cpu_tensors if attn_type == AttentionType.DECODER else decode_meta.encoder_seq_lens_cpu_tensor, + max_context_len=decode_meta.max_query_len if attn_type == AttentionType.DECODER else decode_meta.max_encoder_seq_len, + block_table=decode_meta.block_tables if attn_type == AttentionType.DECODER else decode_meta.cross_block_tables, + cache_seqlens=decode_meta.seq_lens_tensor if attn_type == AttentionType.DECODER else decode_meta.encoder_seq_lens_tensor, + softmax_scale=softmax_scale, + causal=True, + alibi_slopes=alibi_slopes, + softcap=0.0, + out=output[num_prefill_tokens:].unsqueeze(1), + use_sqrt_alibi=use_sqrt_alibi + ).squeeze(1) + + # TODO mv this to flash_attn_with_kvcache when supported. + if logits_soft_cap != 0.0: + output[num_prefill_tokens:] = logits_soft_cap * torch.tanh(output[num_prefill_tokens:] / logits_soft_cap) + + return output.view(-1, num_heads * head_size) + + +# @unified_flash_attention.register_fake +def _( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + num_heads: int, + head_size: int, + num_kv_heads: int, + kv_cache: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + softmax_scale: float, + window_size: Optional[List[int]] = None, + alibi_slopes: Optional[torch.Tensor] = None, + logits_soft_cap: Optional[float] = None, +) -> torch.Tensor: + return torch.empty_like(query) diff --git a/vllm/attention/backends/flashinfer.py b/vllm/attention/backends/flashinfer.py new file mode 100644 index 00000000..be5e7fd6 --- /dev/null +++ b/vllm/attention/backends/flashinfer.py @@ -0,0 +1,901 @@ +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type + +try: + from flashinfer import BatchDecodeWithPagedKVCacheWrapper + from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper + from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper + + from ixformer.contrib.vllm_flash_attn import flash_attn_varlen_func + FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024 +except ImportError: + BatchDecodeWithPagedKVCacheWrapper = None + CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None + BatchPrefillWithPagedKVCacheWrapper = None + FLASHINFER_WORKSPACE_BUFFER_SIZE = 0 + +import torch + +from vllm import _custom_ops as ops +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, + AttentionState, AttentionType) +from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping, + compute_slot_mapping_start_idx, + is_block_tables_empty) +from vllm.attention.ops.paged_attn import PagedAttention +from vllm.forward_context import get_forward_context +from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype, + make_tensor_with_pad) + +if TYPE_CHECKING: + from vllm.worker.model_runner import (ModelInputForGPUBuilder, + ModelInputForGPUWithSamplingMetadata) + + +class FlashInferBackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "flashinfer" + + @staticmethod + def get_impl_cls() -> Type["FlashInferImpl"]: + return FlashInferImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return FlashInferMetadata + + @staticmethod + def get_builder_cls() -> Type["FlashInferMetadataBuilder"]: + return FlashInferMetadataBuilder + + @staticmethod + def get_state_cls() -> Type["FlashInferState"]: + return FlashInferState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return (num_blocks, 2, block_size, num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + PagedAttention.copy_blocks(kv_caches, src_to_dists) + + @staticmethod + def get_supported_head_sizes() -> List[int]: + return [64, 128, 256] + + @staticmethod + def get_fp8_dtype_for_flashinfer(kv_cache_dtype: str) -> torch.dtype: + if kv_cache_dtype in ("fp8", "fp8_e4m3"): + return torch.float8_e4m3fn + elif kv_cache_dtype == "fp8_e5m2": + return torch.float8_e5m2 + else: + raise ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}") + + +class FlashInferState(AttentionState): + + def __init__(self, runner): + self.runner = runner + self._is_graph_capturing = False + self._workspace_buffer = None + self._decode_wrapper = None + self._prefill_wrapper = None + + def _get_workspace_buffer(self): + if self._workspace_buffer is None: + self._workspace_buffer = torch.empty( + FLASHINFER_WORKSPACE_BUFFER_SIZE, + dtype=torch.uint8, + device=self.runner.device) + return self._workspace_buffer + + def _get_prefill_wrapper(self): + if self._prefill_wrapper is None: + self._prefill_wrapper = None + # Unsupported BatchPrefillWithPagedKVCacheWrapper Now + # self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( + # self._get_workspace_buffer(), "NHD") + return self._prefill_wrapper + + def _get_decode_wrapper(self): + if self._decode_wrapper is None: + num_qo_heads = (self.runner.model_config.get_num_attention_heads( + self.runner.parallel_config)) + num_kv_heads = self.runner.model_config.get_num_kv_heads( + self.runner.parallel_config) + use_tensor_cores = num_qo_heads // num_kv_heads > 4 + self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + self._get_workspace_buffer(), + "NHD", + use_tensor_cores=use_tensor_cores) + return self._decode_wrapper + + @contextmanager + def graph_capture(self, max_batch_size: int): + self._is_graph_capturing = True + self._graph_decode_wrapper = None + self._graph_slot_mapping = torch.full((max_batch_size, ), + PAD_SLOT_ID, + dtype=torch.long, + device=self.runner.device) + self._graph_seq_lens = torch.ones(max_batch_size, + dtype=torch.int32, + device=self.runner.device) + self._graph_block_tables = torch.from_numpy( + self.runner.graph_block_tables).to(device=self.runner.device) + self._graph_decode_workspace_buffer = self._get_workspace_buffer() + self._graph_indices_buffer = torch.empty( + max_batch_size * self.runner.cache_config.num_gpu_blocks, + dtype=torch.int32, + device=self.runner.device) + self._graph_indptr_buffer = torch.empty(max_batch_size + 1, + dtype=torch.int32, + device=self.runner.device) + self._graph_last_page_len_buffer = torch.empty( + max_batch_size, dtype=torch.int32, device=self.runner.device) + yield + self._is_graph_capturing = False + del self._graph_slot_mapping + del self._graph_seq_lens + del self._graph_block_tables + del self._graph_decode_workspace_buffer + del self._graph_indices_buffer + del self._graph_indptr_buffer + del self._graph_last_page_len_buffer + del self._graph_decode_wrapper + + def graph_clone(self, batch_size: int): + assert self._is_graph_capturing + state = self.__class__(self.runner) + state._workspace_buffer = self._graph_decode_workspace_buffer + state._decode_wrapper = self._graph_decode_wrapper + state._prefill_wrapper = self._get_prefill_wrapper() + return state + + def graph_capture_get_metadata_for_batch( + self, batch_size: int, is_encoder_decoder_model: bool = False): + assert self._is_graph_capturing + _indptr_buffer = self._graph_indptr_buffer[:batch_size + 1] + _last_page_len_buffer = self._graph_last_page_len_buffer[:batch_size] + + num_qo_heads = (self.runner.model_config.get_num_attention_heads( + self.runner.parallel_config)) + num_kv_heads = self.runner.model_config.get_num_kv_heads( + self.runner.parallel_config) + use_tensor_cores = num_qo_heads // num_kv_heads > 4 + self._graph_decode_wrapper = \ + CUDAGraphBatchDecodeWithPagedKVCacheWrapper( + self._graph_decode_workspace_buffer, _indptr_buffer, + self._graph_indices_buffer, _last_page_len_buffer, "NHD", + use_tensor_cores) + if self.runner.kv_cache_dtype.startswith("fp8"): + kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer( + self.runner.kv_cache_dtype) + else: + kv_cache_dtype = get_kv_cache_torch_dtype( + self.runner.kv_cache_dtype, self.runner.model_config.dtype) + + paged_kv_indptr_tensor_host = torch.arange(0, + batch_size + 1, + dtype=torch.int32) + paged_kv_indices_tensor_host = torch.arange(0, + batch_size, + dtype=torch.int32) + paged_kv_last_page_len_tensor_host = torch.full((batch_size, ), + self.runner.block_size, + dtype=torch.int32) + query_start_loc_host = torch.arange(0, + batch_size + 1, + dtype=torch.int32) + + attn_metadata = self.runner.attn_backend.make_metadata( + num_prefills=0, + slot_mapping=self._graph_slot_mapping[:batch_size], + num_prefill_tokens=0, + num_decode_tokens=batch_size, + max_prefill_seq_len=0, + block_tables=self._graph_block_tables, + paged_kv_indptr=paged_kv_indptr_tensor_host, + paged_kv_indices=paged_kv_indices_tensor_host, + paged_kv_last_page_len=paged_kv_last_page_len_tensor_host, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim=self.runner.model_config.get_head_size(), + page_size=self.runner.block_size, + seq_start_loc=None, + query_start_loc=query_start_loc_host, + device=self.runner.device, + data_type=kv_cache_dtype, + q_data_type=self.runner.model_config.dtype, + use_cuda_graph=True, + decode_wrapper=self._graph_decode_wrapper, + prefill_wrapper=None) + attn_metadata.begin_forward() + return attn_metadata + + def get_graph_input_buffers(self, + attn_metadata, + is_encoder_decoder_model: bool = False): + return { + "slot_mapping": attn_metadata.slot_mapping, + } + + def prepare_graph_input_buffers(self, + input_buffers, + attn_metadata, + is_encoder_decoder_model: bool = False): + return + + def begin_forward(self, model_input): + assert not self._is_graph_capturing + state = self + if model_input.attn_metadata.use_cuda_graph: + batch_size = model_input.input_tokens.shape[0] + state = (self.runner.graph_runners[model_input.virtual_engine] + [batch_size].attn_state) + model_input.attn_metadata.prefill_wrapper = state._get_prefill_wrapper( + ) + model_input.attn_metadata.decode_wrapper = state._get_decode_wrapper() + model_input.attn_metadata.begin_forward() + + +@dataclass +class FlashInferMetadata(AttentionMetadata): + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + max_decode_seq_len: int + + use_cuda_graph: bool = True + + prefill_wrapper: Optional[BatchPrefillWithPagedKVCacheWrapper] = None + decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None + + # Metadata for the prefill stage + seq_start_loc: Optional[torch.Tensor] = None + query_start_loc: Optional[torch.Tensor] = None + block_tables: Optional[torch.Tensor] = None + + # used for GPU in-place advance_step + seq_lens_tensor: Optional[torch.Tensor] = None + block_table_bound: Optional[torch.Tensor] = None + + # An example for paged_kv_indices, paged_kv_indptr: + # request 1, page indices [0, 5, 8] + # request 2, page indices [1, 6, 7] + # request 3, page indices [3, 4] + # paged_kv_indices is a concatenation of page indices of all requests: + # [0, 5, 8, 1, 6, 7, 3, 4] + # paged_kv_indptr is used to index into paged_kv_indices: + # [0, 3, 6, 8] + # The indptr of the paged kv cache, shape: [batch_size + 1] + paged_kv_indptr: Optional[torch.Tensor] = None + # The page indices of the paged kv cache + paged_kv_indices: Optional[torch.Tensor] = None + # The number of entries in the last page of each request in + # the paged kv cache, shape: [batch_size] + paged_kv_last_page_len: Optional[torch.Tensor] = None + # The number of query/output heads + num_qo_heads: Optional[int] = None + # The number of key/value heads + num_kv_heads: Optional[int] = None + # The dimension of the attention heads + head_dim: Optional[int] = None + # Block size of vllm + page_size: Optional[int] = None + # The data type of the paged kv cache + data_type: torch.dtype = None + # The data type of the query + q_data_type: torch.dtype = None + device: torch.device = torch.device("cuda") + is_profile_run: bool = False + + def __post_init__(self): + # Refer to + # https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157 + supported_head_sizes = FlashInferBackend.get_supported_head_sizes() + if self.head_dim is not None and self.head_dim \ + not in supported_head_sizes: + raise ValueError( + f"Only {supported_head_sizes} are supported for head_dim,", + f"received {self.head_dim}.") + + def begin_forward(self): + if self.num_prefill_tokens > 0: + if self.paged_kv_indices is None: + return + + # assert self.prefill_wrapper is not None + assert self.query_start_loc is not None + assert self.paged_kv_indices is not None + assert self.paged_kv_indptr is not None + assert self.paged_kv_last_page_len is not None + assert self.block_table_bound is not None + assert self.seq_lens_tensor is not None + batch_size = self.query_start_loc.shape[0] - 1 + assert batch_size >= 0 + # We will use flash attention for profiling to + # determine the number of blocks. Therefore, + # we don't need to prepare the input for flashinfer for profile run. + if not self.is_profile_run: + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device) + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to( + self.device) + self.block_table_bound = self.block_table_bound.to(self.device) + self.seq_lens_tensor = self.seq_lens_tensor.to(self.device) + self.paged_kv_indices = self.paged_kv_indices.to(self.device) + + # Unsupported BatchPrefillWithPagedKVCacheWrapper Now + # self.prefill_wrapper.end_forward() + # self.prefill_wrapper.begin_forward( + # self.query_start_loc, self.paged_kv_indptr, + # self.paged_kv_indices, self.paged_kv_last_page_len, + # self.num_qo_heads, self.num_kv_heads, self.head_dim, + # self.page_size) + else: + assert self.paged_kv_indices is not None + assert self.paged_kv_indptr is not None + assert self.paged_kv_last_page_len is not None + self.paged_kv_indices = self.paged_kv_indices.to(self.device) + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device) + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to( + self.device) + # handle model warmup path + if self.block_table_bound is not None: + self.block_table_bound = self.block_table_bound.to(self.device) + if self.seq_lens_tensor is not None: + self.seq_lens_tensor = self.seq_lens_tensor.to(self.device) + + assert self.decode_wrapper is not None + self.decode_wrapper.end_forward() + self.decode_wrapper.begin_forward( + self.paged_kv_indptr, + self.paged_kv_indices, + self.paged_kv_last_page_len, + self.num_qo_heads, + self.num_kv_heads, + self.head_dim, + self.page_size, + # Disable flashinfer's pos encoding and use vllm's rope. + pos_encoding_mode="NONE", + # kv-cache data type. + data_type=self.data_type, + # query data type. + q_data_type=self.q_data_type) + + def asdict_zerocopy(self, + skip_fields: Optional[Set[str]] = None + ) -> Dict[str, Any]: + if skip_fields is None: + skip_fields = set() + # We need to skip the prefill/decode_wrapper field since it cannot be + # broadcasted with nccl when TP is enabled. + skip_fields.add('prefill_wrapper') + skip_fields.add('decode_wrapper') + return super().asdict_zerocopy(skip_fields) + + @property + def prefill_metadata(self) -> Optional["FlashInferMetadata"]: + # Currently chunked prefill is not supported + if self.num_decode_tokens == 0: + assert self.num_prefills > 0 + return self + + return None + + @property + def decode_metadata(self) -> Optional["FlashInferMetadata"]: + # Currently chunked prefill is not supported + if self.num_prefills > 0: + assert self.num_decode_tokens == 0, ( + "Chunked prefill is not supported with flashinfer yet.") + return None + + return self + + def advance_step(self, + model_input: "ModelInputForGPUWithSamplingMetadata", + sampled_token_ids: Optional[torch.Tensor], + block_size: int, + num_seqs: int, + num_queries: int, + turn_prefills_into_decodes: bool = False): + """ + Update metadata in-place to advance one decode step. + """ + + assert not turn_prefills_into_decodes, \ + ("Chunked prefill is not supported with flashinfer yet." + "turn_prefills_into_decodes is a Multi-Step + Chunked-Prefill " + "specific parameter.") + + assert num_seqs > 0 + assert num_queries > 0 + assert model_input.attn_metadata is not None + assert sampled_token_ids is not None + + # When using cudagraph, the num_seqs is padded to the next captured + # batch sized, but num_queries tracks the actual number of requests in + # the batch. For --enforce-eager mode, num_seqs == num_queries + if num_seqs != num_queries: + assert num_seqs > num_queries + assert self.use_cuda_graph + + model_input.input_tokens[:num_queries] = sampled_token_ids.flatten() + + # Update GPU tensors + ops.advance_step_flashinfer( + num_seqs=num_seqs, + num_queries=num_queries, + block_size=block_size, + input_tokens=model_input.input_tokens, + sampled_token_ids=model_input.input_tokens, + input_positions=model_input.input_positions, + seq_lens=self.seq_lens_tensor, + slot_mapping=self.slot_mapping, + block_tables=self.block_tables, + paged_kv_indices=self.paged_kv_indices, + paged_kv_indptr=self.paged_kv_indptr, + paged_kv_last_page_len=self.paged_kv_last_page_len, + block_table_bound=self.block_table_bound) + + +class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): + + def __init__(self, input_builder: "ModelInputForGPUBuilder"): + self.slot_mapping: List[int] = [] + self.prefill_seq_lens: List[int] = [] + self.context_lens: List[int] = [] + self.block_tables: List[List[int]] = [] + self.curr_seq_lens: List[int] = [] + self.num_prefills = 0 + self.num_prefill_tokens = 0 + self.num_decode_tokens = 0 + + self.input_builder = input_builder + self.runner = input_builder.runner + + self.sliding_window = input_builder.sliding_window + self.block_size = input_builder.block_size + self.use_v2_block_manager = ( + input_builder.scheduler_config.use_v2_block_manager) + + # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout + # for the precise definition of the following fields. + # An example: + # request 1, page indices [0, 5, 8] + # request 2, page indices [1, 6, 7] + # request 3, page indices [3, 4] + # paged_kv_indices is a concatenation of page indices of all requests: + # [0, 5, 8, 1, 6, 7, 3, 4] + # paged_kv_indptr is used to index into paged_kv_indices: + # [0, 3, 6, 8] + self.paged_kv_indices: List[int] = [] + # 0 at the beginning of paged_kv_indptr indicates the start of the + # first request’s page indices in the paged_kv_indices list. + self.paged_kv_indptr: List[int] = [0] + # paged_kv_last_page_len is the length of the last page of each request + self.paged_kv_last_page_len: List[int] = [] + self.total_blocks = 0 + self.is_profile_run: bool = False + + def _add_seq_group( + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup", + chunked_prefill_enabled: bool): + """Add a sequence group to the metadata. Specifically update/append + 1. context length. + 2. block table. + 3. slot mapping. + """ + is_prompt = inter_data.is_prompt + block_tables = inter_data.block_tables + computed_block_nums = inter_data.computed_block_nums + + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len, + curr_sliding_window_block) in zip( + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens], + inter_data.orig_seq_lens, inter_data.seq_lens, + inter_data.query_lens, inter_data.context_lens, + inter_data.curr_sliding_window_blocks): + self.context_lens.append(context_len) + if is_prompt: + self.num_prefills += 1 + self.num_prefill_tokens += token_len + self.prefill_seq_lens.append(seq_len) + else: + assert query_len == 1, ( + "seq_len: {}, context_len: {}, query_len: {}".format( + seq_len, context_len, query_len)) + self.num_decode_tokens += query_len + self.curr_seq_lens.append(curr_seq_len) + + # Compute block table. + # TODO(sang): Combine chunked prefill and prefix caching by + # only allowing multiple of block_size chunk size. + # NOTE: This only works for oooooooxxx style attention. + block_table = [] + if inter_data.prefix_cache_hit: + block_table = computed_block_nums + elif ((chunked_prefill_enabled or not is_prompt) + and block_tables is not None): + block_table = block_tables[seq_id][-curr_sliding_window_block:] + self.block_tables.append(block_table) + + is_profile_run = is_block_tables_empty(block_tables) + + # Compute slot mapping. + start_idx = compute_slot_mapping_start_idx( + is_prompt, query_len, context_len, self.sliding_window, + self.use_v2_block_manager) + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id, + seq_len, context_len, start_idx, + self.block_size, inter_data.block_tables) + + # It is not necessary to add paged_kv_indices, paged_kv_indptr, + # and paged_kv_last_page_len for profile run because we will + # create dummy inputs. + if is_profile_run: + self.is_profile_run = is_profile_run + return + + block_table = block_tables[seq_id] + self._update_paged_kv_tensors(block_table, seq_len) + + def _update_paged_kv_tensors(self, block_table: List[int], seq_len: int): + # Get the number of valid blocks based on sequence length. + # If seq_len = 16, block_size = 16, + # block_table_bound is 1 with 1 valid block. + # If seq_len = 15, block_size = 16, + # block_table_bound is 0 + 1 with 1 valid block. + self.total_blocks += len(block_table) + block_table_bound = seq_len // self.block_size + 1 \ + if seq_len % self.block_size != 0 \ + else seq_len // self.block_size + self.paged_kv_indices.extend(block_table[:block_table_bound]) + self.paged_kv_indptr.append(self.paged_kv_indptr[-1] + + block_table_bound) + + last_page_len = seq_len % self.block_size + if last_page_len == 0: + last_page_len = self.block_size + self.paged_kv_last_page_len.append(last_page_len) + + def build(self, seq_lens: List[int], query_lens: List[int], + cuda_graph_pad_size: int, batch_size: int): + """Build attention metadata with on-device tensors. + + Args: + seq_lens: The maybe padded sequence lengths of the input sequences. + query_lens: The query lengths of the input sequences. + cuda_graph_pad_size: The padding size for cuda graph. + -1 if cuda graph is not used. + batch_size: The maybe padded batch size. + """ + for inter_data in self.input_builder.inter_data_list: + self._add_seq_group(inter_data, + self.input_builder.chunked_prefill_enabled) + + device = self.runner.device + use_captured_graph = cuda_graph_pad_size != -1 + + max_prefill_seq_len = max(self.prefill_seq_lens, default=0) + num_decode_tokens = self.num_decode_tokens + + if use_captured_graph: + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size) + self.block_tables.extend([] * cuda_graph_pad_size) + num_decode_tokens = batch_size + + # The shape of graph_block_tables is + # [max batch size, max context len // block size]. + input_block_tables = self.runner.graph_block_tables[:batch_size] + max_blocks = input_block_tables.shape[1] + for i, block_table in enumerate(self.block_tables): + if block_table: + num_blocks = len(block_table) + if num_blocks <= max_blocks: + input_block_tables[i, :num_blocks] = block_table + else: + # It may be possible to have more blocks allocated due + # to lookahead slots of multi-step, however, they are + # not used anyway, so can be safely ignored. + input_block_tables[ + i, :max_blocks] = block_table[:max_blocks] + + block_tables = torch.from_numpy(input_block_tables).to( + device, non_blocking=True) + + last_paged_kv_indptr = self.paged_kv_indptr[-1] + self.paged_kv_indptr.extend([last_paged_kv_indptr] * + cuda_graph_pad_size) + self.paged_kv_last_page_len.extend([0] * cuda_graph_pad_size) + else: + block_tables = make_tensor_with_pad( + self.block_tables, + pad=0, + dtype=torch.int, + device=device, + ) + + assert device is not None + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device, + self.runner.pin_memory) + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device, + self.runner.pin_memory) + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long, + device, self.runner.pin_memory) + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + torch.cumsum(seq_lens_tensor, + dim=0, + dtype=seq_start_loc.dtype, + out=seq_start_loc[1:]) + torch.cumsum(query_lens_tensor, + dim=0, + dtype=query_start_loc.dtype, + out=query_start_loc[1:]) + + if len(self.paged_kv_indptr) > 0: + # extend to the maximum number of blocks as returned by the + # scheduler + self.paged_kv_indices.extend( + [0] * (self.total_blocks - len(self.paged_kv_indices))) + paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices, + device="cpu", + dtype=torch.int) + paged_kv_indptr_tensor = torch.tensor(self.paged_kv_indptr, + device="cpu", + dtype=torch.int) + paged_kv_last_page_len_tensor = torch.tensor( + self.paged_kv_last_page_len, device="cpu", dtype=torch.int) + block_table_bound_tensor = torch.zeros(len(self.paged_kv_indptr) - + 1, + device="cpu", + dtype=torch.int) + else: + paged_kv_indices_tensor = None + paged_kv_indptr_tensor = None + paged_kv_last_page_len_tensor = None + block_table_bound_tensor = None + + if self.runner.kv_cache_dtype.startswith("fp8"): + kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer( + self.runner.kv_cache_dtype) + else: + kv_cache_dtype = get_kv_cache_torch_dtype( + self.runner.kv_cache_dtype, self.runner.model_config.dtype) + + return FlashInferMetadata( + num_prefills=self.num_prefills, + slot_mapping=slot_mapping_tensor, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=num_decode_tokens, + max_prefill_seq_len=max_prefill_seq_len, + max_decode_seq_len=max(seq_lens), + block_tables=block_tables, + paged_kv_indptr=paged_kv_indptr_tensor, + paged_kv_indices=paged_kv_indices_tensor, + paged_kv_last_page_len=paged_kv_last_page_len_tensor, + block_table_bound=block_table_bound_tensor, + seq_lens_tensor=seq_lens_tensor, + num_qo_heads=self.runner.model_config.get_num_attention_heads( + self.runner.parallel_config), + num_kv_heads=self.runner.model_config.get_num_kv_heads( + self.runner.parallel_config), + head_dim=self.runner.model_config.get_head_size(), + page_size=self.block_size, + seq_start_loc=seq_start_loc, + query_start_loc=query_start_loc, + device=device, + data_type=kv_cache_dtype, + q_data_type=self.runner.model_config.dtype, + use_cuda_graph=use_captured_graph, + is_profile_run=self.is_profile_run) + + +class FlashInferImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + if sliding_window is not None: + raise ValueError("Sliding window is not supported in FlashInfer.") + self.sliding_window = (-1, -1) + self.kv_cache_dtype = kv_cache_dtype + self.logits_soft_cap = logits_soft_cap + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: FlashInferMetadata, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + assert k_scale == 1.0 and v_scale == 1.0, ( + "key/v_scale is not supported in FlashInfer.") + if attn_type != AttentionType.DECODER: + raise NotImplementedError("Encoder self-attention and " + "encoder/decoder cross-attention " + "are not implemented for " + "FlashInferImpl") + + # return torch.ops.vllm.unified_flash_infer( + return unified_flash_infer( + query, + key, + value, + self.num_heads, + self.head_size, + self.num_kv_heads, + kv_cache, + self.kv_cache_dtype, + k_scale, + v_scale, + self.scale, + self.sliding_window, + self.alibi_slopes, + self.logits_soft_cap, + ) + + +# @torch.library.custom_op("vllm::unified_flash_infer", +# mutates_args=["kv_cache"]) +def unified_flash_infer( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + num_heads: int, + head_size: int, + num_kv_heads: int, + kv_cache: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + softmax_scale: float, + window_size: Optional[List[int]] = None, + alibi_slopes: Optional[torch.Tensor] = None, + logits_soft_cap: Optional[float] = None, +) -> torch.Tensor: + + current_metadata = get_forward_context() + assert current_metadata is not None + assert isinstance(current_metadata, FlashInferMetadata) + attn_metadata: FlashInferMetadata = current_metadata + + num_tokens, hidden_size = query.shape + query = query.view(-1, num_heads, head_size) + key = key.view(-1, num_kv_heads, head_size) + value = value.view(-1, num_kv_heads, head_size) + + if attn_metadata.num_prefill_tokens > 0: + assert attn_metadata.num_decode_tokens == 0, ( + "Chunked prefill is not supported with flashinfer yet.") + if attn_metadata.num_decode_tokens > 0: + assert attn_metadata.num_prefill_tokens == 0, ( + "Chunked prefill is not supported with flashinfer yet.") + if kv_cache.numel() > 0: + # Use the same reshape and cache kernel as flash attention. + ops.reshape_and_cache_flashinfer( + key, + value, + kv_cache[:, 0], + kv_cache[:, 1], + attn_metadata.slot_mapping.flatten(), + kv_cache_dtype, + k_scale, + v_scale, + ) + # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2 + # to process the cache when the kv_cache_dtype is fp8 + if kv_cache_dtype.startswith("fp8"): + torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer( + kv_cache_dtype) + kv_cache = kv_cache.view(torch_dtype) + + query = query.contiguous() # Flashinfer requires query to be contiguous + if prefill_meta := attn_metadata.prefill_metadata: + # We will use flash attention for prefill + # when kv_cache is not provided. + # This happens when vllm runs the profiling to + # determine the number of blocks. + use_infer_inferface = True + if use_infer_inferface or kv_cache.numel() == 0: + output = flash_attn_varlen_func( + q=query, + k=key, + v=value, + cu_seqlens_q=prefill_meta.seq_start_loc, + cu_seqlens_k=prefill_meta.seq_start_loc, + max_seqlen_q=prefill_meta.max_prefill_seq_len, + max_seqlen_k=prefill_meta.max_prefill_seq_len, + softmax_scale=softmax_scale, + causal=True, + window_size=window_size, + alibi_slopes=alibi_slopes, + ) + else: + assert prefill_meta is not None + assert prefill_meta.prefill_wrapper is not None + output = prefill_meta.prefill_wrapper.forward( + query, kv_cache, logits_soft_cap=logits_soft_cap, causal=True) + else: + assert attn_metadata.decode_metadata is not None + assert attn_metadata.decode_metadata.decode_wrapper is not None + output = attn_metadata.decode_metadata.decode_wrapper.forward( + query, + kv_cache, + sm_scale=softmax_scale, + logits_soft_cap=logits_soft_cap, + k_scale=k_scale, + v_scale=v_scale) + return output.view(num_tokens, hidden_size) + + +# @unified_flash_infer.register_fake +def _( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + num_heads: int, + head_size: int, + num_kv_heads: int, + kv_cache: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + softmax_scale: float, + window_size: Optional[List[int]] = None, + alibi_slopes: Optional[torch.Tensor] = None, + logits_soft_cap: Optional[float] = None, +) -> torch.Tensor: + return torch.empty_like(query).contiguous() diff --git a/vllm/attention/backends/ipex_attn.py b/vllm/attention/backends/ipex_attn.py new file mode 100644 index 00000000..7398732d --- /dev/null +++ b/vllm/attention/backends/ipex_attn.py @@ -0,0 +1,381 @@ +""" Attention layer with torch scaled_dot_product_attention + and PagedAttention.""" +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Type + +import torch + +from vllm._ipex_ops import ipex_ops +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import CommonAttentionState +from vllm.attention.ops.paged_attn import (PagedAttention, + PagedAttentionMetadata) + +_PARTITION_SIZE = 512 + + +class IpexAttnBackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "ipex-attn" + + @staticmethod + def get_impl_cls() -> Type["IpexAttnBackendImpl"]: + return IpexAttnBackendImpl + + @staticmethod + def get_metadata_cls() -> Type["IpexAttnMetadata"]: + return IpexAttnMetadata + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return PagedAttention.get_kv_cache_shape(num_blocks, block_size, + num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + from vllm._ipex_ops import ipex_ops as ops + ops.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + from vllm._ipex_ops import ipex_ops as ops + key_caches = [kv_cache[0] for kv_cache in kv_caches] + value_caches = [kv_cache[1] for kv_cache in kv_caches] + ops.copy_blocks(key_caches, value_caches, src_to_dists) + + +@dataclass +class IpexAttnMetadata(AttentionMetadata, PagedAttentionMetadata): + """Metadata for IpexAttnBackend. + """ + # Currently, input sequences can only contain all prompts + # or all decoding. True if all sequences are prompts. + is_prompt: bool + slot_mapping: torch.Tensor + seq_lens: Optional[List[int]] + seqlen_q: Optional[torch.Tensor] + max_seqlen: Optional[int] + + def __post_init__(self): + # Set during the execution of the first attention op. + # It is a list because it is needed to set per prompt + # when alibi slopes is used. It is because of the limitation + # from xformer API. + # will not appear in the __repr__ and __init__ + self.attn_bias: Optional[List[torch.Tensor]] = None + + @property + def prefill_metadata(self) -> Optional["IpexAttnMetadata"]: + # Currently chunked prefill is not supported + if self.num_decode_tokens == 0: + assert self.num_prefills > 0 + return self + + return None + + @property + def decode_metadata(self) -> Optional["IpexAttnMetadata"]: + # Currently chunked prefill is not supported + if self.num_prefills > 0: + assert self.num_decode_tokens == 0 + return None + + return self + + +class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]): + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + if blocksparse_params is not None: + raise ValueError( + "IPEX backend does not support block-sparse attention.") + if logits_soft_cap is not None: + raise ValueError("IPEX backend does not support logits_soft_cap.") + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + self.sliding_window = sliding_window + self.kv_cache_dtype = kv_cache_dtype + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + self.need_mask = (self.alibi_slopes is not None + or self.sliding_window is not None) + + supported_head_sizes = PagedAttention.get_supported_head_sizes() + if head_size not in supported_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by PagedAttention. " + f"Supported head sizes are: {supported_head_sizes}.") + if kv_cache_dtype != "auto": + raise NotImplementedError( + "IPEX backend does not support FP8 KV cache. " + "Please use xFormers backend instead.") + + def split_kv_cache( + self, + kv_cache: torch.Tensor, + num_kv_heads: int, + head_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + x = 1 + num_blocks = kv_cache.shape[1] + + key_cache = kv_cache[0] + key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x, + -1, x) + value_cache = kv_cache[1] + value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1) + return key_cache, value_cache + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: IpexAttnMetadata, # type: ignore + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with IPEX varlen_attention and PagedAttention. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [num_tokens, num_heads * head_size] + """ + assert k_scale == 1.0 and v_scale == 1.0 + if attn_type != AttentionType.DECODER: + raise NotImplementedError("Encoder self-attention and " + "encoder/decoder cross-attention " + "are not implemented for " + "IpexAttnBackendImpl") + num_tokens, hidden_size = query.shape + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + + if kv_cache.numel() > 0: + key_cache, value_cache = self.split_kv_cache( + kv_cache, self.num_kv_heads, self.head_size) + ipex_ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + attn_metadata.slot_mapping.flatten(), + self.kv_cache_dtype, + k_scale, + v_scale, + ) + + if attn_metadata.is_prompt: + assert attn_metadata.seq_lens is not None + if (kv_cache.numel() == 0 + or attn_metadata.block_tables.numel() == 0): + if self.num_kv_heads != self.num_heads: + key = key.repeat_interleave(self.num_queries_per_kv, dim=1) + value = value.repeat_interleave(self.num_queries_per_kv, + dim=1) + + if attn_metadata.attn_bias is None: + if self.alibi_slopes is not None: + att_masks = _make_alibi_bias( + self.alibi_slopes, query.dtype, + attn_metadata.seq_lens) # type: ignore + elif self.sliding_window is not None: + att_masks = _make_sliding_window_bias( + attn_metadata.seq_lens, self.sliding_window, + query.dtype) # type: ignore + else: + att_masks = _make_sliding_window_bias( + attn_metadata.seq_lens, None, dtype=query.dtype) + attn_metadata.attn_bias = att_masks + + output = torch.empty( + (num_tokens, self.num_heads, self.head_size), + dtype=query.dtype, + device=query.device) + ipex_ops.varlen_attention(query, + key, + value, + output, + attn_metadata.seqlen_q, + attn_metadata.seqlen_q, + attn_metadata.max_seqlen, + attn_metadata.max_seqlen, + pdropout=0.0, + softmax_scale=self.scale, + zero_tensors=False, + is_causal=True, + return_softmax=False, + gen_=None) + else: + # prefix-enabled attention + raise RuntimeError( + "IPEX backend doesn't support prefix decoding.") + + else: + # Decoding run. + max_seq_len = attn_metadata.max_decode_seq_len + output = torch.empty_like(query) + block_size = value_cache.shape[3] + num_seqs, num_heads, head_size = query.shape + max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) // + _PARTITION_SIZE) + # NOTE(woosuk): We use a simple heuristic to decide whether to use + # PagedAttention V1 or V2. If the number of partitions is 1, we use + # V1 to avoid the overhead of reduction. Also, if the number of + # sequences or heads is large, we use V1 since there is enough work + # to parallelize. + # TODO(woosuk): Tune this heuristic. + # For context len > 8192, use V2 kernel to avoid shared memory + # shortage. + use_v1 = (max_seq_len <= 8192 and + (max_num_partitions == 1 or num_seqs * num_heads > 512)) + if use_v1: + # Run PagedAttention V1. + ipex_ops.paged_attention_v1( + output, + query, + key_cache, + value_cache, + self.num_kv_heads, + self.scale, + attn_metadata.block_tables, + attn_metadata.seq_lens_tensor, + block_size, + max_seq_len, + self.alibi_slopes, + self.kv_cache_dtype, + k_scale, + v_scale, + ) + else: + # Run PagedAttention V2. + assert _PARTITION_SIZE % block_size == 0 + tmp_output = torch.empty( + size=(num_seqs, num_heads, max_num_partitions, head_size), + dtype=output.dtype, + device=output.device, + ) + exp_sums = torch.empty( + size=(num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=output.device, + ) + max_logits = torch.empty_like(exp_sums) + ipex_ops.paged_attention_v2( + output, + exp_sums, + max_logits, + tmp_output, + query, + key_cache, + value_cache, + self.num_kv_heads, + self.scale, + attn_metadata.block_tables, + attn_metadata.seq_lens_tensor, + block_size, + max_seq_len, + self.alibi_slopes, + self.kv_cache_dtype, + k_scale, + v_scale, + ) + + # Reshape the output tensor. + return output.view(-1, self.num_heads * self.head_size) + + +def _make_alibi_bias( + alibi_slopes: torch.Tensor, + dtype: torch.dtype, + seq_lens: List[int], +) -> List[torch.Tensor]: + attn_biases = [] + for seq_len in seq_lens: + bias = torch.arange(seq_len, dtype=dtype, device=alibi_slopes.device) + # NOTE(zhuohan): HF uses + # `bias = bias[None, :].repeat(seq_len, 1)` + # here. We find that both biases give the same results, but + # the bias below more accurately follows the original ALiBi + # paper. + bias = bias[None, :] - bias[:, None] + + num_heads = alibi_slopes.shape[0] + bias = bias[None, :].repeat((num_heads, 1, 1)) + bias.mul_(alibi_slopes[:, None, None]) + inf_mask = torch.empty( + (1, seq_len, seq_len), + dtype=bias.dtype, + device=alibi_slopes.device).fill_(-torch.inf).triu_(diagonal=1) + attn_biases.append((bias + inf_mask).to(dtype)) + + return attn_biases + + +def _make_sliding_window_bias( + seq_lens: List[int], + window_size: Optional[int], + dtype: torch.dtype, +) -> List[torch.Tensor]: + attn_biases = [] + for seq_len in seq_lens: + tensor = torch.full( + (1, seq_len, seq_len), + dtype=dtype, + fill_value=1, + ) + shift = 0 + mask = torch.tril(tensor, diagonal=shift).to(dtype) # type: ignore + if window_size is not None: + mask = torch.triu(mask, diagonal=shift - window_size + 1) + mask = torch.log(mask) + attn_biases.append(mask.to(dtype)) + + return attn_biases diff --git a/vllm/attention/backends/openvino.py b/vllm/attention/backends/openvino.py new file mode 100644 index 00000000..8b362307 --- /dev/null +++ b/vllm/attention/backends/openvino.py @@ -0,0 +1,130 @@ +from dataclasses import dataclass +from typing import List, Tuple, Type + +import openvino as ov +import torch + +from vllm.attention.backends.abstract import (AttentionBackend, + AttentionMetadata) +from vllm.attention.backends.utils import CommonAttentionState + + +def copy_cache_block(src_tensor: ov.Tensor, dst_tensor: ov.Tensor, + src_offset: int, dst_offset: int) -> None: + + def create_roi_tensor( + tensor: ov.Tensor, + block_number: int, + ) -> ov.Tensor: + roi_begin = ov.runtime.Coordinate([0, 0, 0, 0]) + roi_end = ov.runtime.Coordinate(tensor.get_shape()) + + roi_begin[0] = block_number + roi_end[0] = block_number + 1 + + if isinstance(tensor, ov.Tensor): + return ov.Tensor(tensor, roi_begin, roi_end) + else: + return ov.RemoteTensor(tensor, roi_begin, roi_end) + + src_roi_tensor = \ + create_roi_tensor(src_tensor, src_offset) + dst_roi_tensor = \ + create_roi_tensor(dst_tensor, dst_offset) + src_roi_tensor.copy_to(dst_roi_tensor) + + +class OpenVINOAttentionBackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "openvino" + + @staticmethod + def get_impl_cls(): + # OpenVINO implements PagedAttention as part of the Optimum + # exported model + raise NotImplementedError + + @staticmethod + def make_metadata(*args, **kwargs) -> "AttentionMetadata": + raise NotImplementedError + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def make_openvino_metadata(*args, **kwargs) -> "OpenVINOAttentionMetadata": + return OpenVINOAttentionMetadata(*args, **kwargs) + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return (2, num_blocks, num_kv_heads, block_size, head_size) + + @staticmethod + def swap_blocks( + src_tensor: ov.Tensor, + dst_tensor: ov.Tensor, + src_to_dists: List[Tuple[int, int]], + ) -> None: + for src, dst in src_to_dists: + copy_cache_block(src_tensor, dst_tensor, src, dst) + + @staticmethod + def copy_blocks( + kv_caches: List[Tuple[ov.Tensor, ov.Tensor]], + src_to_dists: List[Tuple[int, int]], + ) -> None: + for src, dst in src_to_dists: + for key_cache, value_cache in kv_caches: + copy_cache_block(key_cache, key_cache, src, dst) + copy_cache_block(value_cache, value_cache, src, dst) + + +@dataclass +class OpenVINOAttentionMetadata: + """Metadata for OpenVINOAttentionBackend. + + Basic terms used below: + - batch_size_in_sequences - total number of sequences to execute​ + - prompt_lens – per sequence size number of scheduled tokens​ + - batch_size_in_tokens = sum(prompt_lens)​ + - max_context_len = max(context_lens)​ + - max_num_blocks = div_up(max_context_len / BLOCK_SIZE)​ + - num_blocks – total number of blocks in block_indices​ + """ + + # Describes past KV cache size for each sequence within a batch + # Shape: [batch_size_in_sequences] + # Type: i32​ + past_lens: torch.Tensor + + # Describes start indices of input / speculative tokens from + # current sequences within a batch sequence​ + # Shape: [batch_size_in_sequences + 1]​ + # Type: i32 + subsequence_begins: torch.Tensor + + # Describes block tables for each sequence within a batch​ - + # indices along 0th dimension in key_cache and value_cache inputs​ + # Shape: [num_blocks] + # Type: i32​ + block_indices: torch.Tensor + + # Describes block tables for each sequence within a batch​ - + # for i-th element, it is an index in block_indices with the + # first block belonging to i-th sequence​ + # Shape: [batch_size_in_sequences + 1] + # Type: i32​ + block_indices_begins: torch.Tensor + + # Describes max context length + # Shape: scalar + # Type: i32 + max_context_len: torch.Tensor diff --git a/vllm/attention/backends/pallas.py b/vllm/attention/backends/pallas.py new file mode 100644 index 00000000..86716602 --- /dev/null +++ b/vllm/attention/backends/pallas.py @@ -0,0 +1,260 @@ +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Type + +import torch +import torch_xla.experimental.custom_kernel # Required to register custom ops. + +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import CommonAttentionState + + +class PallasAttentionBackend(AttentionBackend): + + @staticmethod + def get_impl_cls() -> Type["PallasAttentionBackendImpl"]: + return PallasAttentionBackendImpl + + @staticmethod + def get_metadata_cls() -> Type["PallasMetadata"]: + return PallasMetadata + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return (num_kv_heads, num_blocks, block_size, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + raise RuntimeError("swap_blocks is not used for the TPU backend.") + + @torch.compile(backend="openxla") + @staticmethod + def copy_blocks( + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]], + src_to_dists: Tuple[torch.Tensor, torch.Tensor], + ) -> None: + src_indices, dst_indices = src_to_dists + for k_cache, v_cache in kv_caches: + torch.ops.xla.dynamo_set_buffer_donor_(k_cache, True) + k_cache[:, dst_indices] = k_cache[:, src_indices] + torch.ops.xla.dynamo_set_buffer_donor_(v_cache, True) + v_cache[:, dst_indices] = v_cache[:, src_indices] + + +@dataclass +class PallasMetadata(AttentionMetadata): + + # Currently, input sequences can only contain all prefills + # or all decoding. + block_tables: Optional[torch.Tensor] = None + context_lens: Optional[torch.Tensor] = None + + @property + def prefill_metadata(self) -> Optional["PallasMetadata"]: + if self.num_prefills == 0: + return None + + assert self.num_decode_tokens == 0 + assert self.block_tables is None + assert self.context_lens is None + return self + + @property + def decode_metadata(self) -> Optional["PallasMetadata"]: + if self.num_decode_tokens == 0: + return None + + assert self.num_prefills == 0 + assert self.num_prefill_tokens == 0 + assert self.block_tables is not None + assert self.context_lens is not None + return self + + +class PallasAttentionBackendImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + if head_size % 128 != 0: + raise NotImplementedError("Head size must be a multiple of 128.") + if alibi_slopes is not None: + raise NotImplementedError("Alibi slopes is not supported.") + if sliding_window is not None: + raise NotImplementedError("Sliding window is not supported.") + if kv_cache_dtype != "auto": + raise NotImplementedError("FP8 KV cache dtype is not supported.") + if blocksparse_params is not None: + raise NotImplementedError("Blocksparse is not supported.") + if logits_soft_cap is not None: + raise NotImplementedError( + "Attention logits soft-capping is not supported.") + + if torch_xla.tpu.version() < 4: + raise NotImplementedError("TPU version must be 4 or higher.") + + self.megacore_mode = None + tpu_env = torch_xla.tpu.get_tpu_env() + tpu_type = (tpu_env.get("ACCELERATOR_TYPE", None) + or tpu_env.get("TYPE", None) + or tpu_env.get("TPU_ACCELERATOR_TYPE", None)) + assert tpu_type is not None + tpu_type = tpu_type.lower() + + if (("lite" not in tpu_type) and ("v6" not in tpu_type)): + if self.num_kv_heads % 2 == 0: + self.megacore_mode = "kv_head" + else: + # NOTE(woosuk): If the batch size is not a multiple of 2, the + # megacore mode will be None. + self.megacore_mode = "batch" + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: Tuple[torch.Tensor, torch.Tensor], + attn_metadata: PallasMetadata, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with Pallas attention. + + Args: + query: shape = [batch_size, seq_len, num_heads * head_size] + key: shape = [batch_size, seq_len, num_kv_heads * head_size] + value: shape = [batch_size, seq_len, num_kv_heads * head_size] + kv_cache[0] = [num_kv_heads, num_blocks, block_size, head_size] + kv_cache[1] = [num_kv_heads, num_blocks, block_size, head_size] + NOTE: kv_cache[0] and kv_cache[1] will be an empty tensor + with shape [0] for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [batch_size, seq_len, num_heads * head_size] + """ + assert k_scale == 1.0 and v_scale == 1.0 + if attn_type != AttentionType.DECODER: + raise NotImplementedError("Encoder self-attention and " + "encoder/decoder cross-attention " + "are not implemented for " + "PallasAttentionBackendImpl") + batch_size, seq_len, hidden_size = query.shape + query = query.view(batch_size, seq_len, self.num_heads, self.head_size) + key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size) + value = value.view(batch_size, seq_len, self.num_kv_heads, + self.head_size) + + if kv_cache[0].numel() > 0: + slot_mapping = attn_metadata.slot_mapping + key_cache, value_cache = kv_cache + write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping) + + query = query * self.scale + if attn_metadata.num_prefills > 0: + assert seq_len % 16 == 0, ( + "Pallas FlashAttention kernel requires seq_len to be a " + f"multiple of 16 but got {seq_len}") + + # Handle GQA/MQA. + if self.num_kv_heads != self.num_heads: + key = key.repeat_interleave(self.num_queries_per_kv, dim=-2) + key = key.view(batch_size, seq_len, self.num_heads, + self.head_size) + value = value.repeat_interleave(self.num_queries_per_kv, + dim=-2) + value = value.view(batch_size, seq_len, self.num_heads, + self.head_size) + # FlashAttention requires [batch_size, num_heads, seq_len, d_model] + # while the input is [batch_size, seq_len, num_heads, d_model]. + # Permute the input to match the required format. + output = torch.ops.xla.flash_attention( + query.permute(0, 2, 1, 3), + key.permute(0, 2, 1, 3), + value.permute(0, 2, 1, 3), + True, + ) + output = output.permute(0, 2, 1, 3) + else: + # Decoding run. + assert kv_cache[0].numel() > 0 + + pages_per_compute_block = 16 # TODO(woosuk): Tune this value. + if self.megacore_mode == "batch" and batch_size % 2 != 0: + megacore_mode = None + else: + megacore_mode = self.megacore_mode + + # NOTE(woosuk): A temporary workaround to avoid the error: + # "xla::paged_attention() Expected a value of type 'str' for + # argument 'megacore_mode' but instead found type 'NoneType'." + if megacore_mode is not None: + output = torch.ops.xla.paged_attention( + query.squeeze(dim=1), + key_cache, + value_cache, + attn_metadata.context_lens, + attn_metadata.block_tables, + pages_per_compute_block, + megacore_mode=megacore_mode, + ) + else: + output = torch.ops.xla.paged_attention( + query.squeeze(dim=1), + key_cache, + value_cache, + attn_metadata.context_lens, + attn_metadata.block_tables, + pages_per_compute_block, + ) + + # Reshape the output tensor. + return output.reshape(batch_size, seq_len, hidden_size) + + +def write_to_kv_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, +) -> None: + torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True) + torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True) + + key = key.flatten(0, 2) + value = value.flatten(0, 2) + key_cache = key_cache.flatten(0, 2) + value_cache = value_cache.flatten(0, 2) + key_cache.index_copy_(0, slot_mapping, key) + value_cache.index_copy_(0, slot_mapping, value) diff --git a/vllm/attention/backends/placeholder_attn.py b/vllm/attention/backends/placeholder_attn.py new file mode 100644 index 00000000..3987986f --- /dev/null +++ b/vllm/attention/backends/placeholder_attn.py @@ -0,0 +1,321 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Tuple, Type + +import torch + +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder) +from vllm.attention.backends.utils import CommonAttentionState + +if TYPE_CHECKING: + from vllm.worker.model_runner import ModelInputForGPUBuilder + +# Placeholder attention backend for models like Mamba and embedding models that +# lack attention. + + +class PlaceholderAttentionBackend(AttentionBackend): + """Placeholder backend for when no attention is needed.""" + + @staticmethod + def get_name() -> str: + return "placeholder-attn" + + @staticmethod + def get_impl_cls() -> Type["PlaceholderAttentionImpl"]: + return PlaceholderAttentionImpl + + @staticmethod + def get_builder_cls() -> Type["PlaceholderAttentionMetadataBuilder"]: + return PlaceholderAttentionMetadataBuilder + + @staticmethod + def get_metadata_cls() -> Type["PlaceholderAttentionMetadata"]: + return PlaceholderAttentionMetadata + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return (1, 1, 1, 1, 1) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + return + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + return + + +@dataclass +class PlaceholderAttentionMetadata(AttentionMetadata): + """Attention metadata for prefill and decode batched together.""" + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] + + # Maximum query length in the batch. + max_query_len: Optional[int] + + # Max number of query tokens among request in the batch. + max_decode_query_len: Optional[int] + + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + # Maximum sequence length among decode batch. 0 if there are prefill + # requests only. + max_decode_seq_len: int + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] + # (batch_size,) A tensor of context lengths (tokens that are computed + # so far). + context_lens_tensor: Optional[torch.Tensor] + + # (batch_size, max_blocks_per_seq). + # Block addresses per sequence. (Seq id -> list of physical block) + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks + # in the kv cache. Each block can contain up to block_size tokens. + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph + # captured. + block_tables: Optional[torch.Tensor] + + # Whether or not if cuda graph is enabled. + # Cuda-graph is currently enabled for decoding only. + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention. + use_cuda_graph: bool + + _cached_prefill_metadata: Optional["PlaceholderAttentionMetadata"] = None + _cached_decode_metadata: Optional["PlaceholderAttentionMetadata"] = None + + @property + def prefill_metadata(self) -> Optional["PlaceholderAttentionMetadata"]: + if self.num_prefills == 0: + return None + + if self._cached_prefill_metadata is not None: + return self._cached_prefill_metadata + + assert self.seq_lens is not None + assert self.seq_lens_tensor is not None + assert self.query_start_loc is not None + assert self.context_lens_tensor is not None + assert self.seq_start_loc is not None + + # Placeholders + slot_mapping = torch.empty(0) + block_tables = torch.empty(0) + + self._cached_prefill_metadata = PlaceholderAttentionMetadata( + num_prefills=self.num_prefills, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=0, + slot_mapping=slot_mapping, + seq_lens=self.seq_lens[:self.num_prefills], + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills], + max_decode_query_len=0, + max_query_len=self.max_query_len, + max_prefill_seq_len=self.max_prefill_seq_len, + max_decode_seq_len=0, + query_start_loc=self.query_start_loc[:self.num_prefills + 1], + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1], + context_lens_tensor=self.context_lens_tensor[:self.num_prefills], + block_tables=block_tables, + use_cuda_graph=False, + ) + return self._cached_prefill_metadata + + @property + def decode_metadata(self) -> Optional["PlaceholderAttentionMetadata"]: + if self.num_decode_tokens == 0: + return None + + if self._cached_decode_metadata is not None: + return self._cached_decode_metadata + assert self.seq_lens_tensor is not None + + # Placeholders + slot_mapping = torch.empty(0) + block_tables = torch.empty(0) + + self._cached_decode_metadata = PlaceholderAttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=self.num_decode_tokens, + slot_mapping=slot_mapping, + seq_lens=None, + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:], + max_decode_query_len=self.max_decode_query_len, + max_query_len=None, + max_prefill_seq_len=0, + max_decode_seq_len=self.max_decode_seq_len, + query_start_loc=None, + seq_start_loc=None, + context_lens_tensor=None, + block_tables=block_tables, + use_cuda_graph=self.use_cuda_graph, + ) + return self._cached_decode_metadata + + +class PlaceholderAttentionMetadataBuilder( + AttentionMetadataBuilder[PlaceholderAttentionMetadata]): + + def __init__(self, input_builder: "ModelInputForGPUBuilder"): + self.prefill_seq_lens: List[int] = [] + self.context_lens: List[int] = [] + self.curr_seq_lens: List[int] = [] + self.num_prefills = 0 + self.num_prefill_tokens = 0 + self.num_decode_tokens = 0 + + self.input_builder = input_builder + self.runner = input_builder.runner + + def _add_seq_group( + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup", + chunked_prefill_enabled: bool): + """Add a sequence group to the metadata. Specifically update/append + 1. context length. + """ + is_prompt = inter_data.is_prompt + + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len, + curr_sliding_window_block) in zip( + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens], + inter_data.orig_seq_lens, inter_data.seq_lens, + inter_data.query_lens, inter_data.context_lens, + inter_data.curr_sliding_window_blocks): + self.context_lens.append(context_len) + + if is_prompt: + self.num_prefills += 1 + self.num_prefill_tokens += token_len + self.prefill_seq_lens.append(seq_len) + else: + assert query_len == 1, ( + "seq_len: {}, context_len: {}, query_len: {}".format( + seq_len, context_len, query_len)) + self.num_decode_tokens += query_len + self.curr_seq_lens.append(curr_seq_len) + + def build(self, seq_lens: List[int], query_lens: List[int], + cuda_graph_pad_size: int, batch_size: int): + """Build attention metadata with on-device tensors. + + Args: + seq_lens: The maybe padded sequence lengths of the input sequences. + query_lens: The query lengths of the input sequences. + cuda_graph_pad_size: The padding size for cuda graph. + -1 if cuda graph is not used. + batch_size: The maybe padded batch size. + """ + for inter_data in self.input_builder.inter_data_list: + self._add_seq_group(inter_data, + self.input_builder.chunked_prefill_enabled) + + device = self.runner.device + use_captured_graph = cuda_graph_pad_size != -1 + + logits_soft_cap = getattr(self.runner.model_config.hf_config, + "attn_logit_softcapping", None) + if logits_soft_cap is not None: + raise ValueError( + "Please use Flashinfer backend for models with logits_soft_cap" + " (i.e., Gemma-2). Otherwise, the output might be wrong." + " Set Flashinfer backend by " + "export VLLM_ATTENTION_BACKEND=FLASHINFER.") + + max_query_len = max(query_lens) + decode_query_lens = query_lens[self.num_prefills:] + if len(decode_query_lens) > 0: + max_decode_query_len = max(decode_query_lens) + else: + max_decode_query_len = 1 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0) + max_decode_seq_len = max(self.curr_seq_lens, default=0) + num_decode_tokens = self.num_decode_tokens + + if use_captured_graph: + num_decode_tokens = batch_size + + assert max_query_len > 0, ("query_lens: {}".format(query_lens)) + + context_lens_tensor = torch.tensor(self.context_lens, + dtype=torch.int, + device=device) + seq_lens_tensor = torch.tensor(seq_lens, + dtype=torch.int, + device=device) + query_lens_tensor = torch.tensor(query_lens, + dtype=torch.long, + device=device) + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + torch.cumsum(seq_lens_tensor, + dim=0, + dtype=seq_start_loc.dtype, + out=seq_start_loc[1:]) + torch.cumsum(query_lens_tensor, + dim=0, + dtype=query_start_loc.dtype, + out=query_start_loc[1:]) + + # Placeholders + slot_mapping = torch.empty(0) + block_tables = torch.empty(0) + + return PlaceholderAttentionMetadata( + num_prefills=self.num_prefills, + slot_mapping=slot_mapping, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=num_decode_tokens, + seq_lens=seq_lens, + seq_lens_tensor=seq_lens_tensor, + max_query_len=max_query_len, + max_decode_query_len=max_decode_query_len, + max_prefill_seq_len=max_prefill_seq_len, + max_decode_seq_len=max_decode_seq_len, + query_start_loc=query_start_loc, + seq_start_loc=seq_start_loc, + context_lens_tensor=context_lens_tensor, + block_tables=block_tables, + use_cuda_graph=use_captured_graph, + ) + + +class PlaceholderAttentionImpl(AttentionImpl): + + def __init__(self, *args, **kwargs) -> None: + return + + def forward(self, *args, **kwargs) -> torch.Tensor: + raise NotImplementedError diff --git a/vllm/attention/backends/rocm_flash_attn.py b/vllm/attention/backends/rocm_flash_attn.py new file mode 100644 index 00000000..682eac50 --- /dev/null +++ b/vllm/attention/backends/rocm_flash_attn.py @@ -0,0 +1,668 @@ +"""Attention layer ROCm GPUs.""" +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type + +import torch + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import (CommonAttentionState, + CommonMetadataBuilder) +from vllm.attention.ops.paged_attn import (PagedAttention, + PagedAttentionMetadata) +from vllm.logger import init_logger +from vllm.platforms import current_platform + +if TYPE_CHECKING: + from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata + +logger = init_logger(__name__) + +_PARTITION_SIZE_ROCM = 512 +_ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName + + +class ROCmFlashAttentionBackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "rocm-flash-attn" + + @staticmethod + def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]: + return ROCmFlashAttentionImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return ROCmFlashAttentionMetadata + + @staticmethod + def get_builder_cls() -> Type["ROCmFlashAttentionMetadataBuilder"]: + return ROCmFlashAttentionMetadataBuilder + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return PagedAttention.get_kv_cache_shape(num_blocks, block_size, + num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + PagedAttention.copy_blocks(kv_caches, src_to_dists) + + +@dataclass +class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata): + """Metadata for FlashAttentionBackend. + + NOTE: Any python object stored here is not updated when it is + cuda-graph replayed. If you have values that need to be changed + dynamically, it should be stored in tensor. The tensor has to be + updated from `CUDAGraphRunner.forward` API. + """ + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] + + # NOTE(sang): Definition of context_len, query_len, and seq_len. + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ----------------------| + # |-- query_len ---| + + # Maximum query length in the batch. None for decoding. + max_query_len: Optional[int] + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + # Maximum sequence length among decode batch. 0 if there are prefill + # requests only. + max_decode_seq_len: int + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] + + # Whether or not if cuda graph is enabled. + # Cuda-graph is currently enabled for decoding only. + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention. + use_cuda_graph: bool + + # (batch_size,) A tensor of context lengths (tokens that are computed + # so far). + context_lens_tensor: Optional[torch.Tensor] + + # Max number of query tokens among request in the batch. + max_decode_query_len: Optional[int] = None + + _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None + _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None + + @property + def prefill_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]: + if self.num_prefills == 0: + return None + + if self._cached_prefill_metadata is not None: + return self._cached_prefill_metadata + + assert self.seq_lens is not None + assert self.seq_lens_tensor is not None + assert self.query_start_loc is not None + assert self.context_lens_tensor is not None + assert self.block_tables is not None + assert self.seq_start_loc is not None + + self._cached_prefill_metadata = ROCmFlashAttentionMetadata( + num_prefills=self.num_prefills, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=0, + slot_mapping=self.slot_mapping[:self.num_prefill_tokens], + seq_lens=self.seq_lens[:self.num_prefills], + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills], + max_query_len=self.max_query_len, + max_prefill_seq_len=self.max_prefill_seq_len, + max_decode_seq_len=0, + query_start_loc=self.query_start_loc[:self.num_prefills + 1], + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1], + context_lens_tensor=self.context_lens_tensor[:self.num_prefills], + block_tables=self.block_tables[:self.num_prefills], + use_cuda_graph=False, + ) + return self._cached_prefill_metadata + + @property + def decode_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]: + if self.num_decode_tokens == 0: + return None + + if self._cached_decode_metadata is not None: + return self._cached_decode_metadata + assert self.block_tables is not None + assert self.seq_lens_tensor is not None + + self._cached_decode_metadata = ROCmFlashAttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=self.num_decode_tokens, + slot_mapping=self.slot_mapping[self.num_prefill_tokens:], + seq_lens=None, + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:], + max_query_len=None, + max_prefill_seq_len=0, + max_decode_seq_len=self.max_decode_seq_len, + query_start_loc=None, + seq_start_loc=None, + context_lens_tensor=None, + block_tables=self.block_tables[self.num_prefills:], + use_cuda_graph=self.use_cuda_graph, + ) + return self._cached_decode_metadata + + def advance_step(self, + model_input: "ModelInputForGPUWithSamplingMetadata", + sampled_token_ids: Optional[torch.Tensor], + block_size: int, + num_seqs: int, + num_queries: int, + turn_prefills_into_decodes: bool = False): + """ + Update metadata in-place to advance one decode step. + """ + + assert not turn_prefills_into_decodes, \ + ("Chunked prefill is not supported with rocm_flash_attn yet." + "turn_prefills_into_decodes is a Multi-Step + Chunked-Prefill " + "specific parameter.") + + # When using cudagraph, the num_seqs is padded to the next captured + # batch sized, but num_queries tracks the actual number of requests in + # the batch. For --enforce-eager mode, num_seqs == num_queries + if num_seqs != num_queries: + assert num_seqs > num_queries + assert self.use_cuda_graph + + assert self.num_prefills == 0 + assert self.num_prefill_tokens == 0 + assert self.num_decode_tokens == num_seqs + assert self.slot_mapping.shape == (num_seqs, ) + + assert self.seq_lens is not None + assert len(self.seq_lens) == num_seqs + assert self.seq_lens_tensor is not None + assert self.seq_lens_tensor.shape == (num_seqs, ) + assert self.max_query_len == 1 + assert self.max_prefill_seq_len == 0 + assert self.max_decode_seq_len == max(self.seq_lens) + + assert self.query_start_loc is not None + assert self.query_start_loc.shape == (num_queries + 1, ) + assert self.seq_start_loc is not None + assert self.seq_start_loc.shape == (num_seqs + 1, ) + + assert self.context_lens_tensor is not None + assert self.context_lens_tensor.shape == (num_queries, ) + + assert self.block_tables is not None + assert self.block_tables.shape[0] == num_seqs + + # Update query lengths. Note that we update only queries and not seqs, + # since tensors may be padded due to captured cuda graph batch size + for i in range(num_queries): + self.seq_lens[i] += 1 + self.max_decode_seq_len = max(self.seq_lens) + + ops.advance_step_flashattn(num_seqs=num_seqs, + num_queries=num_queries, + block_size=block_size, + input_tokens=model_input.input_tokens, + sampled_token_ids=sampled_token_ids, + input_positions=model_input.input_positions, + seq_lens=self.seq_lens_tensor, + slot_mapping=self.slot_mapping, + block_tables=self.block_tables) + + +class ROCmFlashAttentionMetadataBuilder( + CommonMetadataBuilder[ROCmFlashAttentionMetadata]): + + _metadata_cls = ROCmFlashAttentionMetadata + + +def _make_alibi_bias(alibi_slopes: torch.Tensor, + dtype: torch.dtype, + seq_lens: Optional[List[int]], + make_attn_mask: bool = True) -> List[torch.Tensor]: + attn_biases = [] + if seq_lens: + for seq_len in seq_lens: + bias = torch.arange(seq_len, dtype=dtype) + # NOTE(zhuohan): HF uses + # `bias = bias[None, :].repeat(seq_len, 1)` + # here. We find that both biases give the same results, but + # the bias below more accurately follows the original ALiBi + # paper. + bias = bias[None, :] - bias[:, None] + + num_heads = alibi_slopes.shape[0] + bias = bias[None, :].repeat( + (num_heads, 1, 1)).to(alibi_slopes.device) + bias.mul_(alibi_slopes[:, None, None]) + if make_attn_mask: + inf_mask = torch.empty( + (1, seq_len, seq_len), + dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1).to( + alibi_slopes.device) + attn_biases.append((bias + inf_mask).to(dtype)) + else: + attn_biases.append(bias.to(dtype)) + + return attn_biases + + +class ROCmFlashAttentionImpl(AttentionImpl): + """ + If the input tensors contain prompt tokens, the layout is as follows: + |<--------------- num_prompt_tokens -------------->| + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->| + + Otherwise, the layout is as follows: + |<------------------ num_generation_tokens (M) ----------------->| + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->| + + Generation tokens can contain padding when cuda-graph is used. + Currently, prompt tokens don't contain any padding. + + The prompts might have different lengths, while the generation tokens + always have length 1. + + If chunked prefill is enabled, prefill tokens and decode tokens can be + batched together in a flattened 1D query. + + |<----- num_prefill_tokens ---->|<------- num_decode_tokens ----------->| + |<-prompt_0->|...|<-prompt_N-1->|<-generation_0->|...|<-generation_M-1->| + + Currently, cuda graph is disabled for chunked prefill, meaning there's no + padding between prefill and decode tokens. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + if blocksparse_params is not None: + raise ValueError( + "ROCmFlashAttention does not support blocksparse attention.") + if logits_soft_cap is not None: + raise ValueError( + "ROCmFlashAttention does not support attention logits soft " + "capping.") + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + self.sliding_window = ((sliding_window, sliding_window) + if sliding_window is not None else (-1, -1)) + self.kv_cache_dtype = kv_cache_dtype + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + supported_head_sizes = PagedAttention.get_supported_head_sizes() + if head_size not in supported_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by PagedAttention. " + f"Supported head sizes are: {supported_head_sizes}.") + + self.use_naive_attn = False + # NOTE: Allow for switching between Triton and CK. Defaulting to triton. + self.use_triton_flash_attn = envs.VLLM_USE_TRITON_FLASH_ATTN + if self.use_triton_flash_attn: + from vllm.attention.ops.triton_flash_attention import ( # noqa: F401 + triton_attention) + self.attn_func = triton_attention + logger.debug("Using Triton FA in ROCmBackend") + if self.sliding_window != (-1, -1): + logger.warning("ROCm Triton FA does not currently support " + "sliding window attention. If using half " + "precision, please try using the ROCm CK " + "FA backend instead by setting the env var " + "`VLLM_USE_TRITON_FLASH_ATTN=0`") + else: + # if not using triton, navi3x/navi21/navi10 do not use flash-attn + # either + if not current_platform.has_device_capability(90): + self.use_naive_attn = True + else: + try: + from flash_attn import flash_attn_varlen_func # noqa: F401 + self.attn_func = flash_attn_varlen_func + logger.debug("Using CK FA in ROCmBackend") + except ModuleNotFoundError: + self.use_naive_attn = True + + if self.use_naive_attn: + self.attn_func = _sdpa_attention + logger.debug("Using naive attention in ROCmBackend") + + def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor: + """torch.repeat_interleave(x, dim=1, repeats=n_rep)""" + tokens, n_kv_heads, head_dim = x.shape + return (x[:, :, + None, :].expand(tokens, n_kv_heads, n_rep, + head_dim).reshape(tokens, n_kv_heads * n_rep, + head_dim)) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: ROCmFlashAttentionMetadata, + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with FlashAttention and PagedAttention. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [num_tokens, num_heads * head_size] + """ + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if attn_type != AttentionType.DECODER: + raise NotImplementedError("Encoder self-attention and " + "encoder/decoder cross-attention " + "are not implemented for " + "ROCmFlashAttentionImpl") + + num_tokens, hidden_size = query.shape + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + + if kv_cache.numel() > 0: + key_cache, value_cache = PagedAttention.split_kv_cache( + kv_cache, self.num_kv_heads, self.head_size) + + # Reshape the input keys and values and store them in the cache. + # If kv_cache is not provided, the new key and value tensors are + # not cached. This happens during the initial memory profiling run. + PagedAttention.write_to_paged_cache( + key, + value, + key_cache, + value_cache, + attn_metadata.slot_mapping, + self.kv_cache_dtype, + k_scale, + v_scale, + ) + + num_prefill_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + assert key.shape[0] == num_prefill_tokens + num_decode_tokens + assert value.shape[0] == num_prefill_tokens + num_decode_tokens + + output = torch.empty_like(query) + # Query for decode. KV is not needed because it is already cached. + decode_query = query[num_prefill_tokens:] + # QKV for prefill. + query = query[:num_prefill_tokens] + key = key[:num_prefill_tokens] + value = value[:num_prefill_tokens] + + assert query.shape[0] == num_prefill_tokens + assert decode_query.shape[0] == num_decode_tokens + + if prefill_meta := attn_metadata.prefill_metadata: + # Prompt run. + assert prefill_meta.seq_lens is not None + if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0: + # triton attention + # When block_tables are not filled, it means q and k are the + # prompt, and they have the same length. + attn_masks = None + if self.use_triton_flash_attn: + if self.alibi_slopes is not None: + attn_masks = _make_alibi_bias( + self.alibi_slopes, + query.dtype, + attn_metadata.seq_lens, + make_attn_mask=False) # type: ignore + out, _ = self.attn_func( + query, + key, + value, + None, + prefill_meta.seq_start_loc, + prefill_meta.seq_start_loc, + prefill_meta.max_prefill_seq_len, + prefill_meta.max_prefill_seq_len, + True, + self.scale, + attn_masks[0][None] + if attn_masks is not None else None, + ) + elif self.use_naive_attn: + if self.num_kv_heads != self.num_heads: + # Interleave for MQA workaround. + key = self.repeat_kv(key, self.num_queries_per_kv) + value = self.repeat_kv(value, self.num_queries_per_kv) + if self.alibi_slopes is not None: + attn_masks = _make_alibi_bias( + self.alibi_slopes, + query.dtype, + attn_metadata.seq_lens, + make_attn_mask=True) # type: ignore + query = query.movedim(0, query.dim() - 2) + key = key.movedim(0, key.dim() - 2) + value = value.movedim(0, value.dim() - 2) + # sdpa math backend attention + out = self.attn_func( + query, + key, + value, + prefill_meta.seq_lens, + num_tokens, + self.num_heads, + self.head_size, + self.scale, + attn_masks, + ) + else: + out = self.attn_func( + q=query, + k=key, + v=value, + cu_seqlens_q=prefill_meta.seq_start_loc, + cu_seqlens_k=prefill_meta.seq_start_loc, + max_seqlen_q=prefill_meta.max_prefill_seq_len, + max_seqlen_k=prefill_meta.max_prefill_seq_len, + softmax_scale=self.scale, + causal=True, + window_size=self.sliding_window, + alibi_slopes=self.alibi_slopes, + ) + + # common code for prefill + assert output[:num_prefill_tokens].shape == out.shape + output[:num_prefill_tokens] = out + else: + # prefix-enabled attention + output[:num_prefill_tokens] = PagedAttention.forward_prefix( + query, + key, + value, + self.kv_cache_dtype, + key_cache, + value_cache, + prefill_meta.block_tables, + prefill_meta.query_start_loc, + prefill_meta.seq_lens_tensor, + prefill_meta.context_lens_tensor, + prefill_meta.max_query_len, + self.alibi_slopes, + self.sliding_window[0], + k_scale, + v_scale, + ) + + if decode_meta := attn_metadata.decode_metadata: + # Decoding run. + # Whether to use rocm custom paged attention or not + num_seqs, num_heads, head_size = decode_query.shape + block_size = value_cache.shape[3] + gqa_ratio = num_heads // self.num_kv_heads + use_custom = _use_rocm_custom_paged_attention( + decode_query.dtype, head_size, block_size, gqa_ratio, + decode_meta.max_decode_seq_len) + if use_custom: + max_seq_len = decode_meta.max_decode_seq_len + max_num_partitions = ( + (max_seq_len + _PARTITION_SIZE_ROCM - 1) // + _PARTITION_SIZE_ROCM) + assert _PARTITION_SIZE_ROCM % block_size == 0 + tmp_output = torch.empty( + size=(num_seqs, num_heads, max_num_partitions, head_size), + dtype=output.dtype, + device=output.device, + ) + exp_sums = torch.empty( + size=(num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=output.device, + ) + max_logits = torch.empty_like(exp_sums) + ops.paged_attention_rocm( + output[num_prefill_tokens:], + exp_sums, + max_logits, + tmp_output, + decode_query, + key_cache, + value_cache, + self.num_kv_heads, + self.scale, + decode_meta.block_tables, + decode_meta.seq_lens_tensor, + block_size, + max_seq_len, + self.alibi_slopes, + self.kv_cache_dtype, + k_scale, + v_scale, + ) + else: + output[num_prefill_tokens:] = PagedAttention.forward_decode( + decode_query, + key_cache, + value_cache, + decode_meta.block_tables, + decode_meta.seq_lens_tensor, + decode_meta.max_decode_seq_len, + self.kv_cache_dtype, + self.num_kv_heads, + self.scale, + self.alibi_slopes, + k_scale, + v_scale, + ) + + # Reshape the output tensor. + return output.view(num_tokens, hidden_size) + + +def _sdpa_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + seq_lens: List[int], + num_tokens: int, + num_heads: int, + head_size: int, + scale: float, + attn_masks: Optional[List[torch.Tensor]] = None, +) -> torch.Tensor: + start = 0 + output = torch.empty((num_tokens, num_heads, head_size), + dtype=query.dtype, + device=query.device) + + for i, seq_len in enumerate(seq_lens): + end = start + seq_len + with torch.backends.cuda.sdp_kernel(enable_math=True, + enable_flash=False, + enable_mem_efficient=False): + sub_out = torch.nn.functional.scaled_dot_product_attention( + query[:, start:end, :], + key[:, start:end, :], + value[:, start:end, :], + dropout_p=0.0, + is_causal=attn_masks is None, + attn_mask=attn_masks[i] if attn_masks else None, + scale=scale).movedim(query.dim() - 2, 0) + output[start:end, :, :] = sub_out + start = end + + return output + + +def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int, + block_size: int, gqa_ratio: int, + max_seq_len: int) -> bool: + # rocm custom page attention not support on navi (gfx1*) + return (not _ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16) + and (head_size == 64 or head_size == 128) + and (block_size == 16 or block_size == 32) + and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768) diff --git a/vllm/attention/backends/torch_sdpa.py b/vllm/attention/backends/torch_sdpa.py new file mode 100644 index 00000000..ef8d5766 --- /dev/null +++ b/vllm/attention/backends/torch_sdpa.py @@ -0,0 +1,547 @@ +""" Attention layer with torch scaled_dot_product_attention + and PagedAttention.""" +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Type + +import torch +from torch.nn.functional import scaled_dot_product_attention + +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import CommonAttentionState +from vllm.attention.ops.paged_attn import PagedAttentionMetadata +from vllm.utils import is_cpu + +if is_cpu(): + try: + from vllm.attention.ops.ipex_attn import PagedAttention + except ImportError: + from vllm.attention.ops.paged_attn import PagedAttention +else: + from vllm.attention.ops.paged_attn import PagedAttention + + +class TorchSDPABackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "torch-sdpa" + + @staticmethod + def get_impl_cls() -> Type["TorchSDPABackendImpl"]: + return TorchSDPABackendImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return TorchSDPAMetadata + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return PagedAttention.get_kv_cache_shape(num_blocks, block_size, + num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + PagedAttention.copy_blocks(kv_caches, src_to_dists) + + +@dataclass +class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata): + """Metadata for TorchSDPABackend. + """ + # Currently, input sequences can only contain all prompts + # or all decoding. True if all sequences are prompts. + is_prompt: bool + slot_mapping: torch.Tensor + seq_lens: Optional[List[int]] + + # Begin encoder attn & enc/dec cross-attn fields... + # Encoder sequence lengths representation + encoder_seq_lens: Optional[List[int]] = None + encoder_seq_lens_tensor: Optional[torch.Tensor] = None + + # Maximum sequence length among encoder sequences + max_encoder_seq_len: Optional[int] = None + + # Number of tokens input to encoder + num_encoder_tokens: Optional[int] = None + + # Cross-attention memory-mapping data structures: slot mapping + # and block tables + cross_slot_mapping: Optional[torch.Tensor] = None + cross_block_tables: Optional[torch.Tensor] = None + + def __post_init__(self): + # Set during the execution of the first attention op. + # It is a list because it is needed to set per prompt + # when alibi slopes is used. It is because of the limitation + # from xformer API. + # will not appear in the __repr__ and __init__ + self.attn_bias: Optional[List[torch.Tensor]] = None + self.encoder_attn_bias: Optional[List[torch.Tensor]] = None + self.cross_attn_bias: Optional[List[torch.Tensor]] = None + + @property + def is_all_encoder_attn_metadata_set(self): + ''' + All attention metadata required for encoder attention is set. + ''' + return ((self.encoder_seq_lens is not None) + and (self.encoder_seq_lens_tensor is not None) + and (self.max_encoder_seq_len is not None)) + + @property + def is_all_cross_attn_metadata_set(self): + ''' + All attention metadata required for enc/dec cross-attention is set. + + Superset of encoder attention required metadata. + ''' + return (self.is_all_encoder_attn_metadata_set + and (self.cross_slot_mapping is not None) + and (self.cross_block_tables is not None)) + + @property + def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]: + # Currently chunked prefill is not supported + if self.num_decode_tokens == 0: + assert self.num_prefills > 0 + return self + + return None + + @property + def decode_metadata(self) -> Optional["TorchSDPAMetadata"]: + # Currently chunked prefill is not supported + if self.num_prefills > 0: + assert self.num_decode_tokens == 0 + return None + + return self + + def get_seq_lens( + self, + attn_type: AttentionType, + ): + ''' + Extract appropriate sequence lengths from attention metadata + according to attention type. + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + + Returns: + * Appropriate sequence lengths tensor for query + * Appropriate sequence lengths tensor for key & value + ''' + + if attn_type == AttentionType.DECODER: + seq_lens_q = self.seq_lens + seq_lens_kv = self.seq_lens + elif attn_type == AttentionType.ENCODER: + seq_lens_q = self.encoder_seq_lens + seq_lens_kv = self.encoder_seq_lens + elif attn_type == AttentionType.ENCODER_DECODER: + seq_lens_q = self.seq_lens + seq_lens_kv = self.encoder_seq_lens + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + return seq_lens_q, seq_lens_kv + + def get_attn_bias( + self, + attn_type: AttentionType, + ) -> Optional[List[torch.Tensor]]: + ''' + Extract appropriate attention bias from attention metadata + according to attention type. + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + + Returns: + * Appropriate attention bias value given the attention type + ''' + + if attn_type == AttentionType.DECODER: + return self.attn_bias + elif attn_type == AttentionType.ENCODER: + return self.encoder_attn_bias + elif attn_type == AttentionType.ENCODER_DECODER: + return self.cross_attn_bias + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + + def set_attn_bias( + self, + attn_bias: List[torch.Tensor], + attn_type: AttentionType, + ) -> None: + ''' + Update appropriate attention bias field of attention metadata, + according to attention type. + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * attn_bias: The desired attention bias value + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + ''' + + if attn_type == AttentionType.DECODER: + self.attn_bias = attn_bias + elif attn_type == AttentionType.ENCODER: + self.encoder_attn_bias = attn_bias + elif attn_type == AttentionType.ENCODER_DECODER: + self.cross_attn_bias = attn_bias + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + + def get_seq_len_block_table_args( + self, + attn_type: AttentionType, + ) -> tuple: + ''' + The particular choice of sequence-length- and block-table-related + attributes which should be extracted from attn_metadata is dependent + on the type of attention operation. + + Decoder attn -> select entirely decoder self-attention-related fields + Encoder/decoder cross-attn -> select encoder sequence lengths & + cross-attn block-tables fields + Encoder attn -> select encoder sequence lengths fields & no block tables + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * is_prompt: True if prefill, False otherwise + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + + Returns: + + * Appropriate sequence-lengths tensor + * Appropriate max sequence-length scalar + * Appropriate block tables (or None) + ''' + + if attn_type == AttentionType.DECODER: + # Decoder self-attention + # Choose max_seq_len based on whether we are in prompt_run + return (self.seq_lens_tensor, self.max_decode_seq_len, + self.block_tables) + elif attn_type == AttentionType.ENCODER_DECODER: + # Enc/dec cross-attention KVs match encoder sequence length; + # cross-attention utilizes special "cross" block tables + return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len, + self.cross_block_tables) + elif attn_type == AttentionType.ENCODER: + # No block tables associated with encoder attention + return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len, + None) + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + + +class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]): + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + if blocksparse_params is not None: + raise ValueError( + "Torch SPDA does not support block-sparse attention.") + if logits_soft_cap is not None: + raise ValueError("Torch SPDA does not support logits soft cap.") + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + self.sliding_window = sliding_window + self.kv_cache_dtype = kv_cache_dtype + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + self.need_mask = (self.alibi_slopes is not None + or self.sliding_window is not None) + + supported_head_sizes = PagedAttention.get_supported_head_sizes() + if head_size not in supported_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by PagedAttention. " + f"Supported head sizes are: {supported_head_sizes}.") + if kv_cache_dtype != "auto": + raise NotImplementedError( + "Torch SDPA backend does not support FP8 KV cache. " + "Please use xFormers backend instead.") + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: TorchSDPAMetadata, # type: ignore + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with torch SDPA and PagedAttention. + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + Returns: + shape = [num_tokens, num_heads * head_size] + """ + assert k_scale == 1.0 and v_scale == 1.0 + if (attn_type == AttentionType.ENCODER + and (not attn_metadata.is_all_encoder_attn_metadata_set)): + raise AttributeError("Encoder attention requires setting " + "encoder metadata attributes.") + elif (attn_type == AttentionType.ENCODER_DECODER + and (not attn_metadata.is_all_cross_attn_metadata_set)): + raise AttributeError("Encoder/decoder cross-attention " + "requires setting cross-attention " + "metadata attributes.") + + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + if key is not None: + assert value is not None + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + else: + assert value is None + + if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0): + # KV-cache during decoder-self- or + # encoder-decoder-cross-attention, but not + # during encoder attention. + # + # Even if there are no new key/value pairs to cache, + # we still need to break out key_cache and value_cache + # i.e. for later use by paged attention + key_cache, value_cache = PagedAttention.split_kv_cache( + kv_cache, self.num_kv_heads, self.head_size) + + if (key is not None) and (value is not None): + if attn_type == AttentionType.ENCODER_DECODER: + # Update cross-attention KV cache (prefill-only) + # During cross-attention decode, key & value will be None, + # preventing this IF-statement branch from running + updated_slot_mapping = attn_metadata.cross_slot_mapping + else: + # Update self-attention KV cache (prefill/decode) + updated_slot_mapping = attn_metadata.slot_mapping + + PagedAttention.write_to_paged_cache(key, value, key_cache, + value_cache, + updated_slot_mapping, + self.kv_cache_dtype, + k_scale, v_scale) + + if attn_type != AttentionType.ENCODER: + # Decoder self-attention supports chunked prefill. + # Encoder/decoder cross-attention requires no chunked + # prefill (100% prefill or 100% decode tokens, no mix) + num_prefill_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + else: + # Encoder attention - chunked prefill is not applicable; + # derive token-count from query shape & and treat them + # as 100% prefill tokens + assert attn_metadata.num_encoder_tokens is not None + num_prefill_tokens = attn_metadata.num_encoder_tokens + num_decode_tokens = 0 + + if attn_type == AttentionType.DECODER: + # Only enforce this shape-constraint for decoder + # self-attention + assert key.shape[0] == num_prefill_tokens + num_decode_tokens + assert value.shape[0] == num_prefill_tokens + num_decode_tokens + + if prefill_meta := attn_metadata.prefill_metadata: + assert attn_metadata.seq_lens is not None + if (kv_cache.numel() == 0 + or prefill_meta.block_tables.numel() == 0): + output = self._run_sdpa_forward(query, + key, + value, + prefill_meta, + attn_type=attn_type) + else: + # prefix-enabled attention + raise RuntimeError( + "Torch SDPA backend doesn't support prefix decoding.") + + if decode_meta := attn_metadata.decode_metadata: + # Decoding run. + ( + seq_lens_arg, + max_seq_len_arg, + block_tables_arg, + ) = decode_meta.get_seq_len_block_table_args(attn_type) + + output = PagedAttention.forward_decode( + query, + key_cache, + value_cache, + block_tables_arg, + seq_lens_arg, + max_seq_len_arg, + self.kv_cache_dtype, + self.num_kv_heads, + self.scale, + self.alibi_slopes, + k_scale, + v_scale, + ) + + # Reshape the output tensor. + return output.view(-1, self.num_heads * self.head_size) + + def _run_sdpa_forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: TorchSDPAMetadata, + attn_type: AttentionType = AttentionType.DECODER, + ): + if self.num_kv_heads != self.num_heads: + key = key.repeat_interleave(self.num_queries_per_kv, dim=1) + value = value.repeat_interleave(self.num_queries_per_kv, dim=1) + + attn_masks = attn_metadata.get_attn_bias(attn_type) + if attn_masks is None: + if self.alibi_slopes is not None: + attn_masks = _make_alibi_bias( + self.alibi_slopes, query.dtype, + attn_metadata.seq_lens) # type: ignore + elif self.sliding_window is not None: + assert attn_metadata.seq_lens is not None + attn_masks = _make_sliding_window_bias( + attn_metadata.seq_lens, self.sliding_window, + query.dtype) # type: ignore + else: + seq_lens, _ = attn_metadata.get_seq_lens(attn_type) + attn_masks = [None] * len(seq_lens) + attn_metadata.set_attn_bias(attn_masks, attn_type) + + output = torch.empty_like(query) + query = query.movedim(0, query.dim() - 2) + key = key.movedim(0, key.dim() - 2) + value = value.movedim(0, value.dim() - 2) + + causal_attn = (attn_type == AttentionType.DECODER) + + seq_lens_q, seq_lens_kv = attn_metadata.get_seq_lens(attn_type) + start_q, start_kv = 0, 0 + for seq_len_q, seq_len_kv, mask in zip(seq_lens_q, seq_lens_kv, + attn_masks): + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + sub_out = scaled_dot_product_attention( + query[None, :, start_q:end_q, :], + key[None, :, start_kv:end_kv, :], + value[None, :, start_kv:end_kv, :], + attn_mask=mask, + dropout_p=0.0, + is_causal=causal_attn and not self.need_mask, + scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0) + output[start_q:end_q, :, :] = sub_out + start_q, start_kv = end_q, end_kv + return output + + +def _make_alibi_bias( + alibi_slopes: torch.Tensor, + dtype: torch.dtype, + seq_lens: List[int], +) -> List[torch.Tensor]: + attn_biases: List[torch.Tensor] = [] + for seq_len in seq_lens: + bias = torch.arange(seq_len, dtype=dtype) + # NOTE(zhuohan): HF uses + # `bias = bias[None, :].repeat(seq_len, 1)` + # here. We find that both biases give the same results, but + # the bias below more accurately follows the original ALiBi + # paper. + bias = bias[None, :] - bias[:, None] + + num_heads = alibi_slopes.shape[0] + bias = bias[None, :].repeat((num_heads, 1, 1)) + bias.mul_(alibi_slopes[:, None, None]).unsqueeze_(0) + inf_mask = torch.empty( + (1, seq_len, seq_len), + dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1) + attn_biases.append((bias + inf_mask).to(dtype)) + + return attn_biases + + +def _make_sliding_window_bias( + seq_lens: List[int], + window_size: Optional[int], + dtype: torch.dtype, +) -> List[torch.Tensor]: + attn_biases: List[torch.Tensor] = [] + for seq_len in seq_lens: + tensor = torch.full( + (1, seq_len, seq_len), + dtype=dtype, + fill_value=1, + ) + shift = 0 + mask = torch.tril(tensor, diagonal=shift).to(dtype) # type: ignore + if window_size is not None: + mask = torch.triu(mask, diagonal=shift - window_size + 1) + mask = torch.log(mask) + attn_biases.append(mask.to(dtype)) + + return attn_biases diff --git a/vllm/attention/backends/utils.py b/vllm/attention/backends/utils.py new file mode 100644 index 00000000..53e3a53b --- /dev/null +++ b/vllm/attention/backends/utils.py @@ -0,0 +1,442 @@ +"""Attention backend utils""" +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +import numpy as np +import torch + +from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder, + AttentionState) +from vllm.utils import async_tensor_h2d, make_tensor_with_pad + +if TYPE_CHECKING: + from vllm.worker.model_runner_base import ModelRunnerBase + +# Error string(s) for encoder/decoder +# unsupported attention scenarios +STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported " + "with encoder/decoder models.") + +PAD_SLOT_ID = -1 + +# Switch to numpy implementation of compute_slot_mapping +# if we have at least this many elements. Could be tuned further. +_COMPUTE_SLOT_MAPPING_NUMPY_NUMEL = 256 + +if TYPE_CHECKING: + from vllm.worker.model_runner import ModelInputForGPUBuilder + + +def is_block_tables_empty(block_tables: Union[None, Dict]): + """ + Check if block_tables is None or a dictionary with all None values. + """ + if block_tables is None: + return True + return (isinstance(block_tables, dict) + and all(value is None for value in block_tables.values())) + + +def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int, + context_len: int, sliding_window: int, + use_v2_block_manager: bool): + """ + Compute the start index of slot mapping. + """ + start_idx = 0 + if is_prompt and sliding_window is not None: + assert use_v2_block_manager or context_len == 0, ( + "Prefix caching is currently not supported with " + "sliding window attention in V1 block manager") + # When prefill, we use it to not write slots to kv cache + # to save memory. + start_idx = max(0, query_len - sliding_window) + return start_idx + + +def _compute_slot_mapping_python(slot_mapping: List[int], + block_table: List[int], range_start: int, + range_end: int, block_size: int): + for i in range(range_start, range_end): + block_number = block_table[i // block_size] + block_offset = i % block_size + slot = block_number * block_size + block_offset + slot_mapping.append(slot) + + +def _compute_slot_mapping_numpy(slot_mapping: List[int], + block_table: List[int], range_start: int, + range_end: int, block_size: int): + block_table_array = np.array(block_table) + idx = np.arange(range_start, range_end) + block_offset = idx % block_size + idx //= block_size + seq_slot_mapping_array = block_table_array[idx] + seq_slot_mapping_array *= block_size + seq_slot_mapping_array += block_offset + slot_mapping.extend(seq_slot_mapping_array) + + +def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int], + seq_id: int, seq_len: int, context_len: int, + start_idx: int, block_size: int, + block_tables: Dict[int, List[int]]): + """ + Compute slot mapping. + """ + if is_profile_run: + # During memory profiling, the block tables are not + # initialized yet. In this case, we just use a dummy + # slot mapping. + # In embeddings, the block tables are {seq_id: None}. + slot_mapping.extend([PAD_SLOT_ID] * seq_len) + return + + # Mask the [0, start_idx) tokens of the prompt with + # PAD_SLOT_ID, where start_idx is max(0, seq_len - + # sliding_window). For example, if the prompt len is 10, + # sliding window is 8, and block size is 4, the first two + # tokens are masked and the slot mapping will be + # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1]. + padding_mask_len = max(0, start_idx - context_len) + slot_mapping.extend([PAD_SLOT_ID] * padding_mask_len) + + range_start = max(start_idx, context_len) + range_end = seq_len + numel = range_end - range_start + block_table = block_tables[seq_id] + + # numpy implementation will be faster than python if we have + # many elements, otherwise it will be slower. + if numel < _COMPUTE_SLOT_MAPPING_NUMPY_NUMEL: + _compute_slot_mapping_python(slot_mapping, block_table, range_start, + range_end, block_size) + else: + _compute_slot_mapping_numpy(slot_mapping, block_table, range_start, + range_end, block_size) + + +TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata') + + +class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]): + + _metadata_cls: Type[TAttentionMetadata] + + def __init__(self, input_builder: "ModelInputForGPUBuilder"): + self.slot_mapping: List[int] = [] + self.prefill_seq_lens: List[int] = [] + self.context_lens: List[int] = [] + self.block_tables: List[List[int]] = [] + self.curr_seq_lens: List[int] = [] + self.num_prefills = 0 + self.num_prefill_tokens = 0 + self.num_decode_tokens = 0 + + self.input_builder = input_builder + self.runner = input_builder.runner + + self.sliding_window = input_builder.sliding_window + self.block_size = input_builder.block_size + self.use_v2_block_manager = ( + input_builder.scheduler_config.use_v2_block_manager) + + def _add_seq_group( + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup", + chunked_prefill_enabled: bool): + is_prompt = inter_data.is_prompt + block_tables = inter_data.block_tables + computed_block_nums = inter_data.computed_block_nums + + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len, + curr_sliding_window_block) in zip( + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens], + inter_data.orig_seq_lens, inter_data.seq_lens, + inter_data.query_lens, inter_data.context_lens, + inter_data.curr_sliding_window_blocks): + self.context_lens.append(context_len) + if is_prompt: + self.num_prefills += 1 + self.num_prefill_tokens += token_len + self.prefill_seq_lens.append(seq_len) + else: + assert query_len == 1, ( + "seq_len: {}, context_len: {}, query_len: {}".format( + seq_len, context_len, query_len)) + self.num_decode_tokens += query_len + self.curr_seq_lens.append(curr_seq_len) + + # Compute block table. + # TODO(sang): Combine chunked prefill and prefix caching by + # only allowing multiple of block_size chunk size. + # NOTE: This only works for oooooooxxx style attention. + block_table = [] + if inter_data.prefix_cache_hit: + block_table = computed_block_nums + elif ((chunked_prefill_enabled or not is_prompt) + and block_tables is not None): + block_table = block_tables[seq_id][-curr_sliding_window_block:] + self.block_tables.append(block_table) + + # Compute slot mapping. + is_profile_run = is_block_tables_empty(block_tables) + start_idx = compute_slot_mapping_start_idx( + is_prompt, query_len, context_len, self.sliding_window, + self.use_v2_block_manager) + compute_slot_mapping(is_profile_run, self.slot_mapping, seq_id, + seq_len, context_len, start_idx, + self.block_size, inter_data.block_tables) + + def build(self, seq_lens: List[int], query_lens: List[int], + cuda_graph_pad_size: int, batch_size: int): + """Build attention metadata with on-device tensors. + + Args: + seq_lens: The maybe padded sequence lengths of the input sequences. + query_lens: The query lengths of the input sequences. + cuda_graph_pad_size: The padding size for cuda graph. + -1 if cuda graph is not used. + batch_size: The maybe padded batch size. + """ + for inter_data in self.input_builder.inter_data_list: + self._add_seq_group(inter_data, + self.input_builder.chunked_prefill_enabled) + + device = self.runner.device + use_captured_graph = cuda_graph_pad_size != -1 + + max_query_len = max(query_lens) + max_prefill_seq_len = max(self.prefill_seq_lens, default=0) + max_decode_seq_len = max(self.curr_seq_lens, default=0) + num_decode_tokens = self.num_decode_tokens + + if use_captured_graph: + self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size) + self.block_tables.extend([] * cuda_graph_pad_size) + num_decode_tokens = batch_size + + # The shape of graph_block_tables is + # [max batch size, max context len // block size]. + input_block_tables = self.runner.graph_block_tables[:batch_size] + for i, block_table in enumerate(self.block_tables): + if block_table: + input_block_tables[i, :len(block_table)] = block_table + block_tables = torch.from_numpy(input_block_tables).to( + device, non_blocking=True) + else: + block_tables = make_tensor_with_pad( + self.block_tables, + pad=0, + dtype=torch.int, + device=device, + ) + assert max_query_len > 0, "query_lens: {}".format(query_lens) + + assert device is not None + context_lens_tensor = async_tensor_h2d(self.context_lens, torch.int, + device, self.runner.pin_memory) + seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device, + self.runner.pin_memory) + query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device, + self.runner.pin_memory) + slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long, + device, self.runner.pin_memory) + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1, + dtype=torch.int32, + device=device) + torch.cumsum(seq_lens_tensor, + dim=0, + dtype=seq_start_loc.dtype, + out=seq_start_loc[1:]) + torch.cumsum(query_lens_tensor, + dim=0, + dtype=query_start_loc.dtype, + out=query_start_loc[1:]) + + return self._metadata_cls( # type: ignore + num_prefills=self.num_prefills, + slot_mapping=slot_mapping_tensor, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=num_decode_tokens, + seq_lens=seq_lens, + seq_lens_tensor=seq_lens_tensor, + max_query_len=max_query_len, + max_prefill_seq_len=max_prefill_seq_len, + max_decode_seq_len=max_decode_seq_len, + query_start_loc=query_start_loc, + seq_start_loc=seq_start_loc, + context_lens_tensor=context_lens_tensor, + block_tables=block_tables, + use_cuda_graph=use_captured_graph, + ) + + +class CommonAttentionState(AttentionState): + + def __init__(self, runner: "ModelRunnerBase"): + self.runner = runner + self._is_graph_capturing = False + + @contextmanager + def graph_capture(self, max_batch_size: int): + self._is_graph_capturing = True + self._graph_slot_mapping = torch.full((max_batch_size, ), + PAD_SLOT_ID, + dtype=torch.long, + device=self.runner.device) + self._graph_seq_lens = torch.ones(max_batch_size, + dtype=torch.int32, + device=self.runner.device) + self._graph_block_tables = torch.from_numpy( + self.runner.graph_block_tables).to(device=self.runner.device) + yield + self._is_graph_capturing = False + del self._graph_slot_mapping + del self._graph_seq_lens + del self._graph_block_tables + + def graph_clone(self, batch_size: int) -> "CommonAttentionState": + assert self._is_graph_capturing + return self.__class__(self.runner) + + def graph_capture_get_metadata_for_batch( + self, batch_size: int, is_encoder_decoder_model: bool = False): + assert self._is_graph_capturing + attn_metadata = self.runner.attn_backend.make_metadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=batch_size, + slot_mapping=self._graph_slot_mapping[:batch_size], + seq_lens=None, + seq_lens_tensor=self._graph_seq_lens[:batch_size], + max_query_len=1, + max_decode_query_len=1, + max_prefill_seq_len=0, + max_decode_seq_len=self.runner.max_seq_len_to_capture, + query_start_loc=None, + seq_start_loc=None, + context_lens_tensor=None, + block_tables=self._graph_block_tables[:batch_size], + use_cuda_graph=True, + ) + if is_encoder_decoder_model: + # The encoder decoder model works only with XFormers backend. + # Assert the same. + assert self.runner.attn_backend.get_name() == "xformers", \ + f"Expected attn_backend name to be 'xformers', but "\ + f" got '{self.runner.attn_backend.get_name()}'" + self._update_captured_metadata_for_enc_dec_model( + batch_size=batch_size, attn_metadata=attn_metadata) + + return attn_metadata + + def get_graph_input_buffers( + self, + attn_metadata, + is_encoder_decoder_model: bool = False) -> Dict[str, Any]: + input_buffers = { + "slot_mapping": attn_metadata.slot_mapping, + "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor, + "block_tables": attn_metadata.decode_metadata.block_tables, + } + if is_encoder_decoder_model: + # The encoder decoder model works only with XFormers backend. + # Assert the same. + assert self.runner.attn_backend.get_name() == "xformers", \ + f"Expected attn_backend name to be 'xformers', but "\ + f" got '{self.runner.attn_backend.get_name()}'" + self._add_additonal_input_buffers_for_enc_dec_model( + attn_metadata=attn_metadata, input_buffers=input_buffers) + return input_buffers + + def prepare_graph_input_buffers( + self, + input_buffers, + attn_metadata, + is_encoder_decoder_model: bool = False) -> None: + input_buffers["seq_lens_tensor"].copy_( + attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True) + input_buffers["block_tables"].copy_( + attn_metadata.decode_metadata.block_tables, non_blocking=True) + if is_encoder_decoder_model: + # The encoder decoder model works only with XFormers backend. + # Assert the same. + assert self.runner.attn_backend.get_name() == "xformers", \ + f"Expected attn_backend name to be 'xformers', but "\ + f" got '{self.runner.attn_backend.get_name()}'" + self._prepare_input_buffers_for_enc_dec_model( + attn_metadata, input_buffers) + + def begin_forward(self, model_input) -> None: + return + + def _update_captured_metadata_for_enc_dec_model(self, batch_size: int, + attn_metadata): + """ + Updates the attention metadata parameters for CUDA graph capture in an + encoder-decoder model. + + This method modifies attention-related tensors and metadata required + for CUDA graph capture in encoder-decoder models. Specifically, it + updates the cross-attention and encoder sequence tensors in the + AttentionMetadata object. + """ + # During decode phase the cross_slot_mapping will be empty. Hence set + # an empty tensor for CUDA Graph capture. + attn_metadata.cross_slot_mapping = torch.tensor( + [], dtype=torch.int).cuda() + attn_metadata.cross_block_tables = torch.full( + (batch_size, self.runner.get_max_block_per_batch()), + 1, + dtype=torch.int).cuda() + attn_metadata.encoder_seq_lens = torch.full((batch_size, ), + 1, + dtype=torch.int).cuda() + attn_metadata.encoder_seq_lens_tensor = torch.full( + (batch_size, ), 1, dtype=torch.int).cuda() + attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture + + def _add_additonal_input_buffers_for_enc_dec_model( + self, attn_metadata, input_buffers: Dict[str, Any]): + """ + Saves additional input buffers specific to the encoder-decoder model + from the attention metadata. + + This method extracts and stores encoder-decoder related input buffers + from the `attn_metadata` into the `input_buffers` dictionary. The + buffers include encoder sequence lengths, cross-slot mappings, and + cross-block tables, which are essential for the encoder-decoder model + during CUDA graph replay. + """ + input_buffers["encoder_seq_lens_tensor"] = ( + attn_metadata.decode_metadata.encoder_seq_lens_tensor) + input_buffers["cross_slot_mapping"] = ( + attn_metadata.decode_metadata.cross_slot_mapping) + input_buffers["cross_block_tables"] = ( + attn_metadata.decode_metadata.cross_block_tables) + + def _prepare_input_buffers_for_enc_dec_model(self, attn_metadata, + input_buffers: Dict[str, + Any]): + """ + Populates input buffers with data from the encoder-decoder model's + attention metadata. + + This method fills the input buffers with encoder-decoder specific + tensors. It copies data from the `attn_metadata` and keyword arguments + (`kwargs`) into corresponding buffers in the `input_buffers` dictionary. + The copied data includes attention-related metadata as well as input + IDs and positional information for the encoder. + """ + input_buffers["encoder_seq_lens_tensor"].copy_( + attn_metadata.decode_metadata.encoder_seq_lens_tensor, + non_blocking=True) + input_buffers["cross_slot_mapping"].copy_( + attn_metadata.decode_metadata.cross_slot_mapping, + non_blocking=True) + input_buffers["cross_block_tables"].copy_( + attn_metadata.decode_metadata.cross_block_tables, + non_blocking=True) diff --git a/vllm/attention/backends/xformers.py b/vllm/attention/backends/xformers.py new file mode 100644 index 00000000..7c7dbfa8 --- /dev/null +++ b/vllm/attention/backends/xformers.py @@ -0,0 +1,822 @@ +"""Attention layer with xFormers and PagedAttention.""" +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Type + +import torch +# from xformers import ops as xops +from ixformer.contrib.xformers import ops as xops +from xformers.ops.fmha.attn_bias import (AttentionBias, + BlockDiagonalMask,) +from ixformer.contrib.xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask, + LowerTriangularMaskWithTensorBias) + +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl, + AttentionMetadata, AttentionType) +from vllm.attention.backends.utils import (CommonAttentionState, + CommonMetadataBuilder) +from vllm.attention.ops.paged_attn import (PagedAttention, + PagedAttentionMetadata) +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class XFormersBackend(AttentionBackend): + + @staticmethod + def get_name() -> str: + return "xformers" + + @staticmethod + def get_impl_cls() -> Type["XFormersImpl"]: + return XFormersImpl + + @staticmethod + def get_metadata_cls() -> Type["AttentionMetadata"]: + return XFormersMetadata + + @staticmethod + def get_builder_cls() -> Type["XFormersMetadataBuilder"]: + return XFormersMetadataBuilder + + @staticmethod + def get_state_cls() -> Type["CommonAttentionState"]: + return CommonAttentionState + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return PagedAttention.get_kv_cache_shape(num_blocks, block_size, + num_kv_heads, head_size) + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: Dict[int, int], + ) -> None: + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + PagedAttention.copy_blocks(kv_caches, src_to_dists) + + +@dataclass +class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata): + """Metadata for XFormersbackend. + + NOTE: Any python object stored here is not updated when it is + cuda-graph replayed. If you have values that need to be changed + dynamically, it should be stored in tensor. The tensor has to be + updated from `CUDAGraphRunner.forward` API. + """ + + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ----------------------| + # |-- query_len ---| + + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] + + # FIXME: It is for flash attn. + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_prefill_seq_len: int + # Maximum sequence length among decode batch. 0 if there are prefill + # requests only. + max_decode_seq_len: int + + # Whether or not if cuda graph is enabled. + # Cuda-graph is currently enabled for decoding only. + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention. + use_cuda_graph: bool + + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] = None + + # FIXME: It is for flash attn. + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] = None + + # (batch_size,) A tensor of context lengths (tokens that are computed + # so far). + context_lens_tensor: Optional[torch.Tensor] = None + + # Maximum query length in the batch. None for decoding. + max_query_len: Optional[int] = None + + # Max number of query tokens among request in the batch. + max_decode_query_len: Optional[int] = None + + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] = None + + # Self-attention prefill/decode metadata cache + _cached_prefill_metadata: Optional["XFormersMetadata"] = None + _cached_decode_metadata: Optional["XFormersMetadata"] = None + + # Begin encoder attn & enc/dec cross-attn fields... + + # Encoder sequence lengths representation + encoder_seq_lens: Optional[List[int]] = None + encoder_seq_lens_tensor: Optional[torch.Tensor] = None + + # Maximum sequence length among encoder sequences + max_encoder_seq_len: Optional[int] = None + + # Number of tokens input to encoder + num_encoder_tokens: Optional[int] = None + + # Cross-attention memory-mapping data structures: slot mapping + # and block tables + cross_slot_mapping: Optional[torch.Tensor] = None + cross_block_tables: Optional[torch.Tensor] = None + + def __post_init__(self): + # Set during the execution of the first attention op. + # It is a list because it is needed to set per prompt + # when alibi slopes is used. It is because of the limitation + # from xformer API. + # will not appear in the __repr__ and __init__ + self.attn_bias: Optional[List[AttentionBias]] = None + self.encoder_attn_bias: Optional[List[AttentionBias]] = None + self.cross_attn_bias: Optional[List[AttentionBias]] = None + + @property + def is_all_encoder_attn_metadata_set(self): + ''' + All attention metadata required for encoder attention is set. + ''' + return ((self.encoder_seq_lens is not None) + and (self.encoder_seq_lens_tensor is not None) + and (self.max_encoder_seq_len is not None)) + + @property + def is_all_cross_attn_metadata_set(self): + ''' + All attention metadata required for enc/dec cross-attention is set. + + Superset of encoder attention required metadata. + ''' + return (self.is_all_encoder_attn_metadata_set + and (self.cross_slot_mapping is not None) + and (self.cross_block_tables is not None)) + + @property + def prefill_metadata(self) -> Optional["XFormersMetadata"]: + if self.num_prefills == 0: + return None + + if self._cached_prefill_metadata is not None: + # Recover cached prefill-phase attention + # metadata structure + return self._cached_prefill_metadata + + assert ((self.seq_lens is not None) + or (self.encoder_seq_lens is not None)) + assert ((self.seq_lens_tensor is not None) + or (self.encoder_seq_lens_tensor is not None)) + + # Compute some attn_metadata fields which default to None + query_start_loc = (None if self.query_start_loc is None else + self.query_start_loc[:self.num_prefills + 1]) + slot_mapping = (None if self.slot_mapping is None else + self.slot_mapping[:self.num_prefill_tokens]) + seq_lens = (None if self.seq_lens is None else + self.seq_lens[:self.num_prefills]) + seq_lens_tensor = (None if self.seq_lens_tensor is None else + self.seq_lens_tensor[:self.num_prefills]) + context_lens_tensor = (None if self.context_lens_tensor is None else + self.context_lens_tensor[:self.num_prefills]) + block_tables = (None if self.block_tables is None else + self.block_tables[:self.num_prefills]) + + # Construct & cache prefill-phase attention metadata structure + self._cached_prefill_metadata = XFormersMetadata( + num_prefills=self.num_prefills, + num_prefill_tokens=self.num_prefill_tokens, + num_decode_tokens=0, + slot_mapping=slot_mapping, + seq_lens=seq_lens, + seq_lens_tensor=seq_lens_tensor, + max_query_len=self.max_query_len, + max_prefill_seq_len=self.max_prefill_seq_len, + max_decode_seq_len=0, + query_start_loc=query_start_loc, + context_lens_tensor=context_lens_tensor, + block_tables=block_tables, + use_cuda_graph=False, + # Begin encoder & cross attn fields below... + encoder_seq_lens=self.encoder_seq_lens, + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor, + max_encoder_seq_len=self.max_encoder_seq_len, + cross_slot_mapping=self.cross_slot_mapping, + cross_block_tables=self.cross_block_tables) + return self._cached_prefill_metadata + + @property + def decode_metadata(self) -> Optional["XFormersMetadata"]: + if self.num_decode_tokens == 0: + return None + + if self._cached_decode_metadata is not None: + # Recover cached decode-phase attention + # metadata structure + return self._cached_decode_metadata + assert ((self.seq_lens_tensor is not None) + or (self.encoder_seq_lens_tensor is not None)) + + # Compute some attn_metadata fields which default to None + slot_mapping = (None if self.slot_mapping is None else + self.slot_mapping[self.num_prefill_tokens:]) + seq_lens_tensor = (None if self.seq_lens_tensor is None else + self.seq_lens_tensor[self.num_prefills:]) + block_tables = (None if self.block_tables is None else + self.block_tables[self.num_prefills:]) + + # Construct & cache decode-phase attention metadata structure + self._cached_decode_metadata = XFormersMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=self.num_decode_tokens, + slot_mapping=slot_mapping, + seq_lens_tensor=seq_lens_tensor, + max_prefill_seq_len=0, + max_decode_seq_len=self.max_decode_seq_len, + block_tables=block_tables, + use_cuda_graph=self.use_cuda_graph, + # Begin encoder & cross attn fields below... + encoder_seq_lens=self.encoder_seq_lens, + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor, + max_encoder_seq_len=self.max_encoder_seq_len, + cross_slot_mapping=self.cross_slot_mapping, + cross_block_tables=self.cross_block_tables) + return self._cached_decode_metadata + + +def _get_attn_bias( + attn_metadata: XFormersMetadata, + attn_type: AttentionType, +) -> Optional[AttentionBias]: + ''' + Extract appropriate attention bias from attention metadata + according to attention type. + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + + Returns: + * Appropriate attention bias value given the attention type + ''' + + if attn_type == AttentionType.DECODER: + return attn_metadata.attn_bias + elif attn_type == AttentionType.ENCODER: + return attn_metadata.encoder_attn_bias + else: + # attn_type == AttentionType.ENCODER_DECODER + return attn_metadata.cross_attn_bias + + +def _set_attn_bias( + attn_metadata: XFormersMetadata, + attn_bias: List[Optional[AttentionBias]], + attn_type: AttentionType, +) -> None: + ''' + Update appropriate attention bias field of attention metadata, + according to attention type. + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention + * attn_bias: The desired attention bias value + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + ''' + + if attn_type == AttentionType.DECODER: + attn_metadata.attn_bias = attn_bias + elif attn_type == AttentionType.ENCODER: + attn_metadata.encoder_attn_bias = attn_bias + elif attn_type == AttentionType.ENCODER_DECODER: + attn_metadata.cross_attn_bias = attn_bias + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + + +def _get_seq_len_block_table_args( + attn_metadata: XFormersMetadata, + is_prompt: bool, + attn_type: AttentionType, +) -> tuple: + ''' + The particular choice of sequence-length- and block-table-related + attributes which should be extracted from attn_metadata is dependent + on the type of attention operation. + + Decoder attn -> select entirely decoder self-attention-related fields + Encoder/decoder cross-attn -> select encoder sequence lengths & + cross-attn block-tables fields + Encoder attn -> select encoder sequence lengths fields & no block tables + + Arguments: + + * attn_metadata: Attention metadata structure associated with attention op + * is_prompt: True if prefill, False otherwise + * attn_type: encoder attention, decoder self-attention, + encoder/decoder cross-attention + + Returns: + + * Appropriate sequence-lengths tensor + * Appropriate max sequence-length scalar + * Appropriate block tables (or None) + ''' + + if attn_type == AttentionType.DECODER: + # Decoder self-attention + # Choose max_seq_len based on whether we are in prompt_run + if is_prompt: + max_seq_len = attn_metadata.max_prefill_seq_len + else: + max_seq_len = attn_metadata.max_decode_seq_len + return (attn_metadata.seq_lens_tensor, max_seq_len, + attn_metadata.block_tables) + elif attn_type == AttentionType.ENCODER_DECODER: + # Enc/dec cross-attention KVs match encoder sequence length; + # cross-attention utilizes special "cross" block tables + return (attn_metadata.encoder_seq_lens_tensor, + attn_metadata.max_encoder_seq_len, + attn_metadata.cross_block_tables) + elif attn_type == AttentionType.ENCODER: + # No block tables associated with encoder attention + return (attn_metadata.encoder_seq_lens_tensor, + attn_metadata.max_encoder_seq_len, None) + else: + raise AttributeError(f"Invalid attention type {str(attn_type)}") + + +class XFormersMetadataBuilder(CommonMetadataBuilder[XFormersMetadata]): + + _metadata_cls = XFormersMetadata + + +class XFormersImpl(AttentionImpl[XFormersMetadata]): + """ + If the input tensors contain prompt tokens, the layout is as follows: + |<--------------- num_prefill_tokens ----------------->| + |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->| + + Otherwise, the layout is as follows: + |<----------------- num_decode_tokens ------------------>| + |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->| + + Generation tokens can contain padding when cuda-graph is used. + Currently, prompt tokens don't contain any padding. + + The prompts might have different lengths, while the generation tokens + always have length 1. + + If chunked prefill is enabled, prefill tokens and decode tokens can be + batched together in a flattened 1D query. + + |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->| + |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->| + + Currently, cuda graph is disabled for chunked prefill, meaning there's no + padding between prefill and decode tokens. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: Optional[List[float]], + sliding_window: Optional[int], + kv_cache_dtype: str, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + ) -> None: + if blocksparse_params is not None: + raise ValueError( + "XFormers does not support block-sparse attention.") + if logits_soft_cap is not None: + raise ValueError( + "XFormers does not support attention logits soft capping.") + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + if alibi_slopes is not None: + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32) + self.alibi_slopes = alibi_slopes + self.sliding_window = sliding_window + self.kv_cache_dtype = kv_cache_dtype + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + suppored_head_sizes = PagedAttention.get_supported_head_sizes() + if head_size not in suppored_head_sizes: + raise ValueError( + f"Head size {head_size} is not supported by PagedAttention. " + f"Supported head sizes are: {suppored_head_sizes}.") + self.head_mapping = torch.repeat_interleave( + torch.arange(self.num_kv_heads, dtype=torch.int32), + self.num_queries_per_kv) + + def forward( + self, + query: torch.Tensor, + key: Optional[torch.Tensor], + value: Optional[torch.Tensor], + kv_cache: torch.Tensor, + attn_metadata: "XFormersMetadata", + k_scale: float = 1.0, + v_scale: float = 1.0, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Forward pass with xFormers and PagedAttention. + + For decoder-only models: query, key and value must be non-None. + + For encoder/decoder models: + * XFormersImpl.forward() may be invoked for both self- and cross- + attention layers. + * For self-attention: query, key and value must be non-None. + * For cross-attention: + * Query must be non-None + * During prefill, key and value must be non-None; key and value + get cached for use during decode. + * During decode, key and value may be None, since: + (1) key and value tensors were cached during prefill, and + (2) cross-attention key and value tensors do not grow during + decode + + A note on how the attn_type (attention type enum) argument impacts + attention forward() behavior: + + * DECODER: normal decoder-only behavior; + use decoder self-attention block table + * ENCODER: no KV caching; pass encoder sequence + attributes (encoder_seq_lens/encoder_seq_lens_tensor/ + max_encoder_seq_len) to kernel, in lieu of decoder + sequence attributes (seq_lens/seq_lens_tensor/max_seq_len) + * ENCODER_DECODER: cross-attention behavior; + use cross-attention block table for caching KVs derived + from encoder hidden states; since KV sequence lengths + will match encoder sequence lengths, pass encoder sequence + attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/ + max_encoder_seq_len) + + Args: + query: shape = [num_tokens, num_heads * head_size] + key: shape = [num_tokens, num_kv_heads * head_size] + value: shape = [num_tokens, num_kv_heads * head_size] + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] + NOTE: kv_cache will be an empty tensor with shape [0] + for profiling run. + attn_metadata: Metadata for attention. + attn_type: Select attention type, between encoder attention, + decoder self-attention, or encoder/decoder cross- + attention. Defaults to decoder self-attention, + which is the vLLM default generally + Returns: + shape = [num_tokens, num_heads * head_size] + """ + + # Check that appropriate attention metadata attributes are + # selected for the desired attention type + if (attn_type == AttentionType.ENCODER + and (not attn_metadata.is_all_encoder_attn_metadata_set)): + raise AttributeError("Encoder attention requires setting " + "encoder metadata attributes.") + elif (attn_type == AttentionType.ENCODER_DECODER + and (not attn_metadata.is_all_cross_attn_metadata_set)): + raise AttributeError("Encoder/decoder cross-attention " + "requires setting cross-attention " + "metadata attributes.") + + query = query.view(-1, self.num_heads, self.head_size) + if key is not None: + assert value is not None + key = key.view(-1, self.num_kv_heads, self.head_size) + value = value.view(-1, self.num_kv_heads, self.head_size) + else: + assert value is None + + # Self-attention vs. cross-attention will impact + # which KV cache memory-mapping & which + # seqlen datastructures we utilize + + if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0): + # KV-cache during decoder-self- or + # encoder-decoder-cross-attention, but not + # during encoder attention. + # + # Even if there are no new key/value pairs to cache, + # we still need to break out key_cache and value_cache + # i.e. for later use by paged attention + key_cache, value_cache = PagedAttention.split_kv_cache( + kv_cache, self.num_kv_heads, self.head_size) + + if (key is not None) and (value is not None): + + if attn_type == AttentionType.ENCODER_DECODER: + # Update cross-attention KV cache (prefill-only) + # During cross-attention decode, key & value will be None, + # preventing this IF-statement branch from running + updated_slot_mapping = attn_metadata.cross_slot_mapping + else: + # Update self-attention KV cache (prefill/decode) + updated_slot_mapping = attn_metadata.slot_mapping + + # Reshape the input keys and values and store them in the cache. + # If kv_cache is not provided, the new key and value tensors are + # not cached. This happens during the initial memory + # profiling run. + PagedAttention.write_to_paged_cache(key, value, key_cache, + value_cache, + updated_slot_mapping, + self.kv_cache_dtype, + k_scale, v_scale) + + if attn_type == AttentionType.ENCODER: + # Encoder attention - chunked prefill is not applicable; + # derive token-count from query shape & and treat them + # as 100% prefill tokens + assert attn_metadata.num_encoder_tokens is not None + num_prefill_tokens = attn_metadata.num_encoder_tokens + num_encoder_tokens = attn_metadata.num_encoder_tokens + num_decode_tokens = 0 + elif attn_type == AttentionType.DECODER: + # Decoder self-attention supports chunked prefill. + num_prefill_tokens = attn_metadata.num_prefill_tokens + num_encoder_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + # Only enforce this shape-constraint for decoder + # self-attention + assert key.shape[0] == num_prefill_tokens + num_decode_tokens + assert value.shape[0] == num_prefill_tokens + num_decode_tokens + else: # attn_type == AttentionType.ENCODER_DECODER + # Encoder/decoder cross-attention requires no chunked + # prefill (100% prefill or 100% decode tokens, no mix) + num_prefill_tokens = attn_metadata.num_prefill_tokens + if attn_metadata.num_encoder_tokens is not None: + num_encoder_tokens = attn_metadata.num_encoder_tokens + else: + num_encoder_tokens = attn_metadata.num_prefill_tokens + num_decode_tokens = attn_metadata.num_decode_tokens + output = torch.empty_like(query) + # Query for decode. KV is not needed because it is already cached. + decode_query = query[num_prefill_tokens:] + # QKV for prefill. + query = query[:num_prefill_tokens] + if key is not None and value is not None: + key = key[:num_encoder_tokens] + value = value[:num_encoder_tokens] + assert query.shape[0] == num_prefill_tokens + assert decode_query.shape[0] == num_decode_tokens + + if prefill_meta := attn_metadata.prefill_metadata: + # Prompt run. + if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0: + # normal attention. + # block tables are empty if the prompt does not have a cached + # prefix. + out = self._run_memory_efficient_xformers_forward( + query, key, value, prefill_meta, attn_type=attn_type) + assert out.shape == output[:num_prefill_tokens].shape + output[:num_prefill_tokens] = out + else: + + assert prefill_meta.query_start_loc is not None + assert prefill_meta.max_query_len is not None + + # prefix-enabled attention + # TODO(Hai) this triton kernel has regression issue (broke) to + # deal with different data types between KV and FP8 KV cache, + # to be addressed separately. + out = PagedAttention.forward_prefix( + query, + key, + value, + self.kv_cache_dtype, + key_cache, + value_cache, + prefill_meta.block_tables, + prefill_meta.query_start_loc, + prefill_meta.seq_lens_tensor, + prefill_meta.context_lens_tensor, + prefill_meta.max_query_len, + self.alibi_slopes, + self.sliding_window, + k_scale, + v_scale, + ) + assert output[:num_prefill_tokens].shape == out.shape + output[:num_prefill_tokens] = out + + if decode_meta := attn_metadata.decode_metadata: + + ( + seq_lens_arg, + max_seq_len_arg, + block_tables_arg, + ) = _get_seq_len_block_table_args(decode_meta, False, attn_type) + + output[num_prefill_tokens:] = PagedAttention.forward_decode( + decode_query, + key_cache, + value_cache, + block_tables_arg, + seq_lens_arg, + max_seq_len_arg, + self.kv_cache_dtype, + self.head_mapping, + self.scale, + self.alibi_slopes, + k_scale, + v_scale, + ) + + # Reshape the output tensor. + return output.view(-1, self.num_heads * self.head_size) + + def _run_memory_efficient_xformers_forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: XFormersMetadata, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + """Attention for 1D query of multiple prompts. Multiple prompt + tokens are flattened in to `query` input. + + See https://facebookresearch.github.io/xformers/components/ops.html + for API spec. + + Args: + output: shape = [num_prefill_tokens, num_heads, head_size] + query: shape = [num_prefill_tokens, num_heads, head_size] + key: shape = [num_prefill_tokens, num_kv_heads, head_size] + value: shape = [num_prefill_tokens, num_kv_heads, head_size] + attn_metadata: Metadata for attention. + attn_type: Select attention type, between encoder attention, + decoder self-attention, or encoder/decoder cross- + attention. Defaults to decoder self-attention, + which is the vLLM default generally + """ + + original_query = query + # if self.num_kv_heads != self.num_heads: + # # GQA/MQA requires the shape [B, M, G, H, K]. + # # Note that the output also has the same shape (which is different + # # from a spec from the doc). + # query = query.view(query.shape[0], self.num_kv_heads, + # self.num_queries_per_kv, query.shape[-1]) + # print(f"5555555555555 q shape {query.shape}") + # key = key[:, :, + # None, :].expand(key.shape[0], self.num_kv_heads, + # self.num_queries_per_kv, key.shape[-1]) + # value = value[:, :, + # None, :].expand(value.shape[0], self.num_kv_heads, + # self.num_queries_per_kv, + # value.shape[-1]) + # Set attention bias if not provided. This typically happens at + # the very attention layer of every iteration. + # FIXME(woosuk): This is a hack. + attn_bias = _get_attn_bias(attn_metadata, attn_type) + if attn_bias is None: + if self.alibi_slopes is None: + if (attn_type == AttentionType.ENCODER_DECODER): + assert attn_metadata.seq_lens is not None + assert attn_metadata.encoder_seq_lens is not None + + # Default enc/dec cross-attention mask is non-causal + attn_bias = BlockDiagonalMask.from_seqlens( + attn_metadata.seq_lens, attn_metadata.encoder_seq_lens) + elif attn_type == AttentionType.ENCODER: + assert attn_metadata.encoder_seq_lens is not None + + # Default encoder self-attention mask is non-causal + attn_bias = BlockDiagonalMask.from_seqlens( + attn_metadata.encoder_seq_lens) + else: + assert attn_metadata.seq_lens is not None + + # Default decoder self-attention mask is causal + attn_bias = BlockDiagonalCausalMask.from_seqlens( + attn_metadata.seq_lens) + if self.sliding_window is not None: + attn_bias = attn_bias.make_local_attention( + self.sliding_window) + attn_bias = [attn_bias] + else: + assert attn_metadata.seq_lens is not None + attn_bias = _make_alibi_bias(self.alibi_slopes, + self.num_kv_heads, query.dtype, + attn_metadata.seq_lens) + + _set_attn_bias(attn_metadata, attn_bias, attn_type) + + # No alibi slopes. + # TODO(woosuk): Too many view operations. Let's try to reduce + # them in the future for code readability. + self.attn_op = xops.fmha.flash.FwOp() + if self.alibi_slopes is None: + # Add the batch dimension. + query = query.unsqueeze(0) + key = key.unsqueeze(0) + value = value.unsqueeze(0) + out = xops.memory_efficient_attention_forward( + query, + key, + value, + attn_bias=attn_bias[0], + p=0.0, + scale=self.scale, + op = self.attn_op + ) + return out.view_as(original_query) + + # Attention with alibi slopes. + # FIXME(woosuk): Because xformers does not support dynamic sequence + # lengths with custom attention bias, we process each prompt one by + # one. This is inefficient, especially when we have many short prompts. + assert attn_metadata.seq_lens is not None + output = torch.empty_like(original_query) + start = 0 + for i, seq_len in enumerate(attn_metadata.seq_lens): + end = start + seq_len + out = xops.memory_efficient_attention_forward( + query[None, start:end], + key[None, start:end], + value[None, start:end], + attn_bias=attn_bias[i], + p=0.0, + scale=self.scale, + ) + # TODO(woosuk): Unnecessary copy. Optimize. + output[start:end].copy_(out.view_as(original_query[start:end])) + start += seq_len + return output + + +def _make_alibi_bias( + alibi_slopes: torch.Tensor, + num_kv_heads: int, + dtype: torch.dtype, + seq_lens: List[int], +) -> List[AttentionBias]: + attn_biases: List[AttentionBias] = [] + for seq_len in seq_lens: + bias = torch.arange(seq_len, dtype=dtype) + # NOTE(zhuohan): HF uses + # `bias = bias[None, :].repeat(seq_len, 1)` + # here. We find that both biases give the same results, but + # the bias below more accurately follows the original ALiBi + # paper. + # Calculate a matrix where each element represents ith element- jth + # element. + bias = bias[None, :] - bias[:, None] + + padded_len = (seq_len + 7) // 8 * 8 + num_heads = alibi_slopes.shape[0] + bias = torch.empty( + 1, # batch size + num_heads, + seq_len, + padded_len, + device=alibi_slopes.device, + dtype=dtype, + )[:, :, :, :seq_len].copy_(bias) + bias.mul_(alibi_slopes[:, None, None]) + if num_heads != num_kv_heads: + bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads)) + attn_biases.append(LowerTriangularMaskWithTensorBias(bias)) + + return attn_biases \ No newline at end of file diff --git a/vllm/attention/layer.py b/vllm/attention/layer.py new file mode 100644 index 00000000..5ac21e4b --- /dev/null +++ b/vllm/attention/layer.py @@ -0,0 +1,116 @@ +"""Attention layer.""" +from typing import Any, Dict, List, Optional + +import torch +import torch.nn as nn + +from vllm.attention import AttentionMetadata, AttentionType +from vllm.attention.selector import get_attn_backend +from vllm.config import CacheConfig +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod + + +class Attention(nn.Module): + """Attention layer. + + This class takes query, key, and value tensors as input. The input tensors + can either contain prompt tokens or generation tokens. + The class does the following: + + 1. Store the input key and value tensors in the KV cache. + 2. Perform (multi-head/multi-query/grouped-query) attention. + 3. Return the output tensor. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: Optional[int] = None, + alibi_slopes: Optional[List[float]] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + blocksparse_params: Optional[Dict[str, Any]] = None, + logits_soft_cap: Optional[float] = None, + prefix: str = "", + use_sqrt_alibi: Optional[bool] = False + ) -> None: + super().__init__() + if cache_config is not None: + kv_cache_dtype = cache_config.cache_dtype + block_size = cache_config.block_size + sliding_window = cache_config.sliding_window + is_attention_free = cache_config.is_attention_free + else: + kv_cache_dtype = "auto" + block_size = 16 + sliding_window = None + is_attention_free = False + if num_kv_heads is None: + num_kv_heads = num_heads + + # The default k/v_scale is set to 1.0. This is ignored + # when kv-cache is not fp8, and should be used with + # kv-cache in fp8_e5m2. For kv-cache in fp8_e4m3, we + # expect the pre-quantized k/v_scale to be loaded along + # with the model weights. + self.kv_cache_dtype = kv_cache_dtype + self._k_scale = 1.0 + self._v_scale = 1.0 + quant_method = quant_config.get_quant_method( + self, prefix=prefix) if quant_config else None + if quant_method is not None: + assert isinstance(quant_method, BaseKVCacheMethod) + # TODO (mgoin): kv cache dtype should be specified in the FP8 + # checkpoint config and become the "auto" behavior + if self.kv_cache_dtype == "fp8_e5m2": + raise ValueError("fp8_e5m2 kv-cache is not supported with " + "fp8 checkpoints.") + # If quantization is enabled, we make "k_scale" and "v_scale" + # parameters so that it can be loaded from the model checkpoint. + # The k/v_scale will then be converted back to native float32 + # values after weight loading. + self.quant_method = quant_method + self.quant_method.create_weights(self) + + # During model initialization, the default dtype is set as the model + # weight and activation dtype. + dtype = torch.get_default_dtype() + attn_backend = get_attn_backend(head_size, sliding_window, dtype, + kv_cache_dtype, block_size, + is_attention_free, blocksparse_params + is not None) + impl_cls = attn_backend.get_impl_cls() + self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads, + alibi_slopes, sliding_window, kv_cache_dtype, + blocksparse_params, logits_soft_cap) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: Optional[torch.Tensor], + attn_metadata: AttentionMetadata, + attn_type: AttentionType = AttentionType.DECODER, + ) -> torch.Tensor: + + return self.impl.forward(query, + key, + value, + kv_cache, + attn_metadata, + self._k_scale, + self._v_scale, + attn_type=attn_type) + + def extra_repr(self) -> str: + s = f"head_size={self.impl.head_size}" # type: ignore + s += f", num_heads={self.impl.num_heads}" # type: ignore + s += f", num_kv_heads={self.impl.num_kv_heads}" # type: ignore + s += f", scale={self.impl.scale}" # type: ignore + s += f", backend={self.impl.__class__.__name__}" + return s diff --git a/vllm/attention/ops/__init__.py b/vllm/attention/ops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/attention/ops/__pycache__/__init__.cpython-310.pyc b/vllm/attention/ops/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c36349ba37d05183ce5c2483e77bec5e9b6046e GIT binary patch literal 159 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HMeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_equ>UYFU||3Nqf;gI literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/__pycache__/ipex_attn.cpython-310.pyc b/vllm/attention/ops/__pycache__/ipex_attn.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34922a6ee7c20e8948133d03df3c55e42ad9a25e GIT binary patch literal 3800 zcma)8OLN@D5uO5xB=Lw?9ybK=E^IB^nLw$eS=7b&VL*qU$W(cRP2(~rs8 zZI8fr^DlqfyxAe-?>LzKEEwE@R(uX1h@c5cX+TpJu+$2yln0#RJWK4dp@mETSBmos9VA2v)a%nG-btagw9`T1Xa9|eZ zgbmGs)__JOqPxz3(i2VL8A=x@Te7+3ouGp;&KX))_@ZrSTR__}v~vdM4K5h$8ayX` zu_)W(JTxjUtarsr;-ca00PoUsT+3$WWrMGPcF2$VUIPACrP*bPOeG@S`m(qzt{9o~ z6PdGGp^RD0>OHQSJzksH?;7kg-Gh!~zb@V|vKK)1n}+(j!M9{rf;G`1=47tuiMI{q zIiP$8ZUA(HwRa8O=srfYj<_LK4DF)00N${wJD*1zQhZpJa)2(N9VN@58a#L73~u{l zym1#`19tv11%9>$4s{1+j?r0(lRPGB4*FAg^6_e!=Bs%>zfneNLzXu_ zNQay0&2)D)>EB1s6AY88BNwV%reAxsTc@;!YK7{g+i=b0*{2sdRU z!lJ(~ixzBFpt3I8zkc#;D=*^F-e8!9_meE%F0^O; z4vs&CR&)Ubtw=>Hwn!>#k*r(BP{*j0F+XB$OzouRak7g#g9dRF zZ_2RPjB<(QG$-u3`&Z96;Yn)kj>j=3mhsnsjswl;{1J32vITlCkpo^qYJq>dvJYv+ zD=Mst9dQ`f>rUmIn0G3>a@H}m@U>&OaH0^7pTXB-+BPa{zSEbxy@p!C^~sUdOE^SZ z=;pTEs}HGLJ5e%}_29HQIz^uJrv^cBg%990TsBW#if#YWDWja^(H3l_ZL^7b6&1LQ z3iw5mm0=p?d4I5>J7c4TqTI`+Zfu7|93{}%sXJ>i#=-b&LI=mjnpdDxk}Z13YA>j) ziXU1>v$=n`)~;D$JwBi56Uo+gF?@x(SvYi2He3|(T^b1`yrS-mqm;hIQ~$} zHZ)_&1Ue-FZzD(apk1*oW<0b5*0lty8p47e7yOU>(1wwH#15T`9Ab}Fjb-3EbdSiv zTnP#BuvN8T%|9}cSX1?`fPkdn|B--08`f3Em{v-`*4{h1s7-qwwH?Q7B&Z!UFz~KtQDuVF_6Ke0_ z@Fv1Z=g>|zEc0P$5>eor>^2I3=i+WXQ@fE8p^$MVfRX^dlf}B5s|;*w{3wLtRAfreqdQG#!NX3t z4!RAd9(0|rF&~%Y4^&-;&SR?2t!3qBDLidbsB=<$qEx1O?b?LXfHF2aWl=n<`VoS$ z?T@i%jCodkb;c*lm0a&X_ze)7RL$pKXoH#(@{fuqiC+7nm}WWc;Tps&OrA*Mc*yaI z)lX0{JgvF~(BtZ-*uR7DA;MYewu;>-9~HhWAgXn3>@S~+{~0pK)T3$y!9Lat-jGF% z^=r?C;LnuE(@zHVOQ0^WVSVp79onVOUFhsi=d~M#gD90@sJ$?(%MtYb z5HeAej8=RI&9Wb-5}s#5;Zs7rjj(`#SFJ80yblmKUzm7RS74y6qEy;mPqL^~?;r_Y zr1}8iHUipTeT0CD&m^9ywHmO1CARnqz$J`9g0W1}dBzy~9{S%g&YpUX|AaLmRaZpC#=E#%Ut#!=K0^Sx&MQ%$^Pwn>>Ji%|6)G-C-3rtp~i;`rKD=q~CP c2x#xKE<(GvYUbM*oP`H)k#w0$eb%-92hxR_^8f$< literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/__pycache__/paged_attn.cpython-310.pyc b/vllm/attention/ops/__pycache__/paged_attn.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fe7b1e37fa432dafc584e8f5cafb7a52b745dd8 GIT binary patch literal 5232 zcmaJ_OLN=S6$UOI1VM_VXju;@X=po%%{-*o&a-t~IZf+0lgMtY^npo=!n{`&Es%sS zK*`cTGo9MmwTrGhow2&hqWi8o{T-MtI@6W9NYW;0(!AX7T!<7!+lA&H-p9G;9Go}Q zo0xDlJg5Hjm(`aa)wF-mV)QYwSj32L0&tD%q1MqoU8gw+i{XPm!hQI>jq(FE%&2PECH?LxB79?>-fDW z#?pQL{1t!s?TgEo=*GdZb5iEo4y=e6a4&u-CH6F=smU`9?TjmJ=r*j zG@n&tal;QqH})ZB+>>6t6WsK`#s8+3&4XRhye zdb}UfyzKio`a!sNQt*A=Yx%w;>t0(ng&&%D^_uXav#HWtG6)rDv?S?zTf*2A!~Fx2n@46zWA7D(&e z`Djy;-_4kRU7B>SVF)ewr~oHnF>6eq_p zj$=HA@i>M~cUKFaQWM=#$#cFZ72l?UPb&BnDDLxp8WKRqL;g9h z27iXntM{7(uLX7QSp`oiNIG!%X?{lG)PeIHYy~Mn>G>gs&0pXzDvU#X8nNmVt|dr+^6!CG9dMB<&q- zM?0cz8>x0(=ZL{=Gu60(Un{llu+*ftNsS{Y8mX~kVHc$Zr*>)&PPw^bVr`}N3d8$a zOS_agc40(K64}hZz@G)#Wk^+>me(x z2Vr02i*cW*#8KGZ*V(fm7Lz&3wyQg=qMN#T*I_l?(YGEM)84_GnE{5GrC7-Yl8I<# zNniQdA`s�JFsO{LkL|b-vg-KBl$%QBcN~b0CpfH~Ktj70Jyqx*Rrhiwi!;-ZLJlaO)nJcP+ChO*SF@ZGe+wZ1No;a6Wn?56Av z+_4kZh=_H{22PFGn>xqs)(Qv1pvg-yfAb2Jb zY{t9B_Mz>m?Zc^+m@^2)SuHUUh|?*G;hNsNkWPaqi!P*x(y1x*Uu)*}40pF@(i!<) zV%@RRdY)?1Lh6torIYZY;to2ww0;G&hH{BFG?LERVKtzUOAioEn&V|#Z0 zY{kIb9iGf7+2R< zekfnUu2;|FWY{qI>!&hHU6mJz*6W0Xxa+G({kcZhIyBz#AKQqtN{$>InIJ*6fX*qi{*E09=(7_uI zTABYeXYa}K2W8HxDnB6sKp&J|WQi169(;RHU!P5o3UWNGvr4>|H8$E^-rHzZabxgM zy(51j!^C9NJi{YystICWmC&k+QI+TMm8HLrB8s>zB5EjN5IMJMh?uHs7*wz{4dtWq zBIrq~A-Q?#9W$i)k$LEWYR~DRjLO!qvB}Cj-Y?miOJ^ua*u1A{u^@xe?II>6#Bh}Tf zsdP>GkL?)HVBVfsI8|+mIwh^EQ&nA!rMyy6T52*C;aMr(2%^0bgKp#sj^r|dcL2Nz zCHJ5c@g|hu!)29DN179_O6MaRV``QZGJnU2DG4~X3L1n?UEew~R%S<}LMyb_5yrV5 zNmuy9j$71S4jvPep|<9+z~>IT4XwPU-O{)588)&qly1=6r8;R&>Bd}YQH~xwM4CvF zK0cx5;2KE1XK(9JuQsRsj@_V+PtWy9&h!Dx*!;73H#g>7?%5FHwtAiD+$ThiXisTd z^JA~v>P4G-hL}IADt~+qlfl&oiYI>#dJshw8^zkg#<=R=#onj1{~lovj6cE?9r{n{ z{AU23JM2DVuRLPV@ti1s0qhSkVjG}hRqkiYQ6^TdF;aCtqpnHKnp1Lu<`n{zV_=6wq ztxeOx3;@xOXCb1aF*c<$L*;IVO|VaJ|KH&D zFIHn8vq|=mSucOcf`i;iQsQocy9w?lxSK`xxw%Kn0WEukmRT!*%ubBcz*|i)DsCua zq=DKuN=wbjtTg<$mxv{Kb!OwU01aktETwp5Co+*|(*Is0`AObeOwxKDEo_2+b@h{W z7|R0T9aa0omSY}Y1M_(r_0>-ps-hs@AWAL)GTQyDnK@_ke>I&We~Zb$Wo-)IMP*h- MJ;pHbI(ohKKSP`KSpWb4 literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/__pycache__/prefix_prefill.cpython-310.pyc b/vllm/attention/ops/__pycache__/prefix_prefill.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b31f3a18b07380e0572d63d51231df04fa8fb62a GIT binary patch literal 11976 zcmeHN>vJ1fR`0%QNiA8HWm$0&Co_|IB(~zjuT0{+C#8rj{BWyzA< zlAQ;)i(Rq{)KWtYU-+~ik}8T~sDdiKP<+}C`~g5wz|JoHsNzxB)kpVn?zxY1Zr|TIw=3=KaS4CJzxuUR>%Js?n=-+_c4Wrz)V>3vOS)W= za?Y=ulSQ7bhjNNcWvmolkK`hzGI}q>hVJ2&n;noFD0ttU;78 z-o}sZl~6h)a&b2|EOIe7H^PVc&_2mJ!F%~YMzUT-ZAh2-(Z|d>>87(tpW*|&H$%Ns zZSP~YSF%R=2p@&cr7tsGT4DR8y%O)tNEkypL+MjgN3~SPyQs`M4GBs^F7GSCW-y=7 zqi|hXi(s^8@XNTwPea0)CJ9Y_@e@TDrLO4Vw?o!hw2AWGJ2HM%J@PiI$Mon!8S^@~ zv&K5_l<@2w=5bpW>T%3!h#!3znv~v=-};Jm5o2uQ7sWatjS+?UG2S8Ob&;RM*b+Sw zPX+Qrn29V#bBSMS8uxQplR^G`V@E#Y)rYVY3$1wM z47ho}E*BN_{7a(U&@OA-HTHcH?=MPxQjd`wwCX^b!M>-EPC@Pz&DERxKH5FgsOjX> zg6j&>X?~7Rp~Y3v;vBy!xMoCd%FSKlGyFWXagARQt2>L->;cJ|jKVK_n(?*j&dwVx>Vuk@=ku;+nAykYea)zuhGrI^nR#o0PeWG= zLRZrq{Xrwsd_idB0-v?c^Q+dmjbCrJ1D+Aqwx?lj-Z*%-nDdy!@0acs{QDaG{yg>z z{>xk@>$4U)qq$nIVH^s7mG-&whOlmkU3!BL=y>};S}z{4W8eCy#eQAH+vy_K1mpBp z&L7JzyJ&iwJd>q*HYi5f{ASuEy%=t?UdL*gx&16{|JPdVYwr3(@80_R8sBZX!MFJ{ z+2cEG<0~!BpB~u6aeknMl^qmTc9SRhO<33K*i~Kpb?moKKV#x=hX~sM6|~KpG7jP= zf-t1s$0PpUWwx@OuNCcZrM_yI@5**quivj4cBoKZ`A|6yTKM3-?d^AEli*Y~W3zh6 zj#tgf9W%eK6?9uERr0zW$(#A|9m5Xag%kr!6%(^Vfnio^c60@7^%Em@SS0MQ z!``;OS*jPRB}1#Mm??B%M+!A6N-2TF>9JL&m+T9ja}v+fif->qvxa^@?d^ zZ`tx|wmfUgx9#YxwwzyHHSFkZHxVA!<|@m!T3gp@%lQ&&ie6&NO|*?`H|u$`?l2-_ zY?O?$t&VFG^?Q_xFO@3GXjXV&*fK`4uvxd0wYpi*4Q)w7owl?iU$f(G{l-$wj(O?T zMpE}ViUE#dgQM8s*a~oLH8{2!9F+h^rNL3r?M}DvqO%?v7*jinR(F-OXjaV|otp}gFp&rmphd}4LF^bXthSV|FR$7vC07lYTC^r~wr;mk zQE|&D?etl-<@!C1v==C<85^SLV1uhr*7Bv2<_Ig4b>m*Ht5Hk%_`Yb;;o5-c6e_nP zm6es6R(5}3`XUdb#ILqtt9oHw6I)8&fB|a-J5(Z)UtacxUbfj%+2N;+hh|%LqzHS$ z?Bt@&Hj3DkT4BwWtJq$Yh$5?%jk|U@U#hO=ZDq-*=j|x!s5BR?x)vQnFK#lj-Bb%%iExjs zpx=~yP>#sTw#wpicS7_;9Oz$pN<0ZUAx4_QNHa)hBpA_pA7v756tPuUrqR5jXpen zw-4c!32&6~5I@`C^ID^X=H{$4`KVZFc(mlBI;(dcsJ@L~5MB)Vwh4ZcU&N|kg8X*i{R_>$r0{9Rz}LaYL>uxCiSH!8B11iKwY8^#rWt})xdsx)Rh#mC@#O|5lemyB_=dKBYhKisp*NvxuSX6~1^mJ{*`YUqP_1+DKcE-*V~j>@k*^nr z-|p=J5v73FM>c$wA9Z~Y>_5kz-}=oK);x{fNO24&9j8bnckI?(^5J&` zECKWR@3DX@?kYIGqB}}YFIb)CRd^NkND#b;y39%Z&H}lY$HPFuh~JQlSrCK7f80t%usp{k!~V;i5wtu zkjQgHo+t7Gkr#>VBk~fF!yx2G)K|@yDgO$Q{X_`Un@2#hvt}=)=^!$*M2-^aCqjWk z^Ei%5BXXX|1tJ%TTp}_?|D$WR%kudQl8aAj?4L;^0Wr9tBTP2h|r0ADh4H=s%bnneiw0CEu+%gAYfpC%xt zv;<<&oj^>;c$x_{;1~6E5O6DnUq#8VH~}U(@-zW3*a}=JfMLpZ17M}aC;{y}fF)oZ zz-5zE2S_z|698iZNC^Pxz)uHAb)#=3WZuKlH3hgyue7V&$*>Uk^6laK*Oh1QS}Y(w(BQ1^~dViM&7Y+2(@Kfbh=ML$lsyID&( zs2hEF{B9pd#Nj|;KLTgtC>8qgxLS-X7-&T}+7E%3_@p0bjbwM0dM;LrQC;_7-7)$_i!qbQcmc8}yW{3Kq zMiwZTEiNUPd6F;q8gWA3jz$)tkp%>suR%wPLPys)^_K)JSkz-Wf_|9wd43(haK<{j z@mtN-Dd6Ow3n!Z`+JTef4!<`-PS9k~g(l6G=|GbSz)x77^%~$O=8IrAY3hXcHgH4W zH&Oa2dzlZpd-rHHB)&u!b%`3u?hzX>aAhB`ypD@25|+YOE!! z+F#3{%_=+IdAs&LpliU|G2;L0R(`|PfpbC9d_K6VyD+ z`4%T0In>9w<4`wStZ=5S*Leqj9hQ0vyC%bLL2C1;n@-Sx)wKEGI`PcldO9oT;s`TV z?rQaw6C)1R-qR9m3p6b-a}OZJCqwTP)gB`9A`u^dn|CSuG7$m41?ay0ETDUU+6)pI zBI3jET;eifeFd#ns#J~I|2!}rq0#=Q5&4ghloqf&*OqV0KG*JML=^6wCoA9A5QVxEYzwuJHocU3&`P$Q!0!1^TO zk_u`>EEOC59yq!0!q-rhhOc2pbyk-3kd@LEtCItD)I)2^ zCj5Aw(!=mKQhJ0dTkvmrSk#4!toGtH=_h3#@$*@cSM(_OhS5G^rOWMH(bXa|&mbQ~ zK3rGz7*`%Lv@1f+kiA=!KrQsR*ECR(>Wl+jkghg1oCR3p2;%=rz4 zSjg(KGS(j6#WQHJ$uoQp#(I~=R&PVtFM}St^>*H+C%YuP`SJRncIx;{?W@tBlbuWGdg_N<)&E~nZ$&%Anr6PL zdTW~9Gyh}}FNP;yATmZo1rY&1JA`ZWEHl4=f}D!9X1uz7!cCoAA9>Jyt6bZxRx4)R z&<{;7o<1aAL1%3xzge$5SZsbXUFfCK-b0&#Hu$LU(2hS6Gkg2c1YN}+a_;~SIahyA zw((~^z-TChg}-lI5+5aq5ap8)3ZCH2CS{{LV{ZO2Z^!7%#k+a4id%L{;_mj2QOl~I z*h1VQn_q@l^QTC5r^)b>+q0UlX}YmhST^wXt>%{sr9%C_^Zh{>UluH{ilApMyopa3 zPUqUiCkxK{t(0?ZUKWdJ$8YCLo5qxBR!m#jD$osWZ8eYYCnEStpk68GI?Bd9+^kma z;zP^IiZ+r{mi5AxNyk7g;@roJ8@lX42ZRX3K&QVgSIi$LE_}Bjj)+{;Ju7ly@$Esb zZOOg8#;tTNhKzH`O-c6(JlBpdCbUAiipyQxh&PUxTp|E&YPn>sRL~3MJK9}bgjeq7 z!s5HnTx|ZMLuBcV>x^CzqglmVL`+9}$>V;x# z|Y?{pIICGCp7*? z*3Q1KCfPrz9qfB7#Xe9w+25;a_IGL*`&%``{zl!y{#xy3-&OarC+a@-SL%NDzS_gS zqaI*?sUBp1p+3jHtv=8GTz!H4nRW9jJta$J z%GX&xe*J&-ca#&j;r@){R^4$kj9dLL8n^P27{xc$SJ)q`huI&|*p*%?Q@+7y?16vv zca;(I9hhfU5qnyY`T1r+R0!_<>)(ZFt2%9h$gSpgTFI7|Q77>qiHc=9vDB+W%GWPO%MaPFr*i_DS z1r0A?b|MZgLg4CoaBh~2i!d~NoeUxXCkzW0@uUhUOmP(Ub>%4gL#3bnfpRSI`_fM+ qb<|>M6lZC`m3R-O7g5@hKXMNSCYDY4o-jvYa;fi2U-bT!-~M-yb7LC- literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/__pycache__/triton_flash_attention.cpython-310.pyc b/vllm/attention/ops/__pycache__/triton_flash_attention.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8bd2f51275a0b25ff5aee7fcca8d9b702d9ad46 GIT binary patch literal 10385 zcmc&)Yj7OLao(Msd+pr;2M_>3@S)=y#}`D3B1Kvv2@(KFkpxHpq@=Z^t;6jCINTGn zdwgZC$6+E>#XpjAstD(YOO@nLl0QkRl2m@>S1NIy-o&X??8IT>RHYmzwpFs2 zuV?RY03_Q#vxn{Nnd#~2ndzD7?m;b`HZ?p)f9nqy7fYJ&LD(b{kDy33! zC!tbll`@^AN_DAJ%1NtKw@P(6-71w)sSLH@^eEV?;3ft86zo@UK*7xl4l1}s!6Ab4 z1I|`w+d~eydcd(4?t z)1luB{`;Nb7XOX;pQPUb`hBZ>I{b4^?o;y*I0rViyO3DWoP%G{okNe98o9%242~!` zqTsU%9#!y|f(X1KK`EAZ|=Xn}SjAmbAIKk-WoOVt)Cm-ro|3%Gt!8!F% zb521&29`EIFswm$aA}J*Qo6q`-Zg!%a_ZB;RvbT?A^W}vasw`Hr^P*Ov>e;b;%^Y6z z{knI0WW*OmCto{KaYyQnxpJ|f_<1Aq#OYY;bENLf=gbSjb&9@sIy+T&tD~2*C0A74 za@KeAl|$LlYh&4KSF;l$Uvcl$M9FIlO2wu*;pY8@aJ{V8sMl-4cb(Iwc{G=uxZ`B+ zK)P%p-|+I~Y$flNiq!@4SS~wSy_fae<%U}=xY@E>UGNt@MO$n5*?Qh_K$jgZG=#t? zl<#Hd!Kx)vty;d9GoQ<4XTsL9^EHumiu3a>(SDLE6)$J=RWLA*=dzQo2LaKss)vHn z>zA{gR-76-QLTYVi?t(ab&?2_bH%)O#55;S+NrRi!;_e-dhL!Y)Rf(+Wf$E%w7gjH zO!EotK9fPxuZhBtRNN@KBdv~p*pT$~%J-}4u1)sfrfa%Cjn*29rn z-6I8o!r4k8?A4+p^?O0A;1qA$^Htg?JrXR#L;{~r2u(sLp}T2?_0B!9CuG;Eig18l;0PTkesjh1491vf7jJ;YK zE36r9;T(8O>N4tJM}M0++C1B$$w;5JS(7o!qg=|{91f_U0nIUy#3zX6MZUV==HfxJ zzE~{R?%JN~IzhUXsLjuNt{)^?I$_rXy%OlvAnMU7a|2fK0#@}lcCEWjAJ+GV(UeF1 zO0kqatcyXQ>-rf(2ED4E`+)dL6(o~}}ect$`C`ySl1JpvDKcC9&PMNGb*E!l{n@s*6x9>_~hb0MO0IX+VJZQ--;7na^3ny znNw|uRNES$e+Yc$&s*+hqNL_SEXd@}M#A=j=!yXw;F8K_V z4Yl%(b@@!}YvuE``J|*$!^ElDxya-POevr+&`^#AU30ZY)ya!{c45&il&r4SMrhaS zbPnY7P+j#FeWUE`CZ=dX_2W7AXDr*6&$ z1JmQPH>W4<@yQEdIA+gGP0U^!eKkm4A04CgrSZ|R!gdJLehvenZAq<`-^V@$9ZSiy z?0;a4Y47Q8TyJjob*!6ha!aj8W*ohQ34FktJLI6)iINENw}<(1n<{yRR+ro)`xh86 z?ii0)Gb^`|R$ zjnx~IyJgmoJFqZT47AcKnyB5iS!?c*18eE!p0ylHk>*}id+)m1S>z?KOYB%-CpB>E zakPdmeigm!aj2i~uj&6jnU(vTXrD%usAo#__Hj~Xo!AP)F1jD(``1b^sOXkbkJB;> z6&?n4mmDVQW=;-???@ACc~c+uS+(14$4K>~&H=w$CgcGbrxBI02hrw1xmA1%wKFma z7?TI(upCZmE5_p=6_uM@%}p*xA+0XiynzEgF{u`K!0)@F_JJNm zl$G!yKU}QNPkj6ImkzoY`@eryt^RMf^*`4iMAWgRmT0&oM5w+O#OATMvH1slzELg* z5p@y-kvoepl(0uoVPH5lKZxPm`?|&5yY*v%;USau$ROh3L~+Fy!a3P4 zf$4@Os_ny8$UC?5u;g-w10!E31V-5|;;;wMmxJULyR$VVZ>>qhJQyp!U9r0J1;3Fm zwSVAEXQhrP7^4Bi5>vF5mr?bYUeG95Y%w6?)Yu^P0|{M{f;hniir zLqjx3(|LgEs+t#cRc-gKFY>luE5R%b;$-X5PAYao1hI#}UIKI)i~R)X6c;%H2M8P_ zaEJiyM^>w2)j%%=`fZCXmn?I2Z?9NUuXx|Jt7H^;C969W%5D=_$yNrH2ApKEIL2}* z(CZe*$S$b^+~RcN1(AHYzL?**t#^|x2ea;uV;8GcS6qcu-tPkR?qmjzY~94mKjP6g zPN}{q=Y3He<~rA-ANLu3dM~iJp2auA;;0kHxtWUe>3w=iPZ>Pq^AtRuGR8j+ITJ_v z@eg^NC{PFGrp|wqiEzSEmw%iAm#CskDab!c#W)+%Gx(W!qw2?>vd@`JyaK_WXfV~D zoFj~k(TN-n=Y^)kO5Mrdb}?hFu-!&DEp((lAp<1GEVJ6E*wD2F*Rv9Y-^q)*_vCX# zLWlnoBA$JcC6tsdXM_{4OeC6k@QC3koi8qUfl+er1??y-53_IAdR_Xxur|0B8F^#k#x5}fNX?Hx{NjacK&hzql zc^o_SiIA5(QR3ny2lla|+}5ghvXg4ndqKV+PomzbYTN-|yeH)zaM{~D<#TyT9(cf- zr{#X}Ris9!4oNCc;d=ru4V8YzkI2*VGm7psPG((vRUTDm$Z^@PPLXG2AAOrI!ki*o z@kOdFbMp*m$pE10J?+IfLy$`mnMcv)OQ4QX&H|IsrT9|9iQ)`DLw%Av<*p&Ec~+iv zSjiB-BcGLLhqS6LpKYE)4@ODuk|zEbC)O$SZL}JvoB>VzY4`=jZka0?&GV2HCmrg? zm88dILP>f-Nvg>UO48ksEFsMwE9n@xB*EpJOeroWWm<6=S6p;C4lcXpDb(+hY4k59 zbMgSban;WW)b3WbGp*V^s`f=yo5_o)eM*uHJ+fETKCot#F5#4fHKfie8eIc@8B}~u zs1~=#p$&3V%i#!LuEMu!UY3VZ`%U=_)|&ZN9_C$g2V>lv z^wV<8Ny+n08us6moRX7$8f=&`(h#hJDLG2x(Y!9Nr(ikO{sa;Z55VLvvE~iAAHDc9 z=+uEBP2Nb6PEJ>uyxyFK4o}O|&|yG14V{@muV;ZDK$|xuQYHQ#6Pv=)Fc&t_8F@`! zg$xt&5_*3MR^JZ!vb-W^)Q!S>gU=9%WT=2dyK z(+*Z=W(8ZMoNl#u7ZNkE7d0`?gOIlhp3PgJ>yZ=SK0}fyn`J`fT?%trcFiTU4%-nN zURI@-*VVX;Ic4IeD!D=>iqDmw+$(tt<1{KS%l+$mmHI@lFfwbcx2+mx@G#o@qCDd; zxqVPmIjcKL^OaD0ny-dEZN4VYU{=+-$E==F#8M*t_pbAa$j$rsD@hrtH3 zE@}&i5!yvZbR%ETL8gL6Fb0cNtE-YPxgA?gJWs{sd5Uuc$UzsU37jB6E~hd$)%kCw z9e1v=V9(`+C_>y(Dt&>#DFSBza&HFujX=K==(ofy(m~wAv0uoS-9W!;8B>T9B*LMv zm+yDb#SU8Opruu`)S)ePXiL=&L5KEshxT@d_VyZrPPJNxw$`Dob!4b@Xy-b#a~;~b z4((huNEI3_1KP8f*QM6l)4l5~X$&Xyq>22#~o4Zs-x|s~D%G3jegd6uV@lik@Am zIgN77u2jV&k)&bc!?$a7QCuNbWb2D-RGPq0*t8n&TFI8Hr4pHzgAwLYaSwvrHu)PJ zF9eGf$x2Id&q@;sd?Ls|eu;j6qt6jcY*6&a3kW%Z6FK| z0V8*S>n6Dgu|BvKCXX9kF#P#b&O!&kR8B(l!+|(Y$KjChmKzcWr!M2uD7)fob6KnY zr{_}3)IwB2xGEp=1UV(-9Kof*8-2=o6mAPiwU%m-krM=I8TtgdqI$+aq|6{^2ab{v zIyIpq^a)5Y)AsLEar2+}F7%L@YyhqFK^|&>esGA^)@XD2$=}r)=0ErCjFRgXQR4@G z=mYAKyO+Vpcd3yd)Vnfk$FMyIeMuoPrv-AV+7CsN$%7*G_0C|*rS@1`K}3Tf;$)dt znuTev_hs$#h{lPtVLsVb-Zqto@G9(@cr(#7rLOQ~Gu2E}B)_41M-&Yr_`r`7?!&T& zEyxg2A|7WbuezVy%snQDCl7T~zX8M$NX1gfJEU*JY z&?H2W?AX3OM2$X?>$B`aEw^{FSvz=}V>b4NH4bYP3q}OQrM;OE;mEw@T?t zrSuI;`xZ4P{_A>kGh*JGZ|mYA{IWs0c}#o5Y;KWT5Uu7y$}P=79Uj24kb`OsJs^#P z|B`MFzN5XPzk~RrlW@$3@#YZfZ*`K*ZBDAWT_%ZI{Dw@n+NCe`kiKYNePgPJJK(FO zn>+lN9HMojMlmT(2V=bulRaP2WiQ@Mc>D0~z#DlV_jK>;(;7URZSa4#zRP5{Ouwrm zR-C~2531HR@pUHx--SEf_+`G_{fj!hoFS>%aYURoJAr;{wh1p=vy@+xP zb{FCvnvI%(8QcPFou_+wSkGkb&LYn9v-ozFM#qL$MzvatXOyPfL`Ta|yhh-4fC8=0 z_Ct6CFI}jqdlWbYI=TY)af`OX-evDdLj3#RKH6S#0qa%lXQjlRxuaT57cZy3`54e{_FOXC)lL#Ww<7&XE^!F_bTR zXMlv&iqn6jYsK?dwqsLSWAIZS!X(-D6S?kA7*^b&vbzN42+)e%XdCXPJY^v6C2Wzv zsxg@B2_ov|+cMos-M?p-i@46G9Zc;2YF86)QIakq)j<>R1-LE3qcm|0qM_NcMk;lY zNm5w@!o|NJAdb5}Fr0fPFbe+N4SS^c9PwP;5XB?nux`jp;(a1fZIde@K2P8wfwu|# z9D#QLti&>mP~~1(39?69=_`z|FR_eAC_X@V`=u^tc!n_ zb*~ya>r9kTUE|-3W<#^41LOMr-cPD<&S-7xxkwPV?JE8a#kR5QhvqQhDcfGg4MM9V z1$Q{B6ba4IJBzs%u3rs7SP0BWm%4WpKRVCrvb} z1-dI%J;3EayqxI{VG2%^&kQ$hM39Bj3%w!S}F#;CtEM^L;Gf``Op|F#9{6V}HvJ pu)pC4*g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;x_Seo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_equ>UYF pkXTfl8jp~PkI&4@EQycTE2zB1VUwGmQks)$2ePr42}rOo006qpF9-kt literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/blocksparse_attention/__pycache__/blocksparse_attention_kernel.cpython-310.pyc b/vllm/attention/ops/blocksparse_attention/__pycache__/blocksparse_attention_kernel.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45bec7c65b57a8046d34358f51045ea27bc9d678 GIT binary patch literal 6619 zcmbtYS!^6fdhV+3Y0eC1hS%^EDUBps(#RsI!?G-`F3OZonar-V-lW$~V>sO;n{)KA zx`#Tp1_&YAkOA`GhrH)04|&V;0y#R#B0vDg%CXsG69foiE8kx|7bPLt zU}w;ORsHqXUw{2o^&jPDvuO>#Gk^HUn{%f$?eD13{~JZ)I=Rn3Y^W--;~ zHQh3p=5UAMum4DQ3@7o3TM3tKnU3iiTLaF3f=LBa3Z@ktB)Fb&GEVl9@sL?bm+hvU zK_}-7J<>e3#}tp@zHuX}9&;L1^_X`*zhP(0$rC?tOgrqhRvdo{>9v)o*8z)qlwPa) z<%%7658NoTZTGd(wbDI1aJQ>&y*xl;@YC_l;LEn_feB2gNi$?2U*`{bka}PHO(wN1 z{bRkAZVk3FQr|Tki0|+zt(ASOHKygjZ)sB7P3&os5j4QI!o_LH`&*7);lVlL#gX8`(~kL&K_6`$NTWj$ z{GOU0D{-_>`Lsr}Is;Uj(sl>;wAPT+yM6j|eZ!%-n~_7b4mm7`DtwRcX7{wGxrQEVlUVzd_842(8o`Q&oRpJ(q=mzBT-U)NjH;iQ~~9&jckmwB1!X#};7mb#kg)6S?IbJ#`(vP|u1zmJRe zIh~PSo%hxAab1psW&+=MyFJk^bDy*CXii3E9&2(yPRgmzm=I33&d{Ee8R(lZKX7=R z;bQ8}(2r8TgYzl#xLCt0g5g>`Re&zaQw~t%QNjuDH{8f{+#Rp%KKNd_*}hhRyeV+2rMgpa+_LYWvGDX6eh-#$ zT{pr_BHS(nn|`zE6kIRdbOp_Frf{M|Uo7~d5R__ep%fG1}>n`$GGIMF~-=Z@5Jx;tQuE=0&U?F7w;pw znW&&q5`k;?dR~gPE3T-!)rH3WD1Eu=1!38*HLhNV2tTL2Iifu{-#>1>S_(FaXWiZ@ z2`JKDzw0bqhJh?qgR7`?IPB98ZFK_Q7`{IJv>0>)vkTprEiUls9a9iXkq5X*m2rT#pAykiA}OY#?7K_1?HfyX-g?(we(%G`T(;l6 zxwIS&+_<&+)(7^7oytn5vK$SyyX;%5OUv)Ayc3x>Kf1YMFI%a4vu58diAG?hsTzc( z4L2xGM`S)!=Ze)-KqD0tiRNW{MY?g4W4dps?$E%^k7v;^!EUBFrDH57oYLxz{ zRBgI9h44kBmmAF}UG+CSXw@b}salP6I2+5vHV43@Tu9)GQs_b~_pa^M8XQ1IEz+@f(E-}M_6e}pN@01^_Uh}dfID+7 zGXMw2EzU%2J7SfH?O2(x84B9`c*E!j2nrFbfx5lv zmYl#!QK4L`m5v2ax=L-^6&tJbe9PRnA!=ZmmAFjSV-FlegYd<%%uYa#89N|C@Ip(+ z#MW3WfL#{;U98K2WeKd&4$`l7aRaeScdTsZ=G%U~dOtEYpn&+8O_1nx)}q7z=XEea z7W@goXfDHZEXmIDSvJLn`7q1#Brpc!%mg%8^1t(@$&!3ZpVepC4Cl;vX6VD9H9$=i z=GR7|Q_Gl1a7*$u`g6<#b?2XE@xR^vcQea>&U5_V%pv}-e3<{t9O3`MNBK|9G5*gy z&;N;!^M5oa_&=DF{3)L@zT(q8P9$3foRdE>UJ;jR$(dVAx9`{=*y>`4gFuN{0u<(m zLquVIXy3YHuapVX`5i@t&i7mRg5QNP3AJ4&O*tqB@^G_oUr8$W*e*<@a*#7}Mo#3l z)&ZFpt8z*#0?%Yh9*{$7L>5jS|4$J{!3`b&kI@Q3oF}|BTfg>;(8^pq-q93?$%=Np zGyV|f`@WK}J0BeKY<=uGzAtcy=LeM`$K}KZttH`b$TTPCz|WMroZaIWG-}I{i&W<2 z^bJk?f$ATSnSC;iVO0kZMD?w7QmZ5!Jxn@?QTKG>1imSWnRe51&PnVcxM>~4yn_gN z=H$VCepAqI>kufj@(@wB3T-+r530Vy2qzBrkAVI)c@XB9$-{X~7ACb8lZSRWf}Y&g z5OI@vIC#?6Gw@#YU|ZRNz{aNhat1z?g?I6m*-5!x0aw zu*%1i?)b3%cM(az_A6vG0ITiQnl*uRNux=^P0Mia zH9#ByOS4&Nx9Zm5zU}aS@hFUaxeW7cd$9PnC(hA0_U&k>(#|}}WVxJsQLfLDH9`?} z%^mt<(`|3Q3|Wg{N-+Q~Vka&W{`xv-O%#ERZdKOXr4yyv+$v$Wx)X{wNUkK(B9;C` z$wny%$wG@O=y;it5LeY&JInAKYaq7Rb@4j&@a>9aV8vSzYgh)>xD_Qz)y8H?oF|Um zs3$>7{u{&0vm75~=CeG^ zEsUyevSZ3Dn_uS#W?=wfteL?xpfVMg7kQXn9>aQ?i#1HWFY2K~M9xncK#EBuj19%_ z4$M~t!D3Ng5eI;ettlUB}f&#@U)z*mR6-Ey_jt z3g&H(LSF*ee##LH~w)L%xpk0#<)cW?=qalUXutZ7rN9Lnlwkkv5LBabD&XZoG{nt(=onYV33y zr^^d+5r&=C`=&f0zJf+L<~WVV4+93%PzDq8f#xjX2&-^8h1*Z}I-Fht4(4Kg;g{v} z+-yf87i%Q6J1}8X`x2b&uiHT_LIZPssrd z+BxwV7a7oHan|;q4V=3d);^J8PlAOF;WrzuGfzpeqE>GlDsB)-7~d6f4Y0_?brey;r-ufyM0FkU#l~Xwr_MX$ z|9S20JH?#10koBF2!BJA;0hfcDYuTCEC&~@TAkibn|j5YG}A@m(uoVU z*Lu~kSKaJYoo=}W;*)-=ogURrkIL`O>-XmMd$hh|4RwMf$^jAd zh}*;w*4K!591tAbmNF zgw#MvnrZ)4w58JCJff!zo8U8ux@Hh#<#`@4Ri0%)Z?Kby#3&{^8OL2;r;!L7&qfoV zMO>C=r}!zxRYE*!bduBmUzz)o&-F%_-DJ6kdwDJWbrod~FR}K{0 zK1C~)tLKK~<(ls_tL{~bCIi};q?UwQd4`0~{D;xRn*p+U1KGTRY~GmSUm4Tjje+(r^rl5``VV;9i^6xkXnTTkD?SSqC%-a>;a&~Z zw&rMzo@t@pE;xm@;TX8r{X$r5n~vEoIitv0E4ML$@A>+3`)Dx=rr+ZW0E55N*0i$~)OHdYp7Tf$CnGMsC|n zww~RgZdi6^BaG`?qqmLFOPYC|9Z!TjavKl*I!$i&pzWd~i6FQ_TjZ!EhF>n3fAopc zV7}(-k4ui`7kuNf;^I=|G8?&jvU7C=V)l_FCV<{F(S$v@L9D zqpL1{L-(%&v0G}g`;qdOjP6e=gKEFhIyE@buMA)BAMw=_z;VBdl6FEl(9vUMc-q&o zvP$|!zp}>$ov6^W>q96o5JH+u!MXC3*8!=DpC{FF9+v&st z*R~giT-e*LM3_|#!$P1l%}XIGq+nsGapy=MzvJq>nvuARb*Hn|{VItyP{*LhM`siz%C zE~4y6uIdxr#ilhqjI8ej{9&dg-L^1j!>s#W*yT5ch(%TzsdVV=#@)0wpXmt?8{$>e z6W=4TNa6&EQzVX&c#Xtq5^s=roy18JKY+-pqb^6&a*pKGK@`;H0xSAHYCUCH$#v@> zlEihNDqs9+C2eY+}0;iv6xvwf-ka3u^jRywdQHOdlnEs`uSxi-_M@#RWeF^I zsG$X7z{!XpT2Sy7$e{%yXn_)1MW6ZV=W|eWZJ-b2)8rPOn^N6xrJ#qFnOb{la-P&X z1*NpiJ#Z}){zp>basR3M8T(90!J1E4LnR~i*z9XJz^+hj8nGyD^$YuKU->Ig0Y=e4 zHid>JRou}x0xzi;WfohMK($zJ>GxS~oj7NFP?@adb)3ncK%`1yd<5@DvcAz-@YR$J0o0-9 zYX|Iry{5nh>JqpTFl57~z$GC1`sWrvRPg=xnDKP!YqEjQ>oGZ;(;d5qjf|sk*RFdBw_$_q zh~vv)>yr+z2aR3YZESsHZ&S&$+rcAR;$*a$Ie3w8e|?0c?ItkNw2ruu!g$;Ab_rGU zMPjT=E6KJ<&&^_LGC)pcmu*;`1Z$@6LspeGWP7@f-rC1}yBh#K08zWsO*vN7@iu}m zNO!5x+t?!8+lY6#eePNtB#3njEEfJvjg|{{VXO%9dQSvt9NAki>^xllcjz6C|9DrB zt*B`^>q^5*R=QRa3%F!@+e_iz&|)BgI6)DOG|OLU@w6y;Cu7?g+sfEZR-XO3&)scj z=6DZtHj+Ik(w*vqgJ7y)Hc?i=NQD8|Xs8`11D5J(p)W%YGHEEG zOr%Wze^w*Ba`eJ+ht_>L2P%h_#~*rtuf7|gi@WH_!23$mFu zU)V_Qoiw$p5;W2suUT0Mm~(l%lkSQiWB7CNk959r6iGCZeS^!DYGmKwFtHJe;5^Ns=rj04WkVXZzB<>h-1{e}tIP<^ed8RlgkwE>Dp2?a`K_g*5LO zIi;&PaEcNm;u@ahka>?vq=w=KiJK6$qWF-?A3!+Oe8gykaETJ}5xxFX62BxNnPf^Q zuRQ<7x)BiYcc_<;3vU`GtHSfv56`P_F`e~RzWukCJw4~u5RDQ7Xobl~@I|5Rtg9qd zd@p@VB_t$Ocsbw&#I|e*o&c+;`4vv|DZJAs?+T-LxFV`1CtJGRvfa$Plgei*@&@nv zMyd`q`L;-mK^_wrLqAYHQ}z{N6G9A1%$E$as0<5u$M&_B(L&y!Rov2&b$Msv&O~G% z?rXymy`g)V0VocN`C=__2KG7Pw%t4u8 zQLUj;NB}%rgSm9BRc=*URpcKmTe)B$T+x0u2EZO%j zUA#z!E{4Yh8TybP-1#J#TF%wm_insK*I6Ht;0Y_d6zJN5mGv3ff7pViz z+lkE|rGmFP+y}}OEZf(cF=&d+DUf56zYULZW~kno>t2-m8%;1w$l=1kT2sOS?jeo7 zEYaROQ~$<#q#)vtNf4I47S52Nl2Ipx1)CJkSOa{(A!vrSdtW`!|3=-0d5}s_W8xNd zbphh3E)__SukDn!-7QyonR%QrRTk??7$hk<8(04cH4_@pP*k;gXdL!_{4(93ti()2 zGwB9N#1dI@nfOEUfV?JT1T-XRa+UuHh7dm`A?<|*n$LrjY4l>#Te*~@C}nCCIaZ?^ zA*&HbUg(^Xo%*gHBX=oczgw3%j67@?x6!0l$rpN=I{G<@DD}MsHX(Co7Nl2#fM5TTJe5=0DfJj)C6k?E zRkaFoL6YF`v^uZOGefn$G<8Z3z}(p1w5`u^4Dkzn6(Y%m=o0(^UZG52Vvd4vCSFBE zl!y7FXbFM6aTkC=&`VKL)S5K+T`a*pZiP(z8tjf_hWQ*ESA?*~|AE!zwV?T?HxJqK$ooEe%Zyuc n&*bc87Alu4t&@2r-&OkWNl9qpredmPSf696rRi#!Eo%P@YKN5) literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/blocksparse_attention/__pycache__/utils.cpython-310.pyc b/vllm/attention/ops/blocksparse_attention/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f9fde49c9bb4d82e2438c125f2248d0877bf9b6 GIT binary patch literal 6719 zcmdT|&2t>bb)W9(nVp?o01E;Hzhr7eQ8w2h0n)OqGELF6MA4QULykyE@ka7!ussV5 zusgHpnI#F-Y?W+KB`Q@(?31f}36Pr(_MebLD*r)pnrluxvi|M_xkM zmZ_hjCG^v3S>G|FDa8XriY?xe>SQ4)Je$9whmTV6TIYbWvD zP^C0*FUnSuLFT3HsJ}7qdidR0iBeC6{cg|><^Dk04^$e?A8d4?>Gf53H%bO+cf%W` zp`35ic}Kr0DixGhQPN+5WJb<1w#v77>Sm^7TkIz+XR9JNe3Vu39v||dxoivtiY56n z8`_zb^DTDMc>nycg1V{eb3-R*c_nu`{FI@t=3?7EXz#!atL0x z&t`1f$;A%Cu2o&Pbluf;Ti0vSfK<)J%`>mZeMqJ^NHcFK^a5{j(EGuL7bq2MC|m}k zSuflujp8&5;&xc@xZkS8QjZtzJ3)64zNS>73M*X+`e9+IAYKj&JLvbrSQchCj0>xr zEo6`dMJ0-5)DF|aMx~#rX0;G$*zL4x5uL=}pR1C4X(8H4H&u0iu zHH*gY-d@Oh{e^zizmx@PIm|A-)ElhyuJrCLbfcw({s#0OFWl{RdkaC9!BAig3rRm+ zSn4M2wNl^y!N?$sy6Jp>qi}pbilfZ;8`va02U2GabGXX{{x+-gI;*o|%w_z;hHf?3 z=FEZN%#ZjNVYyf=4tk;Q7q0L35;^Enz3%(#gP^-VQt^G6w0&RAVK4n^hsbg2GwBd0 zsSV;7g4IpkSFrnS($nZ?p(0Rde)=yUusdVOB>M#Ru+4UiAupej-!;d9T3~r}} zS+GXEj-YkOGKV6^?Z$rZ=BBJ;KlJLS$jxmlhaJP>^|M2cWMcD!I|CI*@iGkbD>oNj zyy<0$YOi=uX__eS!u=*X^yMeJSs=6VU|!*Uu$1?G@y$E0U4>8RLb2NZynY}huKuMf z+8;dH!>`F7(e-{rvQu9GS$s^iG)CgRjgopG%Mj4}^&z8cPomxCDCQ0X@H4)SgKHf& zMX4vTlqB84Z1)F+6%Ts-jZb;O=Pwuheso0Fg?p_VrCB@a^{-#1y=Y_4*nC0abwfW( z{5D?G$IHxLgB=Im^g1ddj?X>vugSyeBRh2DZSqWf=dq)jg{1{sxE+=Be0@xP>(n{$ z9yz9Z0ex3ceof~)X?*U+>Wd@hrm4O}^$SGwmHxbSzd|#bAWy8S4I2X*Fn&xLsFWJm zV(%Dmhg`chw%BBfK>ukcFRIs;!#KR(SJz*pmFT|k59f@{^M6XhBbFylR0I1Atuqgz z1w1nn@IwfBn?HnotRjXu1u5;Y6UOpb>^P)$vO>!R))8c_Y8>A4H$#=Aioiv`z^An`8THCN?7MK~?X|+hdIist zrlro%HgtHk08&_FUa6X<4)U_ z@tnSfQN!0hY+cL!h5Zf zKhc+uguMd^4B_$6j~L<<`0-??5Y(NdYbth7J7ILpp_N&yb`JEF=HGGPFc6gHYy3S2 zET%U2bo+Cur5-#Q)Y_*DrWMz+N}C`pE4m(F7)w&GaEM>CYPBAt>13TpZf`L? zaw6nVv|7J~S?O62$1D|QLxty{$OitG8Q*ib1ISY^Vaj8dfL?{}kk%Pl4CzcV$#B(_N!heQHas0{{}M8Ar* zMNN*{BKdXL>|0>hm${CEs&Al$5CUV0VEi#FBa5?8^odWS0v8VJKqO{v11l-^+%d_P zw63(8G$5$2V=lZHiJ(>KhpIQoLd2#CLW|`A;zW#H#hBi)RLap)(}C!)W3m4==BMOB zPn+r*9*;gGLh`&-SnEDTP%hp5G+akr+ehK*0LD+F&9JCz4Did4rl=w4&HNN`vc!-I zGXQ6)-Zojz=tq~E+FAf-p+EwJ9pfJsnFf1^LO^*AdIg1<5DEJTVL6%iuzJvomOF0o z6Gm>&Av0Rz!&+X=-8G9SuAQ}JbfWx{D2M{t~r-{tY>-j_tX7dsD z*sc(Y4kzD5TfnxqOlHK+6(fE79ztIF{}3gwsK3ST4mj)E)m-GFgLSdm6ng1gHM|Gf z*s-2ZQQ$jTf7r;UMlD=y>`8ByH)f6DR6dFIraIg>S+f3$dm!#UTj9nn(e_)GdUrR|qWpAZ<=7a#iXE5ptcS&-^yk zULit8t!{!8E<%{m&A?rPODU{1{g@^Z`iCxyEE?sd(gRw|Mg4%?UoTD^9ruw`B4?rN ziM2|WT00n|K^LiNx^~p%KIECay=Q(aY6AUrCUza*uGpJIInD7yulOUcluz|8N#bpg&-d3JMsWoY zngCPbw^85-GQMioj2(_kMm~_2if!Uk1-)1({Z7<{8^gzx5}gV&iaT)BwAazso>_|G zKy8#S3$R&=g7jyYSEe!u3j|X?M7yZOS`2AX9gE~od+nSnA%QI!KGOha@D^v_Hnu2d zFc(X&tsOR31fz7u3=X~ppTP|NE?3IjU*BXIMcBIbIjD|6{Y|xs`t@; zbOYp0VVAF0Olbqye@hB$C@viR2(c&}HBr*D@bW z2*HW{kmuDc=RFgj4r;^taAG)_*C?DoA_bqiDx`S|YJ$J5k9+Im-g;hDuji9AvWof3 zET3Ez|HR%h{@%bkt`rEUUdKm=N%FNh*68qjBA@J-?;0E;9eLi{2=mj{Vb8mN+8^Px z_I3rQwRZR?lvFqmk-Pyu@dsY;4o;`NaoKO7)yS&3@erR#$`tWq%+cS?oXn*_K_`mS z+ILTr-QLdK)OycvA#5*z;IE2|Qut>UgVJNVz&{ zOU%Nac=vpkWDiu?mo|1_BY9^t`OFq8`_0E-c zch0+TnSv-UU5ol93S9KA=pOB%)8^P~V-lEx6fBo;U7z5(>|w@XrvL9ycK~l=84&3G zIZ#*1;Rw7nx$<#=3qexccQ64u;2)p#@(vL#vNqMoQ5UmMlgvM2B>fPieo_!BYpf-l9sfV8@6FZb0B68u z=IC^(&br0R(=V+4lie< zzW}KVN-y#4z-*irXWU&#V^45*w=pA}gBGuq(iWfLxd$EW3X}g)2>u6d@!xZs{|>Rw u|8U3stnnguoZWJkyUU$2Cfeuz&4`=tbBo8g`DeV!f6HC|-@MlN&Hn<>gAiu` literal 0 HcmV?d00001 diff --git a/vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py b/vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py new file mode 100644 index 00000000..ec1c37c5 --- /dev/null +++ b/vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py @@ -0,0 +1,423 @@ +import torch +import triton +import triton.language as tl + + +def blocksparse_flash_attn_varlen_fwd( + q, + k, + v, # (#tokens, n_heads, head_size) + cu_seqlens_k, + cu_seqlens_q, + sm_scale, + sparse_layout, + *, + block_size=64, + q_block_size=None, + max_seqlen=None): + # split q to blocks + + assert isinstance(sparse_layout, (list, tuple)) + + _, n_heads, head_size = q.shape + batch_size = cu_seqlens_k.size(0) - 1 + q_block_size = q_block_size or block_size + + assert q.dim() == k.dim() == v.dim() == 3 + assert q.size(1) % k.size(1) == 0 + assert q.size(2) == k.size(2) + # TODO(linxihui): allow k, v to have different head_size + assert k.shape == v.shape + assert cu_seqlens_k.dim() == 1 + + q_k_ratio = q.size(1) // k.size(1) + + if cu_seqlens_q is None: + if q.size(0) == batch_size: # decoding only + cu_seqlens_q = torch.arange( + 0, + batch_size + 1, + dtype=cu_seqlens_k.dtype, + device=cu_seqlens_k.device, + ) + elif q.size(0) == k.size(0): + cu_seqlens_q = cu_seqlens_k + else: + raise ValueError("cu_seqlens_q must be specified\ + if it mix of prefilling and decoding.") + else: + assert cu_seqlens_k.size(0) == cu_seqlens_q.size(0) + + # switch to use cpu to avoid too many kernel launches when iterated over + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).cpu() + k_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).cpu() + + assert torch.logical_or(q_lens == 1, k_lens == q_lens).all(), ( + "length of q should either be 1 (decoding) or same as k (prefilling).") + + if max_seqlen: + assert k_lens.max() <= max_seqlen + + n_blocks = (q_lens + q_block_size - 1) // q_block_size + + q_batch_ids = torch.tensor( + [i for i, n in enumerate(n_blocks) for _ in range(n)], + dtype=cu_seqlens_q.dtype, + device=cu_seqlens_q.device, + ) + q_start_sids = torch.tensor( + [i * q_block_size for n in n_blocks for i in range(n)], + dtype=cu_seqlens_q.dtype, + device=cu_seqlens_q.device, + ) + + out = q.new_empty(q.shape) + cu_seqlens_q = cu_seqlens_q.contiguous() + cu_seqlens_k = cu_seqlens_k.contiguous() + + layout_crow_indices, layout_col_indices = sparse_layout + block_d = triton.next_power_of_2(head_size) + + decoding_only = (q_lens == 1).all().item() + grid = (len(q_start_sids), n_heads, 1) + + _fwd_kernel_batch_inference[grid]( + q, + k, + v, + out, + sm_scale, + cu_seqlens_q[:-1], + cu_seqlens_q[1:], + cu_seqlens_k[:-1], + cu_seqlens_k[1:], + q_batch_ids, + q_start_sids, + 0, + *q.stride(), + 0, + *k.stride(), + 0, + *v.stride(), + 0, + *out.stride(), + layout_crow_indices, + layout_col_indices, + *layout_crow_indices.stride(), + *layout_col_indices.stride(), + q_k_ratio, + HAS_BATCH_DIM=False, + D_HEAD=head_size, + BLOCK_M=q_block_size, + BLOCK_N=block_size, + BLOCK_D=block_d, + BLOCK_M_LOADING=(16 if decoding_only else + q_block_size), # smaller for decoding + EVEN_D=block_d == head_size, + num_warps=1 if decoding_only else 4, + num_stages=3) + + return out + + +@triton.jit +def _fwd_kernel_inner( + acc, + l_i, + m_i, + q, + Q, + k_block_col_idx, + layout_col_ptr, + layout_col_stride_h, + layout_col_stride_m, + k_ptrs, + v_ptrs, + off_h, + offs_m, + offs_n, + offs_d, + stride_kt, + stride_vt, + sm_scale, + k_seqlen, + past_len, + LAST_K_BLOCK: tl.constexpr, + BLOCK_M_LOADING: tl.constexpr, + BLOCK_N: tl.constexpr, + D_HEAD: tl.constexpr, + EVEN_D: tl.constexpr, + M_LT_N: tl.constexpr, +): + k_block_id = tl.load(layout_col_ptr + off_h * layout_col_stride_h + + k_block_col_idx * layout_col_stride_m).to(tl.int32) + start_n = k_block_id * BLOCK_N + if LAST_K_BLOCK: + if EVEN_D: + k = tl.load( + k_ptrs + start_n * stride_kt, + mask=offs_n[None, :] + start_n < k_seqlen, + ) + else: + k = tl.load( + k_ptrs + start_n * stride_kt, + mask=(offs_n[None, :] + start_n < k_seqlen) & + (offs_d[:, None] < D_HEAD), + ) + else: + if EVEN_D: + k = tl.load(k_ptrs + start_n * stride_kt) + else: + k = tl.load(k_ptrs + start_n * stride_kt, + mask=offs_d[:, None] < D_HEAD) + + qk = tl.zeros([BLOCK_M_LOADING, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + + # the following is needed only when LAST_K_BLOCK or BLOCK_M < BLOCK_N + if LAST_K_BLOCK | M_LT_N: + qk += tl.where( + offs_m[:, None] + past_len >= (start_n + offs_n[None, :]), + 0, + float("-inf"), + ) + + # flash-attn2 + m_ij = tl.maximum(m_i, tl.max(qk, 1)) + p = tl.math.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, 1) + alpha = tl.math.exp2(m_i - m_ij) + acc = acc * alpha[:, None] + # update m_i + m_i = m_ij + l_i = l_i * alpha + l_ij + + p = p.to(Q.dtype.element_ty) + # update acc + if LAST_K_BLOCK: + if EVEN_D: + v = tl.load( + v_ptrs + start_n * stride_vt, + mask=offs_n[:, None] + start_n < k_seqlen, + ) + else: + v = tl.load( + v_ptrs + start_n * stride_vt, + mask=(offs_n[:, None] + start_n < k_seqlen) & + (offs_d[None, :] < D_HEAD), + ) + else: + if EVEN_D: + v = tl.load(v_ptrs + start_n * stride_vt) + else: + v = tl.load(v_ptrs + start_n * stride_vt, + mask=offs_d[None, :] < D_HEAD) + + acc += tl.dot(p, v) + + return acc, l_i, m_i + + +@triton.heuristics({ + "M_LT_N": + lambda kwargs: kwargs["BLOCK_M"] < kwargs["BLOCK_N"], +}) +@triton.jit +def _fwd_kernel_batch_inference( + Q, + K, + V, + Out, + sm_scale, + q_batch_starts, + q_batch_ends, + k_batch_starts, + k_batch_ends, + q_batch_ids, + q_start_sids, + stride_qb, + stride_qt, + stride_qh, + stride_qd, + stride_kb, + stride_kt, + stride_kh, + stride_kd, + stride_vb, + stride_vt, + stride_vh, + stride_vd, + stride_ob, + stride_ot, + stride_oh, + stride_od, + layout_crow_ptr, + layout_col_ptr, + layout_crow_stride_h, + layout_crow_stride_m, + layout_col_stride_h, + layout_col_stride_m, + q_k_ratio, + HAS_BATCH_DIM: tl.constexpr, + D_HEAD: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_M_LOADING: tl.constexpr, + EVEN_D: tl.constexpr, + M_LT_N: tl.constexpr, +): + """ + NOTATION: + pid: position id + sid: storage id + sbid: storage block id + pbid: position block id + offs_m, offs_n: storage offsets of m-dim(q, row) and n-dim(k, col) + + TODO(linxihui): + Optimize grouped-attn + """ + off_zm = tl.program_id(0) + off_h = tl.program_id(1) + + off_h_for_kv = off_h // q_k_ratio + + if HAS_BATCH_DIM: + off_z = tl.program_id(2) + Q += off_z * stride_qb + K += off_z * stride_kb + V += off_z * stride_vb + Out += off_z * stride_ob + start_m = off_zm + q_start_sid = start_m * BLOCK_M # always 0 for decoding + else: + off_z = tl.load(q_batch_ids + off_zm).to(tl.int32) # [0, 0, 0, 1] + q_start_sid = tl.load(q_start_sids + off_zm) + start_m = q_start_sid // BLOCK_M # q_sbid + + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M_LOADING) + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + + q_cu_start = tl.load(q_batch_starts + off_z).to(tl.int32) + q_seqlen = tl.load(q_batch_ends + off_z).to(tl.int32) - q_cu_start + k_cu_start = tl.load(k_batch_starts + off_z).to(tl.int32) + k_seqlen = tl.load(k_batch_ends + off_z).to(tl.int32) - k_cu_start + past_len = k_seqlen - q_seqlen + + Q += q_cu_start * stride_qt + off_h * stride_qh + K += k_cu_start * stride_kt + off_h_for_kv * stride_kh + V += k_cu_start * stride_vt + off_h_for_kv * stride_vh + Out += q_cu_start * stride_ot + off_h * stride_oh + + q_pbid = (past_len + q_start_sid) // BLOCK_M + + if EVEN_D: + q = tl.load( + Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd, + mask=offs_m[:, None] < q_seqlen, + ) + else: + q = tl.load( + Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd, + mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD), + other=0, + ) + + sparse_crow_ptr = (layout_crow_ptr + off_h * layout_crow_stride_h + + q_pbid * layout_crow_stride_m) + + # TODO(linxihui): load at once, with any Triton version + # that supports `tl.split`, e.g., Triton 3.0 + k_block_start = tl.load(sparse_crow_ptr).to(tl.int32) + k_block_end = tl.load(sparse_crow_ptr + 1).to(tl.int32) + + m_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32) + acc = tl.zeros([BLOCK_M_LOADING, BLOCK_D], dtype=tl.float32) + + k_ptrs = K + offs_n[None, :] * stride_kt + offs_d[:, None] * stride_kd + v_ptrs = V + offs_n[:, None] * stride_vt + offs_d[None, :] * stride_vd + + sm_scale *= ( + 1.44269504 # 1/log2 as we use base2 for exponential and logarithm + ) + + for k_block_col_idx in range(k_block_start, k_block_end - 1): + acc, l_i, m_i = _fwd_kernel_inner( + acc, + l_i, + m_i, + q, + Q, + k_block_col_idx, + layout_col_ptr, + layout_col_stride_h, + layout_col_stride_m, + k_ptrs, + v_ptrs, + off_h, + offs_m, + offs_n, + offs_d, + stride_kt, + stride_vt, + sm_scale, + k_seqlen, + past_len, + False, + BLOCK_M_LOADING, + BLOCK_N, + D_HEAD, + EVEN_D, + M_LT_N, + ) + + acc, l_i, m_i = _fwd_kernel_inner( + acc, + l_i, + m_i, + q, + Q, + k_block_end - 1, + layout_col_ptr, + layout_col_stride_h, + layout_col_stride_m, + k_ptrs, + v_ptrs, + off_h, + offs_m, + offs_n, + offs_d, + stride_kt, + stride_vt, + sm_scale, + k_seqlen, + past_len, + True, + BLOCK_M_LOADING, + BLOCK_N, + D_HEAD, + EVEN_D, + M_LT_N, + ) + + # flash-attn 2 + m_i += tl.math.log2(l_i) + acc = acc / l_i[:, None] + + # write output + if EVEN_D: + tl.store( + Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od, + acc, + mask=offs_m[:, None] < q_seqlen, + ) + else: + tl.store( + Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od, + acc, + mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD), + ) diff --git a/vllm/attention/ops/blocksparse_attention/interface.py b/vllm/attention/ops/blocksparse_attention/interface.py new file mode 100644 index 00000000..1ead541f --- /dev/null +++ b/vllm/attention/ops/blocksparse_attention/interface.py @@ -0,0 +1,238 @@ +import math + +import torch + +from vllm.platforms import current_platform +from vllm.utils import is_cpu, is_hip + +from .utils import (dense_to_crow_col, get_head_sliding_step, + get_sparse_attn_mask) + +IS_COMPUTE_8_OR_ABOVE = current_platform.has_device_capability(80) + +if IS_COMPUTE_8_OR_ABOVE: + from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd + + +class LocalStridedBlockSparseAttn(torch.nn.Module): + + def __init__( + self, + n_heads, + max_seqlen, + local_blocks, + vert_stride, + block_size, + device=None, + dtype=None, + homo_head=False, + active_head_range=None, + q_block_size=None, + use_spda=None, + ): + super().__init__() + if use_spda is None: + use_spda = is_hip() or is_cpu() or not \ + IS_COMPUTE_8_OR_ABOVE + device = device or (torch.cuda.current_device() + if current_platform.is_cuda_alike() else "cpu") + device = torch.device(device) + # NOTE: vllm CPU backend support BF16 instead of FP16. + dtype = dtype or (torch.bfloat16 if IS_COMPUTE_8_OR_ABOVE + or device.type == "cpu" else torch.half) + + self.n_heads = n_heads + self.max_seqlen = max_seqlen + self.local_blocks = local_blocks + self.vert_stride = vert_stride + self.use_spda = use_spda + self.dtype = dtype + self.device = device + self.block_size = block_size + self.q_block_size = q_block_size + self.homo_head = homo_head + self.active_head_range = active_head_range + self.head_sliding_step = get_head_sliding_step(n_heads, vert_stride, + homo_head) + + sparse_layout, sparse_pattern, self.dense_attn_mask = ( + self.get_attn_pattern(dtype, device)) + + if q_block_size is not None and q_block_size != block_size: + if q_block_size > block_size: + assert q_block_size % block_size == 0 + blocks_to_merge = q_block_size // block_size + shape = sparse_pattern.shape + sparse_pattern = sparse_pattern.view(shape[0], -1, + blocks_to_merge, + shape[-1]) + sparse_pattern = sparse_pattern.sum(2) + sparse_layout = dense_to_crow_col(sparse_pattern) + else: + raise ValueError( + "Does not support smaller q_block_size. It will be slower." + ) + + self.sparse_layout = sparse_layout + + def get_attn_pattern(self, dtype, device): + sparse_layout, sparse_pattern, dense_attn_mask = get_sparse_attn_mask( + self.n_heads, + self.max_seqlen, + self.max_seqlen, + dtype, + device, + block_size=self.block_size, + local_blocks=self.local_blocks, + vert_stride=self.vert_stride, + homo_head=self.homo_head, + return_dense=self.use_spda, + dense_mask_type="bias", + ) + if (not self.homo_head) and (self.active_head_range is not None): + assert isinstance(self.active_head_range, tuple) + assert (len(self.active_head_range) == 2) + h_start, h_end = self.active_head_range + sparse_layout = tuple(x[h_start:h_end] for x in sparse_layout) + if self.use_spda: + dense_attn_mask = dense_attn_mask[h_start:h_end] + return sparse_layout, sparse_pattern, dense_attn_mask + + def varlen_attn(self, + q, + k, + v, + cu_seqlens_k, + cu_seqlens_q=None, + sm_scale=None): + """ + q, k, v: shape = (num_tokens, num_heads_q/kv, head_size). + Support grouped attention, with `q[:, i*r:(i*r + r)]` + is correspondent to `k[:, i]`, where `r` is the q/k ratio. + cu_seqlens_k: shape=(batch_size + 1,), + indicating segment of samples, + e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i + cu_seqlens_q: shape=(batch_size + 1, ). + Default None: same as cu_seqlens_k for prefilling or + [0, 1, .., batch_size] for decoding. + The only case you need to specify is when q is a mix of + prefilling and decoding. + sm_scale: softmax scale, default to 1/sqrt(head_size). + + return: tensor of shape as q. + """ + assert ( + IS_COMPUTE_8_OR_ABOVE + ), "Requires compute capability of 8 or above (Ampere or newer) to use \ + Triton kernel." + + sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1)) + + return blocksparse_flash_attn_varlen_fwd( + q, + k, + v, + cu_seqlens_k, + cu_seqlens_q, + sm_scale, + self.sparse_layout, + block_size=self.block_size, + q_block_size=self.q_block_size, + max_seqlen=self.max_seqlen, + ) + + @staticmethod + def transpose_and_pad(x, cu_seqlens, maxlen, head_repeats=1): + """ + :param x: (total_tokens, n_heads, head_size) + :return: (batch, n_heads, length, head_size) + """ + x_padded = x.new_empty( + len(cu_seqlens) - 1, x.size(1), head_repeats, maxlen, x.size(2)) + cu_seqlens = cu_seqlens.cpu() + for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])): + x_padded[i, :, :, :e - s].copy_(x[s:e].transpose(0, + 1).unsqueeze(1)) + return x_padded.flatten(1, 2) + + @staticmethod + def transpose_and_unpad(x_padded, cu_seqlens): + """ + :param x_padded: (batch, n_heads, length, head_size) + :return: (total_tokens, n_heads, head_size) + """ + cu_seqlens = cu_seqlens.cpu() + total_n_tokens = cu_seqlens[-1] + x = x_padded.new_empty(total_n_tokens, x_padded.size(1), + x_padded.size(3)) + for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])): + x[s:e].copy_(x_padded[i, :, :e - s].transpose(0, 1)) + return x + + def spda(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None): + """For CPU, V100 or other older GPUs. + NOTE: torch SPDA supports nested tensor, + but seems extremely slow. Choose to pad instead. + """ + assert (cu_seqlens_q is None or + (cu_seqlens_q + == cu_seqlens_k).all()), "Can only handle prompt with SPDA." + assert q.size(0) == k.size(0), "can only handle prompt with SPDA." + + assert q.size(1) % k.size(1) == 0 + q_k_ratio = q.size(1) // k.size(1) + sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1)) + cu_seqlens = cu_seqlens_k.cpu() + maxlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + + if (self.dense_attn_mask.dtype != q.dtype + or self.dense_attn_mask.device != q.device): + _, _, self.dense_attn_mask = self.get_attn_pattern( + q.dtype, q.device) + attn_mask = self.dense_attn_mask[None, :, :maxlen, :maxlen] + + q2 = self.transpose_and_pad(q, cu_seqlens, maxlen, 1) + k2, v2 = [ + self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio) + for x in [k, v] + ] + spda_output = torch.nn.functional.scaled_dot_product_attention( + q2, k2, v2, attn_mask=attn_mask, scale=sm_scale) + return self.transpose_and_unpad(spda_output, cu_seqlens) + + def forward(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None): + """Dispatch to `varlen_attn` (Ampere or newer) or + `self.spda`(cpu, Volta, Turing or older)based on + the type of device used and cuda compute capability. + + q, k, v: shape = (num_tokens, num_heads_q/kv, head_size). + Support grouped attention, with `q[:, i*r:(i*r + r)]` + is correspondent to `k[:, i]`, where `r` is the q/k ratio. + cu_seqlens_k: shape=(batch_size + 1,), indicating segment of samples, + e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i + cu_seqlens_q: shape=(batch_size + 1, ). + Default None: same as cu_seqlens_k for prefilling or + [0, 1, .., batch_size] for decoding. + The only case you need to specify + is when q is a mix of prefilling + and decoding. + sm_scale: softmax scale, default to 1/sqrt(head_size). + + return: tensor of shape as q. + """ + assert k.dim() == 3 + if self.use_spda: + return self.spda( + q, + k, + v, + cu_seqlens_k, + cu_seqlens_q=cu_seqlens_q, + sm_scale=sm_scale, + ) + return self.varlen_attn(q, + k, + v, + cu_seqlens_k, + cu_seqlens_q=cu_seqlens_q, + sm_scale=sm_scale) diff --git a/vllm/attention/ops/blocksparse_attention/utils.py b/vllm/attention/ops/blocksparse_attention/utils.py new file mode 100644 index 00000000..78d75223 --- /dev/null +++ b/vllm/attention/ops/blocksparse_attention/utils.py @@ -0,0 +1,242 @@ +# Helper functions for 3D sparse pattern +# These function are not optimized and very inefficient. +# Avoid calling them too frequent or use a cache mechanism. + +from functools import lru_cache + +import numpy as np +import torch +import triton + + +class csr_matrix: + """Simple implementation of CSR matrix conversion without scipy. + This replaced scipy.sparse.csr_matrix() previously used.""" + + def __init__(self, input_array): + if not isinstance(input_array, np.ndarray): + raise ValueError("Input must be a NumPy array") + + self.shape = input_array.shape + rows, cols = self.shape + data = [] + indices = [] + indptr = [0] + + for i in range(rows): + for j in range(cols): + if input_array[i, j]: + data.append(input_array[i, j]) + indices.append(j) + indptr.append(len(indices)) + + self.data = np.array(data) + self.indices = np.array(indices) + self.indptr = np.array(indptr) + + +def dense_to_crow_col(x: torch.Tensor): + """Turning a 2D/3D torch tensor (x) to CSR rows/cols indexing. + NOTE: col_indices padded -1 + """ + device = x.device + pad = -1 + dim = x.dim() + assert x.dim() in (2, 3) + if x.dim() == 2: + x = x[None] + x = [csr_matrix(xi.bool().cpu().numpy()) for xi in x] + crows = torch.vstack([torch.from_numpy(xi.indptr) for xi in x]) + cols = [torch.from_numpy(xi.indices) for xi in x] + max_cols = max(len(xi) for xi in cols) + cols = [ + torch.cat([xi, pad + xi.new_zeros(max_cols - xi.shape[0])]) + for xi in cols + ] + cols = torch.vstack(cols) + if dim == 2: + crows = crows[0] + cols = cols[0] + return crows.to(device), cols.to(device) + + +def crow_col_to_dense(crows: torch.Tensor, + cols: torch.Tensor, + dtype: torch.dtype = torch.float16): + dim = crows.dim() + if dim == 1: + crows = crows[None] + cols = cols[None] + device = crows.device + crows, cols = crows.cpu(), cols.cpu() # faster in cpu + shape = (crows.shape[0], crows.shape[1] - 1, cols.max() + 1) + x = torch.zeros(shape, dtype=dtype) + for i in range(shape[0]): + for j in range(shape[1]): + x[i, j, cols[i, crows[i, j]:crows[i, j + 1]]] = 1 + if dim == 1: + x = x[0] + return x.to(device) + + +def dense_to_ccol_row(x: torch.Tensor): + """Similar, but to CSC format""" + x = x.transpose(-2, -1) + return dense_to_crow_col(x) + + +def ccol_row_to_dense(ccol: torch.Tensor, + rows: torch.Tensor, + dtype: torch.dtype = torch.float16): + return crow_col_to_dense(ccol, rows, dtype).permute(0, 2, 1).contiguous() + + +def _get_sparse_attn_mask_homo_head( + q_len: int, + max_seqlen: int, + dtype: torch.dtype, + device: torch.device, + block_size: int = 128, + local_blocks: int = 4, + vert_stride: int = 4, + return_dense: bool = False, +): + """ + :return: a tuple of 3: + - tuple of crow_indices, col_indices representation + of CSR format. + - block dense mask + - all token dense mask (be aware that it can be + OOM if it is too big) if `return_dense==True`, + otherwise, None + """ + with torch.no_grad(): + num_blocks = triton.cdiv(max_seqlen, block_size) + q_pos = torch.arange(num_blocks)[:, None] + k_pos = torch.arange(num_blocks)[None] + mask_vert_strided = (torch.arange(num_blocks) + 1) % vert_stride == 0 + block_mask_dense = (((q_pos >= k_pos) + & ((q_pos - k_pos < local_blocks) + | mask_vert_strided)).to(device).to(dtype)) + num_blocks_q = triton.cdiv(q_len, block_size) + block_mask_dense_output = (dense_to_crow_col( + block_mask_dense[-num_blocks_q:].contiguous())) + if return_dense: + mask_dense = torch.kron( + block_mask_dense, + block_mask_dense.new_ones((block_size, block_size)), + ) + causal_mask = torch.tril(torch.ones( + max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:] + mask_dense = mask_dense[-q_len:, :max_seqlen] * causal_mask + return ( + block_mask_dense_output, + block_mask_dense, + mask_dense, + ) + else: + return ( + block_mask_dense_output, + block_mask_dense, + None, + ) + + +def binary_mask_to_bias(mask_dense: torch.Tensor): + mask_dense = 1 - mask_dense + mask_dense.masked_fill_(mask_dense.bool(), -torch.inf) + return mask_dense + + +def get_head_sliding_step(n_heads: int, + vert_stride: int, + homo_head: bool = False): + if homo_head: + return 0 + return max(1, int(vert_stride / n_heads)) + + +@lru_cache +def get_sparse_attn_mask( + n_heads: int, + q_len: int, + max_seqlen: int, + dtype: torch.dtype, + device: torch.device, + block_size: int = 64, + local_blocks: int = 4, + vert_stride: int = 4, + homo_head: bool = True, + return_dense: bool = False, + dense_mask_type: str = "binary", +): + """ + :param dense_mask_type: "binary" (0 for skip token, 1 for others) + or "bias" (-inf for skip token, 0 or others) + :return: a tuple of 3: + - tuple of crow_indices, col_indices representation + of CSR format. + - block dense mask + - all token dense mask (be aware that it can be OOM if it + is too big) if `return_dense==True`, otherwise, None + """ + assert dense_mask_type in ("binary", "bias") + if homo_head: + with torch.no_grad(): + (crow, col), block_mask_dense, mask_dense = ( + _get_sparse_attn_mask_homo_head( + q_len, + max_seqlen, + dtype, + device, + block_size, + local_blocks, + vert_stride, + return_dense, + )) + crow = crow[None].expand(n_heads, crow.shape[0]) + col = col[None].expand(n_heads, col.shape[0]) + if return_dense: + mask_dense = mask_dense[None].expand(n_heads, + *mask_dense.shape) + if dense_mask_type == "bias": + mask_dense = binary_mask_to_bias(mask_dense) + return (crow, col), block_mask_dense, mask_dense + + with torch.no_grad(): + num_blocks = triton.cdiv(max_seqlen, block_size) + q_pos = torch.arange(num_blocks)[None, :, None] + k_pos = torch.arange(num_blocks)[None, None] + head_sliding_step = get_head_sliding_step(n_heads, vert_stride) + mask_vert_strided = [ + (torch.arange(num_blocks) + h * head_sliding_step + 1) % + vert_stride == 0 for h in range(n_heads) + ] + mask_vert_strided = torch.vstack(mask_vert_strided).unsqueeze(1) + block_mask_dense = (((q_pos >= k_pos) + & ((q_pos - k_pos < local_blocks) + | mask_vert_strided)).to(device).to(dtype)) + num_blocks_q = triton.cdiv(q_len, block_size) + block_mask_dense_output = block_mask_dense[:, -num_blocks_q:] + if return_dense: + mask_dense = torch.kron( + block_mask_dense, + block_mask_dense.new_ones((block_size, block_size)), + ) + causal_mask = torch.tril(torch.ones( + max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:] + mask_dense = mask_dense[..., -q_len:, :max_seqlen] * causal_mask[None] + if dense_mask_type == "bias": + mask_dense = binary_mask_to_bias(mask_dense) + + return ( + dense_to_crow_col(block_mask_dense_output), + block_mask_dense, + mask_dense, + ) + else: + return ( + dense_to_crow_col(block_mask_dense_output), + block_mask_dense, + None, + ) diff --git a/vllm/attention/ops/ipex_attn.py b/vllm/attention/ops/ipex_attn.py new file mode 100644 index 00000000..6b270ffd --- /dev/null +++ b/vllm/attention/ops/ipex_attn.py @@ -0,0 +1,123 @@ +from typing import Dict, List, Optional, Tuple + +import intel_extension_for_pytorch.llm.modules as ipex_modules +import torch + +from vllm import _custom_ops as ops + + +class PagedAttention: + + @staticmethod + def get_supported_head_sizes() -> List[int]: + return [64, 80, 96, 112, 128, 256] + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + *args, + ) -> Tuple[int, ...]: + return (2, num_blocks, block_size * num_kv_heads * head_size) + + @staticmethod + def split_kv_cache( + kv_cache: torch.Tensor, + num_kv_heads: int, + head_size: int, + *args, + ) -> Tuple[torch.Tensor, torch.Tensor]: + num_blocks = kv_cache.shape[1] + + key_cache = kv_cache[0] + key_cache = key_cache.view(num_blocks, num_kv_heads, -1, head_size) + value_cache = kv_cache[1] + value_cache = value_cache.view(num_blocks, num_kv_heads, -1, head_size) + return key_cache, value_cache + + @staticmethod + def write_to_paged_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + *args, + ) -> None: + ipex_modules.PagedAttention.reshape_and_cache( + key, value, key_cache, value_cache, + slot_mapping.flatten().int()) + + @staticmethod + def forward_decode( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_tables: torch.Tensor, + context_lens: torch.Tensor, + max_context_len: int, + kv_cache_dtype: str, + num_kv_heads: int, + scale: float, + alibi_slopes: Optional[torch.Tensor], + k_scale: float, + v_scale: float, + *args, + ) -> torch.Tensor: + output = torch.empty_like(query) + block_size = value_cache.shape[2] + head_mapping = torch.arange( + 0, + num_kv_heads, + device="cpu", + dtype=torch.int32, + ).view(num_kv_heads, + 1).repeat_interleave(query.size(1) // num_kv_heads).flatten() + ipex_modules.PagedAttention.single_query_cached_kv_attention( + output, query.contiguous(), key_cache, value_cache, head_mapping, + scale, block_tables, context_lens, block_size, max_context_len, + alibi_slopes) + + return output + + @staticmethod + def forward_prefix( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache_dtype: str, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_tables: torch.Tensor, + subquery_start_loc: torch.Tensor, + prompt_lens_tensor: torch.Tensor, + context_lens: torch.Tensor, + max_subquery_len: int, + alibi_slopes: Optional[torch.Tensor], + *args, + ) -> torch.Tensor: + raise NotImplementedError + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: Dict[int, int], + *args, + ) -> None: + raise NotImplementedError + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: Dict[int, List[int]], + *args, + ) -> None: + key_caches = [kv_cache[0] for kv_cache in kv_caches] + value_caches = [kv_cache[1] for kv_cache in kv_caches] + ops.copy_blocks(key_caches, value_caches, src_to_dists) diff --git a/vllm/attention/ops/paged_attn.py b/vllm/attention/ops/paged_attn.py new file mode 100644 index 00000000..c90e8dd8 --- /dev/null +++ b/vllm/attention/ops/paged_attn.py @@ -0,0 +1,245 @@ +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import torch + +from vllm import _custom_ops as ops +from vllm.triton_utils import HAS_TRITON + +if HAS_TRITON: + from vllm.attention.ops.prefix_prefill import context_attention_fwd + +# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`. +_PARTITION_SIZE = 512 + + +@dataclass +class PagedAttentionMetadata: + """Metadata for PagedAttention.""" + # (batch_size,). The length of sequences (entire tokens seen so far) per + # sequence. + seq_lens_tensor: Optional[torch.Tensor] + # Maximum sequence length in the batch. 0 if it is prefill-only batch. + max_decode_seq_len: int + # (batch_size, max_blocks_per_seq). + # Block addresses per sequence. (Seq id -> list of physical block) + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks + # in the kv cache. Each block can contain up to block_size tokens. + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph + # captured. + block_tables: Optional[torch.Tensor] + + +class PagedAttention: + + @staticmethod + def get_supported_head_sizes() -> List[int]: + return [64, 80, 96, 112, 120, 128, 192, 256] + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + return (2, num_blocks, block_size * num_kv_heads * head_size) + + @staticmethod + def split_kv_cache( + kv_cache: torch.Tensor, + num_kv_heads: int, + head_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + x = 16 // kv_cache.element_size() + num_blocks = kv_cache.shape[1] + + key_cache = kv_cache[0] + key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x, + -1, x) + value_cache = kv_cache[1] + value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1) + return key_cache, value_cache + + @staticmethod + def write_to_paged_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + ) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping.flatten(), + kv_cache_dtype, + k_scale, + v_scale, + ) + + @staticmethod + def forward_decode( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + max_seq_len: int, + kv_cache_dtype: str, + num_kv_heads: int, + scale: float, + alibi_slopes: Optional[torch.Tensor], + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, + ) -> torch.Tensor: + if blocksparse_vert_stride is not None and blocksparse_vert_stride > 1: + # use blocksparse paged attention + block_size = value_cache.size(-1) + assert (blocksparse_block_size > 0 and + blocksparse_block_size % block_size == 0), \ + (f"{blocksparse_block_size=} needs to be a multiple of" + f"{block_size=} used in block_tables.") + + output = torch.empty_like(query) + block_size = value_cache.shape[3] + num_seqs, num_heads, head_size = query.shape + max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) // + _PARTITION_SIZE) + # NOTE(woosuk): We use a simple heuristic to decide whether to use + # PagedAttention V1 or V2. If the number of partitions is 1, we use + # V1 to avoid the overhead of reduction. Also, if the number of + # sequences or heads is large, we use V1 since there is enough work + # to parallelize. + # TODO(woosuk): Tune this heuristic. + # For context len > 8192, use V2 kernel to avoid shared memory shortage. + use_v1 = (max_seq_len <= 8192 + and (max_num_partitions == 1 or num_seqs * num_heads > 512)) + use_v1 = True + if use_v1: + # Run PagedAttention V1. + ops.paged_attention_v1( + output, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + ) + else: + # Run PagedAttention V2. + assert _PARTITION_SIZE % block_size == 0 + tmp_output = torch.empty( + size=(num_seqs, num_heads, max_num_partitions, head_size), + dtype=output.dtype, + device=output.device, + ) + exp_sums = torch.empty( + size=(num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=output.device, + ) + max_logits = torch.empty_like(exp_sums) + ops.paged_attention_v2( + output, + exp_sums, + max_logits, + tmp_output, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + return output + + @staticmethod + def forward_prefix( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache_dtype: str, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_tables: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens_tensor: torch.Tensor, + context_lens: torch.Tensor, + max_query_len: int, + alibi_slopes: Optional[torch.Tensor], + sliding_window: Optional[int], + k_scale: float, + v_scale: float, + ) -> torch.Tensor: + output = torch.empty_like(query) + context_attention_fwd( + query, + key, + value, + output, + kv_cache_dtype, + key_cache, + value_cache, + block_tables, + # query_start_loc is (batch_size + 1,) + query_start_loc[:-1], + seq_lens_tensor, + context_lens, + max_query_len, + k_scale, + v_scale, + alibi_slopes, + sliding_window, + ) + return output + + @staticmethod + def swap_blocks( + src_kv_cache: torch.Tensor, + dst_kv_cache: torch.Tensor, + src_to_dst: torch.Tensor, + ) -> None: + src_key_cache = src_kv_cache[0] + dst_key_cache = dst_kv_cache[0] + ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst) + + src_value_cache = src_kv_cache[1] + dst_value_cache = dst_kv_cache[1] + ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst) + + @staticmethod + def copy_blocks( + kv_caches: List[torch.Tensor], + src_to_dists: torch.Tensor, + ) -> None: + key_caches = [kv_cache[0] for kv_cache in kv_caches] + value_caches = [kv_cache[1] for kv_cache in kv_caches] + ops.copy_blocks(key_caches, value_caches, src_to_dists) diff --git a/vllm/attention/ops/prefix_prefill.py b/vllm/attention/ops/prefix_prefill.py new file mode 100644 index 00000000..a2a649c8 --- /dev/null +++ b/vllm/attention/ops/prefix_prefill.py @@ -0,0 +1,861 @@ +# The kernels in this file are adapted from LightLLM's context_attention_fwd: +# https://github.com/ModelTC/lightllm/blob/main/lightllm/models/llama/triton_kernel/context_flashattention_nopad.py + +import torch +import triton +import triton.language as tl + +from vllm.platforms import current_platform + +if triton.__version__ >= "2.1.0": + + @triton.jit + def _fwd_kernel( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + k_scale, + v_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, # head size + BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2 + BLOCK_N: tl.constexpr, + SLIDING_WINDOW: tl.constexpr, + ): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + cur_batch_query_len = cur_batch_seq_len - cur_batch_ctx_len + + # start position inside of the query + # generally, N goes over kv, while M goes over query_len + block_start_loc = BLOCK_M * start_m + + # initialize offsets + # [N]; starts at 0 + offs_n = tl.arange(0, BLOCK_N) + # [D]; starts at 0 + offs_d = tl.arange(0, BLOCK_DMODEL_PADDED) + # [M]; starts at current position in query + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + # [M,D] + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + dim_mask = tl.where( + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, + 0).to(tl.int1) # [D] + + q = tl.load(Q + off_q, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_query_len), + other=0.0) # [M,D] + + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") # [M] + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) # [M] + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], + dtype=tl.float32) # [M,D] + + # compute query against context (no causal mask here) + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) # [N] + # [D,N] + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + # [N,D] + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k_load = tl.load(K_cache + off_k, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_ctx_len), + other=0.0) # [D,N] + + if k_load.dtype.is_fp8(): + k = (k_load.to(tl.float32) * k_scale).to(q.dtype) + else: + k = k_load + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N] + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + if SLIDING_WINDOW > 0: + # (cur_batch_ctx_len + offs_m[:, None]) are the positions of + # Q entries in sequence + # (start_n + offs_n[None, :]) are the positions of + # KV entries in sequence + # So the condition makes sure each entry in Q only attends + # to KV entries not more than SLIDING_WINDOW away. + # + # We can't use -inf here, because the + # sliding window may lead to the entire row being masked. + # This then makes m_ij contain -inf, which causes NaNs in + # exp(). + qk = tl.where((cur_batch_ctx_len + offs_m[:, None]) - + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, + -10000) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) # [M] + p = tl.exp(qk - m_ij[:, None]) # [M,N] + l_ij = tl.sum(p, 1) # [M] + # -- update m_i and l_i + m_i_new = tl.maximum(m_i, m_ij) # [M] + alpha = tl.exp(m_i - m_i_new) # [M] + beta = tl.exp(m_ij - m_i_new) # [M] + l_i_new = alpha * l_i + beta * l_ij # [M] + + # -- update output accumulator -- + # scale p + p_scale = beta / l_i_new + p = p * p_scale[:, None] + # scale acc + acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v_load = tl.load(V_cache + off_v, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_ctx_len), + other=0.0) # [N,D] + if v_load.dtype.is_fp8(): + v = (v_load.to(tl.float32) * v_scale).to(q.dtype) + else: + v = v_load + p = p.to(v.dtype) + + acc += tl.dot(p, v) + # # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + # block_mask is 0 when we're already past the current query length + block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0) + + # compute query against itself (with causal mask) + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_query_len), + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + # apply causal mask + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + if SLIDING_WINDOW > 0: + qk = tl.where( + offs_m[:, None] - + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, -10000) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + m_i_new = tl.maximum(m_i, m_ij) + alpha = tl.exp(m_i - m_i_new) + beta = tl.exp(m_ij - m_i_new) + l_i_new = alpha * l_i + beta * l_ij + # -- update output accumulator -- + # scale p + p_scale = beta / l_i_new + p = p * p_scale[:, None] + # scale acc + acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_query_len), + other=0.0) + p = p.to(v.dtype) + + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_query_len)) + return + + @triton.jit + def _fwd_kernel_flash_attn_v2( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + ): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + + block_start_loc = BLOCK_M * start_m + + # initialize offsets + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + q = tl.load( + Q + off_q, + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + # # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) + + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k = tl.load(K_cache + off_k, + mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len, + other=0.0) + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(V_cache + off_v, + mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len, + other=0.0) + + p = p.to(v.dtype) + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + block_mask = tl.where( + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0) + + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=(start_n + offs_n[None, :]) < + cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=(start_n + offs_n[:, None]) < + cur_batch_seq_len - cur_batch_ctx_len, + other=0.0) + + p = p.to(v.dtype) + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + # acc /= l_i[:, None] + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len) + return + + @triton.jit + def _fwd_kernel_alibi( + Q, + K, + V, + K_cache, + V_cache, + B_Loc, + sm_scale, + k_scale, + v_scale, + B_Start_Loc, + B_Seqlen, + B_Ctxlen, + Alibi_slopes, + block_size, + x, + Out, + stride_b_loc_b, + stride_b_loc_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_obs, + stride_oh, + stride_od, + stride_k_cache_bs, + stride_k_cache_h, + stride_k_cache_d, + stride_k_cache_bl, + stride_k_cache_x, + stride_v_cache_bs, + stride_v_cache_h, + stride_v_cache_d, + stride_v_cache_bl, + num_queries_per_kv: int, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, # head size + BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2 + BLOCK_N: tl.constexpr, + ): + # attn_bias[] + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // num_queries_per_kv + + # cur_batch_seq_len: the length of prompts + # cur_batch_ctx_len: the length of prefix + # cur_batch_in_all_start_index: the start id of the dim=0 + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + + block_start_loc = BLOCK_M * start_m + + # initialize offsets + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL_PADDED) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + offs_d[None, :] * stride_qd) + + dim_mask = tl.where( + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1) + + q = tl.load(Q + off_q, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + + # # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32) + + alibi_slope = tl.load(Alibi_slopes + cur_head) + alibi_start_q = tl.arange( + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len + alibi_start_k = 0 + for start_n in range(0, cur_batch_ctx_len, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b + + ((start_n + offs_n) // block_size) * stride_b_loc_s, + mask=(start_n + offs_n) < cur_batch_ctx_len, + other=0) + off_k = (bn[None, :] * stride_k_cache_bs + + cur_kv_head * stride_k_cache_h + + (offs_d[:, None] // x) * stride_k_cache_d + + ((start_n + offs_n[None, :]) % block_size) * + stride_k_cache_bl + + (offs_d[:, None] % x) * stride_k_cache_x) + off_v = ( + bn[:, None] * stride_v_cache_bs + + cur_kv_head * stride_v_cache_h + + offs_d[None, :] * stride_v_cache_d + + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl) + k_load = tl.load(K_cache + off_k, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < cur_batch_ctx_len), + other=0.0) # [D,N] + + if k_load.dtype.is_fp8(): + k = (k_load.to(tl.float32) * k_scale).to(q.dtype) + else: + k = k_load + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk, + float("-inf")) + qk *= sm_scale + + # load alibi + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k - + alibi_start_q[:, None]) * alibi_slope + alibi = tl.where( + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len), + alibi, float("-inf")) + qk += alibi + alibi_start_k += BLOCK_N + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v_load = tl.load(V_cache + off_v, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < cur_batch_ctx_len), + other=0.0) + if v_load.dtype.is_fp8(): + v = (v_load.to(tl.float32) * v_scale).to(q.dtype) + else: + v = v_load + p = p.to(v.dtype) + + acc += tl.dot(p, v, allow_tf32=False) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + + offs_d[:, None] * stride_kd) + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + + offs_d[None, :] * stride_vd) + k_ptrs = K + off_k + v_ptrs = V + off_v + + block_mask = tl.where( + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0) + + # init alibi + alibi_slope = tl.load(Alibi_slopes + cur_head) + alibi_start_q = tl.arange( + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len + alibi_start_k = cur_batch_ctx_len + # # init debugger + # offset_db_q = tl.arange(0, BLOCK_M) + block_start_loc + # offset_db_k = tl.arange(0, BLOCK_N) + # calc q[BLOCK_M, BLOCK_MODEL] mul k[prefix_len: , BLOCK_DMODEL] + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(k_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=dim_mask[:, None] & + ((start_n + offs_n[None, :]) < + cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k, allow_tf32=False) + qk *= sm_scale + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, + float("-inf")) + + # load alibi + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k - + alibi_start_q[:, None]) * alibi_slope + alibi = tl.where( + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len), + alibi, float("-inf")) + qk += alibi + alibi_start_k += BLOCK_N + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + m_i_new = tl.maximum(m_i, m_ij) + p = tl.math.exp(qk - m_i_new[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + + alpha = tl.math.exp(m_i - m_i_new) + l_i_new = alpha * l_i + l_ij + # -- update output accumulator -- + # scale p + # scale acc + acc_scale = alpha + # acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load(v_ptrs + + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=dim_mask[None, :] & + ((start_n + offs_n[:, None]) < + cur_batch_seq_len - cur_batch_ctx_len), + other=0.0) + p = p.to(v.dtype) + + acc += tl.dot(p, v, allow_tf32=False) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + + acc = acc / l_i[:, None] + + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + offs_d[None, :] * stride_od) + out_ptrs = Out + off_o + tl.store(out_ptrs, + acc, + mask=dim_mask[None, :] & + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)) + return + + @torch.inference_mode() + def context_attention_fwd(q, + k, + v, + o, + kv_cache_dtype: str, + k_cache, + v_cache, + b_loc, + b_start_loc, + b_seq_len, + b_ctx_len, + max_input_len, + k_scale: float = 1.0, + v_scale: float = 1.0, + alibi_slopes=None, + sliding_window=None): + + BLOCK = 128 if current_platform.has_device_capability(80) else 64 + NUM_WARPS = 8 + + # need to reduce num. blocks when using fp32 + # due to increased use of GPU shared memory + if q.dtype is torch.float32: + BLOCK = BLOCK // 2 + + # Conversion of FP8 Tensor from uint8 storage to + # appropriate torch.dtype for interpretation by Triton + if "fp8" in kv_cache_dtype: + assert (k_cache.dtype == torch.uint8) + assert (v_cache.dtype == torch.uint8) + + if kv_cache_dtype in ("fp8", "fp8_e4m3"): + target_dtype = torch.float8_e4m3fn + elif kv_cache_dtype == "fp8_e5m2": + target_dtype = torch.float8_e5m2 + else: + raise ValueError("Unsupported FP8 dtype:", kv_cache_dtype) + + k_cache = k_cache.view(target_dtype) + v_cache = v_cache.view(target_dtype) + + if (k_cache.dtype == torch.uint8 + or v_cache.dtype == torch.uint8 and kv_cache_dtype == "auto"): + raise ValueError("kv_cache_dtype='auto' unsupported for\ + FP8 KV Cache prefill kernel") + + # shape constraints + Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1] + assert Lq == Lk and Lk == Lv + # round up Lk to a power of 2 - this is required for Triton block size + Lk_padded = triton.next_power_of_2(Lk) + + sm_scale = 1.0 / (Lq**0.5) + batch, head = b_seq_len.shape[0], q.shape[1] + num_queries_per_kv = q.shape[1] // k.shape[1] + + grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head, + + # 0 means "disable" + if sliding_window is None or sliding_window <= 0: + sliding_window = 0 + + if alibi_slopes is not None: + _fwd_kernel_alibi[grid]( + q, + k, + v, + k_cache, + v_cache, + b_loc, + sm_scale, + k_scale, + v_scale, + b_start_loc, + b_seq_len, + b_ctx_len, + alibi_slopes, + v_cache.shape[3], + k_cache.shape[4], + o, + b_loc.stride(0), + b_loc.stride(1), + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + k_cache.stride( + 4 + ), #[num_blocks, num_kv_heads, head_size/x, block_size, x] + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride( + 3), #[num_blocks, num_kv_heads, head_size, block_size] + num_queries_per_kv=num_queries_per_kv, + BLOCK_M=BLOCK, + BLOCK_DMODEL=Lk, + BLOCK_DMODEL_PADDED=Lk_padded, + BLOCK_N=BLOCK, + num_warps=NUM_WARPS, + num_stages=1, + ) + return + + _fwd_kernel[grid]( + q, + k, + v, + k_cache, + v_cache, + b_loc, + sm_scale, + k_scale, + v_scale, + b_start_loc, + b_seq_len, + b_ctx_len, + v_cache.shape[3], + k_cache.shape[4], + o, + b_loc.stride(0), + b_loc.stride(1), + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + k_cache.stride( + 4), #[num_blocks, num_kv_heads, head_size/x, block_size, x] + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride( + 3), #[num_blocks, num_kv_heads, head_size, block_size] + num_queries_per_kv=num_queries_per_kv, + BLOCK_M=BLOCK, + BLOCK_DMODEL=Lk, + BLOCK_DMODEL_PADDED=Lk_padded, + BLOCK_N=BLOCK, + SLIDING_WINDOW=sliding_window, + num_warps=NUM_WARPS, + num_stages=1, + ) + return diff --git a/vllm/attention/ops/triton_flash_attention.py b/vllm/attention/ops/triton_flash_attention.py new file mode 100644 index 00000000..f9421111 --- /dev/null +++ b/vllm/attention/ops/triton_flash_attention.py @@ -0,0 +1,820 @@ +#!/usr/bin/env python +""" +Fused Attention +=============== + +This is a Triton implementation of the Flash Attention v2 algorithm from Tri Dao +(https://tridao.me/publications/flash2/flash2.pdf) +Credits: OpenAI kernel team, AMD ML Frameworks Triton team + +Features supported: + +1) Fwd with causal masking +2) Any sequence lengths without padding (currently fwd kernel only) +3) Support for different sequence lengths for q and k +4) Nested tensor API currently does not support dropout or bias. + +Not currently supported: + +1) Non power of two head dims + +""" + +import torch +import triton +import triton.language as tl + +torch_dtype: tl.constexpr = torch.float16 + + +@triton.jit +def cdiv_fn(x, y): + return (x + y - 1) // y + + +@triton.jit +def max_fn(x, y): + return tl.math.max(x, y) + + +@triton.jit +def dropout_offsets(philox_seed, philox_offset, dropout_p, m, n, stride): + ms = tl.arange(0, m) + ns = tl.arange(0, n) + return philox_offset + ms[:, None] * stride + ns[None, :] + + +@triton.jit +def dropout_rng(philox_seed, philox_offset, dropout_p, m, n, stride): + rng_offsets = dropout_offsets(philox_seed, philox_offset, dropout_p, m, n, + stride).to(tl.uint32) + # TODO: use tl.randint for better performance + return tl.rand(philox_seed, rng_offsets) + + +@triton.jit +def dropout_mask(philox_seed, philox_offset, dropout_p, m, n, stride): + rng_output = dropout_rng(philox_seed, philox_offset, dropout_p, m, n, + stride) + rng_keep = rng_output > dropout_p + return rng_keep + + +@triton.jit +def load_fn(block_ptr, first, second, pad): + if first and second: + tensor = tl.load(block_ptr, boundary_check=(0, 1), padding_option=pad) + elif first: + tensor = tl.load(block_ptr, boundary_check=(0, ), padding_option=pad) + elif second: + tensor = tl.load(block_ptr, boundary_check=(1, ), padding_option=pad) + else: + tensor = tl.load(block_ptr) + return tensor + + +@triton.jit +def _attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, + start_m, + actual_seqlen_k, + dropout_p, + philox_seed, + batch_philox_offset, + encoded_softmax_block_ptr, + block_min, + block_max, + offs_n_causal, + masked_blocks, + n_extra_tokens, + bias_ptr, + IS_CAUSAL: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + OFFS_M: tl.constexpr, + OFFS_N: tl.constexpr, + PRE_LOAD_V: tl.constexpr, + MASK_STEPS: tl.constexpr, + ENABLE_DROPOUT: tl.constexpr, + RETURN_ENCODED_SOFTMAX: tl.constexpr, + PADDED_HEAD: tl.constexpr, +): + # loop over k, v, and update accumulator + for start_n in range(block_min, block_max, BLOCK_N): + # For padded blocks, we will overrun the tensor size if + # we load all BLOCK_N. For others, the blocks are all within range. + k = load_fn( + K_block_ptr, + PADDED_HEAD, + MASK_STEPS and (n_extra_tokens != 0), + "zero", + ) + if PRE_LOAD_V: + v = load_fn( + V_block_ptr, + MASK_STEPS and (n_extra_tokens != 0), + PADDED_HEAD, + "zero", + ) + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + # We start from end of seqlen_k so only the first iteration would need + # to be checked for padding if it is not a multiple of block_n + # TODO: This can be optimized to only be true for the padded block. + if MASK_STEPS: # noqa: SIM102 + # If this is the last block / iteration, we want to + # mask if the sequence length is not a multiple of block size + # a solution is to always do BLOCK_M // BLOCK_N + 1 steps + # if not is_modulo_mn. last step might get wasted but that is okay. + # check if this masking works for that case. + if (start_n + BLOCK_N == block_max) and (n_extra_tokens != 0): + boundary_m = tl.full([BLOCK_M], + actual_seqlen_k, + dtype=tl.int32) + size_n = start_n + OFFS_N[None, :] + mask = size_n < boundary_m[:, None] + qk = tl.where(mask, qk, float("-inf")) + if IS_CAUSAL: + causal_boundary = start_n + offs_n_causal + causal_mask = OFFS_M[:, None] >= causal_boundary[None, :] + qk = tl.where(causal_mask, qk, float("-inf")) + # -- compute qk ---- + qk += tl.dot(q, k) + if bias_ptr is not None: + bias = load_fn(bias_ptr, False, MASK_STEPS + and (n_extra_tokens != 0), "zero") + # While bias is added after multiplying qk with sm_scale, our + # optimization to use 2^x instead of e^x results in an additional + # scale factor of log2(e) which we must also multiply the bias with. + qk += bias * 1.44269504089 + m_ij = tl.maximum(m_i, tl.max(qk, 1)) + qk = qk - m_ij[:, None] + p = tl.math.exp2(qk) + + # CAVEAT: Must update l_ij before applying dropout + l_ij = tl.sum(p, 1) + if ENABLE_DROPOUT: + philox_offset = (batch_philox_offset + + start_m * BLOCK_M * actual_seqlen_k + start_n - + BLOCK_N) + keep = dropout_mask( + philox_seed, + philox_offset, + dropout_p, + BLOCK_M, + BLOCK_N, + actual_seqlen_k, + ) + if RETURN_ENCODED_SOFTMAX: + tl.store( + encoded_softmax_block_ptr, + tl.where(keep, p, + -p).to(encoded_softmax_block_ptr.type.element_ty), + ) + p = tl.where(keep, p, 0.0) + elif RETURN_ENCODED_SOFTMAX: + tl.store( + encoded_softmax_block_ptr, + p.to(encoded_softmax_block_ptr.type.element_ty), + ) + # -- update output accumulator -- + alpha = tl.math.exp2(m_i - m_ij) + acc = acc * alpha[:, None] + if not PRE_LOAD_V: + v = load_fn( + V_block_ptr, + MASK_STEPS and (n_extra_tokens != 0), + PADDED_HEAD, + "zero", + ) + # -- update m_i and l_i + l_i = l_i * alpha + l_ij + # update m_i and l_i + m_i = m_ij + acc += tl.dot(p.to(V_block_ptr.type.element_ty), v) + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + if bias_ptr is not None: + bias_ptr = tl.advance(bias_ptr, (0, BLOCK_N)) + if RETURN_ENCODED_SOFTMAX: + encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr, + (0, BLOCK_N)) + return acc, l_i, m_i + + +@triton.autotune( + configs=[ + triton.Config( + { + "BLOCK_M": 256, + "BLOCK_N": 64, + "waves_per_eu": 2, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=8, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "waves_per_eu": 2, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 256, + "BLOCK_N": 128, + "waves_per_eu": 2, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=8, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "waves_per_eu": 1, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "waves_per_eu": 3, + "PRE_LOAD_V": True, + }, + num_stages=1, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "waves_per_eu": 3, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 64, + "waves_per_eu": 4, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=8, + ), + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 32, + "waves_per_eu": 4, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=8, + ), + # TODO: This config fails with head_size not pow2 with data mismatches. + # triton.Config({'BLOCK_M': 32, 'BLOCK_N': 16, 'waves_per_eu': 1, + # 'PRE_LOAD_V': False}, num_stages=1, num_warps=4), + triton.Config( + { + "BLOCK_M": 16, + "BLOCK_N": 16, + "waves_per_eu": 1, + "PRE_LOAD_V": False, + }, + num_stages=1, + num_warps=4, + ), + ], + key=['IS_CAUSAL', 'dropout_p', 'BLOCK_DMODEL'], +) +@triton.jit +def attn_fwd( + Q, + K, + V, + bias, + sm_scale, + L, + Out, + stride_qz, + stride_qh, + stride_qm, + stride_qk, + stride_kz, + stride_kh, + stride_kn, + stride_kk, + stride_vz, + stride_vh, + stride_vk, + stride_vn, + stride_oz, + stride_oh, + stride_om, + stride_on, + stride_bz, + stride_bh, + stride_bm, + stride_bn, + cu_seqlens_q, + cu_seqlens_k, + dropout_p, + philox_seed, + philox_offset_base, + encoded_softmax, + HQ: tl.constexpr, + HK: tl.constexpr, + ACTUAL_BLOCK_DMODEL: tl.constexpr, + MAX_SEQLENS_Q: tl.constexpr, + MAX_SEQLENS_K: tl.constexpr, + VARLEN: tl.constexpr, + IS_CAUSAL: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + PRE_LOAD_V: tl.constexpr, + BIAS_TYPE: tl.constexpr, + ENABLE_DROPOUT: tl.constexpr, + RETURN_ENCODED_SOFTMAX: tl.constexpr, +): + start_m = tl.program_id(0) + off_h_q = tl.program_id(1) + off_z = tl.program_id(2) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + if VARLEN: + cu_seqlens_q_start = tl.load(cu_seqlens_q + off_z) + cu_seqlens_q_end = tl.load(cu_seqlens_q + off_z + 1) + seqlen_q = cu_seqlens_q_end - cu_seqlens_q_start + # We have a one-size-fits-all grid in id(0). Some seqlens might be too + # small for all start_m so for those we return early. + if start_m * BLOCK_M > seqlen_q: + return + cu_seqlens_k_start = tl.load(cu_seqlens_k + off_z) + cu_seqlens_k_end = tl.load(cu_seqlens_k + off_z + 1) + seqlen_k = cu_seqlens_k_end - cu_seqlens_k_start + else: + cu_seqlens_q_start = 0 + cu_seqlens_k_start = 0 + seqlen_q = MAX_SEQLENS_Q + seqlen_k = MAX_SEQLENS_K + + # Now we compute whether we need to exit early due to causal masking. + # This is because for seqlen_q > seqlen_k, M rows of the attn scores + # are completely masked, resulting in 0s written to the output, and + # inf written to LSE. We don't need to do any GEMMs in this case. + # This block of code determines what N is, and if this WG is operating + # on those M rows. + n_blocks = cdiv_fn(seqlen_k, BLOCK_N) + if IS_CAUSAL: + # If seqlen_q == seqlen_k, the attn scores are a square matrix. + # If seqlen_q != seqlen_k, attn scores are rectangular which means + # the causal mask boundary is bottom right aligned, and ends at either + # the top edge (seqlen_q < seqlen_k) or left edge. + # This captures the decrease in n_blocks if we have a rectangular attn + # matrix + n_blocks_seqlen = cdiv_fn( + (start_m + 1) * BLOCK_M + seqlen_k - seqlen_q, BLOCK_N) + # This is what adjusts the block_max for the current WG, only + # if IS_CAUSAL. Otherwise we want to always iterate through all n_blocks + n_blocks = min(n_blocks, n_blocks_seqlen) + # If we have no blocks after adjusting for seqlen deltas, this WG is + # part of the blocks that are all 0. We exit early. + if n_blocks <= 0: + o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om + + off_h_q * stride_oh) + O_block_ptr = tl.make_block_ptr( + base=Out + o_offset, + shape=(seqlen_q, BLOCK_DMODEL), + strides=(stride_om, stride_on), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, BLOCK_DMODEL), + order=(1, 0), + ) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=Out.type.element_ty) + # We still need to write 0s to the result + # tl.store(O_block_ptr, + # acc.to(Out.type.element_ty), boundary_check=(0,1)) + # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + # + offs_m + # We store inf to LSE, not -inf because in the bwd pass, + # we subtract this + # from qk which makes it -inf, such that exp(qk - inf) = 0 + # for these masked blocks. + # l = tl.full([BLOCK_M], value=float("inf"), dtype=tl.float32) + # tl.store(l_ptrs, l) + # TODO: Should dropout and return encoded softmax be handled here? + return + + # If MQA / GQA, set the K and V head offsets appropriately. + GROUP_SIZE: tl.constexpr = HQ // HK + off_h_k = off_h_q // GROUP_SIZE if GROUP_SIZE != 1 else off_h_q + + n_extra_tokens = 0 + if seqlen_k < BLOCK_N: + n_extra_tokens = BLOCK_N - seqlen_k + elif seqlen_k % BLOCK_N: + n_extra_tokens = seqlen_k % BLOCK_N + padded_head = ACTUAL_BLOCK_DMODEL != BLOCK_DMODEL + + # Compute pointers for all the tensors used in this kernel. + q_offset = (off_z * stride_qz + off_h_q * stride_qh + + cu_seqlens_q_start * stride_qm) + Q_block_ptr = tl.make_block_ptr( + base=Q + q_offset, + shape=(seqlen_q, ACTUAL_BLOCK_DMODEL), + strides=(stride_qm, stride_qk), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, BLOCK_DMODEL), + order=(1, 0), + ) + k_offset = (off_z * stride_kz + off_h_k * stride_kh + + cu_seqlens_k_start * stride_kn) + K_block_ptr = tl.make_block_ptr( + base=K + k_offset, + shape=(ACTUAL_BLOCK_DMODEL, seqlen_k), + strides=(stride_kk, stride_kn), + offsets=(0, 0), + block_shape=(BLOCK_DMODEL, BLOCK_N), + order=(0, 1), + ) + v_offset = (off_z * stride_vz + off_h_k * stride_vh + + cu_seqlens_k_start * stride_vk) + V_block_ptr = tl.make_block_ptr( + base=V + v_offset, + shape=(seqlen_k, ACTUAL_BLOCK_DMODEL), + strides=(stride_vk, stride_vn), + offsets=(0, 0), + block_shape=(BLOCK_N, BLOCK_DMODEL), + order=(1, 0), + ) + if BIAS_TYPE != 0: + bias_ptr = tl.make_block_ptr( + base=bias + off_h_q * stride_bh, + shape=(seqlen_q, seqlen_k), + strides=(stride_bm, stride_bn), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, BLOCK_N), + order=(1, 0), + ) + else: + bias_ptr = None + if ENABLE_DROPOUT: + batch_philox_offset = philox_offset_base \ + + (off_z * HQ + off_h_q) \ + * seqlen_q * seqlen_k + else: + batch_philox_offset = 0 + # We can ask to return the dropout mask without actually doing any dropout. + # In this case, we return an invalid pointer so indicate the mask is not i + # valid. + # TODO: Fix encoded softmax. It currently uses just h_q in the base offset. + if RETURN_ENCODED_SOFTMAX: + encoded_softmax_block_ptr = tl.make_block_ptr( + base=encoded_softmax + off_h_q * seqlen_q * seqlen_k, + shape=(seqlen_q, seqlen_k), + strides=(seqlen_k, 1), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, BLOCK_N), + order=(1, 0), + ) + else: + encoded_softmax_block_ptr = 0 + # initialize pointer to m and l + m_i = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) + # scale sm_scale by log_2(e) and use 2^x in the loop as we do not + # have native e^x support in HW. + qk_scale = sm_scale * 1.44269504089 + # Q is loaded once at the beginning and shared by all N blocks. + q = load_fn(Q_block_ptr, True, padded_head, "zero") + q = (q * qk_scale).to(Q_block_ptr.type.element_ty) + + # Here we compute how many full and masked blocks we have. + padded_block_k = n_extra_tokens != 0 + is_modulo_mn = not padded_block_k and (seqlen_q % BLOCK_M == 0) + if IS_CAUSAL: + # There are always at least BLOCK_M // BLOCK_N masked blocks. + # Additionally there might be one more due to dissimilar seqlens. + masked_blocks = BLOCK_M // BLOCK_N + (not is_modulo_mn) + else: + # Padding on Q does not need to be masked in the FA loop. + masked_blocks = padded_block_k + # if IS_CAUSAL, not is_modulo_mn does not always result in an additional + # block. In this case we might exceed n_blocks so pick the min. + masked_blocks = min(masked_blocks, n_blocks) + n_full_blocks = n_blocks - masked_blocks + block_min = 0 + block_max = n_blocks * BLOCK_N + # Compute for full blocks. Here we set causal to false regardless of its + # value because there is no masking. Similarly we do not need padding. + if n_full_blocks > 0: + block_max = (n_blocks - masked_blocks) * BLOCK_N + acc, l_i, m_i = _attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, + start_m, + seqlen_k, + dropout_p, + philox_seed, + batch_philox_offset, + encoded_softmax_block_ptr, + # _, _, offs_n_causal, masked_blocks, n_extra_tokens, _ + block_min, + block_max, + 0, + 0, + 0, + bias_ptr, + # IS_CAUSAL, .... + False, + BLOCK_M, + BLOCK_DMODEL, + BLOCK_N, + offs_m, + offs_n, + # _, MASK_STEPS, ... + PRE_LOAD_V, + False, + ENABLE_DROPOUT, + RETURN_ENCODED_SOFTMAX, + padded_head, + ) + block_min = block_max + block_max = n_blocks * BLOCK_N + + tl.debug_barrier() + # Remaining blocks, if any, are full / not masked. + if masked_blocks > 0: + offs_n_causal = offs_n + (seqlen_q - seqlen_k) if IS_CAUSAL else 0 + K_block_ptr = tl.advance(K_block_ptr, (0, n_full_blocks * BLOCK_N)) + V_block_ptr = tl.advance(V_block_ptr, (n_full_blocks * BLOCK_N, 0)) + if bias_ptr is not None: + bias_ptr = tl.advance(bias_ptr, (0, n_full_blocks * BLOCK_N)) + if RETURN_ENCODED_SOFTMAX: + encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr, + (0, n_full_blocks)) + acc, l_i, m_i = _attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, + start_m, + seqlen_k, + dropout_p, + philox_seed, + batch_philox_offset, + encoded_softmax_block_ptr, + block_min, + block_max, + offs_n_causal, + masked_blocks, + n_extra_tokens, + bias_ptr, + IS_CAUSAL, + BLOCK_M, + BLOCK_DMODEL, + BLOCK_N, + offs_m, + offs_n, + # _, MASK_STEPS, ... + PRE_LOAD_V, + True, + ENABLE_DROPOUT, + RETURN_ENCODED_SOFTMAX, + padded_head, + ) + # epilogue + acc = acc / l_i[:, None] + if ENABLE_DROPOUT: + acc = acc / (1 - dropout_p) + # If seqlen_q > seqlen_k but the delta is not a multiple of BLOCK_M, + # then we have one block with a row of all NaNs which come from computing + # softmax over a row of all -infs (-inf - inf = NaN). We check for that here + # and store 0s where there are NaNs as these rows should've been zeroed out. + end_m_idx = (start_m + 1) * BLOCK_M + start_m_idx = start_m * BLOCK_M + causal_start_idx = seqlen_q - seqlen_k + acc = acc.to(Out.type.element_ty) + if IS_CAUSAL: # noqa: SIM102 + if causal_start_idx > start_m_idx and causal_start_idx < end_m_idx: + out_mask_boundary = tl.full((BLOCK_DMODEL, ), + causal_start_idx, + dtype=tl.int32) + mask_m_offsets = start_m_idx + tl.arange(0, BLOCK_M) + out_ptrs_mask = (mask_m_offsets[:, None] >= + out_mask_boundary[None, :]) + z = 0.0 + acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty)) + # write back LSE + # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m + # If seqlen_q not multiple of BLOCK_M, we need to mask out the last + # few rows. This is only true for the last M block. For others, + # overflow_size will be -ve + # overflow_size = end_m_idx - seqlen_q + # if overflow_size > 0: + # boundary = tl.full((BLOCK_M,), BLOCK_M - overflow_size, dtype=tl.int32) + # # This is a > check because mask being 0 blocks the store. + # l_ptrs_mask = boundary > tl.arange(0, BLOCK_M) + # tl.store(l_ptrs, m_i + tl.math.log2(l_i), mask=l_ptrs_mask) + # else: + # tl.store(l_ptrs, m_i + tl.math.log2(l_i)) + + # write back O + o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om + + off_h_q * stride_oh) + O_block_ptr = tl.make_block_ptr( + base=Out + o_offset, + shape=(seqlen_q, ACTUAL_BLOCK_DMODEL), + strides=(stride_om, stride_on), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, BLOCK_DMODEL), + order=(1, 0), + ) + # Need boundary check on this to make sure the padding from the + # Q and KV tensors in both dims are not part of what we store back. + # TODO: Do the boundary check optionally. + tl.store(O_block_ptr, acc, boundary_check=(0, 1)) + + +def check_args( + q, + k, + v, + o, + varlen=True, + max_seqlens=None, + cu_seqlens_q=None, + cu_seqlens_k=None, +): + assert q.dim() == k.dim() and q.dim() == v.dim() + if varlen: + assert q.dim() == 3 + total_q, nheads_q, head_size = q.shape + total_k, nheads_k, _ = k.shape + assert cu_seqlens_q is not None + assert cu_seqlens_k is not None + assert len(cu_seqlens_q) == len(cu_seqlens_k) + else: + assert q.dim() == 4 + batch, nheads_q, seqlen_q, head_size = q.shape + _, nheads_k, seqlen_k, _ = k.shape + assert max_seqlens > 0 + assert k.shape == v.shape + assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1] + # TODO: Change assert if we support qkl f8 and v f16 + assert q.dtype == k.dtype and q.dtype == v.dtype + assert head_size <= 256 + assert o.shape == q.shape + assert (nheads_q % nheads_k) == 0 + + +class _attention(torch.autograd.Function): + + @staticmethod + def forward( + ctx, + q, + k, + v, + o, + cu_seqlens_q, + cu_seqlens_k, + max_seqlens_q, + max_seqlens_k, + causal=False, + sm_scale=1.0, + bias=None, + ): + if o is None: + o = torch.empty_like(q, dtype=v.dtype) + + check_args( + q, + k, + v, + o, + varlen=True, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + ) + if True: # varlen + total_q, nheads_q, head_size = q.shape + total_k, nheads_k, _ = k.shape + batch = len(cu_seqlens_q) - 1 + q_strides = (0, q.stride(1), q.stride(0), q.stride(2)) + k_strides = (0, k.stride(1), k.stride(0), k.stride(2)) + v_strides = (0, v.stride(1), v.stride(0), v.stride(2)) + o_strides = (0, o.stride(1), o.stride(0), o.stride(2)) + else: + batch, seqlen_q, nheads_q, head_size = q.shape + _, seqlen_k, nheads_k, _ = k.shape + q_strides = (q.stride(0), q.stride(2), q.stride(1), q.stride(3)) + k_strides = (k.stride(0), k.stride(2), k.stride(1), k.stride(3)) + v_strides = (v.stride(0), v.stride(2), v.stride(1), v.stride(3)) + o_strides = (o.stride(0), o.stride(2), o.stride(1), o.stride(3)) + + # Get closest power of 2 over or equal to 32. + unpadded_head_dims = {32, 64, 128, 256} + if head_size not in unpadded_head_dims: + padded_d_model = None + for i in unpadded_head_dims: + if i > head_size: + padded_d_model = i + break + assert padded_d_model is not None + else: + padded_d_model = head_size + + grid = lambda META: ( + triton.cdiv(max_seqlens_q, META["BLOCK_M"]), + nheads_q, + batch, + ) + + encoded_softmax = None + + # Seed the RNG so we get reproducible results for testing. + philox_seed = 0x1BF52 + philox_offset = 0x1D4B42 + + if bias is not None: + bias_strides = ( + bias.stride(0), + bias.stride(1), + bias.stride(2), + bias.stride(3), + ) + else: + bias_strides = (0, 0, 0, 0) + + attn_fwd[grid]( + q, + k, + v, + bias, + sm_scale, + None, + o, + *q_strides, + *k_strides, + *v_strides, + *o_strides, + *bias_strides, + cu_seqlens_q, + cu_seqlens_k, + dropout_p=0.0, + philox_seed=philox_seed, + philox_offset_base=philox_offset, + encoded_softmax=encoded_softmax, + HQ=nheads_q, + HK=nheads_k, + ACTUAL_BLOCK_DMODEL=head_size, + MAX_SEQLENS_Q=max_seqlens_q, + MAX_SEQLENS_K=max_seqlens_k, + IS_CAUSAL=causal, + VARLEN=True, + BLOCK_DMODEL=padded_d_model, + BIAS_TYPE=0 if bias is None else 1, + ENABLE_DROPOUT=False, + RETURN_ENCODED_SOFTMAX=False, + ) + + ctx.grid = grid + ctx.sm_scale = sm_scale + ctx.BLOCK_DMODEL = head_size + ctx.causal = causal + ctx.dropout_p = 0.0 + ctx.philox_seed = philox_seed + ctx.philox_offset = philox_offset + ctx.encoded_softmax = encoded_softmax + ctx.return_encoded_softmax = False + return o, encoded_softmax + + +triton_attention = _attention.apply diff --git a/vllm/attention/selector.py b/vllm/attention/selector.py new file mode 100644 index 00000000..ae90e032 --- /dev/null +++ b/vllm/attention/selector.py @@ -0,0 +1,305 @@ +import enum +import os +from contextlib import contextmanager +from functools import lru_cache +from typing import Generator, Optional, Type + +import torch + +import vllm.envs as envs +from vllm.attention.backends.abstract import AttentionBackend +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils import STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino, is_xpu + +logger = init_logger(__name__) + + +class _Backend(enum.Enum): + FLASH_ATTN = enum.auto() + XFORMERS = enum.auto() + ROCM_FLASH = enum.auto() + TORCH_SDPA = enum.auto() + OPENVINO = enum.auto() + FLASHINFER = enum.auto() + PALLAS = enum.auto() + IPEX = enum.auto() + NO_ATTENTION = enum.auto() + + +def backend_name_to_enum(backend_name: str) -> _Backend: + assert backend_name is not None + + backend_members = _Backend.__members__ + if backend_name not in backend_members: + raise ValueError(f"Invalid attention backend '{backend_name}'. " + f"Available backends: {', '.join(backend_members)} " + "(case-sensitive).") + + return _Backend[backend_name] + + +def get_env_variable_attn_backend() -> Optional[_Backend]: + ''' + Get the backend override specified by the vLLM attention + backend environment variable, if one is specified. + + Returns: + + * _Backend enum value if an override is specified + * None otherwise + ''' + backend_name = os.environ.get(STR_BACKEND_ENV_VAR) + return (None + if backend_name is None else backend_name_to_enum(backend_name)) + + +# Global state allows a particular choice of backend +# to be forced, overriding the logic which auto-selects +# a backend based on system & workload configuration +# (default behavior if this variable is None) +# +# THIS SELECTION TAKES PRECEDENCE OVER THE +# VLLM ATTENTION BACKEND ENVIRONMENT VARIABLE +forced_attn_backend: Optional[_Backend] = None + + +def global_force_attn_backend(attn_backend: Optional[_Backend]) -> None: + ''' + Force all attention operations to use a specified backend. + + Passing `None` for the argument re-enables automatic + backend selection., + + Arguments: + + * attn_backend: backend selection (None to revert to auto) + ''' + global forced_attn_backend + forced_attn_backend = attn_backend + + +def get_global_forced_attn_backend() -> Optional[_Backend]: + ''' + Get the currently-forced choice of attention backend, + or None if auto-selection is currently enabled. + ''' + return forced_attn_backend + + +@lru_cache(maxsize=None) +def get_attn_backend( + head_size: int, + sliding_window: Optional[int], + dtype: torch.dtype, + kv_cache_dtype: Optional[str], + block_size: int, + is_attention_free: bool, + is_blocksparse: bool = False, +) -> Type[AttentionBackend]: + """Selects which attention backend to use and lazily imports it.""" + + # if is_blocksparse: + # logger.info("Using BlocksparseFlashAttention backend.") + # from vllm.attention.backends.blocksparse_attn import ( + # BlocksparseFlashAttentionBackend) + # return BlocksparseFlashAttentionBackend + + backend = which_attn_to_use(head_size, sliding_window, dtype, + kv_cache_dtype, block_size, is_attention_free) + if backend == _Backend.FLASH_ATTN: + from vllm.attention.backends.flash_attn import ( # noqa: F401 + FlashAttentionBackend) + return FlashAttentionBackend + if backend == _Backend.XFORMERS: + logger.info("Using XFormers backend.") + from vllm.attention.backends.xformers import ( # noqa: F401 + XFormersBackend) + return XFormersBackend + elif backend == _Backend.ROCM_FLASH: + logger.info("Using ROCmFlashAttention backend.") + from vllm.attention.backends.rocm_flash_attn import ( # noqa: F401 + ROCmFlashAttentionBackend) + return ROCmFlashAttentionBackend + elif backend == _Backend.TORCH_SDPA: + assert is_cpu(), RuntimeError( + "Torch SDPA backend is only used for the CPU device.") + logger.info("Using Torch SDPA backend.") + from vllm.attention.backends.torch_sdpa import TorchSDPABackend + return TorchSDPABackend + elif backend == _Backend.OPENVINO: + logger.info("Using OpenVINO Attention backend.") + from vllm.attention.backends.openvino import OpenVINOAttentionBackend + return OpenVINOAttentionBackend + elif backend == _Backend.IPEX: + assert is_xpu(), RuntimeError( + "IPEX attention backend is only used for the XPU device.") + logger.info("Using IPEX attention backend.") + from vllm.attention.backends.ipex_attn import IpexAttnBackend + return IpexAttnBackend + elif backend == _Backend.FLASHINFER: + logger.info("Using Flashinfer backend.") + from vllm.attention.backends.flashinfer import FlashInferBackend + return FlashInferBackend + elif backend == _Backend.PALLAS: + logger.info("Using Pallas backend.") + from vllm.attention.backends.pallas import PallasAttentionBackend + return PallasAttentionBackend + elif backend == _Backend.NO_ATTENTION: + from vllm.attention.backends.placeholder_attn import ( + PlaceholderAttentionBackend) + return PlaceholderAttentionBackend + else: + raise ValueError("Invalid attention backend.") + + +def which_attn_to_use( + head_size: int, + sliding_window: Optional[int], + dtype: torch.dtype, + kv_cache_dtype: Optional[str], + block_size: int, + is_attention_free: bool, +) -> _Backend: + """Returns which flash attention backend to use.""" + # Default case. + selected_backend = _Backend.FLASH_ATTN + + # If there are no attention layers (e.g. we are running Mamba), + # use the placeholder NO_ATTENTION + if is_attention_free: + return _Backend.NO_ATTENTION + + # Check whether a particular choice of backend was + # previously forced. + # + # THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND + # ENVIRONMENT VARIABLE. + backend_by_global_setting: Optional[_Backend] = ( + get_global_forced_attn_backend()) + if backend_by_global_setting is not None: + selected_backend = backend_by_global_setting + else: + # Check the environment variable and override if specified + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND + if backend_by_env_var is not None: + selected_backend = backend_name_to_enum(backend_by_env_var) + + if is_cpu(): + if selected_backend != _Backend.TORCH_SDPA: + logger.info("Cannot use %s backend on CPU.", selected_backend) + return _Backend.TORCH_SDPA + + if is_openvino(): + if selected_backend != _Backend.OPENVINO: + logger.info("Cannot use %s backend on OpenVINO.", selected_backend) + return _Backend.OPENVINO + + if is_xpu(): + if selected_backend != _Backend.IPEX: + logger.info("Cannot use %s backend on XPU.", selected_backend) + return _Backend.IPEX + + if current_platform.is_tpu(): + if selected_backend != _Backend.PALLAS: + logger.info("Cannot use %s backend on TPU.", selected_backend) + return _Backend.PALLAS + + if is_hip(): + # AMD GPUs. + selected_backend = (_Backend.ROCM_FLASH if selected_backend + == _Backend.FLASH_ATTN else selected_backend) + if selected_backend == _Backend.ROCM_FLASH: + if not current_platform.has_device_capability(90): + # not Instinct series GPUs. + logger.info("flash_attn is not supported on NAVI GPUs.") + else: + logger.info("%s is not supported in AMD GPUs.", selected_backend) + return _Backend.ROCM_FLASH + + # FlashAttn in NVIDIA GPUs. + if selected_backend == _Backend.FLASH_ATTN: + if not current_platform.has_device_capability(80): + # Volta and Turing NVIDIA GPUs. + logger.info( + "Cannot use FlashAttention-2 backend for Volta and Turing " + "GPUs.") + selected_backend = _Backend.XFORMERS + elif dtype not in (torch.float16, torch.bfloat16): + logger.info( + "Cannot use FlashAttention-2 backend for dtype other than " + "torch.float16 or torch.bfloat16.") + selected_backend = _Backend.XFORMERS + elif kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"): + logger.info( + "Cannot use FlashAttention-2 backend for FP8 KV cache.") + logger.warning( + "Please use FlashInfer backend with FP8 KV Cache for " + "better performance by setting environment variable " + "VLLM_ATTENTION_BACKEND=FLASHINFER") + selected_backend = _Backend.XFORMERS + elif block_size % 16 != 0: + logger.info( + "Cannot use FlashAttention-2 backend for block size not " + "divisible by 16.") + selected_backend = _Backend.XFORMERS + # elif sliding_window is not None: + # logger.info( + # "Cannot use FlashAttention-2 backend due to sliding window.") + # selected_backend = _Backend.XFORMERS + + # FlashAttn is valid for the model, checking if the package is installed. + if selected_backend == _Backend.FLASH_ATTN: + try: + # import vllm_flash_attn # noqa: F401 + + from vllm.attention.backends.flash_attn import ( # noqa: F401 + FlashAttentionBackend) + + supported_sizes = FlashAttentionBackend.get_supported_head_sizes() + if head_size not in supported_sizes: + logger.info( + "Cannot use FlashAttention-2 backend for head size %d.", + head_size) + selected_backend = _Backend.XFORMERS + except ImportError: + logger.info( + "Cannot use FlashAttention-2 backend because the " + "vllm.vllm_flash_attn package is not found. " + "Make sure that vllm_flash_attn was built and installed " + "(on by default).") + selected_backend = _Backend.XFORMERS + + return selected_backend + + +@contextmanager +def global_force_attn_backend_context_manager( + attn_backend: _Backend) -> Generator[None, None, None]: + ''' + Globally force a vLLM attention backend override within a + context manager, reverting the global attention backend + override to its prior state upon exiting the context + manager. + + Arguments: + + * attn_backend: attention backend to force + + Returns: + + * Generator + ''' + + # Save the current state of the global backend override (if any) + original_value = get_global_forced_attn_backend() + + # Globally force the new backend override + global_force_attn_backend(attn_backend) + + # Yield control back to the enclosed code block + try: + yield + finally: + # Revert the original global backend override, if any + global_force_attn_backend(original_value) diff --git a/vllm/beam_search.py b/vllm/beam_search.py new file mode 100644 index 00000000..04624b8b --- /dev/null +++ b/vllm/beam_search.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class BeamSearchSequence: + """A sequence for beam search. + It keeps track of the tokens and the log probability of the sequence. + The text field is optional and will only be filled when the sequence is + about to be returned to the user. + """ + # The tokens includes the prompt. + tokens: List[int] + cum_logprob: float = 0.0 + text: Optional[str] = None + + +@dataclass +class BeamSearchOutput: + """The output of beam search. + It contains the list of the best beam search sequences. + The length of the list is equal to the beam width. + """ + sequences: List[BeamSearchSequence] + + +class BeamSearchInstance: + + def __init__(self, prompt_tokens: List[int]): + self.beams: List[BeamSearchSequence] = [ + BeamSearchSequence(tokens=prompt_tokens) + ] + self.completed: List[BeamSearchSequence] = [] + + +def get_beam_search_score( + tokens: List[int], + cumulative_logprob: float, + eos_token_id: int, + length_penalty: float = 1.0, +) -> float: + """Calculate the beam search score with length penalty. + + Adapted from + + https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938 + """ + seq_len = len(tokens) + if tokens[-1] == eos_token_id: + seq_len -= 1 + + return cumulative_logprob / (seq_len**length_penalty) + + +def create_sort_beams_key_function(eos_token_id: int, length_penalty: float): + + def sort_beams_key(x: BeamSearchSequence) -> float: + return get_beam_search_score(x.tokens, x.cum_logprob, eos_token_id, + length_penalty) + + return sort_beams_key diff --git a/vllm/block.py b/vllm/block.py new file mode 100644 index 00000000..47c381c1 --- /dev/null +++ b/vllm/block.py @@ -0,0 +1,88 @@ +"""Token blocks.""" +from typing import TYPE_CHECKING, Iterator, List, Optional + +from vllm.utils import Device + +DEFAULT_LAST_ACCESSED_TIME: float = -1 + + +class PhysicalTokenBlock: + """Represents the state of a block in the KV cache.""" + + def __init__( + self, + device: Device, + block_number: int, + block_size: int, + block_hash: int, + num_hashed_tokens: int, + ) -> None: + self.device = device + self.block_number = block_number + self.block_size = block_size + self.block_hash = block_hash + self.num_hashed_tokens = num_hashed_tokens + + self.ref_count = 0 + self.last_accessed = DEFAULT_LAST_ACCESSED_TIME + + self.computed = False + + def __repr__(self) -> str: + return (f'PhysicalTokenBlock(device={self.device}, ' + f'block_number={self.block_number}, ' + f'num_hashed_tokens={self.num_hashed_tokens}, ' + f'ref_count={self.ref_count}, ' + f'last_accessed={self.last_accessed}, ' + f'computed={self.computed})') + + +class BlockTable: + """Holds a list of blocks with caching of their associated block_ids + """ + + def __init__(self, blocks: Optional[List[PhysicalTokenBlock]] = None): + self._blocks: List[PhysicalTokenBlock] = [] + self._block_ids: List[int] = [] + + if blocks is not None: + for block in blocks: + self.append(block) + + def append(self, block: PhysicalTokenBlock): + self._blocks.append(block) + self._block_ids.append(block.block_number) + + def __len__(self) -> int: + return len(self._blocks) + + def __getitem__(self, key): + return self._blocks[key] + + if TYPE_CHECKING: + + def __iter__(self) -> Iterator[PhysicalTokenBlock]: + raise RuntimeError("Method should be automatically generated") + + def __setitem__(self, key, value): + if isinstance(key, slice): + blocks = value + self._blocks[key] = blocks + self._block_ids[key] = [b.block_number for b in blocks] + else: + block = value + self._blocks[key] = block + self._block_ids[key] = block.block_number + + def reset(self): + self._blocks = [] + self._block_ids = [] + + def copy(self) -> "BlockTable": + return BlockTable(self._blocks) + + def list(self) -> List[PhysicalTokenBlock]: + return self._blocks + + def ids(self) -> List[int]: + return self._block_ids diff --git a/vllm/compilation/__init__.py b/vllm/compilation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/compilation/__pycache__/__init__.cpython-310.pyc b/vllm/compilation/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f5fb174fd9d28bdf8caf5281e96aa7e5f8ff27f GIT binary patch literal 157 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_esX?pL1s>3NhVMrK0Y%qvm`!V Zub}c4hfQvNN@-529mwQjCLqDW002=UB|88B literal 0 HcmV?d00001 diff --git a/vllm/compilation/__pycache__/backends.cpython-310.pyc b/vllm/compilation/__pycache__/backends.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cdd64f25009c68e06e082ef0a37b33d0d745d52 GIT binary patch literal 5072 zcmb7I-)|g89pBksw|Blf+i~K=4oX%crCezpQiMuNlr~Ko6x7w}jC%d7F>lw!GtfLYx4^5i@}5 zGa69nEO$Gd0EJh7#0C+YFXrdw;!irB=AmhaVrOS;*tUdw9>0$O<=KQ8Kf&+ zBHMe~7}U`&p=G}`G;LWMvO-(cE^F_zw|+O&3w@Pwc0+%k$vt}~|5?@hFP9P3YPvCG<9&kpUvuK1HX80q=s>Jf0T0~~vVGdrvm zwGqxc7?A|d>eT8{#Va7Y#|rB%)-TxZbWx+x%@sAa)%XoH2ETw?*!iHm$JHAD*_YRb zybh$O;az`>oS=EEoF6_$&ZiE@`OgD#(zwPQSf2x_|5`Z1MpEC0)dBl^Q6Dx68!|Tb zWvt}-^<-uy?y#q+vYmtij4d_}m* z$KANk2N3{$SBf+Wy4747xM9!z&c-6~;U-`sk2c(uAai@+Ey1x&n7Lt=4TPdBTeSW@W0JyIY26$Mxq+x> z-&haiayDOD*m;0ZDn%CZK@dp_b06u1Z1gcL7mIpUQ#k#YhUa+1)Rdt55=DS{uR z?ijjznmmKozh63=$LZNLOrNRF<(cQ=!Ag8CUOyX!OJ~!Ke5Ie9U5ldlY?TjH@;kc} zbZkHMz)Z6}DrO@%^SWFMe2TTT6xKFw^?(I;5bXq!57?tkIqc4L~g^#(H; z+Wm&vVEWtCnrvFPEk}2NYnsgpw??=&Yb)#u>x6C_ghfBXY+xPd(^DUt^|n5XF;>M| zdst1AFzgD$d4NG{IR@6fFTVt4yU5r34^>DOn#$xqz<@JUMOw^;$fAX@tKUT~2F!?1 zB@HdWbgm7UoGHv*gG?;40bwRz=vv;kDr61yKq(!@C`Q;IpT*3Bh?mj?!uao z6zkv@ZkW3&?ogO{KN8v6^_@Ilid=k3yCbJbdM}HmlQ@Y`E zlw#iOXAtKe2q(V`n$jMm9PU=yRXJAH5P1?Xrp~5gEAJ@V;#Lr&g!5H6%F0=|8Q7=d z4xow!D%zmHij#JCAf-sCqEkTHN*RCEJwZFS)=|jqlkWnQO{0VPF*KNI6Q-xbOvqT| z9Hu0SYH%z?K9I>FyCIu#(8!iDi7wZ+G*!Kk9W-;cs~5&ymM~O?sOj$5p?1eavvOk> z_C$7*TRVEec2KX~F}Ti++gffHj4HG(cBjV8J8WCqX50F!Hr( z`fX>&A-!#}A=}aA6gTeZJLZn|6UYf2TWd_7E*SagPHj!wu?lux^)p)b zabVmke+#O^uD zRlIsHl!#xLtg_sRiXRaB*+#t7M}n;=ywislphP$l!tX`Fa^|^@MHorxO=T#qGNQ~< z(G4jr352ALxY~j|9r-+~RI4Ju^rQ9kXGoM$YB*}>_syC~8AsP`UD>IFaDW`7Hx1Y< zGF8*qeER=6gicM8*_S58$kO3Lq%=~BmEXXO(rZ~4$sne>M06~rJNY!tsQ^?%P^0Uq ze1Sle(&h8iDVdrFnfW!q*(r3kHH~0q8m7@OZ57mP+lJ4X=B#0!kuL(Jv?99S9EqVI;j#y<~8HsmQNLE zK1jsoa|gdDyYeujKOFee?$QAHHFHt3Nf~l35_f51wC5AqBDSif0ZPyy>M(VDot7Mf zCcWn6H|Hx&DE*XLjH zX6TSdsZ^C-b9>d9lFw5{FqcMyWyEOtGPbMi>nj+Qb#)u*_oGbGxgy8DMBv(J3AKWv zK~?@B&5rbjuhqQ zRpEgeN=F-03ad&CNr~DU-wQ8$&O~16*~(ZlbwZ`%`$>S9<}2lS*0(5_s=B1A29(zr zVUm|dmdlD|rIZJqQjc@Kx=3E2OO^^Oj^;2IL8Zx>I(`^U7}}4VN&SfViEZeg=tkp{ HQ&;{A7p^$O literal 0 HcmV?d00001 diff --git a/vllm/compilation/__pycache__/compile_context.cpython-310.pyc b/vllm/compilation/__pycache__/compile_context.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..237dede535d9ef8b7d59c70702638b11761a9496 GIT binary patch literal 771 zcmaJ zRN!G&w~ZH{y=Dzt3JWL6Y15w%Ppi7sMk)a|1`L@MzOzjo5_biI^A?s1SUrN`cumiQ zr+y`9-B~LdPdCND#$xjaY(w}1vSTn&{W>eXHl>l~$TPbX{^*tNR{B_9mr8yrP48E2 zQ(h^h%RkkamvHb1=G^qdAtXPBAD(f*FiMHt12c>_Q0jnxDpKp1^QH<-Z~E~33g1zDTZZC`XlBlONZb-uO2{f~#26AIF~jw#hs>8W&-Q6-(H z?Im?`9nxFsjHo43;Lc;hxy=AuL2URui>a~VYCERCeA*l?heP)-7gM-!L6NU%mSTc) zJR^1=4vKu3YA843zQngIptN=)V5DSB!~@c56=7ma8*pq`JIQVf)NW7Lc-ra9 zq^ic2P5T8$+VBEDU=_(mysQL&fgjSZJnc_d9@c!Pdd9I6BY0p&Ro!*!^r>_GPGysm z4FkUifB#2+an>;YNsFt$Iu<|2kblDz8X=1fkDg}C64Nsi%d-;Ovst-U@hY0fNvd8| z)2z6b)V;cXx8p`K;Z5lGO598)y-EG<#I0n?o5FjQn|zwve1=RfWACNmW#yrcsu*Suy=6O9oLKC@k0c z0gplZ(RL=f{c?jp7r|h_#q&t@9}T6-lGR8K0@dwv0oGfOQ^iHX!zfUEo2N1p667g4 z91Jp{q#quoL6Q~I`WbcoKO5<_2Ntj?Q#@Aa%7>QM%`M_^SB6xTYL|7HLF@i@zN5ruKt9-!Ikj%|eec(Y5 z54kpUq^gqc|qjlR&)Sp&Q7t zD>|Cx%6U5*S;}|rxzS#l2_C+`e?u;TJ%D`jp(?Wr+Ug~FV4QfD%8Ysbt5O;H`7n|k z>ZsBN5n$;e4qS*#j51?mb9?P$avA93Ad>GcZBef5XTunV$A`g@Bu@;GSfwXQE;k$s zghne#hQv{iD};XR(j*Ice9yhE5)vZK6p8Bw12u%d$hFrJgK~VRbmh8xsM3Dv(ZcG+ z*UQU7ML8I^sJtFxQpOp}jeYZ+ov_!&s`0y-6Ein+wr?FUaa-9T%gwI zjX@w#P6(+i8qX+s*C-0bf&>Pt4IoF%t8b ziuoJmPGUzGMJiK<}%X&I`k@X(+Zr8x$+Iz*(nn+m++r=(PTSp)mo}zya^mFne)DWHui)EiUJZ>Q69d)Ct2lk?)D*wMbNXy1ujbCV$&8oFdT$mxJ9%~g z*4dnvRX?eX_-Z4*r{J6iC^&!yE3aU-)k0o9Wq7N+!4ojCeAN#5d)TljPcbiDwKX@+ zZimh}`|~WL^tGY|R3gO_S=7hM$if0b6^#gO zSgIiHa?dHRP+oO(Z}OVdR?9`y6;+($RYy&(aCDQGVv1B)q^vq zw>qPl2D>r$rpMdBRc%STAwD2E-=pdKG|iC2#xCJH-P)wUz1S#hy1W$@2%b|a0v)M= z`PopR9hCs4p-iI~3J@?D`~ENSwQfEQdz!VZ7F%L9HqDxtn|L-*Wz)>OFf9|{3TVY} zm~(-*Z|7Rzt6-Pi{qBFM#PYy)qp*2;Aibq$>+4Va?aj{~ed<5jeDdqZ>udh{+OxHF zQ6Ups8=E8Y%J$>U4X?TS#m35$O@C#}f4n7XWHds%8%QpFU!#h~2XP-eZCj&n3F3qr znyQgx4DCN4szw#+9zVg5EABkN4_JKJhi9J6UUHpUNaCWx;+uQRJ~kf@Y^1n+{t zP!Jjl7n4_~KGq=V>MXXM8xU;q2+u8biMdD DYbwDK literal 0 HcmV?d00001 diff --git a/vllm/compilation/__pycache__/levels.cpython-310.pyc b/vllm/compilation/__pycache__/levels.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50e2e9f1788849a68123447a84ffc53350a55a21 GIT binary patch literal 444 zcmY*Vy-ve05O#jrMzlq!5<6>$Vs&CcRim^*ky;zkR01zns3RK5ZbFk5b%vK=;1%FK zvNGirm^e3}!dds-eLkP<{Mlqkm^0M1 z7O?-%3}8d>>@h8f%C;^f!tVFHvbM!#qw_lV}^s68cc7 zuPIFp`U|BEK)O!haJ~vv#CdVVxk|)F>b%AI<0h0u-FZNRIT^pAwVux&z6Ll~DNf_Gmxt?zDDPdU z%~JK%6PEEkPIvh-S>dge3NH?lu+*Be^K`e?lC+Mj1>hjtsFq{wE2LY+!-G}Ugu{h% OS!o^UJ2E|%j`a^o-zaatkB_H+|_A`coU-v10VZ+cp=TcPcbQ>}Xyt=RF?Vj_T!)$S} zW#GB`=f4c!?ij}3X>j(bVelzlb%KT)oW+J${xgqZY{q6%^JZ$-@|G}H=Z*M6((yXX_`={0Z$2@2^Rek&Qhn0*LtKRKpFK+A-^$%ENZpAN?w%5Hzml5!AudtvK^Hd& zLjlc5fSFF*-9OF+?YN?aAWA{X-94?Urc;UG=b0w;L6mawXzjYDSS^D3{QNocM7lK1 zwO94q^Zh7|a^K&Z=|UoMkqenbsc?tc5lxV`bJBa~E@aQm4h1gh8}=+2ZWk9TEw#6N#zQln5kwXkC4*2sR$ zPHTC6)R-D02jeDoT7RgWvMHMyehae;BWKh`e{o=-UBccAqt4TdSXq9?m_ao1H%2Q5 zHdZfj1Kw+I%UMjV4nz*S!wF$}iZ0>r2jPK8DPAw*dPJD}T=auUoX=NGui2l(@c`aD zEE?7R%U=61j+6CqJQ+l(l5Obgt?H?gv<}?$*>)utv0hjrJyhPC_qVog`@1`LH@@^Y zc5dIfwYBMQZQkG9Dr_xV(U=KWG-iJ7y;~mHytnJ${N?tI+dKZ}Uu|uDared@khpPY z_p7^`a)CHB;m;IqzF#=#!duW=$z%i}gSec?_D9f@@JX9s=Fwim=f`P~WZoix3yzTc zeHkReyDY^32qmOn`L;jIvV&sr+)y^j01GqecD6S*y-NrlzYK1_a%|wBU04(dT3&57 zxrC$IK+g@iivB-7>E+3|H;%^LJdn7v?nlXFm|RVcdU3Sh8y^E<(;nHncgEJe8Q!dq zk4g737Ty9m^)4F6Y&Yu6VQto8mT57InXLY=y4_}V)?ux>^YzL1cV7?V)+^#6xmER0 zN+Lww!K;w7;GH8xKm?Ef6Z<123wH9i^QcuBr9?az@Ci`d7snY8gQ8G-Lj^o;Oi7>& zMS_*>o!!mr*cJg=CuvA&LL=5NI23Ncc~qto7Y3XVPO7t%XMQyYf*E35(J0L$3IB+X z=ZGAo1A>nK0z@Rt8q?SLye8j-=7j~Os;D0TlLty)Y1w?67RfI)H z)X|}_2E*T@LjX21CQLS{a>&_J^BI7!X-sRQ8Uf-6z;$Zr|N7HL?u?qF)-%B4s&Q&h zEp8o{^8KmJZTe5y!!~v<@H$rY*(p0P)rY?@e#`WHV-@ZPUOyvV;M;z>I5i0#b=BgW zeEcOMnC?`Ox177!Dyq-8f-4@0a6uk=qCMTqAEdqlB-P9M0e2eul=QC^stDlF6&Ly!^U!s+{Q1}9MR zGLf!REdl_qYliZ>n3ELn@_T5!riMUGtx0q>msg$PTiE$IUiCJb7Qn$V0S*9*%YX)x zng49nm)H{KJLbs`U)ND8JCnwjRU6SY18xrRa(EMfw8FY6eHZuJhXZH;xMq^~X zoB@!S!6FWBl;x#6t1sy`=YaCRqxHcMv7Mo&(&%xN$%9Ya&qNqdv2#=B0CzA6WPtKj zZwksHtI8L1)TmT+hk=5Rp!fx7kTWS<j>uaI&RjxpCI`K|cQg$0Zlt>4(UqdC%+(M{N{WXQU?!n#7YQ^S zbSv?@m5I9eHb@mTNCba5329yqq{t^S)yrKeK zR%f!onyte@z5z$l=q6XFN71)tmZ5QtCMds_jRxaRS0|uXv|-CT9g`3K7L(~ zf0dMUL1yg&0X_vF3XzuYCmEk04E>_*`wu5UJX>k_KF>nmmkpeeR7J|~Q1cEo6eXoj zgzb{TRT8!qwJ6QyhctYtmhO>ml0G3X;8g_nj$yHtw)2kJXtCv*xw69a`r5MW(%$#b z6t!SK)Xa+3?As0HM#;&}@!6&%@h|G=BPEtn63&-GQ_lCAx&ov!PZDdduCun%)sbfS epCm1upO$9%IVCYg7v~sMs?cu*Xw=4Ux%O}K-ASzg literal 0 HcmV?d00001 diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py new file mode 100644 index 00000000..4780358c --- /dev/null +++ b/vllm/compilation/backends.py @@ -0,0 +1,269 @@ +import copy +import operator +from typing import Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.fx as fx + +from vllm.logger import init_logger + +from .compile_context import get_compile_context +from .levels import CompilationLevel + +logger = init_logger(__name__) + + +def fix_functionalization(graph: fx.Graph): + """ + Rewrite the graph module to replace the pattern involving + torch._higher_order_ops.auto_functionalize.auto_functionalized + with a direct call to the inplace custom op. + + # TODO: check if PyTorch nightly has fixed this issue + """ + + # debug code, if we want to see the graph before the transformation + # with open("before.py", "w") as f: + # print(graph.python_code(root_module="self", verbose=True).src, file=f) + + nodes_to_remove = [] + + for node in graph.nodes: + # Identify the auto_functionalized node + if node.op == 'call_function' and node.target == torch._higher_order_ops.auto_functionalize.auto_functionalized: # noqa + if node.args[0] == torch.ops._C.rotary_embedding.default: + # manual replace for rotary_embedding + + # Now, collect the arguments + kwargs = node.kwargs + + query = kwargs['query'] + mm_node = query.args[0].args[0] + + # Create a new call to torch.ops._C.rotary_embedding.default + with graph.inserting_before(node): + # just insert the call to the custom op + # NOTE: don't run dead code elimination, + # otherwise this op will be removed + graph.call_function(torch.ops._C.rotary_embedding.default, + kwargs=kwargs) + + # Remove the auto_functionalized node + # Since the node may have outputs, we need to handle its users + # Replace uses of the outputs (getitem nodes) with mm_node + for user in list(node.users): + if user.op == 'call_function' and user.target == operator.getitem: # noqa + # Remove the getitem node + for getitem_user in list(user.users): + if (getitem_user.op == 'call_function' + and getitem_user.target + == torch.ops.aten.slice_scatter.default): + # Replace the uses of slice_scatter node + # with mm_node + getitem_user.replace_all_uses_with(mm_node) + nodes_to_remove.append(getitem_user) + nodes_to_remove.append(user) + nodes_to_remove.append(node) + + elif node.args[0] == torch.ops._C.fused_add_rms_norm.default: + # manual replace for fused_add_rms_norm + # this is the most effective optimization for llama + # failing to do this will result in many unnecessary copies + + kwargs = node.kwargs + + input = kwargs['input'] + residual = kwargs['residual'] + + # Create a new call to torch.ops._C.rotary_embedding.default + with graph.inserting_before(node): + # just insert the call to the custom op + # NOTE: don't run dead code elimination, + # otherwise this op will be removed + graph.call_function( + torch.ops._C.fused_add_rms_norm.default, kwargs=kwargs) + + for user in list(node.users): + if user.op == 'call_function' and user.target == operator.getitem: # noqa + # Remove the getitem node + if user.args[1] == 1: + replace_node = input + elif user.args[1] == 2: + replace_node = residual + user.replace_all_uses_with(replace_node) + nodes_to_remove.append(user) + nodes_to_remove.append(node) + + elif node.args[0] == torch.ops._C.rms_norm.default: + # manual replace for rms_norm + + kwargs = node.kwargs + + input = kwargs['input'] + out = kwargs['out'] + weight = kwargs['weight'] + epsilon = kwargs['epsilon'] + # Create a new call to torch.ops._C.rotary_embedding.default + # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa + with graph.inserting_before(node): + # just insert the call to the custom op + # NOTE: don't run dead code elimination, + # otherwise this op will be removed + graph.call_function( + torch.ops._C.rms_norm.default, + args=(out, input, weight, epsilon), + ) + + replace_node = out + + for user in list(node.users): + if user.op == 'call_function' and user.target == operator.getitem: # noqa + user.replace_all_uses_with(replace_node) + nodes_to_remove.append(user) + nodes_to_remove.append(node) + + elif node.args[0] == torch.ops._C.silu_and_mul.default: + # manual replace for silu_and_mul + + kwargs = node.kwargs + + input = kwargs['input'] + out = kwargs['out'] + + # Create a new call to torch.ops._C.rotary_embedding.default + # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa + with graph.inserting_before(node): + # just insert the call to the custom op + # NOTE: don't run dead code elimination, + # otherwise this op will be removed + graph.call_function( + torch.ops._C.silu_and_mul.default, + args=(out, input), + ) + replace_node = out + + for user in list(node.users): + if user.op == 'call_function' and user.target == operator.getitem: # noqa + user.replace_all_uses_with(replace_node) + nodes_to_remove.append(user) + nodes_to_remove.append(node) + + # Remove the nodes all at once + for node in nodes_to_remove: + graph.erase_node(node) + + # debug code, if we want to see the graph after the transformation + # with open("after.py", "w") as f: + # print(graph.python_code(root_module="self", verbose=True).src, file=f) + + +def wrap_inductor(graph, example_inputs, additional_inductor_config): + from torch._inductor import config + current_config = config.shallow_copy_dict() + from torch._inductor.compile_fx import compile_fx + + if additional_inductor_config is not None: + current_config.update(additional_inductor_config) + if current_config['post_grad_custom_post_pass'] is not None: + logger.warning( + "post_grad_custom_post_pass is already set in the config. " + "Overwriting it with the fix_functionalization") + current_config['post_grad_custom_post_pass'] = fix_functionalization + return compile_fx(graph, example_inputs, config_patches=current_config) + + +def vllm_backend( + graph, + example_inputs, + additional_inductor_config: Optional[Dict] = None) -> Callable: + + context = get_compile_context() + context = copy.deepcopy(context) if context is not None else [] + sizes_to_specialize: List[int] = context + + # flags for all the seen shapes, whether we need to specialize + runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {} + + # if we need to specialize, the compiled graph for that shape + runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {} + + # this is the first compilation, we will compile a graph with + # dynamic shape, as the caller will mark first dimension as dynamic + logger.info("Compiling a graph for general shapes") + graph_for_symbolic_shape = wrap_inductor(graph, example_inputs, + additional_inductor_config) + + # TODO: Dynamo does not pass all dynamic shapes. + # Need to investigate why. It works now because all the dynamic + # shapes have the same value, and either of them can be used. + sym_shape_indices = [ + i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt) + ] + + first_run = True + + # this is the function we return to Dynamo to run finally + def compiled_graph_wrapper(*args): + + runtime_shapes: Tuple[int, + ...] = tuple(args[i] for i in sym_shape_indices) + + nonlocal first_run + nonlocal runtime_shapes_to_compile_flags + nonlocal runtime_shapes_to_compiled_graph + + if first_run: + # the first compilation is for profiling, we directly run it + first_run = False + return graph_for_symbolic_shape(*args) + + if runtime_shapes not in runtime_shapes_to_compile_flags: + # we haven't seen this shape before + # query if we need to specialize for this shape + # we only specialize for the first dimension. + # TODO: investigate if any model needs to specialize + # beyond the first dimension + runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[ + 0] in sizes_to_specialize + + if not runtime_shapes_to_compile_flags[runtime_shapes]: + # we don't need to specialize for this shape + return graph_for_symbolic_shape(*args) + + if runtime_shapes not in runtime_shapes_to_compiled_graph: + # we need to specialize for this shape, and we haven't compiled + # compile the graph for this shape + logger.info("Compiling a graph for shapes %s", runtime_shapes) + runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor( + graph, args, additional_inductor_config) + + return runtime_shapes_to_compiled_graph[runtime_shapes](*args) + + return compiled_graph_wrapper + + +def select_default_backend(level: int) -> Union[str, Callable]: + if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]: + backend = "eager" + return backend + assert level in [ + CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE + ], f"Invalid level {level}" + + from vllm.compilation.backends import vllm_backend + from vllm.plugins import get_inductor_additional_configs + additional_configs = get_inductor_additional_configs() + + if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE: + if "max_autotune" in additional_configs and not additional_configs[ + "max_autotune"]: + logger.warning( + "max_autotune is disabled, but is overridden by level %s", + CompilationLevel.INDUCTOR_MAX_AUTOTUNE) + additional_configs['max_autotune'] = True + + from functools import partial + backend = partial(vllm_backend, + additional_inductor_config=additional_configs) + + return backend diff --git a/vllm/compilation/compile_context.py b/vllm/compilation/compile_context.py new file mode 100644 index 00000000..29db3d4c --- /dev/null +++ b/vllm/compilation/compile_context.py @@ -0,0 +1,23 @@ +from contextlib import contextmanager +from typing import Any + +_compile_context: Any = None + + +def get_compile_context() -> Any: + """Get the current compile context.""" + return _compile_context + + +@contextmanager +def set_compile_context(context: Any): + """A context manager that stores the current compile context, + usually it is a list of sizes to specialize. + """ + global _compile_context + prev_context = _compile_context + _compile_context = context + try: + yield + finally: + _compile_context = prev_context diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py new file mode 100644 index 00000000..655c4c44 --- /dev/null +++ b/vllm/compilation/decorators.py @@ -0,0 +1,113 @@ +import inspect +from typing import Dict, List, Union + +import torch + +import vllm.envs as envs +from vllm.compilation.levels import CompilationLevel +from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher +from vllm.sequence import IntermediateTensors +from vllm.utils import supports_dynamo + + +def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]): + """ + A decorator to add support for compiling the forward method of a class. + + `dynamic_arg_dims` is a dictionary that maps argument names to the dynamic + dimensions of the argument. The dynamic dimensions can be either a single + integer or a list of integers. + + Depending on the value of arguments: + + - if it is a single integer, the corresponding dimension of the argument + will be marked as dynamic. + - if it is `None`, ignored. + - if it is `IntermediateTensors`, all the tensors in the intermediate + tensors will be marked as dynamic. + - otherwise, it will raise an error. + + NOTE: if an argument is `None`, it should always be passed as `None` during + the lifetime of the model, otherwise, it cannot be captured as a single + computation graph. + """ + + def cls_decorator_helper(cls: type): + # helper to pass `dynamic_arg_dims`` to `_support_torch_compile`` + # to avoid too much indentation for `_support_torch_compile`` + sig = inspect.signature(cls.forward) + for k in dynamic_arg_dims: + if k not in sig.parameters: + raise ValueError( + f"Argument {k} not found in the forward method of {cls}") + return _support_torch_compile(cls, dynamic_arg_dims) + + return cls_decorator_helper + + +def _support_torch_compile(cls: type, + dynamic_arg_dims: Dict[str, Union[int, List[int]]]): + """ + A decorator to add support for compiling the forward method of a class. + """ + + # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner + # will handle the compilation, so we don't need to do anything here. + if envs.VLLM_TORCH_COMPILE_LEVEL in [ + CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS + ] or not supports_dynamo(): + return cls + + # take care of method resolution order + # make sure super().__init__ is called on the base class + # other than TorchCompileWrapperWithCustomDispatcher + cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, ) + + old_init = cls.__init__ + + def __init__(self, *args, **kwargs): + old_init(self, *args, **kwargs) + TorchCompileWrapperWithCustomDispatcher.__init__(self) + + cls.__init__ = __init__ + + def __call__(self, *args, **kwargs): + # torch.compiler.is_compiling() means we are inside the compilation + # e.g. TPU has the compilation logic in model runner, so we don't + # need to compile the model inside. + if torch.compiler.is_compiling(): + return self.forward(*args, **kwargs) + + # the first compilation needs to have dynamic shapes marked + if len(self.compiled_codes) < 1: + sig = inspect.signature(self.__class__.forward) + bound_args = sig.bind(self, *args, **kwargs) + bound_args.apply_defaults() + for k, dims in dynamic_arg_dims.items(): + arg = bound_args.arguments.get(k) + if arg is not None: + if isinstance(arg, torch.Tensor): + torch._dynamo.mark_dynamic(arg, dims) + elif isinstance(arg, IntermediateTensors): + for tensor in arg.tensors.values(): + torch._dynamo.mark_dynamic(tensor, dims) + else: + raise ValueError( + "Unsupported dynamic dimensions" + f" {dims} for argument {k} with type {type(arg)}.") + + # if we don't use custom dispatcher, we can directly call the + # compiled function and let torch.compile handle the dispatching, + # with the overhead of guard evaluation and recompilation. + if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher: + return self.compiled_callable(*args, **kwargs) + + # usually, capturing the model once is enough, and then we can + # dispatch to the compiled code directly, without going through + # the Dynamo guard mechanism. + with self.dispatch_to_code(0): + model_output = self.forward(*args, **kwargs) + return model_output + + cls.__call__ = __call__ + return cls diff --git a/vllm/compilation/levels.py b/vllm/compilation/levels.py new file mode 100644 index 00000000..162bf5ae --- /dev/null +++ b/vllm/compilation/levels.py @@ -0,0 +1,9 @@ +# constants for the levels of the compilation process + + +class CompilationLevel: + NO_COMPILATION = 0 + DYNAMO_AS_IS = 1 + DYNAMO_ONCE = 2 + INDUCTOR = 3 + INDUCTOR_MAX_AUTOTUNE = 4 diff --git a/vllm/compilation/wrapper.py b/vllm/compilation/wrapper.py new file mode 100644 index 00000000..1594b64a --- /dev/null +++ b/vllm/compilation/wrapper.py @@ -0,0 +1,102 @@ +import os +import sys +from abc import abstractmethod +from contextlib import contextmanager +from types import CodeType +from typing import Callable, List, Optional + +import torch + +import vllm.envs as envs + +from .levels import CompilationLevel + + +class TorchCompileWrapperWithCustomDispatcher: + """ + A wrapper class for torch.compile, with a custom dispatch logic. + Subclasses should: + 1. Implement the forward method + 2. Implement the dispatch logic in the __call__ method + It can use `self.compiled_codes` to access the compiled bytecode, + and `with self.dispatch_to_code(index):` to dispatch to + the compiled code. + 3. Implement the `__init__` method to determine how to call + `torch.compile` over the forward method. + """ + + def __init__(self, compiled_callable: Optional[Callable] = None): + + if compiled_callable is None: + # default compilation settings + # compiling the forward method + + # choose the compile backend + + # if the user has set the backend, use it + from vllm.plugins import get_torch_compile_backend + backend = get_torch_compile_backend() + if backend is None: + from vllm.compilation.backends import select_default_backend + backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL) + + compiled_callable = torch.compile( + self.forward, + fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE, + backend=backend) + + self.compiled_callable = compiled_callable + self.original_code_object = self.__class__.forward.__code__ + self.compiled_codes: List[CodeType] = [] + torch._dynamo.convert_frame.register_bytecode_hook(self.bytecode_hook) + + # read the env var to determine whether to use the custom dispatcher + # subclasses can use this to switch between the custom dispatcher + # and the default Dynamo guard mechanism. + self.use_custom_dispatcher: bool = \ + envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE + + def __call__(self, *args, **kwargs): + """Implement the dispatch logic here, beyond the torch.compile level. + NOTE: this function can have additional arguments beyond the forward + method, for directly dispatching to the compiled code. + """ + return self.compiled_callable(*args, **kwargs) + + @abstractmethod + def forward(self, *args, **kwargs): + ... + + def bytecode_hook(self, old_code: CodeType, new_code: CodeType): + """Hook to save the compiled bytecode for direct execution.""" + if old_code is not self.original_code_object: + return + # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25 + frame = sys._getframe() + while True: + frame = frame.f_back + code_name = frame.f_code.co_name + file_name = frame.f_code.co_filename.split(os.path.sep)[-1] + if code_name == "_compile" and file_name == "convert_frame.py": + break + frame = frame.f_locals["frame"] + assert frame.f_code == old_code + + if frame.f_locals["self"] is not self: + return + + self.compiled_codes.append(new_code) + + @contextmanager + def dispatch_to_code(self, index: int): + """Context manager to dispatch to the compiled code. + Why does this work? Because Dynamo guarantees that the compiled + bytecode has exactly the same arguments, cell variables, and free + variables as the original code. Therefore we can directly switch + the code object in the function and call it. + + See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details. + """ # noqa + self.__class__.forward.__code__ = self.compiled_codes[index] + yield + self.__class__.forward.__code__ = self.original_code_object diff --git a/vllm/config.py b/vllm/config.py new file mode 100644 index 00000000..3997bf12 --- /dev/null +++ b/vllm/config.py @@ -0,0 +1,1909 @@ +import enum +import json +from dataclasses import dataclass, field, fields +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, + Optional, Tuple, Type, Union) + +import torch +from transformers import PretrainedConfig + +import vllm.envs as envs +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS +from vllm.model_executor.models import ModelRegistry +from vllm.platforms import current_platform +from vllm.tracing import is_otel_available, otel_import_error_traceback +from vllm.transformers_utils.config import (ConfigFormat, get_config, + get_hf_image_processor_config, + get_hf_text_config) +from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory, + is_hip, is_neuron, is_openvino, is_xpu, + print_warning_once) + +if TYPE_CHECKING: + from ray.util.placement_group import PlacementGroup + + from vllm.executor.executor_base import ExecutorBase + from vllm.model_executor.model_loader.loader import BaseModelLoader + from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( + BaseTokenizerGroup) + +logger = init_logger(__name__) + +_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768 +_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120 + + +class ModelConfig: + """Configuration for the model. + + Args: + model: Name or path of the huggingface model to use. + It is also used as the content for `model_name` tag in metrics + output when `served_model_name` is not specified. + tokenizer: Name or path of the huggingface tokenizer to use. + tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if + available, "slow" will always use the slow tokenizer, and + "mistral" will always use the tokenizer from `mistral_common`. + trust_remote_code: Trust remote code (e.g., from HuggingFace) when + downloading the model and tokenizer. + dtype: Data type for model weights and activations. The "auto" option + will use FP16 precision for FP32 and FP16 models, and BF16 precision + for BF16 models. + seed: Random seed for reproducibility. + revision: The specific model version to use. It can be a branch name, + a tag name, or a commit id. If unspecified, will use the default + version. + code_revision: The specific revision to use for the model code on + Hugging Face Hub. It can be a branch name, a tag name, or a + commit id. If unspecified, will use the default version. + rope_scaling: Dictionary containing the scaling configuration for the + RoPE embeddings. When using this flag, don't update + `max_position_embeddings` to the expected new maximum. + tokenizer_revision: The specific tokenizer version to use. It can be a + branch name, a tag name, or a commit id. If unspecified, will use + the default version. + max_model_len: Maximum length of a sequence (including prompt and + output). If None, will be derived from the model. + quantization: Quantization method that was used to quantize the model + weights. If None, we assume the model weights are not quantized. + quantization_param_path: Path to JSON file containing scaling factors. + Used to load KV cache scaling factors into the model when KV cache + type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also + be used to load activation and weight scaling factors when the + model dtype is FP8_E4M3 on ROCm. + enforce_eager: Whether to enforce eager execution. If True, we will + disable CUDA graph and always execute the model in eager mode. + If False, we will use CUDA graph and eager execution in hybrid. + If None, the user did not specify, so default to False. + max_context_len_to_capture: Maximum context len covered by CUDA graphs. + When a sequence has context length larger than this, we fall back + to eager mode (DEPRECATED. Use max_seq_len_to_capture instead). + max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs. + When a sequence has context length larger than this, we fall back + to eager mode. Additionally for encoder-decoder models, if the + sequence length of the encoder input is larger than this, we fall + back to the eager mode. + disable_sliding_window: Whether to disable sliding window. If True, + we will disable the sliding window functionality of the model. + If the model does not support sliding window, this argument is + ignored. + skip_tokenizer_init: If true, skip initialization of tokenizer and + detokenizer. + served_model_name: The model name used in metrics tag `model_name`, + matches the model name exposed via the APIs. If multiple model + names provided, the first name will be used. If not specified, + the model name will be the same as `model`. + limit_mm_per_prompt: Maximum number of data instances per modality + per prompt. Only applicable for multimodal models. + override_neuron_config: Initialize non default neuron config or + override default neuron config that are specific to Neuron devices, + this argument will be used to configure the neuron config that + can not be gathered from the vllm arguments. + config_format: The config format which shall be loaded. + Defaults to 'auto' which defaults to 'hf'. + mm_processor_kwargs: Arguments to be forwarded to the model's processor + for multi-modal data, e.g., image processor. + """ + + def __init__(self, + model: str, + tokenizer: str, + tokenizer_mode: str, + trust_remote_code: bool, + dtype: Union[str, torch.dtype], + seed: int, + revision: Optional[str] = None, + code_revision: Optional[str] = None, + rope_scaling: Optional[dict] = None, + rope_theta: Optional[float] = None, + tokenizer_revision: Optional[str] = None, + max_model_len: Optional[int] = None, + spec_target_max_model_len: Optional[int] = None, + quantization: Optional[str] = None, + quantization_param_path: Optional[str] = None, + enforce_eager: Optional[bool] = None, + max_context_len_to_capture: Optional[int] = None, + max_seq_len_to_capture: Optional[int] = None, + max_logprobs: int = 20, + disable_sliding_window: bool = False, + skip_tokenizer_init: bool = False, + served_model_name: Optional[Union[str, List[str]]] = None, + limit_mm_per_prompt: Optional[Mapping[str, int]] = None, + use_async_output_proc: bool = True, + override_neuron_config: Optional[Dict[str, Any]] = None, + config_format: ConfigFormat = ConfigFormat.AUTO, + mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None: + self.model = model + self.tokenizer = tokenizer + self.tokenizer_mode = tokenizer_mode + self.trust_remote_code = trust_remote_code + self.seed = seed + self.revision = revision + self.code_revision = code_revision + self.rope_scaling = rope_scaling + self.rope_theta = rope_theta + # The tokenizer version is consistent with the model version by default. + if tokenizer_revision is None: + self.tokenizer_revision = revision + else: + self.tokenizer_revision = tokenizer_revision + self.quantization = quantization + self.quantization_param_path = quantization_param_path + self.enforce_eager = enforce_eager + if max_context_len_to_capture is not None: + raise ValueError("`max_context_len_to_capture` is deprecated. " + "Use `max_seq_len_to_capture` instead.") + self.max_seq_len_to_capture = max_seq_len_to_capture + self.max_logprobs = max_logprobs + self.disable_sliding_window = disable_sliding_window + self.skip_tokenizer_init = skip_tokenizer_init + + self.hf_config = get_config(self.model, trust_remote_code, revision, + code_revision, rope_scaling, rope_theta, + config_format) + self.hf_text_config = get_hf_text_config(self.hf_config) + self.hf_image_processor_config = get_hf_image_processor_config( + self.model, revision) + self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) + self.use_async_output_proc = use_async_output_proc + self.mm_processor_kwargs = mm_processor_kwargs + + # Set enforce_eager to False if the value is unset. + if self.enforce_eager is None: + self.enforce_eager = False + + if (not self.disable_sliding_window + and self.hf_text_config.model_type == "gemma2" + and self.hf_text_config.sliding_window is not None): + print_warning_once( + "Gemma 2 uses sliding window attention for every odd layer, " + "which is currently not supported by vLLM. Disabling sliding " + "window and capping the max length to the sliding window size " + f"({self.hf_text_config.sliding_window}).") + self.disable_sliding_window = True + + self.max_model_len = _get_and_verify_max_len( + hf_config=self.hf_text_config, + max_model_len=max_model_len, + disable_sliding_window=self.disable_sliding_window, + sliding_window_len=self.get_hf_config_sliding_window(), + spec_target_max_model_len=spec_target_max_model_len) + self.served_model_name = get_served_model_name(model, + served_model_name) + self.multimodal_config = self._init_multimodal_config( + limit_mm_per_prompt) + if not self.skip_tokenizer_init: + self._verify_tokenizer_mode() + + self.is_attention_free = self._init_attention_free() + self.has_inner_state = self._init_has_inner_state() + + self.override_neuron_config = override_neuron_config if is_neuron( + ) else None + self._verify_embedding_mode() + self._verify_quantization() + self._verify_cuda_graph() + self._verify_bnb_config() + + def _init_multimodal_config( + self, limit_mm_per_prompt: Optional[Mapping[str, int]] + ) -> Optional["MultiModalConfig"]: + architectures = getattr(self.hf_config, "architectures", []) + if ModelRegistry.is_multimodal_model(architectures): + return MultiModalConfig(limit_per_prompt=limit_mm_per_prompt or {}) + + if limit_mm_per_prompt: + raise ValueError("`limit_mm_per_prompt` is only supported for " + "multimodal models.") + + return None + + def _init_attention_free(self) -> bool: + architectures = getattr(self.hf_config, "architectures", []) + return ModelRegistry.is_attention_free_model(architectures) + + def _init_has_inner_state(self) -> bool: + architectures = getattr(self.hf_config, "architectures", []) + return ModelRegistry.model_has_inner_state(architectures) + + def _verify_tokenizer_mode(self) -> None: + tokenizer_mode = self.tokenizer_mode.lower() + if tokenizer_mode not in ["auto", "slow", "mistral"]: + raise ValueError( + f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be " + "either 'auto', 'slow' or 'mistral'.") + self.tokenizer_mode = tokenizer_mode + + def _verify_embedding_mode(self) -> None: + architectures = getattr(self.hf_config, "architectures", []) + self.embedding_mode = ModelRegistry.is_embedding_model(architectures) + + def _parse_quant_hf_config(self): + quant_cfg = getattr(self.hf_config, "quantization_config", None) + if quant_cfg is None: + # compressed-tensors uses a "compression_config" key + quant_cfg = getattr(self.hf_config, "compression_config", None) + return quant_cfg + + def _verify_quantization(self) -> None: + supported_quantization = [*QUANTIZATION_METHODS] + rocm_supported_quantization = [ + "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors", + "fbgemm_fp8", "w8a16" + ] + optimized_quantization_methods = [ + "fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin", + "awq_marlin", "fbgemm_fp8", "compressed_tensors", + "compressed-tensors", "experts_int8" + ] + tpu_supported_quantization = ["tpu_int8"] + neuron_supported_quantization = ["neuron_quant"] + if self.quantization is not None: + self.quantization = self.quantization.lower() + + # Parse quantization method from the HF model config, if available. + quant_cfg = self._parse_quant_hf_config() + + if quant_cfg is not None: + quant_method = quant_cfg.get("quant_method", "").lower() + + # Detect which checkpoint is it + for _, method in QUANTIZATION_METHODS.items(): + quantization_override = method.override_quantization_method( + quant_cfg, self.quantization) + if quantization_override: + quant_method = quantization_override + self.quantization = quantization_override + break + + # Verify quantization configurations. + if self.quantization is None: + self.quantization = quant_method + elif self.quantization != quant_method: + raise ValueError( + "Quantization method specified in the model config " + f"({quant_method}) does not match the quantization " + f"method specified in the `quantization` argument " + f"({self.quantization}).") + + if self.quantization is not None: + if self.quantization not in supported_quantization: + raise ValueError( + f"Unknown quantization method: {self.quantization}. Must " + f"be one of {supported_quantization}.") + if is_hip( + ) and self.quantization not in rocm_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in ROCm.") + if current_platform.is_tpu( + ) and self.quantization not in tpu_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in TPU Backend.") + if self.quantization not in optimized_quantization_methods: + logger.warning( + "%s quantization is not fully " + "optimized yet. The speed can be slower than " + "non-quantized models.", self.quantization) + if (self.quantization == "awq" and is_hip() + and not envs.VLLM_USE_TRITON_AWQ): + logger.warning( + "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ" + " is not set, enabling VLLM_USE_TRITON_AWQ.") + envs.VLLM_USE_TRITON_AWQ = True + if is_neuron( + ) and self.quantization not in neuron_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in Neuron Backend.") + + def _verify_cuda_graph(self) -> None: + if self.max_seq_len_to_capture is None: + self.max_seq_len_to_capture = self.max_model_len + self.max_seq_len_to_capture = min(self.max_seq_len_to_capture, + self.max_model_len) + + def _verify_bnb_config(self) -> None: + """ + The current version of bitsandbytes (0.44.0) with 8-bit models does not + yet support CUDA graph. + """ + is_bitsandbytes = self.quantization == "bitsandbytes" + has_quantization_config = (getattr(self.hf_config, + "quantization_config", None) + is not None) + is_8bit = (self.hf_config.quantization_config.get( + "load_in_8bit", False) if has_quantization_config else False) + if all([ + is_bitsandbytes, + has_quantization_config, + is_8bit, + not self.enforce_eager, + ]): + logger.warning( + "CUDA graph is not supported on BitAndBytes 8bit yet, " + "fallback to the eager mode.") + self.enforce_eager = True + + def verify_async_output_proc(self, parallel_config, speculative_config, + device_config) -> None: + if not self.use_async_output_proc: + # Nothing to check + return + + if parallel_config.pipeline_parallel_size > 1: + logger.warning("Async output processing can not be enabled " + "with pipeline parallel") + self.use_async_output_proc = False + return + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if device_config.device_type not in ("cuda", "tpu", "xpu"): + logger.warning( + "Async output processing is only supported for CUDA, TPU, XPU. " + "Disabling it for other platforms.") + self.use_async_output_proc = False + return + + if envs.VLLM_USE_RAY_SPMD_WORKER: + logger.warning( + "Async output processing can not be enabled with ray spmd") + self.use_async_output_proc = False + return + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if device_config.device_type == "cuda" and self.enforce_eager: + logger.warning( + "To see benefits of async output processing, enable CUDA " + "graph. Since, enforce-eager is enabled, async output " + "processor cannot be used") + self.use_async_output_proc = not self.enforce_eager + return + + # Async postprocessor is not necessary with embedding mode + # since there is no token generation + if self.embedding_mode: + self.use_async_output_proc = False + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if speculative_config: + logger.warning("Async output processing is not supported with" + " speculative decoding currently.") + self.use_async_output_proc = False + + def verify_with_parallel_config( + self, + parallel_config: "ParallelConfig", + ) -> None: + total_num_attention_heads = getattr(self.hf_text_config, + "num_attention_heads", 0) + tensor_parallel_size = parallel_config.tensor_parallel_size + if total_num_attention_heads % tensor_parallel_size != 0: + raise ValueError( + f"Total number of attention heads ({total_num_attention_heads})" + " must be divisible by tensor parallel size " + f"({tensor_parallel_size}).") + + pipeline_parallel_size = parallel_config.pipeline_parallel_size + if pipeline_parallel_size > 1: + architectures = getattr(self.hf_config, "architectures", []) + if not ModelRegistry.is_pp_supported_model(architectures): + raise NotImplementedError( + "Pipeline parallelism is not supported for this model. " + "Supported models implement the `SupportsPP` interface.") + + if self.use_async_output_proc: + logger.warning("Async output processor is not supported with " + "pipeline parallelism currently. Disabling it.") + self.use_async_output_proc = False + + def get_hf_config_sliding_window(self) -> Optional[int]: + """Get the sliding window size, or None if disabled.""" + + # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in + # addition to sliding window size. We check if that field is present + # and if it's False, return None. + if (hasattr(self.hf_text_config, "use_sliding_window") + and not self.hf_text_config.use_sliding_window): + return None + return getattr(self.hf_text_config, "sliding_window", None) + + def get_sliding_window(self) -> Optional[int]: + """Get the sliding window size, or None if disabled. + """ + # If user disables sliding window, return None. + if self.disable_sliding_window: + return None + # Otherwise get the value from the hf config. + return self.get_hf_config_sliding_window() + + def get_vocab_size(self) -> int: + return self.hf_text_config.vocab_size + + def get_hidden_size(self) -> int: + return self.hf_text_config.hidden_size + + def get_head_size(self) -> int: + # TODO remove hard code + if hasattr(self.hf_text_config, "model_type" + ) and self.hf_text_config.model_type == 'deepseek_v2': + # FlashAttention supports only head_size 32, 64, 128, 256, + # we need to pad head_size 192 to 256 + return 256 + + if self.is_attention_free: + return 0 + + if hasattr(self.hf_text_config, "head_dim"): + return self.hf_text_config.head_dim + # FIXME(woosuk): This may not be true for all models. + return (self.hf_text_config.hidden_size // + self.hf_text_config.num_attention_heads) + + def get_total_num_kv_heads(self) -> int: + """Returns the total number of KV heads.""" + # For GPTBigCode & Falcon: + # NOTE: for falcon, when new_decoder_architecture is True, the + # multi_query flag is ignored and we use n_head_kv for the number of + # KV heads. + falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"] + new_decoder_arch_falcon = ( + self.hf_config.model_type in falcon_model_types + and getattr(self.hf_config, "new_decoder_architecture", False)) + if not new_decoder_arch_falcon and getattr(self.hf_text_config, + "multi_query", False): + # Multi-query attention, only one KV head. + # Currently, tensor parallelism is not supported in this case. + return 1 + + # For DBRX and MPT + if self.hf_config.model_type == "mpt": + if "kv_n_heads" in self.hf_config.attn_config: + return self.hf_config.attn_config["kv_n_heads"] + return self.hf_config.num_attention_heads + if self.hf_config.model_type == "dbrx": + return getattr(self.hf_config.attn_config, "kv_n_heads", + self.hf_config.num_attention_heads) + + if self.is_attention_free: + return 0 + + attributes = [ + # For Falcon: + "n_head_kv", + "num_kv_heads", + # For LLaMA-2: + "num_key_value_heads", + # For ChatGLM: + "multi_query_group_num", + ] + for attr in attributes: + num_kv_heads = getattr(self.hf_text_config, attr, None) + if num_kv_heads is not None: + return num_kv_heads + + # For non-grouped-query attention models, the number of KV heads is + # equal to the number of attention heads. + return self.hf_text_config.num_attention_heads + + def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int: + """Returns the number of KV heads per GPU.""" + total_num_kv_heads = self.get_total_num_kv_heads() + # If tensor parallelism is used, we divide the number of KV heads by + # the tensor parallel size. We will replicate the KV heads in the + # case where the number of KV heads is smaller than the tensor + # parallel size so each GPU has at least one KV head. + return max(1, + total_num_kv_heads // parallel_config.tensor_parallel_size) + + def get_num_attention_heads(self, + parallel_config: "ParallelConfig") -> int: + num_heads = getattr(self.hf_text_config, "num_attention_heads", 0) + return num_heads // parallel_config.tensor_parallel_size + + def get_num_layers(self, parallel_config: "ParallelConfig") -> int: + from vllm.distributed.utils import get_pp_indices + total_num_hidden_layers = getattr(self.hf_text_config, + "num_hidden_layers", 0) + pp_rank = parallel_config.rank // parallel_config.tensor_parallel_size + pp_size = parallel_config.pipeline_parallel_size + start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size) + return end - start + + def get_num_attention_layers(self, + parallel_config: "ParallelConfig") -> int: + if self.is_attention_free: + return 0 + + num_layers = self.get_num_layers(parallel_config) + + # Transformers supports layers_block_type @property + layers = getattr(self.hf_config, "layers_block_type", + ["attention"] * num_layers) + return len([t for t in layers if t == "attention"]) + + def get_multimodal_config(self) -> "MultiModalConfig": + """ + Get the multimodal configuration of the model. + + Raises: + ValueError: If the model is not multimodal. + """ + if self.multimodal_config is None: + raise ValueError("The model is not multimodal.") + + return self.multimodal_config + + @property + def is_encoder_decoder_model(self) -> bool: + """Extract the HF encoder/decoder model flag.""" + return getattr(self.hf_config, "is_encoder_decoder", False) or ( + (hasattr(self.hf_config, "text_config") and getattr( + self.hf_config.text_config, "is_encoder_decoder", False))) + + @property + def is_embedding_model(self) -> bool: + """Extract the embedding model flag.""" + return self.embedding_mode + + @property + def is_multimodal_model(self) -> bool: + return self.multimodal_config is not None + + +class CacheConfig: + """Configuration for the KV cache. + + Args: + block_size: Size of a cache block in number of tokens. + gpu_memory_utilization: Fraction of GPU memory to use for the + vLLM execution. + swap_space: Size of the CPU swap space per GPU (in GiB). + cache_dtype: Data type for kv cache storage. + num_gpu_blocks_override: Number of GPU blocks to use. This overrides the + profiled num_gpu_blocks if specified. Does nothing if None. + """ + + def __init__( + self, + block_size: int, + gpu_memory_utilization: float, + swap_space: float, + cache_dtype: str, + is_attention_free: bool = False, + num_gpu_blocks_override: Optional[int] = None, + sliding_window: Optional[int] = None, + enable_prefix_caching: bool = False, + cpu_offload_gb: float = 0, + ) -> None: + self.block_size = block_size + self.gpu_memory_utilization = gpu_memory_utilization + self.swap_space_bytes = swap_space * GiB_bytes + self.num_gpu_blocks_override = num_gpu_blocks_override + self.cache_dtype = cache_dtype + self.is_attention_free = is_attention_free + self.sliding_window = sliding_window + self.enable_prefix_caching = enable_prefix_caching + self.cpu_offload_gb = cpu_offload_gb + self._verify_args() + self._verify_cache_dtype() + self._verify_prefix_caching() + + # Will be set after profiling. + self.num_gpu_blocks = None + self.num_cpu_blocks = None + + def metrics_info(self): + # convert cache_config to dict(key: str, value: str) for prometheus + # metrics info + return {key: str(value) for key, value in self.__dict__.items()} + + def _verify_args(self) -> None: + if self.gpu_memory_utilization > 1.0: + raise ValueError( + "GPU memory utilization must be less than 1.0. Got " + f"{self.gpu_memory_utilization}.") + + def _verify_cache_dtype(self) -> None: + if self.cache_dtype == "auto": + pass + elif self.cache_dtype in ("fp8", "fp8_e4m3", "fp8_e5m2"): + logger.info( + "Using fp8 data type to store kv cache. It reduces the GPU " + "memory footprint and boosts the performance. " + "Meanwhile, it may cause accuracy drop without a proper " + "scaling factor") + else: + raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}") + + def _verify_prefix_caching(self) -> None: + if not self.enable_prefix_caching: + return + + if self.sliding_window is not None: + raise NotImplementedError( + "Prefix caching is not supported with sliding window. " + "Run with --disable-sliding-window to use prefix caching.") + + def verify_with_parallel_config( + self, + parallel_config: "ParallelConfig", + ) -> None: + total_cpu_memory = get_cpu_memory() + # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel + # group are in the same node. However, the GPUs may span multiple nodes. + num_gpus_per_node = parallel_config.tensor_parallel_size + cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node + + msg = (f"{cpu_memory_usage / GiB_bytes:.2f} GiB out of the " + f"{total_cpu_memory / GiB_bytes:.2f} GiB total CPU memory " + "is allocated for the swap space.") + if cpu_memory_usage > 0.7 * total_cpu_memory: + raise ValueError("Too large swap space. " + msg) + elif cpu_memory_usage > 0.4 * total_cpu_memory: + logger.warning("Possibly too large swap space. %s", msg) + + +@dataclass +class TokenizerPoolConfig: + """Configuration for the tokenizer pool. + + Args: + pool_size: Number of tokenizer workers in the pool. + pool_type: Type of the pool. + extra_config: Additional config for the pool. + The way the config will be used depends on the + pool type. + """ + pool_size: int + pool_type: Union[str, Type["BaseTokenizerGroup"]] + extra_config: dict + + def __post_init__(self): + if self.pool_type not in ("ray", ) and not isinstance( + self.pool_type, type): + raise ValueError(f"Unknown pool type: {self.pool_type}") + if not isinstance(self.extra_config, dict): + raise ValueError("extra_config must be a dictionary.") + + @classmethod + def create_config( + cls, tokenizer_pool_size: int, tokenizer_pool_type: str, + tokenizer_pool_extra_config: Optional[Union[str, dict]] + ) -> Optional["TokenizerPoolConfig"]: + """Create a TokenizerPoolConfig from the given parameters. + + If tokenizer_pool_size is 0, return None. + + Args: + tokenizer_pool_size: Number of tokenizer workers in the pool. + tokenizer_pool_type: Type of the pool. + tokenizer_pool_extra_config: Additional config for the pool. + The way the config will be used depends on the + pool type. This can be a JSON string (will be parsed). + """ + if tokenizer_pool_size: + if isinstance(tokenizer_pool_extra_config, str): + tokenizer_pool_extra_config_parsed = json.loads( + tokenizer_pool_extra_config) + else: + tokenizer_pool_extra_config_parsed = ( + tokenizer_pool_extra_config or {}) + tokenizer_pool_config = cls(tokenizer_pool_size, + tokenizer_pool_type, + tokenizer_pool_extra_config_parsed) + else: + tokenizer_pool_config = None + return tokenizer_pool_config + + +class LoadFormat(str, enum.Enum): + AUTO = "auto" + PT = "pt" + SAFETENSORS = "safetensors" + NPCACHE = "npcache" + DUMMY = "dummy" + TENSORIZER = "tensorizer" + SHARDED_STATE = "sharded_state" + GGUF = "gguf" + BITSANDBYTES = "bitsandbytes" + MISTRAL = "mistral" + + +@dataclass +class LoadConfig: + """ + download_dir: Directory to download and load the weights, default to the + default cache directory of huggingface. + load_format: The format of the model weights to load: + "auto" will try to load the weights in the safetensors format and + fall back to the pytorch bin format if safetensors format is + not available. + "pt" will load the weights in the pytorch bin format. + "safetensors" will load the weights in the safetensors format. + "npcache" will load the weights in pytorch format and store + a numpy cache to speed up the loading. + "dummy" will initialize the weights with random values, which is + mainly for profiling. + "tensorizer" will use CoreWeave's tensorizer library for + fast weight loading. + "bitsandbytes" will load nf4 type weights. + ignore_patterns: The list of patterns to ignore when loading the model. + Default to "original/**/*" to avoid repeated loading of llama's + checkpoints. + + """ + + load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO + download_dir: Optional[str] = None + model_loader_extra_config: Optional[Union[str, dict]] = field( + default_factory=dict) + ignore_patterns: Optional[Union[List[str], str]] = None + + def __post_init__(self): + model_loader_extra_config = self.model_loader_extra_config or {} + if isinstance(model_loader_extra_config, str): + self.model_loader_extra_config = json.loads( + model_loader_extra_config) + self._verify_load_format() + + if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: + logger.info( + "Ignoring the following patterns when downloading weights: %s", + self.ignore_patterns) + else: + self.ignore_patterns = ["original/**/*"] + + def _verify_load_format(self) -> None: + if not isinstance(self.load_format, str): + return + + load_format = self.load_format.lower() + self.load_format = LoadFormat(load_format) + + rocm_not_supported_load_format: List[str] = [] + if is_hip() and load_format in rocm_not_supported_load_format: + rocm_supported_load_format = [ + f for f in LoadFormat.__members__ + if (f not in rocm_not_supported_load_format) + ] + raise ValueError( + f"load format '{load_format}' is not supported in ROCm. " + f"Supported load formats are " + f"{rocm_supported_load_format}") + + +class ParallelConfig: + """Configuration for the distributed execution. + + Args: + pipeline_parallel_size: Number of pipeline parallel groups. + tensor_parallel_size: Number of tensor parallel groups. + worker_use_ray: Deprecated, use distributed_executor_backend instead. + max_parallel_loading_workers: Maximum number of multiple batches + when load model sequentially. To avoid RAM OOM when using tensor + parallel and large models. + disable_custom_all_reduce: Disable the custom all-reduce kernel and + fall back to NCCL. + tokenizer_pool_config: Config for the tokenizer pool. + If None, will use synchronous tokenization. + ray_workers_use_nsight: Whether to profile Ray workers with nsight, see + https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler. + placement_group: ray distributed model workers placement group. + distributed_executor_backend: Backend to use for distributed model + workers, either "ray" or "mp" (multiprocessing). If either + pipeline_parallel_size or tensor_parallel_size is greater than 1, + will default to "ray" if Ray is installed or "mp" otherwise. + """ + + def __init__( + self, + pipeline_parallel_size: int, + tensor_parallel_size: int, + worker_use_ray: Optional[bool] = None, + max_parallel_loading_workers: Optional[int] = None, + disable_custom_all_reduce: bool = False, + tokenizer_pool_config: Optional[TokenizerPoolConfig] = None, + ray_workers_use_nsight: bool = False, + placement_group: Optional["PlacementGroup"] = None, + distributed_executor_backend: Optional[Union[ + str, Type["ExecutorBase"]]] = None, + ) -> None: + self.pipeline_parallel_size = pipeline_parallel_size + self.tensor_parallel_size = tensor_parallel_size + self.distributed_executor_backend = distributed_executor_backend + self.max_parallel_loading_workers = max_parallel_loading_workers + self.disable_custom_all_reduce = disable_custom_all_reduce + self.tokenizer_pool_config = tokenizer_pool_config + self.ray_workers_use_nsight = ray_workers_use_nsight + self.placement_group = placement_group + self.world_size = pipeline_parallel_size * self.tensor_parallel_size + + if worker_use_ray: + if self.distributed_executor_backend is None: + self.distributed_executor_backend = "ray" + elif not self.use_ray: + raise ValueError(f"worker-use-ray can't be used with " + f"distributed executor backend " + f"'{self.distributed_executor_backend}'.") + + if current_platform.is_tpu() and self.world_size > 1: + if self.distributed_executor_backend is None: + self.distributed_executor_backend = "ray" + if self.distributed_executor_backend != "ray": + raise ValueError( + "TPU backend only supports Ray for distributed inference.") + + if self.distributed_executor_backend is None and self.world_size > 1: + # We use multiprocessing by default if world_size fits on the + # current node and we aren't in a ray placement group. + + from vllm.executor import ray_utils + backend = "mp" + ray_found = ray_utils.ray_is_available() + if (current_platform.is_cuda() + and cuda_device_count_stateless() < self.world_size): + if not ray_found: + raise ValueError("Unable to load Ray which is " + "required for multi-node inference, " + "please install Ray with `pip install " + "ray`.") from ray_utils.ray_import_err + backend = "ray" + elif ray_found: + if self.placement_group: + backend = "ray" + else: + from ray import is_initialized as ray_is_initialized + if ray_is_initialized(): + from ray.util import get_current_placement_group + if get_current_placement_group(): + backend = "ray" + self.distributed_executor_backend = backend + logger.info("Defaulting to use %s for distributed inference", + backend) + + self._verify_args() + self.rank: int = 0 + + @property + def use_ray(self) -> bool: + return self.distributed_executor_backend == "ray" or ( + isinstance(self.distributed_executor_backend, type) + and self.distributed_executor_backend.uses_ray) + + def _verify_args(self) -> None: + # Lazy import to avoid circular import + from vllm.executor.executor_base import ExecutorBase + + if self.distributed_executor_backend not in ( + "ray", "mp", None) and not (isinstance( + self.distributed_executor_backend, type) and issubclass( + self.distributed_executor_backend, ExecutorBase)): + raise ValueError( + "Unrecognized distributed executor backend " + f"{self.distributed_executor_backend}. Supported " + "values are 'ray', 'mp' or custom ExecutorBase subclass.") + if self.use_ray: + from vllm.executor import ray_utils + ray_utils.assert_ray_available() + if is_hip(): + self.disable_custom_all_reduce = True + logger.info( + "Disabled the custom all-reduce kernel because it is not " + "supported on AMD GPUs.") + if self.ray_workers_use_nsight and not self.use_ray: + raise ValueError("Unable to use nsight profiling unless workers " + "run with Ray.") + + +class SchedulerConfig: + """Scheduler configuration. + + Args: + max_num_batched_tokens: Maximum number of tokens to be processed in + a single iteration. + max_num_seqs: Maximum number of sequences to be processed in a single + iteration. + max_model_len: Maximum length of a sequence (including prompt + and generated text). + use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not. + num_lookahead_slots: The number of slots to allocate per sequence per + step, beyond the known token ids. This is used in speculative + decoding to store KV activations of tokens which may or may not be + accepted. + delay_factor: Apply a delay (of delay factor multiplied by previous + prompt latency) before scheduling next prompt. + enable_chunked_prefill: If True, prefill requests can be chunked based + on the remaining max_num_batched_tokens. + embedding_mode: Whether the running model is for embedding. + preemption_mode: Whether to perform preemption by swapping or + recomputation. If not specified, we determine the mode as follows: + We use recomputation by default since it incurs lower overhead than + swapping. However, when the sequence group has multiple sequences + (e.g., beam search), recomputation is not currently supported. In + such a case, we use swapping instead. + send_delta_data: Private API. If used, scheduler sends delta data to + workers instead of an entire data. It should be enabled only + when SPMD worker architecture is enabled. I.e., + VLLM_USE_RAY_SPMD_WORKER=1 + policy: The scheduling policy to use. "fcfs" (default) or "priority". + """ + + def __init__(self, + max_num_batched_tokens: Optional[int], + max_num_seqs: int, + max_model_len: int, + use_v2_block_manager: bool = True, + num_lookahead_slots: int = 0, + delay_factor: float = 0.0, + enable_chunked_prefill: bool = False, + embedding_mode: bool = False, + is_multimodal_model: bool = False, + preemption_mode: Optional[str] = None, + num_scheduler_steps: int = 1, + multi_step_stream_outputs: bool = False, + send_delta_data: bool = False, + policy: str = "fcfs") -> None: + if max_num_batched_tokens is None: + if enable_chunked_prefill: + if num_scheduler_steps > 1: + # Multi-step Chunked-Prefill doesn't allow prompt-chunking + # for now. Have max_num_batched_tokens set to max_model_len + # so we don't reject sequences on account of a short + # max_num_batched_tokens. + max_num_batched_tokens = max(max_model_len, 2048) + else: + # It is the values that have the best balance between ITL + # and TTFT on A100. Note it is not optimized for throughput. + max_num_batched_tokens = 512 + else: + # If max_model_len is too short, use 2048 as the default value + # for higher throughput. + max_num_batched_tokens = max(max_model_len, 2048) + + if embedding_mode: + # For embedding, choose specific value for higher throughput + max_num_batched_tokens = max( + max_num_batched_tokens, + _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS, + ) + if is_multimodal_model: + # The value needs to be at least the number of multimodal tokens + max_num_batched_tokens = max( + max_num_batched_tokens, + _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS, + ) + + self.max_num_batched_tokens = max_num_batched_tokens + + if enable_chunked_prefill: + logger.info( + "Chunked prefill is enabled with max_num_batched_tokens=%d.", + self.max_num_batched_tokens) + + self.max_num_seqs = max_num_seqs + self.max_model_len = max_model_len + self.use_v2_block_manager = use_v2_block_manager + self.num_lookahead_slots = num_lookahead_slots + self.delay_factor = delay_factor + self.chunked_prefill_enabled = enable_chunked_prefill + self.embedding_mode = embedding_mode + self.preemption_mode = preemption_mode + self.num_scheduler_steps = num_scheduler_steps + self.multi_step_stream_outputs = multi_step_stream_outputs + self.send_delta_data = send_delta_data + self.policy = policy + self._verify_args() + + def _verify_args(self) -> None: + if (self.max_num_batched_tokens < self.max_model_len + and not self.chunked_prefill_enabled): + raise ValueError( + f"max_num_batched_tokens ({self.max_num_batched_tokens}) is " + f"smaller than max_model_len ({self.max_model_len}). " + "This effectively limits the maximum sequence length to " + "max_num_batched_tokens and makes vLLM reject longer " + "sequences. Please increase max_num_batched_tokens or " + "decrease max_model_len.") + + if self.max_num_batched_tokens < self.max_num_seqs: + raise ValueError( + f"max_num_batched_tokens ({self.max_num_batched_tokens}) must " + "be greater than or equal to max_num_seqs " + f"({self.max_num_seqs}).") + + if self.num_lookahead_slots < 0: + raise ValueError( + "num_lookahead_slots " + f"({self.num_lookahead_slots}) must be greater than or " + "equal to 0.") + + if self.num_scheduler_steps < 1: + raise ValueError( + "num_scheduler_steps " + f"({self.num_scheduler_steps}) must be greater than or " + "equal to 1.") + + if (not self.use_v2_block_manager \ + and not envs.VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1): + raise ValueError( + "The use of BlockSpaceManagerV1 is deprecated and will " + "be removed in a future release. Please switch to " + "BlockSpaceManagerV2 by setting --use-v2-block-manager to " + "True. If you wish to suppress this error temporarily, " + "you can set the environment variable " + "`VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1. If your use " + "case is not supported in BlockSpaceManagerV2, please " + "file an issue with detailed information.") + + @property + def is_multi_step(self) -> bool: + return self.num_scheduler_steps > 1 + + +class DeviceConfig: + device: Optional[torch.device] + + def __init__(self, device: str = "auto") -> None: + if device == "auto": + # Automated device type detection + if current_platform.is_cuda_alike(): + self.device_type = "cuda" + elif is_neuron(): + self.device_type = "neuron" + elif is_openvino(): + self.device_type = "openvino" + elif current_platform.is_tpu(): + self.device_type = "tpu" + elif current_platform.is_cpu(): + self.device_type = "cpu" + elif is_xpu(): + self.device_type = "xpu" + else: + raise RuntimeError("Failed to infer device type") + else: + # Device type is assigned explicitly + self.device_type = device + + # Some device types require processing inputs on CPU + if self.device_type in ["neuron", "openvino"]: + self.device = torch.device("cpu") + elif self.device_type in ["tpu"]: + self.device = None + else: + # Set device with device type + self.device = torch.device(self.device_type) + + +class SpeculativeConfig: + """Configuration for speculative decoding. + + The configuration is currently specialized to draft-model speculative + decoding with top-1 proposals. + """ + + @staticmethod + def maybe_create_spec_config( + target_model_config: ModelConfig, + target_parallel_config: ParallelConfig, + target_dtype: str, + speculative_model: Optional[str], + speculative_model_quantization: Optional[str], + speculative_draft_tensor_parallel_size: Optional[int], + num_speculative_tokens: Optional[int], + speculative_disable_mqa_scorer: Optional[bool], + speculative_max_model_len: Optional[int], + enable_chunked_prefill: bool, + use_v2_block_manager: bool, + disable_log_stats: bool, + speculative_disable_by_batch_size: Optional[int], + ngram_prompt_lookup_max: Optional[int], + ngram_prompt_lookup_min: Optional[int], + draft_token_acceptance_method: str, + typical_acceptance_sampler_posterior_threshold: Optional[float], + typical_acceptance_sampler_posterior_alpha: Optional[float], + disable_logprobs: Optional[bool], + ) -> Optional["SpeculativeConfig"]: + """Create a SpeculativeConfig if possible, else return None. + + This function attempts to create a SpeculativeConfig object based on the + provided parameters. If the necessary conditions are met, it returns an + instance of SpeculativeConfig. Otherwise, it returns None. + + Args: + target_model_config (ModelConfig): The configuration of the target + model. + target_parallel_config (ParallelConfig): The parallel configuration + for the target model. + target_dtype (str): The data type used for the target model. + speculative_model (Optional[str]): The name of the speculative + model, if provided. + speculative_model_quantization (Optional[str]): Quantization method + that was used to quantize the speculative model weights. If + None, we assume the model weights are not quantized. + speculative_draft_tensor_parallel_size (Optional[int]): The degree + of the tensor parallelism for the draft model. + num_speculative_tokens (Optional[int]): The number of speculative + tokens, if provided. Will default to the number in the draft + model config if present, otherwise is required. + speculative_disable_mqa_scorer (Optional[bool]): Disable the MQA + scorer for the speculative model and fall back to batch + expansion for scoring. + speculative_max_model_len (Optional[int]): The maximum model len of + the speculative model. Used when testing the ability to skip + speculation for some sequences. + enable_chunked_prefill (bool): Whether vLLM is configured to use + chunked prefill or not. Used for raising an error since its not + yet compatible with spec decode. + use_v2_block_manager (bool): Whether vLLM is configured to use the + v2 block manager or not. Used for raising an error since the v2 + block manager is required with spec decode. + speculative_disable_by_batch_size (Optional[int]): Disable + speculative decoding for new incoming requests when the number + of enqueue requests is larger than this value, if provided. + ngram_prompt_lookup_max (Optional[int]): Max size of ngram token + window, if provided. + ngram_prompt_lookup_min (Optional[int]): Min size of ngram token + window, if provided. + draft_token_acceptance_method (str): The method to use for + accepting draft tokens. This can take two possible + values 'rejection_sampler' and 'typical_acceptance_sampler' + for RejectionSampler and TypicalAcceptanceSampler + respectively. + typical_acceptance_sampler_posterior_threshold (Optional[float]): + A threshold value that sets a lower bound on the posterior + probability of a token in the target model for it to be + accepted. This threshold is used only when we use the + TypicalAcceptanceSampler for token acceptance. + typical_acceptance_sampler_posterior_alpha (Optional[float]): + A scaling factor for the entropy-based threshold in the + TypicalAcceptanceSampler. + disable_logprobs (Optional[bool]): If set to True, token log + probabilities are not returned during speculative decoding. + If set to False, token log probabilities are returned + according to the log probability settings in SamplingParams. + If not specified, it defaults to True. + + Returns: + Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if + the necessary conditions are met, else None. + """ + + if speculative_model is None: + if num_speculative_tokens is not None: + raise ValueError("num_speculative_tokens was provided without " + "speculative_model.") + return None + + if (speculative_disable_by_batch_size is not None + and speculative_disable_by_batch_size < 2): + raise ValueError("Expect the batch size threshold of disabling " + "speculative decoding is > 1, but got " + f"{speculative_disable_by_batch_size=}") + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if enable_chunked_prefill: + raise ValueError( + "Speculative decoding and chunked prefill are " + f"currently mutually exclusive ({enable_chunked_prefill=}).") + + if not use_v2_block_manager: + raise ValueError( + "Speculative decoding requires usage of the V2 " + "block manager. Enable it with --use-v2-block-manager.") + + # TODO: The user should be able to specify revision/max model len + # for the draft model. It is not currently supported. + draft_revision = None + draft_code_revision = None + draft_quantization = speculative_model_quantization + + if speculative_model == "[ngram]": + if ngram_prompt_lookup_min is None: + ngram_prompt_lookup_min = 1 + if ngram_prompt_lookup_max is None or ngram_prompt_lookup_max < 1: + raise ValueError(f"{ngram_prompt_lookup_max=} must be > 0") + if ngram_prompt_lookup_min < 1: + raise ValueError(f"{ngram_prompt_lookup_min=} must be > 0") + if ngram_prompt_lookup_min > ngram_prompt_lookup_max: + raise ValueError(f"{ngram_prompt_lookup_min=} cannot be " + f"larger than {ngram_prompt_lookup_max=}") + + # TODO: current we still need extract vocab_size from target model + # config, in future, we may try refactor it out, and set + # draft related config as None here. + draft_model_config = target_model_config + draft_parallel_config = target_parallel_config + else: + ngram_prompt_lookup_max = 0 + ngram_prompt_lookup_min = 0 + draft_model_config = ModelConfig( + model=speculative_model, + tokenizer=target_model_config.tokenizer, + tokenizer_mode=target_model_config.tokenizer_mode, + trust_remote_code=target_model_config.trust_remote_code, + dtype=target_model_config.dtype, + seed=target_model_config.seed, + revision=draft_revision, + code_revision=draft_code_revision, + tokenizer_revision=target_model_config.tokenizer_revision, + max_model_len=None, + spec_target_max_model_len=target_model_config.max_model_len, + quantization=draft_quantization, + enforce_eager=target_model_config.enforce_eager, + max_seq_len_to_capture=target_model_config. + max_seq_len_to_capture, + max_logprobs=target_model_config.max_logprobs, + ) + + draft_hf_config = draft_model_config.hf_config + + if (num_speculative_tokens is not None + and hasattr(draft_hf_config, "num_lookahead_tokens")): + draft_hf_config.num_lookahead_tokens = num_speculative_tokens + + n_predict = getattr(draft_hf_config, "n_predict", None) + if n_predict is not None: + if num_speculative_tokens is None: + # Default to max value defined in draft model config. + num_speculative_tokens = n_predict + elif num_speculative_tokens > n_predict: + # Verify provided value doesn't exceed the maximum + # supported by the draft model. + raise ValueError( + "This speculative model supports a maximum of " + f"num_speculative_tokens={n_predict}, but " + f"{num_speculative_tokens=} was provided.") + + draft_model_config.max_model_len = ( + SpeculativeConfig._maybe_override_draft_max_model_len( + speculative_max_model_len, + draft_model_config.max_model_len, + target_model_config.max_model_len, + )) + + draft_parallel_config = ( + SpeculativeConfig.create_draft_parallel_config( + target_parallel_config, + speculative_draft_tensor_parallel_size, draft_hf_config)) + + if num_speculative_tokens is None: + raise ValueError( + "num_speculative_tokens must be provided with " + "speculative_model unless the draft model config contains an " + "n_predict parameter.") + + if typical_acceptance_sampler_posterior_threshold is None: + typical_acceptance_sampler_posterior_threshold = 0.09 + if typical_acceptance_sampler_posterior_alpha is None: + typical_acceptance_sampler_posterior_alpha = 0.3 + if disable_logprobs is None: + disable_logprobs = True + + return SpeculativeConfig( + draft_model_config, + draft_parallel_config, + num_speculative_tokens, + speculative_disable_mqa_scorer, + speculative_disable_by_batch_size, + ngram_prompt_lookup_max, + ngram_prompt_lookup_min, + draft_token_acceptance_method=draft_token_acceptance_method, + typical_acceptance_sampler_posterior_threshold=\ + typical_acceptance_sampler_posterior_threshold, + typical_acceptance_sampler_posterior_alpha=\ + typical_acceptance_sampler_posterior_alpha, + disable_logprobs=disable_logprobs, + disable_log_stats=disable_log_stats, + ) + + @staticmethod + def _maybe_override_draft_max_model_len( + speculative_max_model_len: Optional[int], + draft_max_model_len: int, + target_max_model_len: int, + ) -> int: + """Determine the max sequence len for the draft model. This is usually + the draft_max_model_len, but may be the target_max_model_len if it is + less than the draft_max_model_len, or may be speculative_max_model_len + if it is specified. + + This is necessary so that sequences do not exceed the capacity of the + draft model or the target model. + + speculative_max_model_len is mainly used for testing that sequences can + skip speculation. + """ + + if speculative_max_model_len is not None: + + if speculative_max_model_len > draft_max_model_len: + raise ValueError(f"{speculative_max_model_len=} cannot be " + f"larger than {draft_max_model_len=}") + + if speculative_max_model_len > target_max_model_len: + raise ValueError(f"{speculative_max_model_len=} cannot be " + f"larger than {target_max_model_len=}") + + return speculative_max_model_len + + return min( + draft_max_model_len, + target_max_model_len, + ) + + @staticmethod + def create_draft_parallel_config( + target_parallel_config: ParallelConfig, + speculative_draft_tensor_parallel_size: Optional[int], + draft_hf_config: PretrainedConfig, + ) -> ParallelConfig: + """Create a parallel config for use by the draft worker. + + This is mostly a copy of the target parallel config, except the tp_size. + """ + if speculative_draft_tensor_parallel_size is None: + if draft_hf_config.model_type == "mlp_speculator": + speculative_draft_tensor_parallel_size = 1 + if target_parallel_config.tensor_parallel_size > 1: + logger.warning( + "MLPSpeculator cannot currently be run with tp>1; " + "setting speculative_draft_tensor_parallel_size=1") + else: + speculative_draft_tensor_parallel_size = \ + target_parallel_config.tensor_parallel_size + elif speculative_draft_tensor_parallel_size != 1: + # TODO(wooyeon): allow tp values larger than 1 + raise ValueError( + f"{speculative_draft_tensor_parallel_size=} cannot be " + f"other value than 1") + + draft_parallel_config = ParallelConfig( + pipeline_parallel_size=target_parallel_config. + pipeline_parallel_size, + tensor_parallel_size=speculative_draft_tensor_parallel_size, + distributed_executor_backend=target_parallel_config. + distributed_executor_backend, + max_parallel_loading_workers=target_parallel_config. + max_parallel_loading_workers, + disable_custom_all_reduce=target_parallel_config. + disable_custom_all_reduce, + tokenizer_pool_config=target_parallel_config.tokenizer_pool_config, + ray_workers_use_nsight=target_parallel_config. + ray_workers_use_nsight, + placement_group=target_parallel_config.placement_group, + ) + + return draft_parallel_config + + def __init__( + self, + draft_model_config: ModelConfig, + draft_parallel_config: ParallelConfig, + num_speculative_tokens: int, + speculative_disable_mqa_scorer: Optional[bool], + speculative_disable_by_batch_size: Optional[int], + ngram_prompt_lookup_max: Optional[int], + ngram_prompt_lookup_min: Optional[int], + draft_token_acceptance_method: str, + typical_acceptance_sampler_posterior_threshold: float, + typical_acceptance_sampler_posterior_alpha: float, + disable_logprobs: bool, + disable_log_stats: bool, + ): + """Create a SpeculativeConfig object. + + Args: + draft_model_config: ModelConfig for the draft model. + draft_parallel_config: ParallelConfig for the draft model. + num_speculative_tokens: The number of tokens to sample from the + draft model before scoring with the target model. + speculative_disable_by_batch_size: Disable speculative + decoding for new incoming requests when the number of + enqueue requests is larger than this value. + ngram_prompt_lookup_max: Max size of ngram token window. + ngram_prompt_lookup_min: Min size of ngram token window. + draft_token_acceptance_method (str): The method to use for + accepting draft tokens. This can take two possible + values 'rejection_sampler' and 'typical_acceptance_sampler' + for RejectionSampler and TypicalAcceptanceSampler + respectively. + typical_acceptance_sampler_posterior_threshold (Optional[float]): + A threshold value that sets a lower bound on the posterior + probability of a token in the target model for it to be + accepted. This threshold is used only when we use the + TypicalAcceptanceSampler for token acceptance. + typical_acceptance_sampler_posterior_alpha (Optional[float]): + A scaling factor for the entropy-based threshold in the + TypicalAcceptanceSampler. + disable_logprobs: If set to True, token log probabilities will not + be returned even if requested by sampling parameters. This + reduces latency by skipping logprob calculation in proposal + sampling, target sampling, and after accepted tokens are + determined. If set to False, log probabilities will be + returned. + disable_log_stats: Whether to disable periodic printing of stage + times in speculative decoding. + """ + self.draft_model_config = draft_model_config + self.draft_parallel_config = draft_parallel_config + self.num_speculative_tokens = num_speculative_tokens + self.speculative_disable_mqa_scorer = speculative_disable_mqa_scorer + self.speculative_disable_by_batch_size = \ + speculative_disable_by_batch_size + self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0 + self.ngram_prompt_lookup_min = ngram_prompt_lookup_min or 0 + self.draft_token_acceptance_method = draft_token_acceptance_method + self.typical_acceptance_sampler_posterior_threshold = \ + typical_acceptance_sampler_posterior_threshold + self.typical_acceptance_sampler_posterior_alpha = \ + typical_acceptance_sampler_posterior_alpha + self.disable_logprobs = disable_logprobs + self.disable_log_stats = disable_log_stats + + self._verify_args() + + def _verify_args(self) -> None: + if self.num_speculative_tokens <= 0: + raise ValueError("Expected num_speculative_tokens to be greater " + f"than zero ({self.num_speculative_tokens}).") + + if self.draft_model_config: + self.draft_model_config.verify_with_parallel_config( + self.draft_parallel_config) + # Validate and set draft token acceptance related settings. + + if (self.draft_token_acceptance_method is None): + raise ValueError("draft_token_acceptance_method is not set. " + "Expected values are rejection_sampler or " + "typical_acceptance_sampler.") + + if (self.draft_token_acceptance_method != 'rejection_sampler' + and self.draft_token_acceptance_method != + 'typical_acceptance_sampler'): + raise ValueError( + "Expected draft_token_acceptance_method to be either " + "rejection_sampler or typical_acceptance_sampler. Instead it " + f"is {self.draft_token_acceptance_method}") + + if (self.typical_acceptance_sampler_posterior_threshold < 0 + or self.typical_acceptance_sampler_posterior_alpha < 0): + raise ValueError( + "Expected typical_acceptance_sampler_posterior_threshold " + "and typical_acceptance_sampler_posterior_alpha to be > 0. " + "Instead found " + f"typical_acceptance_sampler_posterior_threshold = " + f"{self.typical_acceptance_sampler_posterior_threshold} and " + f"typical_acceptance_sampler_posterior_alpha = " + f"{self.typical_acceptance_sampler_posterior_alpha}") + + @property + def num_lookahead_slots(self) -> int: + """The number of additional slots the scheduler should allocate per + step, in addition to the slots allocated for each known token. + + This is equal to the number of speculative tokens, as each speculative + token must be scored. + """ + return self.num_speculative_tokens + + def __repr__(self) -> str: + if self.ngram_prompt_lookup_max > 0: + draft_model = "[ngram]" + else: + draft_model = self.draft_model_config.model + num_spec_tokens = self.num_speculative_tokens + return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})" + + +@dataclass +class LoRAConfig: + max_lora_rank: int + max_loras: int + fully_sharded_loras: bool = False + max_cpu_loras: Optional[int] = None + lora_dtype: Optional[torch.dtype] = None + lora_extra_vocab_size: int = 256 + # This is a constant. + lora_vocab_padding_size: ClassVar[int] = 256 + long_lora_scaling_factors: Optional[Tuple[float]] = None + + def __post_init__(self): + # Setting the maximum rank to 256 should be able to satisfy the vast + # majority of applications. + possible_max_ranks = (8, 16, 32, 64, 128, 256) + possible_lora_extra_vocab_size = (0, 256, 512) + if self.max_lora_rank not in possible_max_ranks: + raise ValueError( + f"max_lora_rank ({self.max_lora_rank}) must be one of " + f"{possible_max_ranks}.") + if self.lora_extra_vocab_size not in possible_lora_extra_vocab_size: + raise ValueError( + f"lora_extra_vocab_size ({self.lora_extra_vocab_size}) " + f"must be one of {possible_lora_extra_vocab_size}.") + if self.max_loras < 1: + raise ValueError(f"max_loras ({self.max_loras}) must be >= 1.") + if self.max_cpu_loras is None: + self.max_cpu_loras = self.max_loras + elif self.max_cpu_loras < self.max_loras: + raise ValueError( + f"max_cpu_loras ({self.max_cpu_loras}) must be >= " + f"max_loras ({self.max_loras})") + + def verify_with_model_config(self, model_config: ModelConfig): + if self.lora_dtype in (None, "auto"): + self.lora_dtype = model_config.dtype + elif isinstance(self.lora_dtype, str): + self.lora_dtype = getattr(torch, self.lora_dtype) + if model_config.quantization and model_config.quantization not in [ + "awq", "gptq" + ]: + # TODO support marlin + logger.warning("%s quantization is not tested with LoRA yet.", + model_config.quantization) + + def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig): + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if scheduler_config.chunked_prefill_enabled: + raise ValueError("LoRA is not supported with chunked prefill yet.") + + +@dataclass +class PromptAdapterConfig: + max_prompt_adapters: int + max_prompt_adapter_token: int + max_cpu_prompt_adapters: Optional[int] = None + prompt_adapter_dtype: Optional[torch.dtype] = None + + def __post_init__(self): + + if self.max_prompt_adapters < 1: + raise ValueError(f"max_prompt_adapters " + f"({self.max_prompt_adapters}) must be >= 1.") + if self.max_prompt_adapter_token == 0: + raise ValueError("max_prompt_adapter_token must be set.") + if self.max_cpu_prompt_adapters is None: + self.max_cpu_prompt_adapters = self.max_prompt_adapters + + def verify_with_model_config(self, model_config: ModelConfig): + if self.prompt_adapter_dtype in (None, "auto"): + self.prompt_adapter_dtype = model_config.dtype + elif isinstance(self.prompt_adapter_dtype, str): + self.prompt_adapter_dtype = getattr(torch, + self.prompt_adapter_dtype) + + +@dataclass +class MultiModalConfig: + """Controls the behavior of multimodal models.""" + + limit_per_prompt: Mapping[str, int] = field(default_factory=dict) + """ + The maximum number of multi-modal input instances allowed per prompt + for each :class:`~vllm.multimodal.MultiModalPlugin`. + """ + + # TODO: Add configs to init vision tower or not. + + +_STR_DTYPE_TO_TORCH_DTYPE = { + "half": torch.float16, + "float16": torch.float16, + "float": torch.float32, + "float32": torch.float32, + "bfloat16": torch.bfloat16, +} + +_ROCM_NOT_SUPPORTED_DTYPE: List[str] = [] # + + +def _get_and_verify_dtype( + config: PretrainedConfig, + dtype: Union[str, torch.dtype], +) -> torch.dtype: + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct + # because config.torch_dtype can be None. + config_dtype = getattr(config, "torch_dtype", None) + if config_dtype is None: + config_dtype = torch.float32 + + if isinstance(dtype, str): + dtype = dtype.lower() + if dtype == "auto": + if config_dtype == torch.float32: + if config.model_type == "gemma2": + logger.info( + "For Gemma 2, we downcast float32 to bfloat16 instead " + "of float16 by default. Please specify `dtype` if you " + "want to use float16.") + torch_dtype = torch.bfloat16 + else: + # Following the common practice, we use float16 for float32 + # models. + torch_dtype = torch.float16 + else: + torch_dtype = config_dtype + else: + if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: + raise ValueError(f"Unknown dtype: {dtype}") + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] + elif isinstance(dtype, torch.dtype): + torch_dtype = dtype + else: + raise ValueError(f"Unknown dtype: {dtype}") + + # Verify the dtype. + if torch_dtype != config_dtype: + if torch_dtype == torch.float32: + # Upcasting to float32 is allowed. + logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + pass + elif config_dtype == torch.float32: + # Downcasting from float32 to float16 or bfloat16 is allowed. + logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + pass + else: + # Casting between float16 and bfloat16 is allowed with a warning. + logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + + return torch_dtype + + +def _get_and_verify_max_len( + hf_config: PretrainedConfig, + max_model_len: Optional[int], + disable_sliding_window: bool, + sliding_window_len: Optional[int], + spec_target_max_model_len: Optional[int] = None, +) -> int: + """Get and verify the model's maximum length.""" + derived_max_model_len = float("inf") + possible_keys = [ + # OPT + "max_position_embeddings", + # GPT-2 + "n_positions", + # MPT + "max_seq_len", + # ChatGLM2 + "seq_length", + # Command-R + "model_max_length", + # Others + "max_sequence_length", + "max_seq_length", + "seq_len", + ] + # Choose the smallest "max_length" from the possible keys. + max_len_key = None + for key in possible_keys: + max_len = getattr(hf_config, key, None) + if max_len is not None: + max_len_key = key if max_len < derived_max_model_len \ + else max_len_key + derived_max_model_len = min(derived_max_model_len, max_len) + + # If sliding window is manually disabled, max_length should be less + # than the sliding window length in the model config. + if disable_sliding_window and sliding_window_len is not None: + max_len_key = "sliding_window" \ + if sliding_window_len < derived_max_model_len else max_len_key + derived_max_model_len = min(derived_max_model_len, sliding_window_len) + + # If none of the keys were found in the config, use a default and + # log a warning. + if derived_max_model_len == float("inf"): + if max_model_len is not None: + # If max_model_len is specified, we use it. + return max_model_len + + if spec_target_max_model_len is not None: + # If this is a speculative draft model, we use the max model len + # from the target model. + return spec_target_max_model_len + + default_max_len = 2048 + logger.warning( + "The model's config.json does not contain any of the following " + "keys to determine the original maximum length of the model: " + "%s. Assuming the model's maximum length is %d.", possible_keys, + default_max_len) + derived_max_model_len = default_max_len + + rope_scaling = getattr(hf_config, "rope_scaling", None) + + if rope_scaling is not None: + if "type" in rope_scaling: + rope_type = rope_scaling["type"] + elif "rope_type" in rope_scaling: + rope_type = rope_scaling["rope_type"] + else: + raise ValueError( + "rope_scaling must have a 'type' or 'rope_type' key.") + + # The correct one should be "longrope", kept "su" here + # to be backward compatible + if rope_type not in ("su", "longrope", "llama3"): + if disable_sliding_window: + # TODO(robertgshaw): Find a model that supports rope_scaling + # with sliding window to see if this case should be allowed. + raise NotImplementedError( + "Disabling sliding window is not supported for models " + "with rope_scaling. Please raise an issue so we can " + "investigate.") + + if rope_type == "mrope": + scaling_factor = 1 + else: + if rope_type == "default": + rope_type = "mrope" + scaling_factor = 1 + else: + assert "factor" in rope_scaling + scaling_factor = rope_scaling["factor"] + if rope_type == "yarn": + derived_max_model_len = rope_scaling[ + "original_max_position_embeddings"] + derived_max_model_len *= scaling_factor + + # If the user specified a max length, make sure it is smaller than the + # derived length from the HF model config. + if max_model_len is None: + max_model_len = int(derived_max_model_len) + elif max_model_len > derived_max_model_len: + # Some models might have a separate key for specifying model_max_length + # that will be bigger than derived_max_model_len. We compare user input + # with model_max_length and allow this override when it's smaller. + model_max_length = getattr(hf_config, "model_max_length", None) + if model_max_length is not None and max_model_len <= model_max_length: + if disable_sliding_window: + # TODO(robertgshaw): Find a model that has model_max_length + # with sliding window to see if this case should be allowed. + raise NotImplementedError( + "Disabling sliding window is not supported for models " + "model_max_length in the config. Please raise an issue " + "so we can investigate.") + else: + msg = ( + f"User-specified max_model_len ({max_model_len}) is greater " + f"than the derived max_model_len ({max_len_key}=" + f"{derived_max_model_len} or model_max_length=" + f"{model_max_length} in model's config.json). This may lead " + "to incorrect model outputs or CUDA errors.") + if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN: + logger.warning( + "%s Make sure the value is correct and within the " + "model context size.", msg) + else: + raise ValueError( + f"{msg} To allow overriding this maximum, set " + "the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1") + return int(max_model_len) + + +def get_served_model_name(model: str, + served_model_name: Optional[Union[str, List[str]]]): + """ + If the input is a non-empty list, the first model_name in + `served_model_name` is taken. + If the input is a non-empty string, it is used directly. + For cases where the input is either an empty string or an + empty list, the fallback is to use `self.model`. + """ + if not served_model_name: + return model + if isinstance(served_model_name, list): + return served_model_name[0] + return served_model_name + + +@dataclass +class DecodingConfig: + """Dataclass which contains the decoding strategy of the engine""" + + # Which guided decoding algo to use. 'outlines' / 'lm-format-enforcer' + guided_decoding_backend: str = 'outlines' + + def __post_init__(self): + valid_guided_backends = ['outlines', 'lm-format-enforcer'] + backend = self.guided_decoding_backend + if backend not in valid_guided_backends: + raise ValueError(f"Invalid guided_decoding_backend '{backend}," + f"must be one of {valid_guided_backends}") + + +@dataclass +class ObservabilityConfig: + """Configuration for observability.""" + otlp_traces_endpoint: Optional[str] = None + + # Collecting detailed timing information for each request can be expensive. + + # If set, collects the model forward time for the request. + collect_model_forward_time: bool = False + + # If set, collects the model execute time for the request. + collect_model_execute_time: bool = False + + def __post_init__(self): + if not is_otel_available() and self.otlp_traces_endpoint is not None: + raise ValueError( + "OpenTelemetry is not available. Unable to configure " + "'otlp_traces_endpoint'. Ensure OpenTelemetry packages are " + f"installed. Original error:\n{otel_import_error_traceback}") + + if ((self.collect_model_forward_time + or self.collect_model_execute_time) + and self.otlp_traces_endpoint is None): + raise ValueError( + "collect_model_forward_time or collect_model_execute_time " + "requires --otlp-traces-endpoint to be set.") + + +@dataclass(frozen=True) +class EngineConfig: + """Dataclass which contains all engine-related configuration. This + simplifies passing around the distinct configurations in the codebase. + """ + + model_config: ModelConfig + cache_config: CacheConfig + parallel_config: ParallelConfig + scheduler_config: SchedulerConfig + device_config: DeviceConfig + load_config: LoadConfig + lora_config: Optional[LoRAConfig] + speculative_config: Optional[SpeculativeConfig] + decoding_config: Optional[DecodingConfig] + observability_config: Optional[ObservabilityConfig] + prompt_adapter_config: Optional[PromptAdapterConfig] + + def __post_init__(self): + """Verify configs are valid & consistent with each other. + """ + self.model_config.verify_async_output_proc(self.parallel_config, + self.speculative_config, + self.device_config) + self.model_config.verify_with_parallel_config(self.parallel_config) + self.cache_config.verify_with_parallel_config(self.parallel_config) + + if self.lora_config: + self.lora_config.verify_with_model_config(self.model_config) + self.lora_config.verify_with_scheduler_config( + self.scheduler_config) + if self.prompt_adapter_config: + self.prompt_adapter_config.verify_with_model_config( + self.model_config) + + def to_dict(self): + """Return the configs as a dictionary, for use in **kwargs. + """ + return dict( + (field.name, getattr(self, field.name)) for field in fields(self)) diff --git a/vllm/connections.py b/vllm/connections.py new file mode 100644 index 00000000..e785a0b3 --- /dev/null +++ b/vllm/connections.py @@ -0,0 +1,167 @@ +from pathlib import Path +from typing import Mapping, MutableMapping, Optional +from urllib.parse import urlparse + +import aiohttp +import requests + +from vllm.version import __version__ as VLLM_VERSION + + +class HTTPConnection: + """Helper class to send HTTP requests.""" + + def __init__(self, *, reuse_client: bool = True) -> None: + super().__init__() + + self.reuse_client = reuse_client + + self._sync_client: Optional[requests.Session] = None + self._async_client: Optional[aiohttp.ClientSession] = None + + def get_sync_client(self) -> requests.Session: + if self._sync_client is None or not self.reuse_client: + self._sync_client = requests.Session() + + return self._sync_client + + # NOTE: We intentionally use an async function even though it is not + # required, so that the client is only accessible inside async event loop + async def get_async_client(self) -> aiohttp.ClientSession: + if self._async_client is None or not self.reuse_client: + self._async_client = aiohttp.ClientSession() + + return self._async_client + + def _validate_http_url(self, url: str): + parsed_url = urlparse(url) + + if parsed_url.scheme not in ("http", "https"): + raise ValueError("Invalid HTTP URL: A valid HTTP URL " + "must have scheme 'http' or 'https'.") + + def _headers(self, **extras: str) -> MutableMapping[str, str]: + return {"User-Agent": f"vLLM/{VLLM_VERSION}", **extras} + + def get_response( + self, + url: str, + *, + stream: bool = False, + timeout: Optional[float] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ): + self._validate_http_url(url) + + client = self.get_sync_client() + extra_headers = extra_headers or {} + + return client.get(url, + headers=self._headers(**extra_headers), + stream=stream, + timeout=timeout) + + async def get_async_response( + self, + url: str, + *, + timeout: Optional[float] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ): + self._validate_http_url(url) + + client = await self.get_async_client() + extra_headers = extra_headers or {} + + return client.get(url, + headers=self._headers(**extra_headers), + timeout=timeout) + + def get_bytes(self, url: str, *, timeout: Optional[float] = None) -> bytes: + with self.get_response(url, timeout=timeout) as r: + r.raise_for_status() + + return r.content + + async def async_get_bytes( + self, + url: str, + *, + timeout: Optional[float] = None, + ) -> bytes: + async with await self.get_async_response(url, timeout=timeout) as r: + r.raise_for_status() + + return await r.read() + + def get_text(self, url: str, *, timeout: Optional[float] = None) -> str: + with self.get_response(url, timeout=timeout) as r: + r.raise_for_status() + + return r.text + + async def async_get_text( + self, + url: str, + *, + timeout: Optional[float] = None, + ) -> str: + async with await self.get_async_response(url, timeout=timeout) as r: + r.raise_for_status() + + return await r.text() + + def get_json(self, url: str, *, timeout: Optional[float] = None) -> str: + with self.get_response(url, timeout=timeout) as r: + r.raise_for_status() + + return r.json() + + async def async_get_json( + self, + url: str, + *, + timeout: Optional[float] = None, + ) -> str: + async with await self.get_async_response(url, timeout=timeout) as r: + r.raise_for_status() + + return await r.json() + + def download_file( + self, + url: str, + save_path: Path, + *, + timeout: Optional[float] = None, + chunk_size: int = 128, + ) -> Path: + with self.get_response(url, timeout=timeout) as r: + r.raise_for_status() + + with save_path.open("wb") as f: + for chunk in r.iter_content(chunk_size): + f.write(chunk) + + return save_path + + async def async_download_file( + self, + url: str, + save_path: Path, + *, + timeout: Optional[float] = None, + chunk_size: int = 128, + ) -> Path: + async with await self.get_async_response(url, timeout=timeout) as r: + r.raise_for_status() + + with save_path.open("wb") as f: + async for chunk in r.content.iter_chunked(chunk_size): + f.write(chunk) + + return save_path + + +global_http_connection = HTTPConnection() +"""The global :class:`HTTPConnection` instance used by vLLM.""" diff --git a/vllm/core/__init__.py b/vllm/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/core/__pycache__/__init__.cpython-310.pyc b/vllm/core/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41f568833782bb5d1ea2f7ad1340efed190e8b50 GIT binary patch literal 150 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Heeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_esX?Ms(yTYW?p7Ve7s&k8%I9y>cbJG)q50T38~Ac-M?1wcJ0QXok1O^O0x0wkre*To*p8id;yB5%^HP*!=QweW`y@Y<$4T@z%#T!0g zGG3;h@eI#w8+8N!t+v^*>Q=|D+Xn0Hw$pLzZYNvMO4(`WI{A8D@@~7(nW|59iuGb= zx;`y+S=5#4Gm_6CU#^!sv-Mdi=i8OeTz#%HU!Rw9p}nKCP+ySzRC{M+= z9;`oTWc&mELpP23;TJRBtXFwA<5hg?rd6+@Jm<}$Jnz>~ei-E)-U7-CQho&Go!%~# zclrBq|A=Rv%hVS851+1HX!n*cS3Ava^P;b+;iYC67lLZoyX<#$gJ9`<#QB+;8Cj>F zc{VCFF9e}#E{7dIywvkh?JW1!yJ3_Mo0t7-ms)MVW<ORt|@<|^-j%-?B`m` zVPwDB3c@J&+FID^b(?Lp%boMDtoz+%pTnQ?!^kHI{L&)xFs0sYYaKvy{j6vCInT!B_|8hsbG_`l4hSUg+xc8R@9p&pe!d^LuzJFgHh40 zjGhbLPN|)ZYcVEjcX^9aTanm??M{4H19Ek6}s3I5?oHl{rXPSZk-70ukqcfa|OLR|Zq0#~}2*rRrO#8#ZWZsJtW9Rv1)4*1jUyR)zMfxC@F>e#h@h*Oymh zun>?Pag|AYsc1FYe&PTy6A^TAM31N>YV6YJJxPpIJR~j@}PU{YP;Pzw%k+x zF%4rZUm90WEUjISa*c*CN`vDECKB82?|Fu-K1L2pNxL(OASV;P)C?{~vz#zNt7rNEicv16C;vf|G?;cOm`S_H zk=ZVDM#+!)Z2-7!Y5*QV6a83%rR3Q*7ySr|?{NUz?C(rL58Etr@Z`t*b`LZ3!{ia0 z&zZvtHi$}&6%){LjG~s1^dC%lXiA5V3WGYvktZOSPX-KoVY3~B@W~+v58VTV(H9Xn*pp=$-T54!0J~IU3Knf7!Z9gi}-5w>nw1uBqU-O!wFQk2P2*AD+07e>ay8xQQ z4ai%w|HN6*!&NFpzj~<^Ruvw!R6Tm4+VPtZmEB%g4c6DzdJ5vJ3MuNf0%~4w>5N{q zwB=jm@@Wk(>K2eg1#;x40exU1v911*6!^F8c-z$cUfV%Q(QS78Mk6XT8l9fE-e$ho zXk1xuw&Oe5M#JkZHyVn(u4a)$)~m0-p%x_vJ)p?HD$k_E38jVA)n2cynVQbJ7O6m_h-B^dKJA{KQH6h+6Jgk zqkk3}T=ajv6qiKr*JtDzu2=%1^Xs!xgE3$Qh|aH9(1RAhUhh6Uf8NW29`{G{V!3(a zl%e)unyDJ7h8t(Ls4~*mv$z8C%D@<88kvEK)EHPuVJ;!HS{bBv=nP=sAm@hJo23}vrAB4XUumwlL*e4a za;Ve{t7oNS12!{^)dtwSsqRPp2!RQ0kzCK<3d%@a$2IV0x<s%&uaW5OU&EYN(`ZWjb zGB)~q=3^O?2uI7Bb>_As$cu`CCMuN+zS4HYP{n?a+NqXT&)}c7E}lcK?g;iaVb^>* zv_o#&lbSkBa+VS`tQY2O3>=V6i)MisG;F!RsrYl>FFCf+zdse1Uday;GoNS8B72a&N*Zt>}>M4ya zA~>>O?+ca{t-VR)QA_;geM-wLnjxCmc!5E3OL^U zc9P{DHX}*c!X=ADy9DeFQod!VtdOL+N|6Bx4Aq838(2?@qC*ZMLljuNfMUZ4H*%=j zsOTi`)N{SICwgge3bcE^fh#B=DM8`5Mh^7WucjjRUe;%V)Au+MF}h*g8o?xGN4-ee zNKD1Fje*!+-$GMaW*vMUz;a*87^;~M+rPQ%6+`J3-&>os< zz9U8;?!)b)jYOxO_eyjsp$D8qFbiQeii^nTUG z7^tD=(T1Vuz(_Q`MQ=Asvz`m1VQ;kXMt7O}k&j{qY73BzlKdz#ICo*1myjXbj$42i zL4XY_vP3uP z%TAhxQzKF>a#XVmryuDja+_;we%GtH+G-h^F(S_*qa`|QlKV)0zm7{x7S|P{&niM8 z(p(vuEZ^_%Pe<6M0cc>5O-pKv;gayCsAKYGkcFBwL_LyIP1=D_n^prxgxGccBd@KC z1FZuqKwUpjJ<}War73vct~Oh3?iFK|XR9A37_TFVthL^nXh8J}>kbZ+p?UB-s1m8; zS~=)9nrhq5^CKDpWV(-S2pphf-GG!OA&?A2uKo~)f*7R^mkH^qeVN7WRQ7jpGbke2 zRN2sxd$w7+dn;_p(8RDz)HK}WZNIAlNz;$iAAf*LeE~`T;ccG#akR5K&*7+I)Ei9F zo=UC!hrIbFlKvwf^3;!{mER+GhcM?zpYRLxmtO8(ZMIt;T&|wK)(kI+FL*r`BC!`W zrjednx@|;rXL{kw9ULcUx2(3mwo~DMgvY%{sX~s!l0C4^B<`WeE%wGmnWUObE-+bU zB5th-2B6ygue|9a=^xzYf%j6ktm(<$aW(GR3#zSt}*+tI*q0!kxS*Q+rcw z=4ZVIj_IbnIrN|Mi%_xjqd4{H{DSxPu+L81-6h_d5|lKRqyv@7ep7gc4OP7tYO_2s zS{9?-=j}t=S&WqT_WKn)qk`J1cL43@q!xZ2jBwDqA2st6?GAYlpms;xu7Y+CdJmyy zL7v1hHQYP>T{w5LSIouCVg}yr@hmtIiyFBnrFOcQS*S;^q%vc zNAAHGV^}wt;|t!4sC{VoOkMku_cCe^V;;bq_loy9lxlJ5RqyjCJuKs#0bOYfzUG}p z?ITkAL+E9DuY2cEb7bOq=e;*jd(?XqSk^SR6hrcs_XXTJCXnb6zUb9abKLu3{{)`x zV9YOhUq;QN9yR=hXij`^qi30Kp6H)S&f$1E)qzQagVgE;KfLDSKnzEmJc*TawA3TI zrT_J#^S}1ucl+-?b>;=}0YxRAGF)6+kKJavyu4Y?UxUwBb(-pOR0<&Bd2V~H)%AMU zqLSaGZIK-3DjI^zStQAidD?}Pf&rE9Ry%Ou!qMfxPh+UL(9#OUqitx%wFg6oHsHY6 zletAJ(uR$gX=DesXI^%KM`RNQcNo2fv>xX(;!7;RWXP#A;nYUavuHMi(_v|Kh8wk; z@aQzkQl3Q_=kzF7y5(?gq6Nz%EyPB-p?0r#xp@gbu}0ADg~8ZI}}u?bzk_ z_;5N&_yCUCCl1b9-D(QzV2QecgpSn8s?3GCfh5Ys2o2OO7PD`ie);^%XI_kESdZ8O zShZZ(E&KSFN)%Eqv8sa)rq}lE^vp{F~3e9+|IwE^=Nyeg65^6H5OvHu)1s z>brQrpSJS#Ma?K&vI(MyWF1p0xpNtj>65fpTyAy~_Vf|?lnH2e7hG~p!Ls4~weQ++ zG}$~ei|gPbfiE~`@jv(peCC?{G=wgV4t`#{SSzhyh%|&^5P8d^{sQhMNOYnngy1q8 zATW|nXxt>>(35WjB})6yhec}S2KH+HWTrbkus{X{^uaQrHB`R7bLGguRIkWL#>Uhw95m>W zq<0bWYFhhdV>PaRoTvsuGWuYkBxu*t;3jJ+1Jt`rzK#R}9m@zr2I_6ry~8?}^0nnb zrS{gYtFN&BI_nE@%o6S;jwfMzSyWXb;=yG>{~($N>VSX6a%!<1Ys4!9Fq z?v8o4=yK62hHZg!!JB2v{(ibHi}kTTChECF7B1mD3a?2~52T8lm`?6ZF%OhOs9%Pv zQ-2bjYR-uC`}3%XrRM8zoH+xjiQA&thw@JSS+l!=H2@lqdUk&7#V4as?ckO3uN zJ=gWGrJV;va|r9`b$WXAOHu;$cczqqEGs(GI`;W0M>0xNvR8gQg z!=r*5rcj67n}JjXH7v+^USjAVj+95|^(<8+-C#82l3*^*t3E9OVc>*7?oTZJTmxES zb4rbs^>+K`I4>?$!Cu&IHd}j*EjDK}T7ap|)*Q~g!6Qhx=> zDD;Y}GQphDVx!Z%e!)*!kzXL>6oLguWa!Z}iru#N*m`i3;S6XP3%OEP8FUNN9g3AP zRxL5R=MIwuzY+H5fUnUwCFoK&e z!2OJ93n(f4IXPQSrB9@yH4$~1oW@5SBMurKLoM|`P#~!jjjLkSQXhq|U9qLKV2_?m z@Yf5BjZxiXMV@X3ty-y9bsdnUpaGON=YC7L#nNiMd& z&g#sHd66+^R=;*yhvedfU^A4L8-X~47ptWsd-b`2crIZd%X(7XslUMFZ6;#-(0Eb5 z$Yd`Q?haMRM2r}bf!}0Nh)5*gODK++JG6uK14Ik{0*fLDXEcj#OF%_^tfSS%?~-0< z&AKjxpgE6yAEFu{3K7ZRkb-iE@WN4sX^XbZ@s5r79XU`` zabkj|!Eyh|3@An)hvlhY5fov2h6IP$ zS9Z&phcKk0u`lcJ7x4scPD9Osv7)_&TJ=jv>N9~_)|{km#Z>dq#;*SI{~_k6LWNMQxbt(S6$5> zs53UQxcg9iFE;>FkbvRa0b&vfT~I%ZC#Y{Rxrqev{Xt^|q05Xqj$G}N7qd)binf-v z#%zXbkC`JJDRMgj$-Hjrq6?2`uQxpn|Rp11rKJF^zecVP)DThtm+2Hpm#= zaQj15^(?MnfMly7wjpYZ=v>7nXw_o45fEx@*g)A{aJ~JQ*w3qQm_b>`cC+lfxgl~k za-cOAa_mHyUoC(Vu~8cCt3QjnBKFmJ?zn47D9z(3bE$egM97Z>rNZkN`w~Vr-`b`Y zBdDZqV?Ns4+l5?|J@?k>vuB@wPMg^x6KHyBd2)g|?wi$5GP%O!t4#hhk}W%D^~aL7oliehzHs5 z?oYvo*q!kxin1%3CesbwUs1`!I}0P_Z0xli3z{6$L~mo1FrzoVV4K6#3j-UWXFFkb zgIOs1Asui&EYiV%UV$Wml(%qEsf%tHxX=O)w2W+E2PQO+XoD-zJBf17>RqgeX)#7c zZy(0sOil8WP|+(d+SEw6Fr>R7;$CF6^c2{8^24SsLB1M)Z?7 zTqj5W9ORFU|Mp#ONFl$?{sU>K-!je?p+vqSPrWBRMeT03L?p#LguN#)JP$s~jEW=h z3*#$uu<(aM>SX2$bT2(s>Qiy+0tM7WThw89i4cZ&5rN6w=W}pLgQwuD`f&g<7HCC| zr?xmp2uXbmi}-TkOXdwjOgcz`)y#_db@K|%Q<5*juU_ImD>9dk*YiVfZ#_SB2TziD z;UuLVUTj%A^*Ycj=cq2iv&wh-$I_4*)AG_uzO~a1P9c-@uNAfEy$KNXuJ~7&`*kLw zkHjvx#-gzGrsQ>6q2CD*i+YGRWhW>?oK%!5ce61Vr$`y>1f6YHu&tr6HSZMf<{avE z3B>pycUM9)aF&0M360SPG0*E+T*CKQQ7vETp@5eYt18jAvdKMwR@kD69D5siB~zr( zO120xBMPHQ?2EHa=F(da-{54p-~|J&+-==wN%Jv?sa_LmR)3q1{xFJCzcGa8H<|l8 z!y+k3UT9H&54kp=Qhy(rbmwvc%||1Mzr>*@2tsHvoykzOcR>AgPyol30pIXgKq4X6nb{Crfaae&IAS8CsVADPj zxS|fCB@``P0EkXvuy%H{B*vN$O->ug-{M5c;3KxNVC5+5h?>L$Kq*@!02nG=mzlD)*$FJLn!6kZSoCrkk{1~Pqp^Dr<(?N-#$8>9y?^~hq z@HYy%`lqN)pb(KmznmI{LU=a?iU`UKj&$+B{{+TY{|r6*`_e?rX2UW0PN0UQCcA;y zod$BsA{?Dalc*->WZr>kjNKaSe!5$>wAm?lq|GLQIkEo{;Wv0UQs#x1e%pRiiOGuy z*+ZOf+VmXdW9RVG08%_`C5>dg8Cp!6rL-TzG$+W5G$O&!oCXclMG09;EOu*CJ46=z zKWadj4i1wCkke-?Lim~(f1ZGG@G|0d)4-vlX{7xaU?hPdV7pP+1nh>%5JPBZGuF)3 znJtUP?xK{*PNI|@zCEy-8BxWGLKXP^yZD#Jp(0zqnM|UR=|8#MB(@xerixlq7E~+X zf6iwf943Ol9mL0$p}s)?NQ1J+t$kz=#!!O2;SihZEk0I+n>ph4JvIbJz!Hdd5~86c zJ~U8*Uj$4#2JVsV;7RHA(I65vbDY3@I7CD1Dfp$20nui!XsWrq?Bg6eY<2G8SEpf3 z@>8=&6UP#Ci7+@{>#-Zwl_|3HCR^Lub38Pv>eJ#K)6et0WDhF>^xtu|!r+gjEW)lB zMr=qlw*=Vk&~NeANf|}-aGfGf!(nazz_#O80w-L?+YJ$wwM%#T(t7u@yz#tcQ#%xY zYsA8&GX!2_3cZ1e<6&R~__25EO{{PmwmNiY@vV_6aT7Xt^$G4c(Z@5`7GVp!2+gaG z$AO`yu4!biJjnKV{YlPs&SMX=se!%$Na{n{q+z^{H!I`Um35d!PWe}T75JVU@I|?# zQL+IQy#ITSGuLUpgO_i{#;DH^E9k|J(XN;1N6=}Kb9cwh#pgY~NWZ~FqT#@p9L9>XNcX6XH$ZD8PB1q zbTg5q?)I`&{p;b>M+82Az=^keL>y@Op~8m z#0P~MW&CCgZOu@>hYI}r53D~pOhlUHPPUt!3tscocL*CUz_z_i{}8D3NZPS#jnJmU zXa4r^!2-gjEKM2ocL)r*m;{1Y7aKm$JN<3tYRU-wyug)U3r$c3@(Pxad==s=a7Yb5 znOT9~0s0d!xlw=8Kn67o^{L+L_P_GFy!1c%4FwOMF5!1f;6p{2Fuy<&dz&w|t|E2~ zK*-5-Riael&qq}8djo7(Mf8RQQnlOF!*JVFgUhY8VGX__5#YNGI?Onz>F+VH>)4ue z#faWHPHF*(=yjN3U(IZQ=x-S#x*a6&2WXLKcVYKUeZmD1D*YoOpW`lM`p<7ipL(tn zB+65;52aF$O!-MF74L{ZoTDXjF>~2cFdj!>7J5w}{+H-7LBIlWlt8?TlWVgTnLpXa z{zv5NIXR)jo?j0j^X8PJ`^3VKrAwL4?{R2h*{p3;a3GS$t4jux>3tfR0RpS!)n7(0ie7}1mUTAt-#qFZ>Po); zi}$DT{^U0$7z>f*{c;k1uRlTZL=}2!^AJWhnvdZD>M9elkX}a)1aOEIR4Ob+;A2N|8BBn>Me~?8HFe}LYPh3G6sEZ@(;rp`xcs#z|{DJG5-^UjX{@BbF{@=n0 z)3NWsL#DrXU>5F}hwKlA-#Os-53Irml|ANxiB<@kOumm$cpnF6QYw5PUrLC-n}GVl zuztV&LDHjy7v&a~%vnAaC$x9X%EZ_U0#6K+ErA^T4nyJneLE^U_t}+&%1k9!$So8W ztdd=_@Rf!USSrPZidp0gIJP@&B3*D5u~R#$Xt1ar61=f{VOf!<xNb1>Mu*A1hMCL_qc%9MHdc}n@gyw|O{xb)fVnS-E7x?9zB{42E!I8r13sTGP z>F8TaTGY!|1?oE{Z?1ATPBu*4Xv24kzFQFYvwp%zVvcxV)fVk=G zLNr;Zbt-j|7f+{?>GY{BHSKi94{hK2lxb%=)0h4U`_}10CeMD!#g_W}eP?&EAVDYb zG&2DQ2fOF`o$q}2@31#BQ_kV@{6GKu+TUK!<^Gp1*^h;nm+%dKg2Kzya-QLtZKGq> z%uc?R?^rdqC6McrOs4sN}la@xiejxmgi!-(wV8v$aASZ+u2v! z*O{x$$@^4$e`mfnFVE%nfzCp0p>wcyP~NB8hpMHG{okwbq7&-q~ z?NR@D?Zma5H{;FT%Xzc@!Mpj|W8S{&x$4~3vzMLacCWGSblk4H>MJK)bHn&0aKhfY z-_;$0#h+ofxoSSLt~48AWWCu8!l?9CKWz58Zo6tkrR)B!f!}TTcrM%+^xJ$Wzs%Wh zxXbdk@JfGhtv`^u%Wbr{VNbEU)Ktyr;48gOe-QefR0KDa+gSHibofm-2roApeh`c` zVCZv=HNUZ5@Am2qx3SjjuGV|19&EVv75px9@y@EM11!&PV1a2__Il`sgCN>3lU?t- z4gWiO`_;TEV3*NMvX^VBH|R&R$*UfSW$i2e?PkMo5GUEsKD@kyZ!l-(a(=F6fIdvm zsO6{3sr;1Xz2X&o%d_z<;#=|y-jt8GqdBka7rkjOzcTApyqSB2T4`$5yW;Khr@T4z zoc7BrwzuD#N6WN#z^|ZYX2tRryo2|w+N|{BXv;g~9Y)Q*%*Y}}9`TN%c1}j>`S$zs zzU5mh=F!~I9C|$B9Yc=;dTyENQSUfv7BX|6@E$|W!Fc`sgc~b5-s7k}r2F|v?GxUU zs68xwpYjXn_q6v6YK}<1lUUjEPI*<-9QB^Xs0wf}?LQLF`J8teEyuhw*y$p73y5s* ztalEzk7o8ptUT|XNA2-+{p`Lh?*;EGXgv|P7P0DAy|1AL901%-W99MwFM3}`tAiHc z#`}hM0dJ4TZ{PGT;_V6VTmF-%;hrve-$u<-=~=PmCI9L4%*x(n?`53XGhQD2b~QS% zi{#!r@4i*c<#yFO8*9zR+K3g`e8<%+1BP>YE6yA5IV5?1v0QfWc`bB$8{NQBe$elA zgXVIx-3&J!ut8?rX$P#)?}JTFr?uTxr{i~eO8UxF?f`7*uVSX4dE0Mqo>}#~zH-5q zS|S7(-1VGZxCVjYdbeFLz2~g>U@DApf&t)m1DUF?{FUa10;ntsaMtl~w$bZvp6PYZ zY^Y}F>*<1a(?ipljb_*DZ6q*__8jl?##%FQ8f`ZSoMs0?rsH=*NGhPm^MmGUmp={F zTn0qVuE4MshW+5e*|V$7aBZ->*ywf6-fp)$XZotw@*APNob3C#d`#^EJBydvcgs z{<2%VaUzyS=mfEJU3A{<{mM!Ig`aD0_c*Ks$U0R|AaF@=my1K96^3O1b$cQZyAL3pj}^dYoE$VQy1qi1v) zy+PYc=E98Q*rA?r)S&Bh2JNtkBTXT|&Np1;IenZq2rEVeh@_2efb_jV8x+{=-lir2 z@)w<};0O{RRRw5w;!J>w;0WoFiwN#*8847 zY1rJtd2zgR>3L@-UuF*Cdf#sh00fW#j%f5`BREY$I!4(LPBeBb#k`0M;v&&erbCqB@H1MLXn)0MzL8Tj-OWBc)nm(^-;A?+({SPy^^=M+$1 zy&%jf@4%-|N{Go;dM>5|yMs=BwLj42L69AcmSvz8!?PpxA%nz-`mS^B?RUrLYJB;* zUdhh2ffaxdweFl+Y4_alw9`w>E4{aH4OF?-)4ZewLkwpk)<=Mm8?X2)E_h1F`rP8V z=SQuvu&u|!_EciWeE>w>+_i3kQ_^lcGDRlW(X%X$t+&+cLN&3KAHPczFtO2`Q_Hov*Apm#6}R#S=Q0R5+Pjpp;lmweT22l1Le{gZ`%x%NL>_rAUKYE&K( zN;Eru_WFHve$iBV2GQ&|j-uJ|Jw&$hq3^m4DDU)B!jp~fEBFTggkoq6bM@TNL}?83 zD9vF3W!}r7w1x#QH!#!*)ESS!WjLI>QydogE3FpF!qD=}^+Is+_1vEuLmM@ASR7hI zb0K$^bC<%YR(WWv7rlJf@KBFc46pDp$4)mnPesa^aCVq$P0N^l;avK>-^;-;v6p^k z0NPc2M-X%%079}W;B-`P|UuHIFIaxg8AV``}A3-Kt>f^5t96Z zXuhF(K@cyb>lB^6=tv!Afp{MAwDnN?9%`D+$LcUR!n!MZWk-){DY62;y&^l$;6niC z<^=j$nJ5upf=_blB8`E1zfpY1pMXF^x4`r(-uJA<{(^PP>e?dI~?{${w=>uT!+ z`p7@4cM>~l_1otc`n z1pO#s1A56J2ytlKH4T(4BQ#ri&-fT#^U>U$d}y@_!~C{=AAMTIq3Pv6&fl?y#&+p` zPQ8HNsr$J*1Uf;d&0`L1B(h;P3_FGYT%MratTEu(PTm&+Ore zOISUmDyWZ2IM6z97N|MC7T>@8+Ktzit|{WCc9@OCqJmY|RK6EeMl{0)u^{SAFOaFD z`DQ=|w&(e(E~*?J@jmuTDzS<>&PDQ_W>-DVrvu3XE6Vrd$<==81NEgI#glrB#SH5T zOIP2!`c7oN^+r^9^YV?W?^KKG8MZ#lg4>MpSfnNgzZ2x?3>cC0kTpjNR)}2}r#Mxm z;dbjWhx%>Q1Ybu{w&zKah6Nh6KQGy!UeYn%OApEg3%}MwJ5QP|S@LYo3Y~8`lV~@) zg!(3?Z@?c{*ozE!+Pe3rJS_$*ihFJ|6M&-QYE0q(z&yLkjNSliRW z{lCH3TNeRoh4KI^%-l8tH-U!=E|D?Y`*3m};ne0JJM1A=8yG<-DwNRLDYP@g!bQaJ zuFJPtTvDfjlm-P}@*&;u8}Lg-_XF(hrhUgC#@@>rxmza3e_|{#EQHen<08;xE}j#* z+j+867iqMA*jJbSffMo!%#RjwTW2R7VnR2I7a1#S2bUJp(OZuQ`8Yy8a6+K|yiLdd z5K*~6gp@jlsjK_dIsAwSQqSXwd>pDL_++j7n}NE_W>SHAij_sz^GI^)Y3Tw3eLX6Z z<9(>Vrl+1{Ps%s-0t+gftgNV2HryudH!b-{=jtUEud|?z+JO)CZPfiez9Iu=^K(Xt zS_!;fDwH7sX7jVg9BSEO+o{faP_{}YB}CbnHLTAj>#qcRyHpd2UkLv_Ja`5OFd=^P zV*@04NR$T1vV{N_NG(4sKt(~_fKZDR1EQp;a`LyPG@*y()-;F;j4-s<4TuQKeIHbF z#ivCNAKPh)`%SMWfHB+~tinxY;1M>jxP*|(X+(!ES?msuJlNRRi$e@ziNnQft~BA4 zBr?Jg{hV>n-uK1p7Dqe?#SwMZ{DA_MG#eKXqzMKcU%N_S4`Cb3rO9fe*={40B+AKg zQ`5WaZzNF$0&>bnNCigv?WXIz(tF=Qpn^SQit3_+)ggx+1e|Tr#% zkLUs%Rrv704QAkOp)F(AjOrx`O6%AcJ%T6R9QSLV7zJB6Nhq%gNia5q$dw0L#WsL$ z!RJ_@hrx#RE<$RF(<|O(98C%0a7xx$n@$2?2`n5i^QsqhkG;0?@?*>~zp2+>!k zM)RT8byJ@BUSU{1zEUaMHq&jc)i{#iWh zrg7+_vRLWj4^PA13Cie*O&AvJYJh%XFVO2UWbX~ka|0cx6V6fy z8_2^NhOY{1IIpeY;F$DaMkl|t%UI;pVl2}4{{e7uvq)o+Cg{jq6nqF*!kN5+j*+pv zgGtp@R3A&WYJPMmw{>x^t?mkIOt2WMR)$1~{U{H0h}w~-_d06TyDWB`z(L}3*CK~U zbVHgs#M7w$#vn#{XxA~ROw93c#&qB$+-8*78^x&*6t zZJFDaEz`T>mbZ;n@in_EX>*&q;F0BVWZEOlgOdvPMY`XjmuPPz(sQ4N zvf=(E02>*~I_Or|BLtF-I8AW%d;# zZj|EW3>ggSEMTT$50vg==R!1vR1P#6Hav3L;>h&%_bSGjugd+L+o&1jwiiHY645n4MV}*#m z+=UsoZmOfVrqEVwm4+6xU-;sB5RwLk(TMHJ(4s5x!`uq8VqW213Gthm`v^GRWt7I; zo(=a6ZPwg1*bi|W26k=t?H%+hlFDg%xbC6F`>7Sx>o~~U^6(I~sb;e9%bV%>Ckfdk zzLU~?OzvNn%H!d`ErF+SO(Zei<@-Rc>@oj)8Zd`9Z)##owhjH1Zb{=L#X~wbsg)m% zm=RY=P*q|UpX4K3M#Jf%z|tv`TG>N%Q9qBF({h}JI~%7^;r?}o$&-r)iOEQn&>KKB|IQTE zmthhxyA)MXHE?`dMF}a> zCsMSrBvYxrfv2pZs#V~!^&0g!J1ayt;!Mdj3c+CUAL7u162_M+j2o8C$>FV3X<^71 zkrgpC(OPPSaM5B8wOfbheex(NT-xqR1bCkiUU=BJZzoYQ~2PF zeyzL~ilvy-c05I>y2avg7K+6g=L3q~2X zW89vwsv#!Z13D2Cqodplx>TbF|3Sq4DJJ|4U>l86XE9<-AaH}!Jfd~zR%v9y8c9(Z z$8C;sznL3Vetivhc6g&2(Z6RfQYPPFZ>x{c@SlKHf;I@ro|#0Ed(N!RV0eRCG5M77 zP{dc@{Wm-y^;v-FZHxPgjK3s5AIB9mI;+JNZ|9VTCESO|zeFY@LM z(x6s+^QHh(y2zelN|z-!UusR^P7Uq|$laU*)`E%6G${=Dm#{(w@4O>0!#L2efSWjP z@Q#jT2uj;?x)!%|rq&DU57Dw8_i<)h^J1j`BT)ZEP9|wzVlmC`*)No+sT3Mz0Z(c> z^kl{@`<=R*dO7FR6@7c?0(?XMXGy9ysa8(X)RNrpNkpAS@{zJrYd$A+2(+)AxV&;{ ziOEXLAZj2E8zB?f^?ffBr%L3o#EftwnzyOr?4fv>#HkoYy!s!zjH8D5s(mua@0)#+ zahWIk(0fx%N+QYL5=_PlbrR(yT}OFZTn^EBxkd0@^?xsCSD3?COS(4}U-P}4pN$#cL45)RwhfA*l!&o*~J;_UvUAN=c>ruI0@AUXz0(h>} z>$e7OJ8mh~>mKaJx?-LoD#(3n#lT)<(U?_LJSzS-m&V;CK2ezJTV9v>#H%anEf&-Q zingKpE{p3dZm@Wd#UHWwfCaZp78jcnW~Mg(0^v1*+1Ks4Ha z$S0z>c8veLga6`D{-RK)F6;UpU6DOb?(9J=o?0i#%{csv2fcyXBRhK|`r)e;+|3Ep z6gxb%vqMIc^Qx0TA=ilIvT*I-&i;e2*$(8^RBd6@mRW|EFcr_B;Zj$qYL%<{j;K@#Vkc2YRJy1+;u4uf->%3lfHm=lm@uQl6?4vD KjRn7aA^(4@wWi%&-Cb2(^{cAxyxVmR z{Eq(qkI_H7hVd^Ntp98b9^os#Lc%vOP8)th_wA&aw)_?| zer@muZ$3A8Qy8btmE~`N(&7#%jxaHHG2Y@X#;$NNZu7R-;vL}*ZQkX*=eFPB?8NAA z{~OYc)JwxN5$H)QQn?prg$gqjhe~*AB)lk>Qbfx2s-NbMmBnNbC1Ft< z=!xTgt8{`O3sVsUr5glk&L;`=J3;W}Buwg=RuJ$!0@)VOtF*(3%1i6FpML(zkUgAC zH~;;`p-RVx<9NKULitRn{hy|jQF@eK947JLa6E+>v%~WwNe`i%IINWq&W{epQ`rO? zhb&cfY}TTuo{7j;6GiJGU0_z|M~7>b*_u-R3*A64V~Q1Ly6Qt=hP=KYa$nziw$!c*Q86 zB=DM8dXGG$5%V|!ZbAjmXt%(V+A*UrfVqdt94@coCP ziyJ4#D|W2?^oo^sA(G+Bb8;I?2ncZ&t035;U4&fQobA-dIao}V?tqXUjKX46Hl!&xbq z&DqaBon(a$1+fUBFnN~CSdCH-=JhJw*OpLV(b+09$-5i0Y-nEJ!AxZ!Yt~&TsAtNC z=5V_~-ojQW_u*sCi%J>=FTp|Y&*ejk>^OVo#mWo42#Us>VtPFn)PAh3`MnK#S|R7( zC;E5M%-RbbLH#Dx!gt`jlQ>LV9rlro}kVig*Bg)1m6)786N{D^!)FieFqTIn~~WkK#;gP}>Vp1un>U z5I|@^r7#V>>kr=Z>02P{XYeR^^{&Ebf1!W1F5^YNalVEBWIDw zeN=Zn(n;KR>($O8f%|RI!TQE3dHdZub-SoxwdKOU0QkCJ5X3?+5n-YBQA9= zZsf_O5)?#pHaAX<^Q8(*z7MM0kPUQXJ;J0G>$2IK*TmjZ)aXF;=~F9o6wZ8#O9y9X zrfe&dn^){>rYvO>o$iu!CAlQE*o)>%<74BGdw(!5%}Z;}PFr*9v~_04cbSoGVeDY+ zV7x!KF4>p&^&Xt&h5Ek!=WpoSZh zTM7su&%b)@MjC?nz*MxaI`|)E1JCBu4Q+ZtztFYFk4rz%B+x1G?x2hxb zN!N-G>hlI*ST5M@(M$3U3QzP?Kjy8~Ua^gW`q=?PnDK~Ym_LEP5FYelpU&Ii@g Z^Ot0df;KqD4q~-q?piyoo%ik?{T~R(%9j8D literal 0 HcmV?d00001 diff --git a/vllm/core/__pycache__/evictor_v2.cpython-310.pyc b/vllm/core/__pycache__/evictor_v2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0da42db3466c8381c2f562423030a7efc921cce GIT binary patch literal 5077 zcmai2OLH7a5$>w)d1y2mjb&M0v%vQ5vKSdz3WJFaz&fGX)zO3$fNh9cpD%Gnqvn#*M{6^{8TEoI~ z>#u+B{q2%v{evd+kAulwwCp$Nge6#Hb?7gTSj;;-wmUYXxg9xirBjKkohq|_U-Kk@xDr#7%Nsjpn=5?`x`N}EpG%#z3Rm@i93U*gTQ`W_rT#dJFDtLCPE~NIvh&u%V@L_pkF-fOZS?%&`R+k%8 z&`zT~-@rSoXid4=1tvtxyw@yuY1I+yqHR{!id9ItBsR=SORgIYkS;EZO~Hj^3ZsqSs@Wvilh~Kx!R=h)yYC( zfYA+525F*wVBPb!=m2%2&6)Pu_W$`(if}&q z3O@R>>)9l-5NU~)iFSK_)^q(RO`f_Z5a9MpCloKZq4=MhQ*p)htIJEizhbQX9Ck8u z@s-iq#PV%3bK@H8PI%hwS5+Vg`zX2>H!1)?Soji)|&8x!2mckT(1 zmG&dY5FU-9RP9g#3h}9-c6~R%U4?>D&wR=%fmE4u=ocVYeUZ9%sGH`F0$a8(#JgB8 zzRnq~PQDK7fsKvHL*uiy-hR?I7MR4B^;1{`^GavA#V5Nuj3rW~izM#jv|pNN&zZQ) zuPNhQz=V!Ya7oHgI@3g%85jk7Ig-U)Dg;i2Mk7tIKoFCKy;b)zbyY*e}^ z9HI&$fWm<^QP0=zQ^@}mnE42p@jn3cXyAen3<4y#0`?j5GePc14qd{QD`Xa^5cxbH z&z6Z}$IU2^{Gu`=noL$VnIEN{DmhhlgjI(3gD?Xpa8G(+&~wwoYzMtG#liVNYnbHm zLy!^o#8i)Exu-~C0yKjx+;M%gfLCIF2pcnbER`P_Nd*jwEJiV(fgdcSxgGalpb500 znyEs93p00+gwF=DpciT}+ykmHlZVEX#2%n!l*Y$wYxIp`SF5E{Qn;>qh(US$n7+FEesv|fP-Ek!j&yxOLYgv}|KVe_ll_z*3lFd%F; zYMiy+C|W;by)!_=O{~Og>WhwXsc*V*_hB3 z&{^7{m6ED}&Jw0en$lU5El8w!Nl`lMB|&L`&z14+{~i!Gidm$nNEk$9+Njgz15yGhO zNZ>|PdP0)U>n74*#P!0by`{?BV?P?m(#FERLNVeRVePAf5I-|i7!8e_b2)t)Gff~! zTc;Mq%Pdm3qQJ@ z|Hk4uyvwpXKej$)V`LJ1D*s+ zvf?EGLO`J~h4)9d?`e3suTADmkrfRyG#*2GF_|(+^(0A}t4|e*ps{_A%r+ElS%eXz z()0londDc_oHuR0@F5ql#f6}r#Mq<)bG1u)jRi}7ni%RfBE2L9*ts%yar5M{y|m1l z;%aK`S4hhuZ7k|9L1IQ4)eV|n$QOwse;>{K+l6X!y|!U**{!RbHQsDgTQ;w5GDWBI zb#uL7SzhrE7G)A11@Yve@zlu_lVjZQA?2RB=BIF}wgue$XX`H}50Qi4d6FC@=}A%s z|3pRkKb&v5;4j2U)EaeUh9a+SGxc>E8*VWr7FY0mEk+tjbzk-KO`})GGeu zwto;Px_?8)^Qs>9fiP1L3<=L2`ma)+TW!{6 P8+N~?H+x|U&!xB;hIyhY__|HJ7L}gDh@k>0}CnoU~Q<1T*c&cAAOQNhoSu@L` zT!ON0>Y}W972s=r)vSto*{k_=vo1^bB&yTO*AlHT>8queirD~8mDYe$V+88UR6dYe z_5Xkm(te9%%H2;t%Nq7!oN(Joe3tY=+LAN%z8fd-)eqRylm!lha_LbTd8|cv3EE~$ zh2$O&(+I@c=j@s5Fb6fCdSwB*CSgXt_`O-WfA`VeL$kEM z_jvE&D~X?h0TkrFA9fNy>O^j|onUv9?GOC4=kNGO9nU@NM9*Os1)XP}=XactvyK}; zaJw*CaXWg>E1;%^69U(ZE~&Dv>vCNd{HfW4m^%pm7tDZS_ArIYOzjd%sPeV?QZ`FW ztEhCBmROl;Os8d5>1tvnlt5Z#nvj%fwX4z^t&29bAved@0l!2WLb3#sW!4yKT4qhA zGrdbTqzwsBO?pP4&H!o!P^<8wW{s}VvqG|RN;7(Lj-D5i)uE)!PHR=^I^7V6wGpBY zzFwdg1^4WTJK1iNZVALWK%mcgA1~3%f_uK;f;}C)SE)g-2+2A~%GfJwvEUe-rp6Pw~|k1o)6)fUg1n8t}_r$=6;$_B~b(q@+ysW9e_?@8qTY z7r9m0H_5XdjxIosTb=Uq@eas$XNz{TTI9j3?}Z*^959;?#?>iJgY_?iKBT0;81}pP ze%#Wtk{2E^ZeGAO)y6f|KExDg`FZ-Yz{RuR;tv3J`8fo%#u`reX64#q%&3JQdhC(?H~tEhA!y^{Au ze_kn>8Z)oUH$lL+kc^UF-p%^S4W|u&8233<9bB63!Aa=8gSg_sa^VEO3>9u5L0hi~ zzlvqy!Gv+W*u#DXi0_19GPp2pxPZ;=_=HV4tgBcsaT>a@{G zc&5PUEb2i^$$Q;GM`p&|KHEbqK1);kL<{DgWsVgJWvg`rY)VoEjruy+o*7&PvxPp zeUyoAZA=>*Vt$e#v6zISFijqgtj$?U1@I@C)C(<-N9JZMpaQh0sbbNJ={pfkwaXbZ zN7iP|5CAM{iQhs^6DF3&xiYa2J%;;ZK5$p22W|p;YL~^FG%qj7`3jvdeIC_e?!{hO zPogy61LDJx3$58M%xJTyEBqHTVsP3KXXHqCVMoSovYPF{p*6M=*Jq2a-BT&-S2H|Z zA1a0HJPZ;nXiP?5A@~N6!S(4_&1-}k)zf&n4zq;8B$lD6{`Mlle-1_P75+Q79G2?LfeV1WS(4ICp{48o1+88Ww3OU}=) zS2^CMINrT@9Y|J>;dbph`K=%J>4bAUpm97K@iHP5xVnjp8Gj3T?;uY-3`389;D8EY%l<`B9rMK b<#+y1a1O^9EM02CT^oN*IC-xqYmM&#wSQ*G literal 0 HcmV?d00001 diff --git a/vllm/core/__pycache__/placeholder_block_space_manager.cpython-310.pyc b/vllm/core/__pycache__/placeholder_block_space_manager.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..923858d68061d4f3a4c8cdc7f7d624f0cee555ec GIT binary patch literal 4345 zcmbVPNpllN6rN2ZX}n;JF_^_P0Rl>FIY1yOlA;o_7r~)oDhFMr+S4s-;@Rk)F~QgH z6Y?i~$X`f)N9LMS&fHP~@?Otk*+??UROzXgw|sBuH$CypjHAJG{ny_Hzr*NHoD4q} zOg@31dG9fWvg^(;$ewjg+RD%#YuL@%BbSJ^p}3 zbc+k%Xl?Mv8IL@k&s3wYM4TlZJ-0vSJHE#~R6YE(Ve$$5ZufY4%uU4+#|@ElOHx7a0^ohoOS z*%g?bRuXUXr4s#Bb`4g~fZZ8By#ysu>*}y~VCSqt^e*VL*a};Pm2<4i&ja5QtgNy3 zU}c$^P!2cqa}NVZ^B@iw7q6wA{sD>mlx3%N(2;?FF5NG#r!2G;y=+43i1x&;U;IykG4n{Gag6+ zn>0;98i1_!1?Sa9;BRyCfQFm2>nJ^s2B6;ysFWlT@s7{9Bgn*e)5A7>7f~T2?i1ZOK*`YxOb`+RLB2(dz3HSWI?7QU`JB9abI3F8(a5FP zNomTLU_}A9t=-N<;KNcLx>`L|E)bWPLV}2tRe-gRrA|Zvqg6q;0Nl@2?2IE~S4oTy ziz)>O%AT?QsPOw#g{7FP2+S#8l7vTPoG&5MSs=TsN5^1DE;k{6lhJHJVeSW{Qzi#} z9-SOIr2Z;!6_IHik(^g6PkMrCUXvn8L<1kT!ZU@FSh*u6HqReN&_2-8$sTq-q4m;4M)0_LOc0{9f(x%ETle5?aZfJFG2*$+iZ3yI=T{Go51g6~oSIkg zC{Ug99-(^SA(TrPo<#7z|C8%cZ@}FF+{gsnZF-=yXxX=qjPFPVU=}}v*E_npt#vNo zg)f2|y6fhS>%yf!3vfK`x{os&lsiq=WwGbF;yBuG0?AFd(W?uZIE`y(aLw9`<3KFq z94~VMBOpv97m-{+f)@aB4M_*dJ4jZLtRlgep|}AgZ}fwhrs92EP%o+vaMVI_2g!cO z)ChZ@!0%q5~ck%w+ zt-pd~5xNH8?9HfsIM?bczla=$xaX*iEcF9fIF9~}ay)N+TqN!;)|b>B=+&$+X+2ll|?90CN0C4mJmkrWA11aT1(xr70blt!Y4gY91I4CYu= z_X5OfHljp2jt|PRr1*#rEo}Lq<3n*AJ5CfQzTz{9<;1q!wj3!^Ea{6RC!s8vA_?U8 zd)3EWpyBV6lRx6!Z>p=StE%gMzk2W0>(=maE{4C`e)^Y}?%NWNeTFxke@Wath@bNl z@mS1?S@C+Tq_6Q(T<(pMA=gAHfvZtB8_7~K&NzvBs*x_G8<|o@?#+6(kt^jIL#3fc zzLakamxddKQlT+Y8flD{MjK!v-mUhEsd?Et>Vwtw>7qxwu?VkFE(x{-O$)k+R@lq+S$0VbYo*zX;))+ zX}3Hds^8SuQ`*y*EzQPb_GbI$=i;SX?7en;eV>)LhA$hqpR)?3{nkk7)?+bi)Eav} zW{ugg+HKDzO1Il-gpFGp5Vqkt#=S#ACag(>OiIX|5;A2?BV<}a-XI|xtr>*ONXT6h zvdP+vkj)Zuw}fo5wjyM!gxq6ov$j7UE4|SwS~uW&ueHP4iR*nR!Hw1~#M||pQ95A7 zPsL_;e}R2CKO6UwmVJK3o;AFzRdFi|^@`(o$;GN&w`Q|m=1`?xubipdUh*)bdFJ8j zg6n0DyS9>G^O359`^T2uYO7hPdx=xF>m^UGEHlaDO$5#+yrDytg(dq?tGQS`>*XG4 zojmCKM@}K!TB+Mg2cQ%~2kZ6L!YQ}nt~lP3NqNfyp=@pR&ZYBiSKgH~l3WemboUgi{|G#6|yANUTVzTR+fcTBZbmc7ZY+edA; z!dCG%4GcbP*YT_nJky0n9a4ww3st;n_{62h&eZG$SL$Ed`NSiSJX$_|%Jfvv_gIZb~Lm(u%T+%63*7wlyayZ=Wlkv=`8cD;NYQ_r1l#RcE21tm`Zb z-YoejKNDLQ;q_m=x9BPxuWO#|EuJHpHSLSVWf@s$F1FVEY{tu!%gsu|E|a&>v1TTy3ici-KOm8HgQjf?y1)ie8-FS$#t=Kc%ydSm}WOWFIKU<%DG zUsBtU3x7@qP9|o?Gnsr`LiZjlYBk6Ih-yBPJ(lQRjOjm&m-3P>BmQCWXW`G;rZr@nizZji=gm^VDp(^}okpxtJBRg> zPq$n(Gu9@AY_KP+&DNIZV_0J+;ooX)6aSR89W$Zm zjrwa1y7-|L3+-C@66V5b4DZT?N>vuQqEm0VPI0lNihipivOhacKX&-BJx-;0cJ|)l z$?Dl9x7ci5#Hiu)knD<82jf_lp>oyQSGmZ6-LNZ7j;X3!T&OgQ)w4}>nT#Jl(aJK` z7rVHKp``~N##2W&bH(E>Gd^Q4R8}0DQwTXUD)o*6N_NW?cWH0&qFwYeJo)(i{PFo? zMU1&J=9*KiEZG%Hrf{`k@9ipBYo%`C4U&&eZed|S>D?uixrH@T&)cX?ib3{%g&iy% zRmO9dP}qyD=1s2TwuI#b6TZ60lC_%kOU0$i1-rPgw9-6>^#d!(Vzu5`7pVeUN?|2I zOy(y!En;%6ph#gu9XxSdm#Q~26u@qhHla?Jik7`tS*g2v&U$&bbG5^KISrQa2-OZaI7@mVrqUKlO8MHrfr?MX6uWUYq4we8B`TJs_QUsL zLRaSi{PM!SiXrBJv2_&jU*g)Ii<|MhF+QkzgHv8-jV1=w=AN)NeKkU(aUx5l)SFnF zo&mGjSvlh>Ot=BHc@*)5lI4tid^~PmS>4;$S+A2^y#eX3Sue%W8MOW!{SVN&6;4;{ zGn3vz=Yuxn3_ZmN2jL7D+nw>QU6O~GgFgWo+caXsxymFLM+`-m}t^a zJ6^w`x>gn>mbu3HCSy1S5BoV7opIfLZ$xlUS!8UM5iYYSWvuS(dvE{p(l^P27psT) z25E@6uKu7`O0P2FaX72n2EH&{w&rK2)pkZM(z$`o4me(-+H}2}YF4o=j-Am^|I)~|S>ZxOJybOTivaQ@pdgk^FAn8o5m1-6a zS5=31xtM8WvYGf4{<&O^8=2I2B9%7cxvS%;@wAc5#f?NEE~yrfpzAMzy9e=e2>v24 zX9LPvaX_^st_DC@0#L~Gru;Uuty4aN*PEX#hDG_T?1M%%r4-^?cB-rP{A{YXAvfY( z>MeBMOeY$pQ57Gm&H2|MG#IgJtTbd5?EIo(0jmOO4qIc^ zI6@2d2*OA0v6&d2ZLlVuH-SCJ{m=~(I%!Qw=!7+mrr#K%P+L8=;$ny6tj-aFIvT_i zDq8kJ)xox>*r-@O-l+u7Z8hgj_;GUPr`uD4Vrz-3M;s|&3v;3kxvmU7EyE};^LQneg!TJPz~*yWR55G9kkc0Y8~fYrzw zE|-^Ej*E5$Q=D25=OQ6q2U;+goU5w1{Sm(O)2A+7+lZT2-)uqw({pG}pIwMN)j z{?+iWiF~+Fn@WgL*dpplw#}eoXR1z_e1S_Jj7ppWKI3n7CRiVWK{^-2?kgnct@I}s z1D8DtL~0q}H6>m^Z^>Zuy!>8O*LZ?jEjI6p1?D2NcUC~ZEShD z2$2J^$Io`GI$TA)%z=8PamK1V@cqo2FBy%k-q}~5zL79@06dOL-FW~WoxzK7FLBPk zM0O5`jcw_+9nhHSop{an@_iXN29Q@Op1-mhkl>Df?;U>wTZPT}5?e(#B-`dNHnnKR zM0Yc?c5Ht(<4Z+2OK8Za@@v4FY)&h-5?6-yf512&HmjkeS@WIq;ce2vSS|jFyBKspNbg(WT`8l_qX*mZD+PlN3#aG0IAC$9n_FdA@d*PTSzWahH0d( z446J$Z@6X?sv7cmy(UxV5J8VLze%RLCw#CPs0JfObqT(atiAdn+Z>{Au_+^GvH%S7#W= zk*((G5DTdZI_xAhNr&a@XqSlWpX3AloZWDUZ1P41{}b}B{h4^iDC9CpBRLT_(i`K7 zX|4?mIK5s67Jy+Tq@2*ewA11zJP7=R1*U+2YzP$y3wpx{3$i{W2+|T31b#w;9AJR3 zut4kq3@|T8z3H%TPHJ)(Q1{;_bq}~CWT?1B@`W6qX=iDMMy2Re(V|dMpXGE&7&8m4 zri(3X$Sm0ta7~!mO-{f}nJt2A1?(UU-Qsa!)hZj$fp3L_VmDhWXP0~y?cRVRb;f4K zqy*rX>>=kXWSdHk#0<-L1dQPGP{#C2y1<8Dq>eE4c669Gp9xsYg0klw~;Q+@fA z>y#ycBnwxlBQtrRDjXmrioqZK9G0?RZi2^+o$dIy;=Hqr7qYT+YJNvnT|h*2kq(Qf zo}okZ5EV^b;_WINtW_&=@e&Jlr(>uKt;sKtWo&CY`rC--Ou;b>Bi{R;`r7J^*BBvl zotcoO!D`A|XY&JRvCLRvXg!N%+R6kb2=|6WfYY$SB(~JM87o@1qWQ^6{S1B%TTSL? zDxb(>mU4z>4EW=QF{&$n?K$Zp@%7J1%)U%G_x!op6{ewk%F^>PD|49hF^AcfwQ>TZ zhpZuV=CBtblv8?u9&ZLS?&5XMI1bHk&~%ge)iJOHe+l%Gkv>^%f_>>!!ISJ5)Pu(} z*JzGshf8fVo;j8=l2g%0elc~;7pnIn?QC4V2N$(QCmI7nyErbsgr8uq?(v%X`s(g* zr1rM$;IYcrT;Es?hp8qu&m!Tq$0)P?1b)&I!5H<&C(4Y`$r#rjoe$#a>yA!MB}g$c z{$?_Zah9^uGSYHZ292HTSlLhLbtxEUFJ9+3yI84;Sn1j;dj&+mc9y~wJ|q`djibdm z3>6~Mv~4Zrg5(Wrc(`c$1jSiU7;354I+nxkW*t1lnEmX~buWip3DvW7uDJp- z$CvSQSWka+85zCm@mb%vN+VDjW*i<@uz3|K4EQBpNTPAfc`tT!HmNK;_OjS7X#nVD z*!Chr9&sv-%N12=I9^`%++~--3I~ligQOi(HC(Q}O=16)v9gOq8jS2}-LlmP z+m`dn42e$aRrMVR=&M0MD%HF2H0aFP>mrDxg_

5G<=sEjuRkOrG*pCbUGPYy>= z)M*x#%UV>_fKI0VTD^b=eMJsOqh1YN>@B$tK9{f(kVjI&ot#(Sf|Oslia(+7Alcu9 z01#54kwP8^m3MPc-VPQ=P|}?&DYqaXgYnc=_5ML+8%E%R_&Gn9h!&b`JZo4rD`BI2 zV7Vp1cuUR1EXq&Q@Mqx9SjM6zXvq?!sn(E{7dhn+{2Adg4O<0Sf`%moye-Xl0^BS> zPC6nvM6%K-B#@OxA=D|MtaPId>8K5%`iLSbME~h;^eb9o}bJbJl+3zRM5Ujl8*pZ(@z`^j7OOJiSTY%kr7l z?baO#+4DN@nZEl_f;!!0-HkMR{kwatH{x!ee|N8SAMWPl{kk^!#guix zx*wtY{Zi%e-UqBVA>>x1+KraE4Q0Fq{hAPd;DgpdJiFa`$iBlmgdayk$~tTvLFk>< zQ9s1Aj#&>QS4YTOtVa-Xw^g?uwdUcwr{mdU)(M2XQR>d|@J%Sm zZg4k!PO0t<ac^-jci)9d6>ww^|e2d#>|2_<1GpRpDYa?r9cu5@pt(UP{c zh|q^5r?d7UdBTy%5s|Z&tSX)!7TuVpmkoi=7ck*t^WJE%Ca5xYT~tvml;KPU9E$Mt`q43<)zx-Xj8a!CpPxEiPN6Pof)yq`kDa;z;( zvvvZ0h;!Fc!Wm1bG0X7%rZYn(TP>+JTZy%ln_5q|Q9$eJtYz96HQUa#%`Gwb zjaqgs+s>-n+gS<8w$qX}hZtG$4T+n_Vw%Johuy++U?G)9{OhQH9dpMee1pYU6BbHU zo)rHS*h^s0mN&X6Re2L~+oaCAo11Y9Zz^vQck5Z^xlP>d;uhP?=LW>SflMm+cfh|x zJ#S@@5+y6|l$1BRyAZzfx%lDOQ=8Uu@NZk+?cU_>dCnK6*0#(2YQFl_$e=9SKlM-e8gf9@_tE(Q zINqdRRM}oX6O`B+*0uKwpjcur?PFFi?UQmZ?Qi5gQ}*anYx&UpX>Y1ORe1s8!5OUp(O5x=s~0fO!}5AF z(MB$UWU^Lx#ZdjB9|`Fgo5C2K*h8MF?~$(J*r+&{nhRu{07J3q7Q7KDk1R-42rC>f zBWcJMPbz38*f|i%Arb= z5A`Zx9tVz%#+Vncb|6CGj6g9?%R_DQs}F>1(K;XT$t+I&Y*CK}xCcaMOOql{saj zJH8XO7J)dVKM`ket}D*93&Nd}mZXsBA26udh_Z4Fe7|Pq`+83wWTUfQIb$^x^{P2x zoBL7v-i{=?8BBOYL)ciI;SLvL?u@OdI7Ssnt@ku1jiTFNP-|@FKo#~Lmr9+c42d`= z&G!U_GZU-3qW*cE6e6xzV10r=`T@G7z#=JOzXd8bkqU~T))7hl0-P5OJ=!Vx=<7Qw zX@onhA3*ukzd^Llp6iNn&7!D($K3CMgWbnv`uS@0D|C9s%M2ur9WSMH)?aGZ@qRsa z5}`5$#J<;ze|@F-;lFKZ{Pq4v*&4r&#IIi#KelXsy9z{h7EzDPuwRA;13T`XB)F?1hdVT0}Og(b`h?}ns zx6|bUgv}%EwB&RI{!yL7`8&=}=t)ue3|i#i83+WSQW+{h7e#5Jc(Lj(fyZwb&w^Yu zi^mW9jC|?~A6l{(paQK`s4?0^7yVl4grUe({DVRxD31dSK=sc`^HQLQ?LcwG4p^8v zZjqdSKla{8i4p>b?l*>Nlmqi%UI*(=3sQo`t{3{5I4lV3*^?_u3TZ(s@n1JzZl#kUY{bu{lm zb=JAJ>wP{j!E$_Wkqg=J!+RWvGzNA6m}nLDY4pDN*)cB%J&Y#G2E|tOD8{aOgw8sh z%XDN?k>8`|ImfvA2plg%bcVKZa8|(yp_1B+XQk2JNB| z?rgzp+VS-m=4Bo;H@=P)g7cB{PtURTw3=v{`tOnXuytB8KOx+o65?D7&6ZouWn(Iq_>KF;cN(^>Tg2>UdOs_uiQlZIV0 zx4uSBAf+_&Hh6+Y4w`pbhoEVv0T+e#tN(;}13{W>+6Z*I+tQz$pJj`(N4G^vyJ$%k z^qOh93d^+Nh~O@fZ3zk+m%TgY6xgNl)rXGOSFmxzksYjw+{|%F^cRpKRHAr`X1-8u ztvJxv4?3MoyB8$WNmOh2a%u-60VG0X+9Us5I}y6 zt<`8r+lPYNNB|~2-^rb^zCiG=}U#yn1;_U=PC`Q{X4F9$O3zD4eRClSUc~I*EWEI z6u)e)4Y!9GSDkDAWqW9SLh2B!O{(YH`PvlItWRHNT{qTdplbI#b6?-o&I|r|0=aE& z=i7`P@)~kWwdFw^J}A4d~L)^u8pA#lj=|5?qqJxZ~Mm?cD6@rTdmX!alCzW z{YFp+#980f9@Vq~>BiJc2-)2p1u8Vw#@plWO|?C!`>AjU*_Z+ipU{l@k%{QKG&60#q`Y+k?J?`4)c zx;HWVGnHS(@DAov4|?N@;A^M&TagZUbyACTG}tUUOQM;A6F>D8=%{K%4gf_fP)6Yj zA}4zS?odZo{YXtuSdG%0ND)>C&k1QvMv~a zIy7!jBCX{Tk$xtL7KwX{hj4ZW@4%|(ABQP&_2^t2`wG*AIGJB>;q)1$>nu#qDs?Zx z{AslU%!}36-z)HN6a>D(sV6qNAeBmAQGzI-9{vtq@5Fl)kbv>~l0NOXJqb_u(V(Ys zf=i=(U!Q290{&ywy~HD+e&^|vWNeM#4_xqobs+BwUwBefUwe>yFhG?*>U@cN)Nx0l zt_X0h;Xu8v{unE`FkI(PLySsHthqLKsE^W5){U1V@o33uT&*meT&UX>rL}%8Fxo{r zZ>RGN6KpI8q_;=qSxe6^F`~#tFViPj>KB=ALzpf)(k0N}OZ%thy!3Jl=c5;0^>a*s z6IIwLr1>`JFAGD?8|Kk80iO~^J_6L2c~VMq__Xr1K6t?TdK;NMIM3$KU)h4lkp#7X zIEpC7OW@tY;8VXP=``vLsdq@`ak$l=GID%W?Pl!Z?$`ebbNy#@gs(!mt9p>mQ92LM z`4}8% zEVbvTv%Xv!1^5JKI@;8fa#ky-W2q3X57O+4Rr59k1G!RPO8Y1#jlDQit^} zY&5RspamZMgX0;`T+LzAB-~5#KQ*rs=d`%3G7;0fnn@Vob&jXV$;37)o;SIvLO9hE za2-ebJhonlZI~%+!psy-*5L}CX9kBvxGf_W)x^8=jKe&Bzi@jdUP$=w!+XFnMai&X z(>&M|p31=!^TVSH*q9iro5NW+*H^e>(>03jAPdVpAMh@;k%8ghVy1H8DYS{fw$mgYyrpMn1oH@lv*jP_7% zNI=KXdS2K$skMSTTr0E-<&pFEfh3Q%hlC_&*T+Z@&kMl7d&bqfkahzgh+!oFFiJka zn2@`Q+>OY|gLkzN=R#b)%bjdvD}}TZwW)JSq|dES_vVh8Y$TXL4QARyD9I)tf&rKy zH^ccLL}i=XW9?CDE;t`*9&78jT-N1KxB_C0be0);ZAF`yQqS1>Hu#f1I0@P+~R;^yc?CftTPUfw4GtvMPl_&ClS%oj)O15mj%KnX>VmU;_kU~t8W z>BTCTY(6P^n(!jbC8FBJ1NY;g%rkPLBRU-HzXC|VsHL@6ZAd@XZA+T7vw1pN5lNAq$X(U7coMQADe&dT%%QhcHMkd0Dynh~lG z`hO*tz@bq*0|qe6*YYOuhhPWFMM3imkeQCga|woJfF2~DEA$aHfSH#Vm9uM%NmRmr zK1p3Qk(YV30C}tV>$wL4bxu6vKP0oMmj~kpzw}+KKM;{ni03?k8&Df;v(>O%4M$%S zYeqBcQY-|7Aa3$G1LC51Ed@mvxV1E{au1{scXkQ=mT;WB56?VhG_0QX_e8&q3hx|wV{D*)X$J)8t zI1n#ce>o{f9-$i~G%umC^@(<_ZMu`SDPf)&K*)k7+es^RH+BQpE~Rrj>Nu?hts|^r3w?+tAmbWr6&atntBD=Pgwo2_Fv7z_(MdYGu;0p zRPWyd-ay7A9R}ryXTA`m%gu~ae+v19rf#TD5+aKMW$N1BZ$R17_#MK}`5E*e_Vcp8 zm-Q>)8a=`_FLMrZL^~plL zBegb+{J=vf7tWu8vd9QiVEw|*Soi@D%Rqn;tBrvpps+eAlwku@TNB_sFnppmiS-do z{1*)H3P#+i+BDY9XPX}r|3>&fEN{)=t%ZSaecv_S3LZgiMtvNuvI+bG!^(mS@zXvR zVzb=+ynn}7;9H=ExC$@g<7hq2i`ZP-0$#+&vHRcJTQhJ!D4xNpEu4o>VXfbeQjS?U za2`J0e7k-p1S2}{FJt{JY5_^mI%ta<{9GxT@Y{(#^!_-W?m*j&`@D#8gb%aC&0*AO zXIsaV# z+8`;yY!h#Oi>M>t?A`)?1#zPEp*8%1VHM=m(nuv|UJt*b*nqWgvP;V(&e~1b+{7{6Fh9-pob4!R7k4Efj?vQp_uQ4gmG~%XIPRMJ zX(s9B|10Lo;L`j(Z>4z3Zy9x>>$qkFx&)x$l$dZE2u@ZHJn0|N3^*gcsLdC1ipfHe zv9+4Sk~A+HF;P|z92c&lAHHW7%#_l|AauWZ;|J@$FOM9oNoWR;WNGhj(;V={MClF05I^pPv0K1~P2 zH>O^q?<;ivicSy9=YtHBVhC4>ia&}#OF*CcU%dU_bb44rUu75(o)R2+H+{nTftoE@ zE?6~1>x4QdKGGLnkawdfg-08I2*X-^CXQ5|C5YHYf-%WEQeUU@V|1Q}GrJ*zmyvu+ zACr{>v#@a;D{TrHL;(N|MxC#om+uI7ge_DWqg=~p73jgGxPR9yBn}_lNDK4{LVc3H z9vC{_3HgNnIFAy0{(HnVw?cXZlsELHSb-(xiy@P>W*%{$gDx{pFqvx!7u#+?Fw@P;HR%q^ zmFKw%&YR$l;JjDD!shse1Y(Usb7TyVhQh9I!qa$d+@i1xer&Tv|FJ6$z-n!xBZnlO zCTo+=#pOChbu`I~pmv(^0BTc!fg2@n1B|kF3B-)o^l35}1$CbHLFXoD{U9$91x4jD z)ej-9xxPhgF;LTGjP(^rE#XMB)lZYEZIil~>RT3eh;+|*TB=2p^y9u^IZJQs45=e=YwIWy@ zImAk7og}2YLBx>^*&$`xi8FH0Qb(JmYBwT}U4YOKz}0r69H=n_f@wh699R;=m76gg zgTbdc5Rp_XB0V8;BbO?Gk(wz$VgXa!u(}4$0eJQ;nCx!gM4B+}-6h!MJG%to)m-st zRXJ{?Sq(y>Ig`Ma4kTgFdI`{F1Ye*)4*g-;pO`Bi1R+AAVC|21A-=KNWfi=~;6qeAP(VeO!v;xl|{lLk5TD&9cuK#f%~DcdzINm{Tb#T@>M zaNdK7g$XDYt*jV2n4I3&6IlJu8x|EB4XyV9GRoBtw@q9$_3BywvImHupIKH9e38o@ z&;1+EnlRVJZ8WyWS61hO9_-v62L^;A`(Qyf^GAUq{+O>r=YuT59ax!Dc>2;f9G*se zDyb{;UD_ZQ%ZOY^ZP6$!yBF`XGm|?&JxwveHG!>+oDLa}t?s|R;)ZFjSvWOGVj|6j z&nU#yba*RaJm948J%!jNyj*R9=f#{h;dPo%?xgcL9kNffKFwR{6T(!Yk4iPOH;Ls+ zectRULvNvTnT`-sA*-L|tx)$z=zE^dyXmaZc@Lc*rZa|8XnaA+>ZSA~QdtK7B_o7- zJn9wRN+I7s-#Q%0MElCnwLacFeBuxG?}h=2T1=yBx!01_ElIsSGf2gMB@c7tH% z3YAMBl7S#c7)ZfUI^2_-gM`NmWG0cEA{~jggoEh><&&%91_-OMk>nru5b#_rNIp{3GOFa-|Rod4F}V2*|HN*C0%xCS;2r!q52#`fd;vc1I({*bhB8QbD)2`4hj7;K|l9{{()89HbDQFX1F5_*ymt@lh%?R zKIqd!v9MSpJv{Ey!%?umk+;wdsE>e21gZ^cX4WP;s2Ri4e)wchct*mLeeq#!2Kj*2 z_QcOgc%nAd5k3tZfSL(q+StZ3rh`FmQ(WXNez<4>Je8KfGq!}ZiTI?(CTM;V>M2R6 zqnNeLNSCN>fp05(pp=M>TKX;7-J(N6?edix*6CpJxejV~sK?wpG5JOLW`v02`Uuuu zP0~pEU=+UvsSCv1A$5@xLp|h&>K!1UJ}JFzE+lUuQIrIfqz6e$ig$G65zs;Zt#idYJ09w9O30cKmL@1(2UCY&!LJ#WKb;jY zjQsOgOAyEjSzd9WC$K7?%ZR=epuaf))DY$#RE6>qpWcTJLeznVF2YI;lm}=%2*&uA zs&#yH0BW}lY(Fm85n6!xWSk0>%}BA1{S^*$hcE8S+2RKQ!1>b92Jk>Or5&6shzNtC zMGe4K<)0eI%5YjuaBau$+C=tx>DWp@$5!tSMsX*_?~xIAn6UiDkSf4g zTVO&D@$Mn*Ld#Pfw47vIZDVFdI`}?R-!2*l|9u8ixo4a_gi1T#fgFs2rp+mnVwn__ zF?uO(5JuHaO?gu(Qd}Nn?{+tWD(ej{56m0bj2o}v6#T21Bxo&#KM2tj7~5kWsTY^@ zpU+cY?mY5z&60PGLDV@?9N35(Py^6NnrWu0CO>CDF%OjpqT_viL}1p`K<`zy=nl5y z4Rqef+a8pY4a@c2Y(ZiIwp+$n-7sKwE&u_Ae@Xa|M&kzs56h}%T$~xS;udJWn>dGi zglbdL@Zs|du!JrjEHG&eojZH%)1g|jPSKOb2?=#uTc@}UY17!}@5lM$+k8Bbt>u6s z%)kKEYWuK_wGhf8@|hIUDsFpt5cZ<2zjxxRW{IT^oDh&@_AO8NdRlHPT%M_{Ty3u4 z{IKTrQ=0rYsATB9#|KdNquBfi7?oi?r2Z+wU|SkXeXHT6SDIDGH5=Zh!KN@yoP6vd z_0ve9wY-G;_CekXGmd`Y*=y!4O5w0wO}AFIzG><8JqTi{8hzyKn` zL?6r8g=lj$K_#Zt)h_JV`@$}D3XTJ=mx3LsHN%Y|tjWZd(K{Tm6n%knLlvNZjmK!6 zZ^Ollcj@SqhMTa94qTv3Gxn4lX8;7N^xNY27px2qx91Wt%a4iIVeJ`b^$BqL6Lrlq zwgk^lO8Kcw;^N#RMj}O_REailk2A0wnYBdCbjB%mg+Hkkt6lJ-YH3==d4K$=sdpG_ zJTI1!^I}j9p^`O3qzNCh(#lUBCxeEOA5^Yu1*&>L~y-(t8>C6vr!i>c^tnB($lnH9uYm@B> z^)cK};eN{9)&|e?P$H=x&MQ6f{lf{>fRD~us@0{&ZYMZ4DaelY&N z_}Wl&8~S4-PK50f<@O=;@XXq#_NMi@%dz&x_RInF`}u;qAAXdS=g`_v$HA&J^+Fnb zoWkiat^S^bi1ZGOsoSK_lIOQ@+@O#0qLe?eemh4Eep&Vd^1K^u!SQkj#><@?C3;-8 z;~QgZ{EW~Wkaj1zX&f2I`y$?dSEs*rw}0pG3D(=Y+neQcH+LiV$J?73TGoEN1!TD{fZ-{ZmF@3~d$B`rAUXqHMwl}K912~KWqe2pQ zLqoXn3*d~kRQVkAThBVmL5ZOKhi^p2^xJQtkF;*~9#w!J>VURd)XF3{F}{L~9@KLHyEmqoU3Q zwZZxKpj>jINqvElFEh(DEOM&qg5wneV5p=^X154+EQO!zpp*@QetB3Liz540GQ9-8 zCs*1uxUD#7OVigNE2U}ni&%JwY*BHmqJD{`{4$+rB#rk+*29G(r8LwrK-6Bg+dewK z$2{(&PsTSVV`)@OTW~D;TQ~J728(1dXe{6MPpQC{a=7VM_Y8bS>kejjH=Q0pm&-j53ilk^r$^;oF&kH=V5NoG^waujQikULLj&Je23`qaisjTreGiQ9& z9DKVr3Yo7q}7UfrbV64N)R`6KHOBY%TcD90c zMmx$wgrXVwt2=v*dH7~LWTo-VcF;gN(WqzA)&_0_fy(gg9!2VR5bJFyrncqOSx6*g z9lFZb)m}HUd~u>LU;1YdnJI7vVT(i#Se?i2fWn8>o21Oa=}Bq@h;5uos2`wJ3Fcro zM6ag5g}kL~FLr2(>-p7P0h}Whb0R_BEVA;R9tDasWKidV;a_te>*T*fbiYCr&hZhO|QKW!9Kg)GZ_J@r%6u zT{_Z6QS*>0`xDK}hF==g3v_p@c^j|MD9$pQQnpeU1X^ltJPY}~$lD8XXo?!1FKj>5SvCPt8NpF#(jI}0Y9;4cdgxd~FAe*ll3 z)7(X2v_oAUG69Tk$ozTG5p@$6it4DN3EYdLC6Jl(NmA{Ag6Iyh!5Rlz#wW;RpEE&q zeApJt;kYDiPGsDnc1HFhCRAVzNFHP;o!=t{XOopPn1BcQMN}vOk@>4XHyiL>dyQ*p zBVw5L9j3xM*uwqFxgDKh-gQ%ne_xB@ekfCmxI>ADKLH@GL?e1Q@RJ7sM7m zz25eR<&7lTj4JE1LlbuVa#(TCBY+-nmoI?79b97A5No)}XaQb$(3{V1U(rX0KSC5Vkc0(j6y6J8}zBL%a^|iq9jb z6)j|qEPUAk0TW{er#&fV=xY?dNIFgzM6=NctoKz5fYDzdoop%1I}j9noqM_ReNL zm3jG_^a-~o-^uCGl2&qAVJ<%6`3O6;oAFbUPYrP6wBiU=E13(B_4H2j&*MGc#%aDA zp3JD`0uk|$a|-uW!DBdvhQ&n}|2^RV%;B?J?t^{fPWcL3QJeaA#Ju5K< zeHzkp@`u!jYt#~JiRLJ-(AsJ8AUX`p6kob>^|fGP`zne!3re|Hpkl$ZRtTcuh)^1XAqe#i#3D!1=R}6gM`20&GXU(i zVnv%l>Nb$FS3z7 zOJ^G$jx4o{&d<YkJ z$(TM04>_GKC>W_SDKd`+aP&GO!U;0~aYmqv1~k`BltJal;})ospdK6eZeyg~{1yah zw?3+q>7dxMC~$wS6*_;LJ_75~j5gpU9Fj7|M%0E~E%YfS7=+G`;dP&63l$KfU}9^A ze_|Q)wbkvX0u|o5pcT4$>Dn6|y|GwFv*;-mA+@*%F#}HlW49e_b7W5}eb@s+9!B9l z&%zNek0w_)4K5J#6U~sW0wG<5qBvH#qG)stUakRvs(S!hhAu3Ag?P%u9jt$q)91&f z*oV2=X7pk#*0*_pSp;3ONXwf2*T`Nk-0Ybyempf+z?;BA2?9T`3tmQyzho_CKLNub zKG^EmDg_6puU)}6Ba^6tPR_+p)aQ9nX`WBSX+~6tCDoRAb-pm;tsg1Po{gbRnDfW}rY`peb{Sn$C@1UpoEMdGp*J!k65 zg%(x|`6@klZY?#ZL2Aw?a+ELx!@|p-#922m(`2HTGC_E6qsyF-mhWqH8BA~COd2%i z-;4(@v%$F^&nFGp;O8Mf@Eg8M+~to|bq^BttpN#SAZx&~-~0;H1>FE1Ew(sNk+bk2 z8@<`V(MiF!SQSilC;UlurwjaQnWD3bO%Qcfso~l-nA*p|#srk@mPPuNrwjtL3AWkf zI-8o;*6q9YX2I6-6iOr?c@br|lvZ}D#XHp9xM^n-CE&o{L;^2v_b#E-HyZ_dmpM2Q z>#~#Csd+v~I)57T=Esn&`cXPRPKSu6lows~@*+Mq-co9v;aBKfh7&AXZ12(}3`5nc zfQ#kML&2qd=Zkv_JdA`{7MRpWpRyl^cCrX`p+QZtE=0E}Fqz_AP^K$S;G#-3)= zN^kFhryb{RT_myv91o`c5_aswALt>-y7WEQ^vc&zqSFMdq5ZFLzY%PrNZF%(f!yFE z0Q7I{O90cwb6`nEYMoly18m#5;wjkZ_pRbRiGzAzB0c|*s<*y_-d<#S2N8eoM?AemY1=F z1Q;2{(^{0oYO}!smIivS4uImcrr-8W#YK00rz5rQvM`3P!Z^stk0~jhA{3kTp z7oh((ga*`4wgdv+d#K-^A<_&7GUgYI1W8JGlF6gs(~Z&z{O(8vVlXD zVC{(X3iMOm&)qw44M+`ExI41YVZ$s)wllbUqXljJHYBDJrX~4Hm?3HTE{X{`es+pS z-g!(iq22-XO`DVURMf)KtbB&FVON@B?+GtKMFUN}FsEhYPZNk} zJSq}$?s2|TrEEzyf*noaA7d}x>Yw3{TAv!~{;Cp4Cqjt9MX2bTJPco}g^wXFUj25N z{?almLhnwTya1WX&xl?JqU^;=_KfUjx^0IP_ek0zKR7dsh6CRLOPy84)`-SPxQEjm z4o1LQ43kx7v9w7d`NU9h&+6!GahHx2gqX9%`-|A>6CIYu18G`^2||JK_u^^Cf&fg` zDTMq!KD`-^mwTd8U(rZNXk_rU5W&o{4UnZnG@!+D&8L`DP>;NVIR)%Z*valCmY>Aq zySTR0Lf8lB&uLltLjKt4m?f1o3*;oqlk3mh-GSZ`HaF;}hrGefg< z{tKO1I$TG!4!mf$zlXQ4(2*X@(+BnglF@mdx1uq3lD-=lwu6qyoIk@bia53W^j`X= z81?{tzeC?a`u>8xL-dW%`8;olZPe%Ja092fQBp7Sme^Ji0%`vIp9UU)Zh7d-5 z7aYnF9m<$8F(VC0*VozCfW)s+;$}eNmV(61NatQQEXX+$kfh-^ZsI?=Eoto0v2(8# zhQ`f8Vmtxi*{ckL>7&=6(2g8(uf*PMZ0`>pZy2K(Iadqu+*M;J$j5l7KW6ORg+gJe za2j$y$_x>Ie4~+uX{0N$cTa5|PoONR-0=9MkuhJB9Q_&?iQH?(tj-VGdwz{_uNZUT z+vL4`W|+BGQGcWi?{J1Lx*H!&Noo1^)K*;%l!b3H3RiSX-mi9|K|Q18s2kY>qHHj< z0P{PrXivbUPiziR{wkk`vK^IRKaiE+rQA!hMU=6PDH&6w*ZN!GG6-c3$oP|C#=%?~ zx=^n-=Co-&JyT?oaN?9E{AbFZ18f232%o2N^Z`z8eEhW9IGEbNkTFR?Tf1|<_OELrjMHgM!@L$*E1l*1AuAT@dzPve1}hBNeSqQmhBV;w9sSyhfqI{lTT z?R@?c9nNGgg@wazUQj>Dklk#~pJK3i1jJMif0>Vu@v+RV75d&oN2c9((DwqJ_tW7} zA2bKN%mFazR_ZnR->2Kz9k!vXxV V_}z!!EPh4sP$n~(OnyA|e*g+lZlC}F literal 0 HcmV?d00001 diff --git a/vllm/core/block/__init__.py b/vllm/core/block/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/core/block/__pycache__/__init__.cpython-310.pyc b/vllm/core/block/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34aafd7fcbb5d148dfdd4230fea6a45e6bc201b2 GIT binary patch literal 156 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H$eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_esX?Ms(w;VesZ>ce0*kJW=VX! ZUP0w84x8Nkl+v73JCMP}OhAH#0RS3;Bz^z@ literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/block_table.cpython-310.pyc b/vllm/core/block/__pycache__/block_table.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08a2cb613e1a3ca8c7ec178ca4da851027bc727a GIT binary patch literal 13226 zcmd5@-ESP%b)TqG(E#EPE^`HrJ7;*iP(Nu~X|Kc5N4G8Y>B~sj(dGy(EX+ z-C5n4l|*6*6p{PrKOlXQ(I*$^AJCT|PyH7P42q&fUTUCkd23>^zjN-~nc3yCbw2`h zg&oe$+&TB$^Z7gHj^c#{-@@nBzx~zDPp@0nZ|Fz$I zr|-61n|`}JuU~7|_}uH&`*ZENexu#s-?iR+-*5Zkon!6(O|FB!}DF5r23)7zl(43&$eYrt8ELb?FhT=&O7st zcvHC26P~Pznykw?*;upIz&g&oHEXLO=0xMuT6!z4qz(dGPU8@f6X0M$gaV z8e&zP!_!ORJk1Dto>&tX@Z=eBU0f8;;%r%bUtAK;;q0tj0awiBmLr}Q!KWTLdq%vF zb8hZ=;<9)V<5$HuFc(%`%+FkFKC!lMikC3zT!rS#VjWk`^V%KJ60hLb8qazGGk!~4 z;TbRHe1I~pUKJaB^;y2^;mS4fZN74;xbmuajjudcaAG7}7q5%&V3y~LJG2U3BdB}g zyW$Pp4TJ+Ddpq;lR_=!PdNTZ{nq}Pxy1g(-f;0~L;UL_W!9B9IB-n~o5C+LmcB8GR z8zl1nNDjJk!}kMx?(WE-q=ZL@360JN1@RUL?#Thu4Fj?~x~55A2_Ei9B?B3DcbFJo z4wM`!naDv(l)CXCjkZVeC<*$qAFKT!)aDJB$XOf4iX4RJ0yj*_FtEW4<|~1Nh1(l!qmXdFdifkWTofKSr}{w;XnjiO3Dc5IfEpy6pZLub2;5m+sSn% zMpENjog_Mt!Fn`ETi2P*e)uTrkNUx2)W0WHRdzdJ8g#=!a8Cva99xPY8U$fMjz}ip z0PHd4jpB};C>I&{T0e2gIS>4Rkue?^C73LY3xYINZXYKsmqyU1!=#d6-Ndh-Q{4Lb zO|q&_KDiRaCahf#Za{hYufjy+BaFtHCT7k_!bYP(lty8XBuKJms%*@`!zkU^2tL>f z25}k;Rs0|lQe1(e0@)v?`xEpkk3~t)RWeMaD5VF5=v8!*UL?Rw=V3Gu@x#iRpuNep zkV6y-oS3X+eox9_5UO}Y#seXf1F+A8O@bFY_M(F7r~<2^Y{gLS#R!A>*? zat5(_W)Q2P<&*YW<$6dhAEZID6OVc#fK9>o;gDNV%0ir?!RAR0j1~1`A$!J@(^N(G zMyaeAv=P%E-l9AI4~;qZoyq7{4K?Xb9!W?8^eGLm)J_Lx)7cvJda$+_!LJe-3WGAy zv^!D?5wf>mi3iZrgqdi2)Aa9TKL6yHlbzP*g(S1N+*z~pCz*T~DWdq$xr6U5e3LhD z7~5m3V~rgg?XfGYk)@j3)}fm^WA`!LbH^Tz_&%w9ZsDvhtdFfP>^rTw>{P{eGQU)> z@<3QE;E=6#PGT!t?rh7HCC7QD6Nw}{JtfIjE%OrD+ft{9V$+aSi#YqjQmgp$?>Dcd z{o%D?G`yOI3M8+-*&prnU+q7<){E|48}6q&@!;BnUaxVofar2M7=&&w;Q%+`*z(~v=7eSq~OP4YQv0GfLm5JUvhtz%%B&?d*O}@g|>yP zu4Fn=gRYCg=0oT7Wi2e@B(+k=8uE6|K_ibT&)eZE7;-D@bw@qKHl$&4W^g%~XLvaY zGVlFSMA-!amh(&y2HVjCIVkMPYP{wi~iutb)eQvT|P z`^aAfrDCr2UbH7EpY=z*G`gCkh>hP07^zBe6;NO+>h%!BiYM>FnF2tSQ6KAh7^@_` z+6@zA#;zQMDvAqO@jwDD@Y2l62vH2=!-7=q0fZjb<;M0#5N!`)MY){keT1aj-NDc5 z^plyKoVn0lu+AxtazY*wX3GoWEi!rO|3r$p{a*ruVs%82KEPvG!*Osx_huNI2|Tma zNCs4_3OYSDTdvrO!hX!QTgetQF@%((p096>H20)E5+(W+92vkl%PtF(n_gL|gO;Nj z@TAP`$wA9iYdF_Y;{u&gq|k?RH0YcOJ%(@|Mc=~BgyQ4!>cQoT%bzejs|&8+CY+ms zABEXlL<{98g66KJY9Cs^u*Y^X&ws65Tj1B&5zc4sp^M;(aQnG!Td528r?BgBMd^mW zM3A|mK`VSF2X493696G+C#x?6JX@d@ow5`5CBhx~Bs8UQ*7|%@o zRi^ucI-M53!NwEF^ez^Z-$snkMFDZ2%PvP!m9WOvh=Y-d*g%!B6(Y2838dlo#`{0| zNojL;P^bHdHxIP=Xrp@$Do5U0=tPN0iBe?G73aCBb)daY5r4A!Xebco)w6h7tq=)M zJwxYg$V4-%Yu2Z2xnUUoGYm?WaPSuzcHMUHum03`r?QtBtzN{)F4?;I(7B5*8!piy z!+ioXCBtNMbPmLiQQvW10D7nFNv1(!MQjG7F`(T@xv-LTPB9o5DHMi4Kvb3yF4rRi zB>z@17#$Qw59%_Qz&Xo-*qvHB-`~P@cG)HG;7rA3wLG(B3X|QqKLqN{Q*xI))O-k2 z6yEx8_(6+K=03!NAKSmMDJ}0hk8R=XdC5}lyRL9Ywz`cw9)Y4m55Ngml7R2Sn|JFM zh>m;SI;^F0!Wq{dJGZS*RuAh$ZCqco4(G;mV-HkrF_p%6j)Y85i|)T|-M@;ndCX4H z0H8p4pV575*T)ys4;zRXr${x2^J#ND|JVlog+F%={lim4*Q|)jGvgluC721U zn`ncKlw@$qrKrYsn+2_Wh46*23u8mg1z%>r2!%koOHDuxHj+wcybHEC0_c>7$E)$qRYE12x=T+f*tf2 z&}BIeu5#;zUWJO$L(ql$mO_`z^uLVvO^SoL0YHyOsQ+~{BkvVl;bwzrFAV{#t1S?E zjy^xjpUrW(#>c(hl>ohiA`7yuCSf1;e_Vc3MOn$-!KDH4ux>W)gb!dT5>hKXhzq1U zXnvojAY5^r)}jg!YIU-sI#E#(q|V^bzF;2jAU|r{g}Rz2QIwRjnhFQoGV`c+m#xnB zOwORlxw>nvv#jQm)2_C=Y+iydNhY2x=}TooZZ#|Emm%agi93HlRX&qyFCb{@Q`D}H zGM5haz4mGL!OD$hb^gqb+NUZPrc>G#%+kX*`8E!V7nW_`_3b87xo0~^p6wkiIz9^1 zCJIyMFMQ9j8%Isgx0jq1d(}C(c(W{1r&pw20>z1hU&nRdWcrQu9GxO zt{dBXHFcWOKeY-tr@1LI9?p@~4!yKKMgtz%=K|$ATy5mpvXxV4JekK;f0y!7L%3^b zHXxfhl<)38$FQOn{v%NC4>`u>B`3wQaxKA5h#+N}UUV;SYQu?(C{VaP8(Ln!h7ZE1 zM=o0lUsV9y-I3HOk94nWYQ{W`ROzip8*<}HF;zic=b9F!%IOS1s9K^nuCQuSLEh^u z>A@w9`D4Ll>R^LID&UM80E5y*+VpjFBbgZZUT{~4M)yy1kx;y!A(i7neadM~ad{V0 z9q&U_g=82h1H~N<1z#TF^M#|Hhp8!s$xTJ}m$_IGM}%x?I>&NR$BwUFrpFi`b8D%t z4V5mwfYX581wHK4=^Nj5?oFd@o1s-oNYw1?6Wp-!UF{v~3*Z3yHL4DD; zeFxnkr)f8xWuWak9MU_gA9+s?UMS6IV|IAzQs>Bg2zHYvvKQXQ3H-!LxLz#Krw#PE zXgu->qeS7XSwMewTElq4m8cwXoD=S69*~_Ez6`>*xGxTcGs-FX0>DU8YFX#4AEdA0 zzDYy5u^~0}DJ>M{ly@$RXHRmeuQw-qF(n+Hjv-Fz2l`sB(^zTDLR(5Fav2=sjy7fr z-U;MYmQV2{LKzc39AG&@68=R~7JzhOlMAhKq(~qK?!FboI8R~+K&sw8LB!+RTc+9x zP@Hc^1pdm&D`K13UMWXnd*y$qw}Mqh0637ek4dN)#ono2h3%p0oY87+HKy}5rD~+$ zSnVm=e%203Njxvt+6#0K6&GtVnRce{VdSszP3RGSVcA~h42K+tJXiny!Rb;68${@s z!%zvw4nv+z4ub^LI3x$K(MsWp^**I2_lftpWvbWkBlz8WfDNopqqex)9StLdXKXBz zm^8;Q3;tDMWOh{(p=bw!fGweENzr=-k0g7xNWAC3u{WzR1Jf2jqyGWlghHR+ShUfmn9w?e zF`;z|nG`BX>%S$NBdtHNKgVtadgcF$+qpL(*qQB|;m&>%LDG4Qfv4kj3%xIXOw+}B zPu4iP8$+qmeTbi0`V>~pOeomowp2;c!KQt|{33>%ZSi6fn*KO<@)5;gigPnMB-$P) zl~n5D?nwkG-3OyV1Q3EQ3xXWp1t3E}?~s_tV;5j!NM&llVAYHYS^Za`tLULq{{(klIm%GX6g!sYgE9@x|mp!un=maS05s?}Wb^INhLGMdFR2hUebZ`N1_#}ggO zc|2HDX@7$s*q-(9t>IfwdFyr;TZTx0D~EH~wwgI#;iQ5VOxC$8=+-fxcUmGhc< zee5xgtN}aP;~F}c_Gi2UG4=>qB9+$F$Dme6HRtX*$vccb>jE!_p?8VeZ+8yQxuW)& zgI;9gNzuW?EDhbugeTPLWw?CLS{J=A1qR}bHol~wuEED^IBU%-TC4^W4D`Q?pIKc# zN`cRGdtfaGg|ZX%C!l_!-k@ixeg++G8FQUc^v`vwx`F%c#S~Rxsh{=&pi%i#(WcFm zV$}<{RtLLWmwT=U@V|tdMev`_>r|iH-c#?W;d#yyD)c&CJ@Ort3C=;QqIHweWu9&) zHv%Sd9tD{K>AZp;P&p|V4eBu!NJJ}%?0?QS5DE2cN4HX zR_-j!SZlsB4?&kQd5MdY6Axxg7Sm30<_UL%faN09dLmu7NHZe3XA1h;s$ z02>_P;b#ywd;yghg!@C{!YPYU^)@a{n@oUfZxI*l`p&`msu5H-hfbVYeV@p(CUwrs zJPNEcXCo24V`lT6iMwM4XuiiLVtFSnke9=#;KrL=WK{a>=+xxaHtqZc9`N=;blGM5B1}tiNLj?d$h}Z}_6NE!VF+JWdg?jn@Rr5wF zUZX6C^UI|JK-z6Np!y5Y6}!KR8hPlU^+(tQ0k44F)R(Gu#nny72z3hwlfrm2uOFs6 zc-LA+J<~Jd#6g|k_QFJ<$d+`2SNE1_Yo@(a4$!VP6$-MKbR)ZOLVhm8&b|Fk-lpZY z)Q?F2#8eYIxRgHX=r5tC;*#?6wUHa-TT#P6B$ijBaV%WA5FKY5&j(uY=`}hrRqgM7Dq&?m?X_jH@If6uk5T zJ4#93F5Wkr7N}yQUb$IPg=AWi`ZZv^t(Cy=$o}wFvL=0*Urpkjho0m&py?J2udov! zL5-N1n(i{uJ2mKhQS`uj(`CmM9RTW{=dU{j*!-gRR4d#O1gw}eb{(<(Q7)Olwb=mY zih4lz^L5irT{c#&@eVR{VT6jb!w6_%@Bc%(dW{Z~MUEmsAjo3qbpg-!>;^-QMI4vW zi=2iX|LI;x%ee!j%TNq?4Mgd`F{q0)tB*r|ey8;Uy(%|A+pg2W8-E>abi#FU-t2Vl zkHTJlXRgx`aTk|eL{CMAr(8NvB2*Nu)Oj4TCVG%*)YTmib&0MM&{eO~;Tv=ypumt$ zQTtU<+lc!NS;N%g>K(fFE*)N=!)-cH{?EL7aop4C>P5OpxkyozRUgrTq*swB%a+~^ z;`I^?P5ugp2H=D9B9Pth9LIU;xXxF;=ls52^MB_oId1bQUa~miJ5KYdx8^;?YZv)% z|LN*8P6JPVWvyB@t-aSi%n;4$#&coeDT-Acb-QO zUE4o$R_K>+FCJ;VEUG^SmsvgCA0i5C0h6fPtMta>2EFmP!B&k^Y)cRL)y(#V>6_H4 zP+Qn7PqcbSWo@5Fjnb%>XuS}c(g>YwVwUi-#@pH?en2L~fU#jM0V6NjEB3PcAGetw AWB>pF literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/common.cpython-310.pyc b/vllm/core/block/__pycache__/common.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61171accf208778a1fd0efda2a185feb02d06a31 GIT binary patch literal 13450 zcmbtb&5s<%b)W9(>6x9Ko!uQSNs*M~mTXwuh}@OpB#LE9;g@XDu~!mB*}=%ns5#Zk zo#lL~x`)f%3nIlvY#|r6x+J zP+C=MD6OG%Iw+&Ou39L!9@&qq_H*h)u%b4CTCj4awCkvo>eOdW`;0pMVQKTZFEGE- zrWIFI@aZH#W=Z*x-|hQhxLJ)W*I64o*L&S4uHA?N?ceF6{Jma?@_XZ`HyZl=xc0st zMWgPhA6ITo#{FQ^jw@IDqwc-9DSxi?QShUY?sBx*pM%0>JmFO&L8)!2QrlKmP*!%} zfY!2dKx<{DhFo*n?o18&=CrG73G=DPt((E_wb5h|9WdX z8jQEcz467!*LQ>H;v0j>-r)7Y!FIoQXM21Y?Tv=p_xt_9c6X$MZJ`I*!C*Ap8Xv|^ zZ`jo<9G}EE*3s$Xv)E#rXA-X0@S$F3(qgi~gru3|G;*<{g05+I2E~PAu*Lstj4~#F zg2T!Xrxr{D4R$tbajnxC`h%d;iR+!tV5BB}<{O>PrxU-Qys3)x+)!~X?MFYyzRxl_ z$0VPj?<1#W~4N0xkvRma-WvP9(_}xL%T>RQ1enAE5VkaW#s!9OS3#t%GBXG5u z8mg+(UYe<)vO4WL6tSsRky}=45YJXzPiF#E`fEtt6;E`{+Z}0d;1B(~z2RL?2fKj| zhTXvHQj$Y?>w94kd81uVgvaX*Rj(U_TlKn!->tpC%X?1x9QeIqgeUZT4~<1x^`RH- z`4Pq%hsB<}XoOcmPfLdoQyqlk5k^@+(3W>2^2T~}zo&xG8w3z<6|w`$s3^+?kD_F< z(xE?8m{+8G!To^a%uh0z?Ck6n85AlRlM?N4od zVrhOM!VgnxZ@m^*AP&KU8FdhauV3P_!_17|{yMvJk+>z>sy{t?C6|`1PNz5QMV-#p zC7S7e`1V%*9r`ar?x3U)N+XS%Wqp#vzl;QCH&Bytpyr2$dYewa%7N)@YnE%OApbZI!9xR$QiJjFDh z-@r3Vu~oMnyFl?mSCEM{?eOBI06pS{NY5fSB!>HK4$rkE!;YFWjDmZh5BJr3!~S7j zU(gXQU`7e8JQNHnm5dDaxQIHeu!w2yB19*a{SeFRs}Z`tie}<*Yu3?>tJ<1rva@N6 zP)e7Z4NY~{FEDu#NnGu8)TrC(XsVX}3X}6pJSKC#YMXUAHMzrj2`@-n$Yra+{kN7| zZmZr}ZaSLpx_An|3^m zpu9VWoRboAkWAL&<;+?I zz@|p#v89<6HGxHx%bR#a;GuNqZXQ#z%o9Ar7|&AkrO`VD$M$1u)808+y8+9Yvpi71 z&-EYp{YfCOd7&wwhI~X?UqeDbIxjbiL2t?t{ag4<#=Z`)F*~+?gS{>Q!Gaxs?-vfa z(C}rZnpU31_c0vu=b6c|H4Uo4q>Dx+Sp67JNYWc(ZH`SW7J!&4G&S}3FbjY(fbp2)z{-WAiH21w;BBaOrpK@! zJq?IHi!-Dr4jwSpD%h z03cT)t-~NNnie3Tl&{PIlvqNL2g-6uzHEL^6CO}x#p9R z@k%f3nDH5_8KFwxx9ik^Nr6qRqXmkvRrn}wgu0td_$8MoO=QSW<;j z8Q+Rdo02;_4e$I_MuBdq#BUe1TbV%)_Ilktao<=OFYB3o@rM=n7#}OVnB~s=AakU$ z=VXXQJ#mO$NU7)r`LqZ@x$}u#v#cq|Cx+I)!(>hZ04B&$sNTU5Udy(xvDC0OtfLne z=v3ai0J3MDeMlXr*L!HCnvUhO{Qto6UF!$F7Q|K*XJce&N_#_? zn43WEPmCQTJ&1dV+IRC=3w}iybm4?&rgGDepGPs+A0w$BOVBh8TwL%HIlX-w5sXDs zEl3|bNo-X%^mC_Nwhb%EDt3DD0sChCfLq&|Pu4DPAOjdF##9 zh4_b|QGdUTW zKV@X(#TCsKT|9*!JH3o293ziG9ZJ>>-txVu7#%`+is5d+g+HcH9nXj=tI zpWG@L;lBkn8Dn%08mbwRq)mRAHJuPavzNVOSaEp=y?)=s0{u|7bpu4b(-tx)yzDR_ zF}^-DAxwnY8G2)Ea?+1_2)Y;N1ARXP1<*D@Fd!#H{MZ}s9ft6z{Zt9m1?k^dS=f$) zJw`l{5#sTe?l@2v;7`nGaNt5NymR1(J;SbWG9Hh#F-u_aZtnrSe0MJcpqvilYN<0P zBuG$Wk)VWoab0v&SQR(S-q5(zcqw7^NXJVVJG(r9N`J)V6hCpY6hE^GoZ5^bGRD=h zrw&p^yUG#;J3*HlIZ=f*kFZ6&U{F1<|wUGqnfPsOB_N~Z{$^<(+ zfO`Ww_>Ps9h3{M`=P|kODqHwoMZTtR+K}-59FZze=_}Vq6Tmp*EHh6dvP`wh?D!XC z-j4n*(#<9PKK^KK_JybV1Lhb|7AEQ)=DyG5JtPULY_Grt83qS=QQuv8$>l8C-Dg-w3wPTlZm z!f%zPm8sjQPF+M=ZeRc$S(W?2&~Wt!sEDiM5scJsWH@^sRl-l!!zlLDX4MF5HZ!9o zKV-u;6T$IwJgrOATy*E}(MA~B1ZZy|K#LG9gq8pJ*ZsNc!k=!;G89pK`Bb{zIksgh zqetOb3mNcBf*AV-k7p63vP?0q8?Y%8B(;gZUre;(c4k(mc>GeXm~K@68A`BxJ2^l8-sJj{bQ8L7`rU&k$L7u zylr-3rmbLqZX?923$8KNk&d0vfcqJQg7u^;SUn#?NRiJ+Ll4K0I182IL57UdsJ0nH z0!)E935{K;3!kk`XCWnKTO=u*Ey4>34)XR#8mBGesKE4tqcq8gZ*$E|Q|0o}(~MAO zUhJPyZ%*&%!R*9Safvcdm4|G9f!~0t%U#B4cr`my-*`!}GpNIb(6kL-kGf z#CMU`3}4Q1F<$E!PG_T1wdDk=VK=R#vomDRH7u+rpLe-_wD zpPAA7CJKgu@OlV+?OP9>Y1xFr;O~&%x9*koIZ#jVg3{-O)AAl>nraS@#y2JWMJ zvtrzyECn69xQzBn4jdu*bIYUeq215%NTj+BA&0jsw>Zg}m6;zdUdqI%ut;=2nGA`@ z&xpBEPO~(}rga%#>U?36)094Ib5|o;$(8^KCB2Kx>`E9S3%C)@iibEUEJ=-nt>r{Q zWMd1B=LX{`BcMm8v%~>^lLM~jOP{SV?1(7SLh;TpCTJT;%fPfr{V3{~^*{B7jr=Sr+*8=X7I9y6F#I1rj zQRF&6(ouoHtj)8RV+~`@+U{H=>=|8Mv?vwakHK9hvgTXp>Lau`TFHAx{pQI< zAk9Qq2+|BJt1%}BQlc`y&!Vg$4VI|1@gV~i8Hm zdDE$T9id`>9g29xn?${SFFKUyu>n&+Dk%_>*h0WCCq7ETEosPR1e~%9fOWw;K#Sx? zmtdkjL@sdN6)?_ohhEpai@o!tk4Qvxn8i0tGz8HKAh+yu-wP3~K!hdhg9m0q1nhX( z5y;E9N$-K_Si%-2u0r62XiFEvNVtagxET8EArzB!hLd@Mw%rktAqFxb&#NDLZ^?=I za5A`qFbhuDaJdv0!VsH5QKn!fKfHRZWk#84mxNW`y@`g|u;}nSK7HU_{ctnw+I08< znxzrWU5Zu*V|Tdj@eBMMyXfMg;R2;FH9oWW zH9qhCD>OrPqj4F%p_z+mwq9pIN0onMKP&@vG00}! zbF_yXIbKxb12Jhab|Gmg2+rXOHfcyazQ`GiZtAo9j= ziA+mkXYI9IVkkzvQM(b1B0srM8MaT%%M%#dKAC@%B7atI%vL{(@eSaFOlYs9{BPih zmnV|uuAHm_!W;ZsvvEP<`FZ}&cCzz~V*;5l-4M>5J8~#1o752ZvnVWa1%}j*=H-Zz zt{*b_Lne#-(S5vlk(4uVQZG4nW4U3Oe@(lwrujau)43OlZ^-~BF?u`N|O?g;a_5T(W;1Ef!&|>e55Ie1` z*iBv|@O<15hM4l+umtHDkd`n#n~}KG#)=7&ef@5F-2j{BiRQ@wZh7H$0Gv*k1Rb+J zVP5mG5CvuvbHHF+!|^fAW!`+`3vxIxcZ`Q_k4;4gB-QEQ3ok$9{7#Ntlat+iKyqb4 zH^)xmxL-BrZ~2+qBDxBl$pH&{PYzl3fo2m=qD%} zL_A^$ur$;K>{aENA~AB?vXuCw8Ml9{g&DLsgh?LI#}*CS78Y z+n)AW{-292oQ4Nm246@BotF(mGZzUm3UbYFR>^7OOYGw!lW#J)&g2s&)O8_uk)?=h qm(2fVcpEb`j;4Y8USGqZg9G!nR$gnYZ8y&}-)vsDudG>Tzwy5)R(tpW literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/cpu_gpu_block_allocator.cpython-310.pyc b/vllm/core/block/__pycache__/cpu_gpu_block_allocator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe2ad476bc7661fed405021794b8d17c5d593f16 GIT binary patch literal 15402 zcmcgzTaOz@cJ6LAn;gznGo#VnZpjzT*wT#bUEj1?Thhq(%2{ij^(~>-OZTvvnjVpC zs=7JjVW_oZ$yp$P17sfp1VI4lQ4jkE5+Fc+Kz=|PdCODsD&%EjTlvnZzL1=zEE(%? z&^6uFRj1B5b#8U4+Mk}TX!t$%`+wQ|r*oS2pY$;JGx2a9SMWIsp$WaIwe+gqGO9+a zP%X5~s@W=5i#okCnpUe+Ew#$ka%-YG(W+D{TwiETwx+66t?BACKby^&)@*f_?~Bd3 z*1qaKU3*g#mMGoQM9Hn(EL7+5Tow~}o^VTeUJ#XcwUx>L0v6gzF)}ZC^)Q-vL;5|p z{jM8E=Fhz#jLN_0gr47ano;q+Zl~$4m{IX!)30BPX87NQCLWy7mr;qIFN^5VC3oAa zyBBsG=?=G9F`~*9$JOQ@6}& z1uZ5UhL{#Jc$yNk?zB63NE37J48O6&J~59sv+0`!cMgBnAuVk+M?D&4u_*SV<-WA# zl4_U2DvJZ+AX?0eL+&JI>@cu6B95YFK^&tRv?oZWcuYKw+Qqo1g&JENN6r3$niJv) z)GTq$lT4}Au*6g1Y1AGNClxHQ3_4brmV>V4S`C9SeMUTsR)^A7idyP_N}R@h=f#ik=CSmRqU^`w1=Kz+UX0r({a$iyh7JybXD^Fa z(CWB2i&hC&i4XMF6h9H?@b-lJ1bTo*#^a~rRi4YIVqP3WyVt~dw0k-S>kHx{-kcPN zp#kgB;(Dk1W~VzWY0kguns&k7AX&GaM9S@O(+Tam(@v^f)grL{w!PtpoA&xU@7qpW z*l*&0%We5`cePTn@q2I63+#H+2?E<|L6cf;I}8BNwLclFFrV07J9K5ksk;ocBmHep zxB&xfblY{Mn_jqUH+-4k1g+uLEvM~Vr3WSgC^lk@Mv-N3gs@=E4N{DOTOMydv0Dxh z!xh+p+i|25x(Vox?>7U+W`GksfWqPwqecwfQO#oB!pNM$_=8U%=^D^iJ%Syl(;;H~ z#sDTwe>ko*P-x4ZE^k zCySl@CU9fnTE`Xq4n?SidSCAw2eg~|P5p-1HwB)*ght&c_RYR7)VX-W^QTob&M3X`O$(-(F<84aSZ*!GHs*iBm81)aJo0~=W}GE$FC3dBXMp?*@@jB4ZAJ`9 zhh!nMmxEBQoQ1}_+%+96YiEbP-G#)G)`F5DU<@#}APUD-d%Y_q>_v0e4!WI=FGDbF z+i7-bex+-lXp>hxL8`R})QJwe(^_5m(-TTDXb}1htAo7^wT>|#WfY+t)2Z-=!Pqlc zV(}DkRnicdhCR{{d92ABB2(Hy@HC~wdl!_e*@hGi2tepxbK8lrBV2#U`aUwZV~z7! z`@%!2)*d#csy0@|uK3X1VIe!s0;Qz&@+CF#@Fw2XHiQAH8bVe5ZqVF4rR1Irde*w` zw_rW@V6x#t1YP!8Y1>Rfu2hRGSt|&Fl!hz28C7UfxkDfqUv)~hp(C%l;pvxJ-Obi>t(~=|x3SjQg~e>IZ8w{(wK|x;#;jYT zak8OPn5&)L$Wpu~<^|_b%pIT7r}VO6=*ge;wNXm$%U_qRy}A})4(4czqgEv;{CA?B8qx3G+1H>7VFeLd8-3Vp*f z`dZEOG?Ya_rN#H9zP4rFF5l5EX&=x2%D7P=m$^kxC3cwFR;gb+r(IvZeq3m9X2zA? z1+a#r2CyfV=#)JOshsx7(B5>mT@@?^E=NU~(Z#f_OhnV@q9m4tRL+2cY612oDnL)7 z$!t!8Xu3whMy;g+4!mo3lfKb`3`Q;VvtCir3*A<*Vnt@)HX9ieBfS<)r07QFL{C^! zRB;TrMCDqITwJa8F`j}GD9nPX8~4roM#22XvMk;D)+(Yj=Jg6@pWgJ2<<4UQsg4LX z{n33oe^258^MEC0SCg{{V6@wY(89u&c?aPH=8E6GtKHSFtQ6%FXdO*l0TV9r%5Nzq z$E~%xp|jC+6;GI_(njOEx_lb{MoB!J_JfK;Ee7~{9nAw;H&-CiQ@VBEGJ0#d#Epj$ zRbY6w6)usbLfIh^CUK^VMED-*Ke3>XNbNQvwmUk6O?Q4pQNN1zId*EhJ9bQEEP8RA zl9_Nd(gWjz3^98-Sx|q(%=|d^w$#Fd^^OODuZ64PsLZ|IbrGj!Mmh1V?^D!70fe%* zP=SCJ%)<>8a`v+FV)0;cB`De1^xzu@Jv)s_M6;X@5S8m2nFJN+pbw&xWOl?b(=wb^ z40#-!m#0y{Gq9MPp^E3IAnTdw1UK42)jb2n+*I$`2TR0|G)A*b0hAO@o>YD)jHi&E zE*1rH04xdxB?^7x4u#RW^A5%x6^5(Q?{>(zc&#D17zj4g5SB5pQ=iMI*yOtZUu5Wy zBS0fuw3jI@^bx|uL~_aPX2v$75kj@$X2d5_8z5H%xs8!aB@JO%(t`f~uwu(3ufap-7Dz$VFWx9eDb3iSA4B5=pjTiCa zOI-KJF<17PI^4nPgY{!{plIT|l|#M&@KA*#IjWGti;Y&gl1vpKr9YPVlzl&}_59;V ze#)HjGM~|>qVxB?dPt~W zhD_xO^(%Ps_w@256ur}rB+gNhd3&W!&La7djZa*DK*NTJTSNV(Axi=dA2Fx zqyZ^N8t|NZ(*@kZUgoKZU9zfDQq;gWBI7MWu&Ck%Ny0{+0vU3J3JT*QlbR>&J)6E; zrn?b;itPyP_qYOD9#u-FZXwpH=relnP%f(pa#TUeGE}B%JZs5JWvpdJJ_KuttG5r-Z@9P*@Ig)%-!y9S%cb*Su5fYgxPZ3|a6cSo1C@#HwFr7u0?K%FKMl}R9# z3iRHY$!Hy(9;%P6Tx?t(1UENU8ge=pn9#({@Z^sna)_D~bFv%s%fUjuux*Ome}ZY* zD=$c^Ci3!AYQVCTCoiw!?LXt<$b4o>NlRs{v;gjqgpeZA#X%M=xH!ntDag7*OFP}U zg1L{G#R`<=oJjN(!;$FNcDyF7Grm{c^dR#}(Q_ICLp-F?<6$)RFkMa8s(m2{TuG@> zPAu3+?@Py{pxkTNSNt~CqXDv*Xar?Wf#k{t!l~Fqr`gFzWI!_Oi6LH4>#J6c;KYH& zT|)&Hd7<`sOwYgJ3TVP+D!t`ggOCGs?2OCT07*WJBEe}f=^px_6EDUCd4ZaZ7^;h? z`*&Ol8@;#kKx_uVMU&1(;5WOWiMuxFe=Lm%_|@ufcSIh0uJlj)jYi<6+RusV zI_4cwU*fuz-KY33o#q6K#ApnPz+SWSh%zeOCUWmccqp&o4LFNL4wCV~g~60)Ql0kX z1teh}gKB3aB%pRSeW+gS*nRoXvixw&%lup-Ci_WDI-9$JS5Kq_R3wW<7GAO_qcU>| zb^k&2KSI%aW~{{QfgD*%p`%J5>oqkMQANP%sr#XYBklhr3&)^`2cZk&76OS&*#AK) zYbc|jXUj^Y#W<8G474tK+6@c4L1w>r(?9}jKhj~P8?=vOc6E8SUrzo5s@X4{)3C>5 z(f{jD^)1-XqH`45`ueAC9pcXRlk!&Sj1EL2*lCk~%MSdmth+H;si@Lo)Q2di1fQ}4 z%BGQ2)YZO&&4A;CnA*VWbX@FFq(IQPMPR6tgRrR8g^0owenQ!>)uDviCyXH{<Z`P@kgwB^IRb(j+Fle!#pDTJ+m9f7!r889=h^6brwL z>mKDiu0lLNF5W=$YfJlE<2rI4tW(I|QABXph>X=Uk|WAt^Bh&qQRIRhh0d#gAYcS} zeq6jx++96Kr$m~;tGG#ej>S#>4B(V@U#7da(nm&vvZeCpD88Yaw^2}}a~%JxWg4sI zcf)Fl@4R+Y=y@G^0xyyB=Va6C)awl@-lQ(mBpW>Klq0H6(>!O7i?RF<$!~RPun$g$ z40IVWN=OC+qyNT00VVHe=H~P{gxe-^xN``+%ewKoIWC-{o?{)Dd>-HsKv8@(CV)b^ zRvGm;u-x&e0&4AdN!hSwNyUg-Bz98l<#qqV z6`Rf&z@H49DWFMBY-4;9B;k?rZp0w+Wmxl3XdcHkNV*P4Y_80ZJNbyuxMju|*7$-= z7AqYy9)u;FQ@I#!&uonuKS30WFIo-pRtX!fXt7Rws1XZle!Ipj$d8Qw6de!HJW&v0 zOzF}?mMr8E=^|0RN5eAp*t>+N($)~Dr^SIJule0{kWWM~mPV^gM;-|u0NgVv35B_b zBq4zx9pEjc(sJ51}%eVc9S3N0n){)c^EfLF9ohf-%Mt^U$|2M&AfQq zBdkkgXZDT^C(I7^L%`&nbopm=mq+}cwzKk2otRN4C>~;R`PX1ugQi=fbIvM$6 zr&hjC$mA1rEFPl;)l0#qK|o?UtK+Eo2rphvsVUG7Ca8HnS0W!c(vS%Sz*H^+G8ag% z!HC{A?qH#j%uOMhK!At?yI7#c*}!Y=ZXj0)*WsW|K90KA2u(7~Ge+-3E`nI*jo~JB zOKm#AhFQRBXIl$OapDv;5TH1>(>wbPpF6_-Ji-lo(+iOu3Efru^m8~f=HTRD+eb<- zJ@r4A!ZGq%Y=xD3%bZrc410j1N*u+9ama?y z8Emng1_-y^ZAw$6p^4vd)%iSini9r6K4Fz zzt$UuUb#QBSfS%b_vyYe1Ix(mETi&mWzmYC3_Q&*F3rr(&rdBCW@e~;h1wrYdLG_a znM&@LN);T1$>126N#nkS_C{$@pD#RC@NIzEFn+5wbWsrI zt*L5}z7TLre5#zjpD?Rsd>z1FPT*UCid)&5#J2)daf?ZOQ9<7j;OhY~mGG*}NU#9D^pXS)fxrX*`FO3G^I>27#U3 zY2ws5Qh(TEQKscjK8S6(THaxmh&>rPcg27tjy?t04zAP6gRyM=99hixVi|Wj5cw>v!MS~af7nZ*ou=4Jt0Q6AjN^l< z1so>6oP2WP3Uv%N2e)W$6z+8($_|rGjv}ZXnCH=^d$2}Vh4moYX@A{}eMh$Owns>nS$9iW2y8a|?nO`7}-QMF6Oh@Q|$ ze}$1)E2oS{pkSJpsmazD)iR~!8ze5KEv^{^t zpcc03vjCFu>XW0*2G*LuCvHDYO)K2A8+uK3cATS3b+jUr8ond`MCEg8iSR0Tqy_UR Vj%o*uIfKsa&lR-Kw9gjb{y#svySV@W literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/interfaces.cpython-310.pyc b/vllm/core/block/__pycache__/interfaces.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..209e865396714efa17c12a64dcaf83f4e741b85b GIT binary patch literal 10837 zcmb_iO^h5z6`t%PRa_4+zUK#md=AL$TF{s^Gh7 z;9;@;d;VsdN%jA99A#-+KpDz^D8%n zv&-9exaVuHwp`CIzt-_uPTREn@@?UGPSdgc!rgAi;&sI@-rx^fP2R+sd!Hh_tisQI zOUdPUuA#78L*;pdes|tB#nHQ9+ zKwHCnS(0jnl2tYoBK9K#oOGwaMkW^*(>(;venvH?(D$4X)iypEPJ`> z2)?|5W-Pbb9v7Qtlewg&*gr0=<@c2Lat16^=wx;G z%=&A0UwU4@;plD0)4ML$y{(q3uid_-J6+FhF|JD$>vfp5MyO~{l{OxN`r@7yXsDP0 zG6g>uwN|x3n=PiXa5PG?E*!aeoCmUnt9Q4!-tO8PTSl!&_H&&)?Z{x)#$nxbC#A9b7i>vZLP?e6#iLb+g%OzrALG+b}&aLPtGw zND`C#IPji_pIpp+77D7B{z1o7+Dsg=OxH8aCio{XW);O~$xO-~-Iwdv_PrXO+rP!J z>Uyy3PhtIu)|>?1OQGHa8z;KBRrtBrfW#YTMS)FBSw<_Vh}FTy`i+$v#n)abne zo9Zxafd$w)OXRsmc7wTi6SjIs?5v9;P_aNaOzz&VZlK#X+}1AlD`q$tz8va!aL`$a z5g<+6+QSFa(V*}5&h?`7e58&x7FvU+p#*upM7Q{5!)Stf!*~@W?8Ekf5=C9`+|u$JcmRC`EfmDbVyK5O8%M z<(#UjCGFFqs(x0?tN&3o_1|hi{a3N5eyWygpNM4;%3OQ`T{3JQe|VktW9dg7Saxmi z!iI^9Vas&4$lE48?0MZ0`nuERdTUc}d3wu@{48!jJKWP5ETT5l0fNYKyPb|BV7)F? zBhj0XN6!18QqRF<8Pst;#l_oQ7rZ@>>Ov2<6utFv_QOr7&A`UJ2|Xar0@dIHy-C?ml!6Bm)B%mG>&707gN1Hth>q^NmSnNc*L^9sxWNF!%Lfti3* zrb$MoQ4;tdIYFf+lv-XL!O~Rwfklte#wx13toAPYI9lcfh?etA!^jV!RvGouyKwLcx8b;VCafg6AhFmC#5xEvd#f@Re zfjAu^)*u>gj6|Fu5^jtFXNH$aP75R6#u#wtWPO3QdA2~>Y6o%_*B(@QB z9$`n34}CC2{1TrATmaV0$p~F%$Ee&4=fl5>dBSueu|x@oH@uj z6O;?f*ZDm416ojKXW2QDTmZ@QAlciqD50?nY>6ZmL4tag9|rC+yGYz4`?xFYY2qFQ zZqzr>9*tdMmnlbw9F@;P+pn-^h;t0^!BzGw;DcxRali*B*f0dB&xtcIf8?YmFAWnZ zdN7^krGt%%ymY)8$QjNAy8$=su4it5Z>6n*Lu~@%*4bnbm?yLDTZmAf^$58kOL@#o_GK;*RN90Lprm=o&9@#sPctWm5#XoZnjFu z&E7xvt4R&$rT~~45<38{kW(jut&gUF_nDK;Gt_lq5+Au0k zPPSRw6!(K^Zr%TtMd$tW=B$cGb+EK1D$q%k2-{c+S+k1!@I;Lj4Y4qxNp z_9xu?t&S19NL;fTR3AQ=&e+@bG`Y9E1|ovYzm2Ix8)k6z9#s4j6s3(Lx$CCO`!i7; zE`KL(z~grFofH!f2P$otjFFaTe*F|n(26*p82P@%vJFEJwWsc$lP7R@@z~9k=L!gW z0K~{+1&)Vst~`Yc&Q)v>1>DHv1vdJhCF}70OzB~^s@=m`xNSLxOgM7AGSd+NN<{;L*D|JD@>0s=@(XXo11NSKdwcf%UUiD6kF zm~`Gio`Z80>QQLm(met|~=EeW)tK4-ryhx{Aj!>hNc3VG7_k0`yw&3*SS~ zuI3jZr>dw?4VV+u7}cUGAn~wB8lVOZf3nsoEwTwLiXZIaRrtBO8czcNYtaQ4oXzD4 zl;cGm!gjm_3UiA9TPp;}rBoSD@kj~*c_>vuaEo%0q`V+2hu2&U1nMejssLyVuDJq` zjx{4UxaK0DUIS2{XXC&P;X8u$Ddd7C3H7Jh3|TV95t!%sI9dPzG`_k*P>%AG9Iodg z@IA-osT@M{1!*^2VNH|1A`7mq8VJq9TxrQsrc=3D$T=323nh;8L!=2UD61A#n_ugacgmxzNPUXH~ySaO+NA+FBG0iHkO&&i9@wQW;KhY)!Q0_}d0 z(M75}V#SplKA6r3D8QM}1sH+%dmz`dq|lEDbOFD`1Z~F`Gfn7=+y{(q>Opiv#-MK$ z_mITyjVwYh^gHCA0J8h^02U>7MR64h4jOUovk42P+JsK=_n1jGU3z?*9vdXxLv4wk zSq0u7kx#qqg={OL4L8P{18@6gielUZ{uC)*dbDD?zzgC7fu{@uA5Dmq5Qw&;KS%nH zWa~%FEbxe7;O2fFt4;JXZhRjPr0xk%CTQ#Y3u>a(cRIUa|2L#ad-UI;#`PJ?G>j-Q z34**q(V=(+btO6&ql3RApPIdr-Rvg>g5t|5$}uYWC-P|(4X!+dF@nI^ZKz>TA0xh! zVg`mppCF%HN*~9Ii#gOCq7vo`L|p^V^1CSpU^w;=`7{Vuvj-tCARWqy>u5(FNcdp- z5xWtVN%gZB3TjYT+H|GAO&K`EuezdXFxo5qDub(BnI3es!X1H(+(1j4NcK-*_!#H%u0Q00tH@u(H+X>Z6ZY&jO=!qqncb-nUN0MRz^t{6TF*)l!2x_l5{4sLMX)*0N>b>Kph zb>Zt6M|Z6jgA0CryVYjSc6@aR-=G1gnfT!MKq>RUnHV}m=1I< ip@y^$^2;yCj}~5pDUx&Xh^Cg)m0>`+wS28HW&Rfn5&6*o literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/naive_block.cpython-310.pyc b/vllm/core/block/__pycache__/naive_block.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6ab38de37619ff9908459c61e2e922cda385654 GIT binary patch literal 15188 zcmdU0O>i66ea9{q3lIb;lA;2!K{>)6ZfX|h`{pD8ll|tb^S;&4Y6kf(RxQ|2>gwZK< zjfP<`Z+6UXu~F<=4Xayfl)84q?v@+nZlzIC?ZwVix7w(7ryJAVTBFvTY0Rj4t25g@ z);QLkYs_`$8}r?T#)47sjyD!xDTtD=?-zva&D|?DPM};C6_hL9F_f3Y)SHER^?g86 zs2fp9c<=PRx*e5nFcVGPl>Wf$z3GKf-|;&8n5d*Cie}Z%)egG2p)aGdD&G>3eZ#xcZh0+2o&D5Mcp2Z|+h(EQ6&i*p zG)!SMic{v4DP9pp&)O~t%d^#8i3qSiv| zv^ULuqK0w}zcZe-Q4zD^7<$g+dd{Jz>YM95%~>#FUMyh5Y;Ht$PFoxoi|BC-E6i<} z;)GbbZ#Cw{NzW29!rZXLvRFaQg1DvXPJ72yy)B*)Ph#Anc#1s%0oSv{)51aT6R5E< z+Zpi;N=tF+tauislj1pGg6)<#C(fg0*;^48ME!o@p3yjk{ET;+d&l_a#S8Z>)IOo% z0fd&gC@x{lljt?=J>|_2YT^^USH)$tJPj;s+R2L?==GExpu3puMY3(C&gMoOP17 z147p%?1Wox=yctlyXghGF>w41r|Wfnx$o??!!5rbI=j-_Xy2_m__W;CR=c;kT2-a1 zax-{QGdbvAtIv^WSeIz4YMor$w4 zP=i(Ht*v(8;D0~xgtNEh^&Hokuupbr`Q2S#bpzn-UT!)!-bJ#53=%F~0vD7m7pb^Py_5-SO>-e;H@Ki=R;?w=kGagZ=V2dI$kI6aqSpUcVa)BIiQ87Xo>20lv@=-HwB{bx-n;;`Ks5KETm<)zS>w11~#Xl`+o45ysnW z_b}^@*9*qSwngAvNF?gpiKu-Wz}&ul(eYzZ%Tw*8B?q!qNn`+*ZY4cQ=hh7z=B*8< z=M!`Oowo4A#TX8>f!I17XWet$AZTy)aMl4uelKi;TK9AFZAicCd>Znsx9bqP#(s}GOsosUAXxSM7UPhkf>y+VtiDYPrLcbs@VPRHFwP5^s9yM8=;LF$o(I(XL(peS_!96SAI}^2py~Hr-vmq z_f=6mu)`W!D?{`jnnQcBaL>Y+8But%FvRRSSGU&cMq@rzTFrJ(KnVpa!L=^BZ%yJIkEz;vmqDIB9Aj!WO9PZ5|T#EyNl6H zEjivR$g`;Y&9%#6clYvcd-qc4N+|A2U+DI?x>ve;mpkqC%e!DTzjyggr_;UM@}+lK zt#?^*w}y6gcR#8$v9)&CY(9Y*gJ+RcDixz@R*eP2G%cg@$Tm&0g5OHfGOCtsR3BCC z!4nfC&uTJNRE40tqt}S06k|0>Ubax?L1arW?8{z@N0NL>$Q1EaBz6v&VNn?ULSU0{ zKm>)MK>|6jhSmd+)I-An5klA(-G4S9*R>#r>~<+Xop!g|4`~#|d_@7SWZNc0Tg>ZZ z#EVX+-P-|=g?`Fe6oG`l?e}9<5DR&-a9+@}X0+9eptwf$ayyn+`Nprl3Vjl%7z1AAC{U=9lp ziVq8<_jl&dUQXMW&<+8+U4nRoa9k;%gb~a^#99Z&u#C}$8tDwn;1zT2Ju8DFikQay z?$&tAMKI)XMATC`~DaWN-K_a3)J0`A)`CogasQw)EGsH6iqcBqzkjDKJJgHgo!WcuI}Z__GTNd zuZ+Qy_B9O&{`kX!GJZ|2JGEUi3iivNh=1&ULZ`-EcQwlf@DSnRLqi%=Q z_&!-{aI)p{!VRawjzgL_FZ6Dd6n*~@KJ_U{_876Ce38X7Oq66*HWbCHyn-ar>DQ2N z%mjX4wqi3NXv}aKPiTq@v~{6479LhyTe7W(X&Xa-if(~}q%v*sx?#)~XQAG4>44DU zKl_8K4P(qag61(rP7)IXCcfA4{WV1(=p0;m53*KsqN%MLidzCU$WyZB!df& z10_2ons(O%zte~P))qOzz)hthjY=^=m=Fql+pfOWj_j3sF|q?+hMtf-*r)`Lgm*Vl+P{mc@nb&0V?Jbw z{}Xx!JfFGgLH($cJ~ZkO*^wm_JJ7xbGo(t%P$tDBx|1i-^qrJ?Cas$5nAn2?VPe5zSg994?<$=k} zH<6~V(XhNFG+oCKk~pc82zq+8cL4bKMpO6`g)`du9<&qCKm^g739vX@syDMMwGX>H zde~16rJSzk!HE~Ra(~2Fw3U_F=-_Z0Gwq-`!lX8^X#FSlYRiL=YB^lw;0T_5Zk%}aR2U}DaiP*ImB(d%PoI>a_)c169^ zTx8V;#zU}c-5C6PR{rLZr4{m}mxLGK0YOt@7l@&s-R%jd)$v@3r-Sgf5r?u`)T80s zF`Zo0E}UE9Htd+IaF)R`N~YG!9-@i#sOf{H`zQ-mu5om(5RDEt+^8y0d2Yc(IkA|u z@9sDKUQ@AO@GFc|QnFf}tPdv*ii&uSIAq$Vl9EzHz+|YKgP(lEg&m~;2kYa1i)2Y6 zR}V2zE4atwRVaD>$RPPm+|}@0LAFv)Dyimigb&>C0YJS*b>ZC~o8_Xjhj+P=Qxbjk zb5p{4Yl>o%`-5^6(@Qc=$0xghUK28y1^;8{{~LXkY`4mT7ba2d5#w_5meiV{O#T51 z6VSz{C|Di;N6;1Dn-#jwrsz^u-#DfczzytBk(XtEEKDWiqLa!qoD>sb0yp?Vu+!dc zj&2{c$VhO>hxr7pzh+@mi-6Cr`D0Y#YcI+YaB`V|Byy1tCTD+LU)jK%%mZA=2dSJVL z*H7P?wK&o;?L7m}<$idJ&*BtBo=|S5#9E!;LyJ!7**v(qn`1uom4sL-#$a{Q=8hgJ zKZgzEwny6MS7}6g)xyF(dY%k;DysA}Z5#5~(8NWhsJz@IJU-$%Qe9cT13feJFY$Z?ruG3l;fgt~br78y^wG2?e)*Fh7#Cpm1R(_{yA8^<(> zCX0{i2~wMK^c}0<6`*RE?}F|oT49fwB}a3`EX^&Wps1}3s-ve*y*s`G&u8LIFqGMu zsA2AD;wCCQk|rqV3L)k_j7~{7@+Q8cCY^JD@qvP0#SGs@MgjH$9kJU5TxQ_pK#nE2 z^L}U^6nitMgIf}T z6!Dr{cqPX@&$~s~D0SgfpZMT(fgBW>d~!!KfTZ5k*Bv3Q2daBuS`%0{I+sBrQoVjy|H#GpA1H7<0TGDT0lx zb%Y(rt1N$w$=8{@%47{mRDqbrbuQdjjsQs^EPtQL8%(Ha*kS6;~V@8Nd=fSPr(288_P65GK=PixLbWtEt&sg z+tuHg=d2I$@W-%(k8HLTs~?&x)(0$CQMS?7wyGbX?xUoiX;wd)J+V+(t}az(&&(DV zislT2MRCzsD4tqDOd;ko5!kRaRjqIgA|5`nF~%&T*CU2JEKDsGmrg8M3yTWY7QoDX z_;~X&zQJ8k$Ds(v=NaMX!41wBN3V?El2=BYT?O%tmMT|t`$kPni7Fxnr_!F&Dh945 zW)R<29jj%eqt%%9YKVlKK@{Aam{*a2v)=6XG4X;}NXzWO$VJ96GFEWjn@5ims%|N* zV=n~{dY%-^s^@~YfSxNcoEn-_XWeO$p(#`q`2Q>tcB)u>&g_Gv~wV*L|o zKNbCmnx(WxML?qFq^hyRb1H68MM0u=S=SP4n2!;WoNq-%{#}f!Baz_#RS|(9z0nmb z_R(lZ$2A%NxEdqMN?QTxl`L-|Ds@N!0Oc2}f7n%-z`J-3=|NZou%GB3Qfg7TR2WpI z;wV%Eii=>4yRa}Afrhvb?GOU1F$OF^upt6=5N}4Oj`nLPg_}-+yEL`0o}3r^BwjIg z2Pja-eNu=gV-!ionWDPo%@=i3(eviKO{^bxf{LT$;!|iuK zX+Wip6x!QrBbKr2xjniS5*If^OSj;%SogeMZoaN7;CW08Jo;OjI)W?%=`}LQVOaC7 zr+C!ZzB%!s07rO1MFwt$yJ-;CGKK_0QlhC8DLtzY0{}2$kZ3Bs z%A(^4Qa-6h%dwYmGyzVE<|L&bK%XSTB*Vyg%9`1H8t!)tAB&Fh&Ze9v&7k9lf!t%QjW||>?#Uqv<9Ov+e2;jgeOa?DW5Jv= z2WLkbef22zI?~uz?z0dPILhpn-{z{XFv%ZjR7spPKf?OjFPi7Er2Wt-qQAaYe3#Sm zO;@cF(@#{o<97Pq*mCk+bo((z%I_g_B=9mni#5|Gy7P+n(GqH<%-QP z__$k2JVJlUJ?8Nz@1yRg$>K+ED18jX%e!K_31^|?;m>GvJbF!BnIQcQR#taZ zFwm*Y+G9L<2nGz()SR9a_DIH^xDrA5JMK}P)M93xRDK^?A;G6?Vlds57YUQC*ncH!%RABS}p&O;^9OF z51L|W?yC#@$6MzvrkjZGb7=U*#C1mY$&pJ07DlIzE6oNIHiJJJ@!lbKy*d&q6HLNw6!SgW6sJNT6yCS4{yCS@jcU}*QmE_1YLw9A6(RO?~M+&fI( zMS>U#FT_iL3|K=6qfI4pmgYOspZoz6o`5#4{(`v?kE`6XtP7ER1KSDs^r2h~r|4ox zT?Db4!H(SwOLI%J<_v=EiZwV2_&cW@23B!dm03pzg8H?R{Ia3eS*xFnrdxiegZM!h z;eq~h6Fc1Jo2(?GYnMuiInt;`5%peUiGQ%M%0Jjx)n+wR51tXUkB85~>aR~!{8r=C zcwd-Dc!YQHLaFhsu&i7^6e^OKZ&coh|7=BhG8DX%5EfOwtbI~10cnL`0ZFak7|Z4| PyfJgd!gmYbUA*zX8!i*Q literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/prefix_caching_block.cpython-310.pyc b/vllm/core/block/__pycache__/prefix_caching_block.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef023112acba34c1dc517d1d4868b4408c4879d1 GIT binary patch literal 27516 zcmbt-TaX;rdERtS&+P2%?CfH(SlkG*3EnUyupoHRg(4~9DnTNb2mzATP}FEJ-3!cO zE{oH%0C&8w6oa{`#5hhRRgRK4fNd#(D~?i09LFk^W9Q<;Nt`&Taw_S{L#k4F(3>mF zgC$Z#zVAQXr@Loo7nHKp9CV*P*Z=(Izn`Q2^mI9gzhl4gJD30Gd%4{I;!FC^!OKhd z1%HCX%hhw9<=GvpYu9ay&-qTi>(rfYp&5OweWF{cm%5Yn$!@t`mNHIfs#~d7 zy3_UPZna*Oyh3NDJ6oUa?x^qR&ei9-JL@|ozu1}Y?yB$VF4PygyX(6pZ=$oOySKi# zyRW{lyT88Q%K7)z4|t{e{h!NulV17VoL9bS*B`*!lvlx9#ouu=Uw837?N#w!^=I+^ zAl_%ZS-j7p^g+Ds@aFI~hqs5koqoxm-J4ssy?JlfyH5RKZ{d8dwtF~zp?}5ixfeVA z)|FuK2LM8@W`&hjzuWEiR+Ycpey27OI;YyLK{$Cv^@o1%ygvv_X9vD&UhMdx^Kv`D z`>U&icE8u`gn520Tv%J}__cgEbF$gG?7!jv!6%^>9Mo(Q>8Ryy7?5DEj%qUHr8Omh_*E zmzVGhoJ~7^_N;o|v+ItR_ndcg^@3OMiukwaO?V~zJE7OIUh<{@`AS%k(M2#A zzWjRNd+wm`4hR%?aM^c*L36Mcxcz0fsn^GC_kydd6IsW>~7aA`gKYyWd{N%~g=g*%$)wpo>mD6e;>Z$!m?pUD{_?_iDIpyN@ zM_)WT=&l}JZLb~~G!>Tm$S1pNm%GQh*N=AE7mu#44=(q6N3V4{-J`9(@{eBRav#<6 zYee(Yn6bFJ9u|}z_=CM@_kC{NqE)trb7@E}N`bAXH-}m>F--B6bvuV0hzY9ukbTEi zY*-y+az7K-BoE>#EHxVKUVG4J9Kc(k`>$BTU77ylGFuOUeQo4ca;or!+|O$$X9fdk zUr-OT>v4l9`aNzu4(RO*LFh8~9zw9i5Ck4uk72@V2OPws9ONM+!}(12 z(dVVwBw^NTcKt>pEH@h6zPHxlbEVO^y4LJOUnUw2uirxUWTO#u`h%d+(2MX0+kcEn zjftC02m-YS$vl2R6-g=QSdLw?OHRqD6y)2cy*q<)FR{W)`O&@WJ1YfH$h=?l9Q+FY z1gK_lZ*FhSEBelINp^CfJ~?SmPI_PV%J^Q!w<&*$rKC>9uXxkQ=X=$k_GbL@a@Cvl zcD!4tS0`=n?M=1jPjKc z$J>h%bKX9`3=Uy0JKlcpKJb8@shk7e{m7Y51V*&qJIoPbZk&nZ9q|@X`aZxY z8GjM^N4+Oe`T$B6F_L56Qz&)6_cT_BIga;?_bhTA@Ljw=NXVnqC%jL->)f>J2a)#> z-aqAi8f6~VWAmp`>pAZ;$ay5nSwPP7-V4ZiG?nwB_gUmThBl`BL;eCGL2N*KFL}rD zr3M_)Yk$IjD4N?z?-af~ZoX)UpZ3lm_X)ubz)$0bz~Xb>=kevR`J&68^}c}IBgo|} zIo>aNFC%Ah@)fV)Ek(V?y}v4$CL;kxob&z+;C0k{4W(kd;S2~F$2;#`Ks!%*uOo-p zt6}(t_a<_WCAoqpKjVD~xlfJc*1flo`?P0+k9{@Ve@>(XGJ0?#Ap?fZHv3>J$0P1# zcoY~|bKthS5D~h5ZxFayX0})sCUyZVaGR#xZnNDR;1_^Bl1=#Jlo!c70{3E*+_c{# zt86Q2uGQ}i(3yL=8C+g;&kkJWuLVAe>KfPEgUg!hf>^of!eq+LYyGx|8s1vZYxV}x zVAt>V)q2c%AxA8_9C9+tW+&*o!P@F-Utx|d$UR5;y(8Bt`MCX6h))zgf@rqK)urG$ zX_pMVx7Lk>A@>kwS$j^J9`pyz4!>RWRcf*rU$@z7Uh)@BeVJ4vXb=5#eR&6vd_c=c z6qp_aKn#QZ@=w=od%>Ytrh6+E_TIu+-hTTqMz4iEjO%#Myt$Z|$b=Z1Y0b3go;?)+ z?#phkKfr?buR)UW4oAblH}`tG({V2XfI-l{)Wh-x{?#?CSQ`*s&$PF!`d#-)u1g9| zN8Dq24ohJ<$qvg2D8flIwQxqnJHWrOD$-x80FDs)bjohh~DxPKrvv{7lwc~cq%fH8ObAz4I z`#fcsnzQr+$|SWyz0y*CbKq+nQcogN9b@tol6nM^RmjN@^~>13t^OJ|hpO#@c-8D( z^1~@zQ!ps3iWKI15k`i22w>q%W6)p2rZc(KsEWw1z7pQ6cA&p-x{)4@X`B74oh-1k z!A;qLgi0+y%du*KUnU|cTs_OTDQE`qSk)AA>Ql`Cc}=h1>#RejX!Wl*;{K`jg;QP8 zJQ^NUkJ^NK0C^+OP*1a5DaMR?#eWAaG?pPg_f<;Z5Ltub$Y<~i2*=Vy$tv1Kr&K8B zOIFo#3S|p9cWuY9Z28ZzN*|QV<$`UOtkT^|e)#Y<^lUMf8_$Gg(8FsQoZ+Nw&jz+* zIGF&4m?^S$SXBOCP4!yD*Z7amCHw?`DIaa*ZaBnU!Lh=Ivr+V{_lm$6d#P4br%*zj zW+FHuZJ*&Qg?V7e+~!f=iC{TWRce{BjY98vG&jI6pa|}iZCgW0r^O!)Yn%Hy9;3rz z7Ke%+5yNj0-Q&L^9{7`4{~8{!Ypu0hP^6Hpcdo@fX|31-iQEdlW4AdsoDFBiL0K!f zk7ZG&aI2{EZreAohxtu?8u=3|C2Du?S!fAxY)nFjC~r8o`R<%&>nu@-N}#V>Q_t3#0ZY z?9VeaQjj@t+sp21AH)hko*&5i*HAahb7v5=%Ugx)KJYSg+8}Z zY=^BGe02}eG9WC&@GqJgNb5nmLFof*G0^#_IcYmJ(ANa9Q%sSq%dj*zjCKPX$O0^3 zWlMLQ5$vw-SiQU+v|G){*no|70Vtlo+#Z~%Rl|qCe0()%V8SqbWVO#mUp0FDp08d2 zZiQu01ZB+Xlgxn0L<9_>W$IJR%0t;zC-_tVCsXT+s0k&y-D@dB$}+??EH5iRBHQ6Y zJi|tW4-rY2ZLY5RJr4vJ;3Q?h9u~@$?8Yxhr!untOu3f;4u=hN_+?e_0horE(o%49NI@9D!B$|X>; z4I4am2Ky2!GrPoUIn^(Ov-c|>5Cf!xpFgA&LU zWlVu4iY~JyArA=mZZ(tOyZ9{D6Q=+^ARM6hQ6&$^Jp0o?CTQi+VEjk<#89^-yllYU zw3)iyf<*)JNp8hn$!}PfNRb@CC#G@*b-z$JX9XMpwI9)MfK|SEJWGe%3=tB^4Z^g7 ze1>1I?-EfpJpdsZTyARt&d?xX-0oTrGPW1dr>Jyli*HYhPLKrs6cmVbVhJ17#h55; za>1@+6l=Xs`wDWQdIXfqAw;)e9g8J!R{QugCkq`j?Ira^tPMnsSMX0b7t1rzmd0ik z>Y=!RLNOz2}fgNINB)VSs0q_sDLoIboXuz;z^%9cMA<&i zR%PqfH?4}nX>v@$%wY3m1WoG}@H@h0Kwa$t3MMCh75 z{G~H(6%63cXhW){2_+J7UE|E~z(b(OgR6*9hWcMzs;>?klG^M+!dqssg!yRxOuj>b zIPOC897xb(*iK3)giqI5x$I0 zhWkJwpr3dk>F-%w*cBU_06NhvN6S!n2FZ?*e8YYdB*@7PiYpWAk66GyEB7TK-PdV% z0NLhNI-|aVv{n+-R%BBe@hLH@^Gupd2sqV6az}$oGdbBJEBIY4-j(?R%z z%2>Qh3IRGHBB*_L_`pz(l%WXv5d>5&?-6pd>UHK)Yq%&VAXwAsmnZyN1z4 zR09-ZuNJ?^@A9{?TMWg9IELpZ;*zA+q6CsRP8g7v61u`YM!JIO$AFy_iYyk4dO>hW z)9IH`LvO`$gnpg-VugkcY= z0I0J(vzCaq2MKA9Fw8fqDL&}H+ze`H#%o3u4K^AaN_jA@lIpUkfW5CE!oyaJ42ZqbUp%U0P!SQq=*g81 zEiTqzm4#h=6GwaDf?5;#8j5sB_e2w%R5ODz!*pRgKNLWK)2?d`U(Fh0FpDwXb1$yT zh^19kGc7F*A0?5csbdW>24W&aueu>X3RZdO>#!LE;fX(5q`AbW6()Tqu1Oxm6Ret? zqe1@TM0DY&rCG{>`LZ)-4fviAg8&9GcBlK?IC9;{@(A*)wM%w!^TGgOL6lsaX_TCkd zMJf?Rh+PvK6Hp@Z!K2vrp(h#hq22?m+Gtb zr8dyfw79Kio50IGbRwnMGC#&-O%t|jcdUXcb~b7AZEYI+Nv5giWTr?hn19VJ=cvC! z#TJQo_&}@Kdkm=AS62i-AB*-SBqa9*_h)a2LacTJ6KZ9}4VIFH$*962s&<=$lwy3E zrD#-BUqVuw)M}l=bA3AKuc=nFZG-wWyNzYCK-+qZ`lni~QArmW!G1{JsNW>|)6P|z za%_lN5>G+?Cc=i;y~OhMVYvuZ#JROCFxc#d zh;_$W0ud9{U?;<(sCOmXK1c)rSh7Uh6;pwWcJhAF9cEoH< zUm4K|t(j@OU&c#3bi~fA;T6M?LC?nxiI%9zYcMX3MZJSs-$1kP6FP|EdGg~7FWqW5 z)w~$=JBVfiu5E%jsqabr#3BbG1ZsOTqPHLnR`W@89qlIB^Fv?tkMx(9;oOS1qj->8 zum{7-9OW4fX*Nt`ln2#_M`NmQOAq4G+$Sj?Z=7u!;m%6&0@H824KcLqm-}!Qn4zV* z9ZN3Y#MNX%-D}R;%aW*0j4KP^%|;;EG`cg)GYAW9P|J5>oE!q~qg!nk5o43_{XJqS z5ny&|cnXk~kI@|N{C?T~nw0VDTBZcf{# zWXD_sMT#dtBc-StcOuFogw|!QBoE>#oM{IUR%l81HU=$$a`Y1npV)4NM`~^wuG!Fk z5gkQhe#(?4azr&HM?g`40%l)vg1wR(o6IcrB$WtY2Q=N4-1D3$jb~CG<{kVi#*IWN zlX83UrGZ+vprVPSA?2W4yaqVFO4+s<%lq-(4cz6mPUkQ!&R~+%CJ^_~?PL3hK_unr zGrL*@DykunYP)_D!7VygMgmB5087*qPy_}M^ea;H@*2Fru|3|jcDZ?tGP4nBRt2w8 z^kE5uiF#Oeuakz*o4yWv%r(wOuVFMD#Q~(igETi?mUl)=r}ZKtDk4$nnbqbz6b(T`A0l_-{$V5X{EZOYArykuNJD170UUl zRW20m;U~7+uUj{p*}55<${F;-h{MZ>searpD&&<5=AKG{|@YNv)02s#!;z5yHDY#^}UmDKe0HB~UFR zNYpC>b;-PqV)A~*_Xr>w?Pd7l5!WtWT&~$45tV1rV6j5;lmi_ePHG$C`nJBDZCc!( zt=FvK|456ESx{*$<;aVkPr4?7eunDNg42SEuMnXzxEE@OYA1&&-a4TP)L|V1an^eq z%HDCe1IlTd*4=JeNrO@dnE^xT!?KhNEAuf~Rfux3o@E&-S8c2mAI`_$SJxYerjzK# zfUZ1|>dF(_i}m9whxuOrdd84K5>mjAl3ipPLp)s^n!OXDQ?|Ep=ELFR=+^4NEvel7E`hzCJ z){-R|KAns;Jygj5Z3J{eVxYJCqTlHwgil0_&z*an0bb&&lamc{M&q*pV_4L~3CnLZ zJ8QmnTtwF5r|@0<1ttXe=qB9V1;F$!#7@WmB7prLuK64)lu9tq!yPqiF~ZBa>-=c= z^mcq?i}o^1B#(BDVV(0mrn1aoz4i*Kq5mqTsU`Sh75VVU; zV=~>3Gt>C!OdbMXP((vz*a)iDPc*x@jSPNscKmS=!Q)88L|`Q*f(>hLbXG;2>iEvt z(jlRR^SoZ-$52(f&m+Hz%!*SUSwwLTOvclosblsY!ug1+J+nAy$Cw5JW8oM({A9?s z9x{)0#QKDc)-|JjkWb4U}^}!8BmFBf+P{$WIVZP({)F&_i z^_Q8XZF;}J4B3Ex1yA)^nJW?o-!WRfUcyO09fh1%h&r~~V`Leb6d|)23AH*9 z;7y1)oM@A5#%KcKQHu3GsHh~zeiBQ{sjgFFz20BzK-Gl0h|`fh3`5Q$>O%|(j}n`( zEPO_M!a@jzJCA<@p4MVy+S^6?ofTXlzX*5!;w#DiVaS{5&Nd zj_So1xV+-BfomB5eMEch@RjZ8L$sg(;23mSe4dV_PVgcg;yxe-{US%j3E){Qu75oq zrXHIN>%g`xAK^%8G}XacHlo3|57pSVHC0DT8D!%&H(UJ-Uf^f}Sg~R6V>}j48oJo7 z7CqAHKD@)NkBANAAxIE!Fm1pX0%6I8!$1E?XlDqZ#DdhE=Fftwr^Q^DqoczoG;L(C zDn7pvoeO|3&qz1H^+@Ay_m1}0(n9YR`W5`Dlzh`J8XYmrhJ4rqw5qd3z@8X5DVqIs zqmA1h8MSQ4lopB8EQm4`6Hg2W*;to)jA}L`j9a_TP!+gz+m!pHtz>lZ=7hhNhbofk z$xhPLv{ncwz*Y!$brVah4Fe`jE7qxq8}&JQ4YKDm#sw0F`-x{P%T{BVE~De*-O3v9h8`1vu~~-9!s!EMT$!H^|JZl3NJ}+K^xRmW z`*{=h;|k1_8r*I5Q%Zn0bV5p z_}Dk{Jsu;%uIAoHzz_C4?I|mu2(+iH6ycb~;RP5sbq-24iU2;HsLt>Qn}82h#lZgL zWu6%la0?>CV21@jWS0ScADqLuC#ZmetHXAwLv!Ugzwoovxipl4L%$tdP8vycEqypb zD=-MDFx~u=i`niaYY7^EipzJJNe>B*Fsr}87lw_ezDaP?>&z3eHF3B{VY~(j>XSJA zuJ_hh*=0i37r_z4iG66WR4NuItd#CLc?TS=A|a6_+xe5>L)-DTs2zQBTbvHonVNN> z_Kc!Rp`KuJn8^_)+}ZW~%dfp2=0PqMUG|Cyr5<52&4dW5gWQ~pJgfCQ-)Umeuy~74 z0*2T4bcu;rB4_wSn~#KVNf?o~12NV>4VYYGLIzr^HQO#UX5zs1DOCS=~~C=x;@ z_-{yfgxsD3srvxx>qmCp{&Cr{KeP+w5A4rEo&6zV&^{_67p5iT+8-2)_K)ld`-gU^ z{HOLF=f?>AM}%5bCSU%s{kZcX-^-{|M4h5j{?MLrK8nlPcKM^Zy>qj3`>WNtnYrn? z-Fp@m%6kr0C+70@^t@Hg&*kTwxhMAREl)zoxocbHy9+zYMe|&obL`2aUyiuQe&-?K z-i0yu?!vBx-TTTi%N8b>`Qt*rgkSJ=F#7aG7C7Rb_ZD%a-SH7nfy2y2A7Vd_!B2QN z6z`Rg&-Y1xa=GG_y(t_xE_*oo{w@yX;Rt*c`8Ezj&&na}X@7d9ieub6%sWfWpv0WF zQ%cPGvnVkydAs5~(`!`@BlDoUQ{N%2OYf-9qwPHAJ?GC!5Btn}JO(-HjuJbi-URB% zQFr9bo16!D)EzmyOpY9PN6v!e$lOs=j=Upxx8w?BkSoXDk-I0#1zeCTN8gdV*Wc&W z{7LWe2x3n}kgWO>II4ZvJ0d;r7jx8;>hoBO=F4D=@u6vRN?IEcb)-ssrdVrABc>@D zkvG!`ebfQ*WvCO#q3bvfmjQ6`kuvR)(aj23rQ1R~W>%XQ+nx3R(Yy#G8@Sh1bCp*2 zUc$SV*!=C0D-$2@dZ47mJK@0?@o^b6ZT0_K`7|XS1iveluj22SuUEo`y zsc11t+B2@BEC>=V6KO$NmVm<1xysZ@w{xOyp41aPK_lz7$KDLLq4*GMUI_y+hq5*w zf8&WU9A-{I8(apWcEX)? zaME!dWQS;G?O98qgdUe0W1Svh0(hR^?zh@7bwX#yNcXsOCdIhrjO!U}GtG@@&e?Q$mfY|p zegXLl6nAG}-L!eq#bX4Cql3`~h&a7z(P4~(*bYus^vyqLV20 zn7Eu^qU~4WF9$H}$8g__*Je={-Os^NjH?p{@b;#bPU5w|2w)!85(^F`CE`%_bZ~7J zXCM*OiCc=`gck0i0#wx}pQd<(GCspRAD?kdM<9oL;#{q)@Dc)jv;-H$7<5zzIGU<9 zbH%1+ToXeB0Ix9$CqOisgMm`F*|!KUG(w|wEgy<{lR3C@mIYmthy=iTf0Y8*r|?iA zpXGmdosUWmj(8&=O#fGQ^szpEg^2cnGoef3`NK+aIOZk{bE#6>X=n|(8zd2I}WEvKix%kJ2}`o~CO)7L*? zhHDbpI~>Fxa}fUs51SgKKrd)cWlB(-=T7wjII+9a$_x|-DV1Y`^VIh@tl)q|{z_1P z6#2^|6*}@JcdJN}7`0}pXYoYUHFulqG07JAgW~#MkBfs??};c<&&WF95k!5B@{~E0 ze+fn>ACB2nezI8-K|~)%ifUejU%LyxDs3n!+iODvgPKPN)VD_SudzEpb6XFR0~(9#{P2H? zW?#p{*gYEy6`ODnwKNV^@SusSK2oU>{5O%S{vi|BBrnovS zAub!1L}(8ip2yk+%qmYp{)90QF-?$0?T;!GuusYLIgmVIRT@4p2J_Oww)$K_W0&+fry=qCC;WC;B`gf$p*Mk z5fk}RUyhQw%@i!xtnz3Xz5_ytZlwu8HtnIHrqAaI7U{mA_ehuytO%wlRD=6CR!>v- zvh?bO)6cml^+-|}$sW8uICzX|(VcDX%l+4mr|O?;Dx3?Ve2`q!b_CLNdR^T^G?hYW zKJG4enlN4<%m)T;v15#c3vr72_+Ptc22bqchs}nCBZNn!Q3=M3=+2R%p59`tkX+j% z?y3ai*^|gIfFb^uj7ET+$Qnne8>I1CWR1+-K2a2!F(gqKfrEfeM!POEKiz@cXqVlT z7zT_7YY0{(+F%2Zh*XgyG6VhiEJG}?r!kSXB@SX_M%>*F;e@@J-4~+*z(2!oc$S-x zQta$JqEE)$;I+u6(f|s?1P|^^jb*8J-&p%QO$;eB``_kkmB}wNd4!3GYJZ1M)EaQ| z!XE(n)Gg*v=+*+=Nj?cdB8%6WneazpT4Wm{5r!3BC){oUfiL$x#hImM>+)6T4r$rX zVQ&YtrHd_m@77lD@()Xg>=~RGghFTYJQ9L6W8DIRE0i#9Hx zc0}wjuBd?M`PfMDepqy0m7QoB3f3;_;iE@%jmvw`w>BP`0kCBgbj*%mJGd^_FGz@^ zX^W%LW)?k~IE+Mx`eGYz8lJJ+%F%!XR`GD7X{4y`g-y3_#*jj&hGy-}&Lspj4=#5B zIb69J!$&;sn2Jh>9x04_@KSU2pdnC^At@T_qD!g5f2YaINdX!;6(Oe~BKW>wCjZv4 z!%rQ4`tUP{pM~9$kU*@bl+^uW_PWH|jMXGW%^WpUWb6s$NA2rW|(y*(Nk%{*H+D2v}*25pgH#*L-zMD(Y_P$)XGFx=)P zoQX@V(vmqy5|Q-I`r)xLN)i&QZ$?Gq1FdS15Am#tmrC@Scv}MN{!C{so>>T!0K9oB*D1 z!aiq0z6S@SH9RnW9#Pp%0|x`qW;ybyXyA#}2RSc@h31YuJaB%{2O5y(jv4Wi`d4T` z{c9$!No1TjPrzHCIMgc`>ic@AIAjJ1PuqsGAWw=*z+pxNCa)>rcM`wgn@GkDc`tAM z$nye6HOCVu2#Z7*rDyF)!YAc&HRQuuixA3!S~Smga9F}AQ4ec|xC;iP%T8ndJ_}J)-vA(V1RDmoL-bpalyCzwx4CRJ(4A(xlh~`0SUg-yq!?g{ zAfeyIKiV)PMj-mh{ZHxql@#11h9?l zA}Ni4aa>)fnZ{M3*yHG{T5xd8S^^$mTH^VQS)AY~YU{dP{IF7jR~w#eCqMkm_|^Sb z{lPfK-qc7a1oy2H0qpnr^ao6eO#VF+H=Br%^gGCVg@fON2MS50xw*m|JOy+bP^`h_ zG=845d9zpRBF`63SZ`jiMdJHp*BJAaJZHkCrQ8_aiNI9EeS%EdCQ)V8XR@>mzUeE zHY0rWO0x~))4|~eK?QfTCylyW5)<{$0kRCGo)yZ>W};&V>i1a0HHoYi*Z134E%gu{ zhVB+XFf>`oJ`mY`(@@ZgCKC7IAxn_u90Kw{j4f)&>R+M=3GN@_-*FU75!>#d#fZw{ zB*6n)jqP4^R{a}x@^6{^$E~^}UVj(esegxu;lZuCO6X+hP(aKOO3t6ZllaMfZ6ky- zVMwtfM#d9I#tCHq6Y7pEQ8*Vl>rzYko9IVFhG#4u-)gvg1 zm5ZD5yqZDHMOQt^C2;!4q4DuxDzc#8;Viy^Bo4eZ)HAdPo9N+zXqaTk5gANP^|v|I zGTN%zwD34KZk~De9upqlQmyi_Dcmhi;E$LHt@|&0a808u68~18=m(3d z&B0}f><){Ab)HC8sF~9-XcPxDWGEb{5);Oq)ff2xRI|wcr9v&Hxueg44DY;y7 zQr|aPOrm3OtvI}+(<~~={tTZekkD@{KBLE56C`udH?_b literal 0 HcmV?d00001 diff --git a/vllm/core/block/__pycache__/utils.cpython-310.pyc b/vllm/core/block/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf15d4d89cb0cb686f3460d8a840fc25e3ea8f30 GIT binary patch literal 1644 zcma)+&2Aev5P+Afl`Jc^V5DiAO98frBJ02s>`MS62kjq(ehBKURnD6g*JZ#6m{}C@+p7*B} z-fsa4Utrfya1h?a6GZq4N&SgW?7WdQ(qIy#%}EpGKs4Wala^>nU+jqXED*b*^EQ~^ zeC&mH-rV~#$)lGvX=R%Ve~kXHtHWHdFg zq`W8~FF>I=T3fe9OD*Xv2P&kIkg5z1a5fgPL?xhTnkr3Y#;1uCy^dQvgL#?a;rg_e z59!qAiKTNm-3;=WqV0{FIH4-jhGW{Csp?*>x(bff!BzKDe#N$}ym{fQ-6F9zhL@>n zCY5t~n%UHtv>)Xl`%b@KGhHmJ&lsw#Vn(BE z^G2BUX9rKw_NdNYi*!hfw0+pK-tB7nip^~qR4tIEglrV~xUg<^Zo32e1Si+z7w=aO zJ~ZUoyCH&<1iHpI4q`+)fR9mXTBHTyZhPqSr)#u_86p!&#EKhQfSjo-8u2K$bbh|( z^k_S$F6$g5jY_bzBl%^f%c97^a(<`_8L63yc#eBVNN?}{ly1C8wBTCT zrF)Bti+&WgCU=!)tH9VQusW3=YzYPIt2*h{MaWh+4fcf!eb~hh?AyVTQ4e=;Ry~Yj ziA9;^YYmnoRSV8Ac{OJiGjsya{{$6u7l(FtkF-h4Z6}GD##zY=5HrcoM z_Ap}i@PcJ((sBPB$Id6`tqaDfzvq^7(D~MWj9Dxb`NTf8`YsNEw?__0JMcg9iT}2RR-k= literal 0 HcmV?d00001 diff --git a/vllm/core/block/block_table.py b/vllm/core/block/block_table.py new file mode 100644 index 00000000..d10cb29e --- /dev/null +++ b/vllm/core/block/block_table.py @@ -0,0 +1,374 @@ +import math +from typing import List, Optional + +from vllm.core.block.common import BlockList +from vllm.core.block.interfaces import Block, DeviceAwareBlockAllocator +from vllm.utils import Device, cdiv, chunk_list + + +class BlockTable: + """A class to manage blocks for a specific sequence. + + The BlockTable maps a sequence of tokens to a list of blocks, where each + block represents a contiguous memory allocation for a portion of the + sequence. The blocks are managed by a DeviceAwareBlockAllocator, which is + responsible for allocating and freeing memory for the blocks. + + Args: + block_size (int): The maximum number of tokens that can be stored in a + single block. + block_allocator (DeviceAwareBlockAllocator): The block allocator used to + manage memory for the blocks. + _blocks (Optional[List[Block]], optional): An optional list of existing + blocks to initialize the BlockTable with. If not provided, an empty + BlockTable is created. + max_block_sliding_window (Optional[int], optional): The number of + blocks to keep around for each sequance. If None, all blocks + are kept (eg., when sliding window is not used). + It should at least fit the sliding window size of the model. + + Attributes: + _block_size (int): The maximum number of tokens that can be stored in a + single block. + _allocator (DeviceAwareBlockAllocator): The block allocator used to + manage memory for the blocks. + _blocks (Optional[List[Block]]): The list of blocks managed by this + BlockTable. + _num_full_slots (int): The number of tokens currently stored in the + blocks. + """ + + def __init__( + self, + block_size: int, + block_allocator: DeviceAwareBlockAllocator, + _blocks: Optional[List[Block]] = None, + max_block_sliding_window: Optional[int] = None, + ): + self._block_size = block_size + self._allocator = block_allocator + if _blocks is None: + _blocks = [] + self._blocks: BlockList = BlockList(_blocks) + + self._max_block_sliding_window = max_block_sliding_window + self._num_full_slots = self._get_num_token_ids() + + @staticmethod + def get_num_required_blocks(token_ids: List[int], + block_size: int, + num_lookahead_slots: int = 0) -> int: + """Calculates the minimum number of blocks required to store a given + sequence of token IDs along with any look-ahead slots that may be + required (like in multi-step + chunked-prefill). + + This assumes worst-case scenario, where every block requires a new + allocation (e.g. ignoring prefix caching). + + Args: + token_ids (List[int]): The sequence of token IDs to be stored. + block_size (int): The maximum number of tokens that can be stored in + a single block. + num_lookahead_slots (int): look-ahead slots that the sequence may + require. + + Returns: + int: The minimum number of blocks required to store the given + sequence of token IDs along with any required look-ahead slots. + """ + return cdiv(len(token_ids) + num_lookahead_slots, block_size) + + def allocate(self, + token_ids: List[int], + device: Device = Device.GPU) -> None: + """Allocates memory blocks for storing the given sequence of token IDs. + + This method allocates the required number of blocks to store the given + sequence of token IDs. + + Args: + token_ids (List[int]): The sequence of token IDs to be stored. + device (Device, optional): The device on which the blocks should be + allocated. Defaults to Device.GPU. + """ + assert not self._is_allocated + assert token_ids + blocks = self._allocate_blocks_for_token_ids(prev_block=None, + token_ids=token_ids, + device=device) + self.update(blocks) + self._num_full_slots = len(token_ids) + + def update(self, blocks: List[Block]) -> None: + """Resets the table to the newly provided blocks + (with their corresponding block ids) + """ + self._blocks.update(blocks) + + def append_token_ids(self, + token_ids: List[int], + num_lookahead_slots: int = 0, + num_computed_slots: Optional[int] = None) -> None: + """Appends a sequence of token IDs to the existing blocks in the + BlockTable. + + This method appends the given sequence of token IDs to the existing + blocks in the BlockTable. If there is not enough space in the existing + blocks, new blocks are allocated using the `ensure_num_empty_slots` + method to accommodate the additional tokens. + + The token IDs are divided into chunks of size `block_size` (except for + the first chunk, which may be smaller), and each chunk is appended to a + separate block. + + Args: + token_ids (List[int]): The sequence of token IDs to be appended. + num_computed_slots (Optional[int]): The number of KV cache slots + that are already filled (computed). + When sliding window is enabled, this is used to compute how many + blocks to drop at the front of the sequence. + Without sliding window, None can be passed. + Without chunked prefill, it should be the same as + _num_full_slots. + """ + assert self._is_allocated, "no blocks have been allocated" + assert len(self._blocks) > 0 + + # Drop blocks that are no longer needed due to sliding window + if self._max_block_sliding_window is not None: + null_block = self._allocator.allocate_or_get_null_block() + assert num_computed_slots is not None + end_block_idx = (num_computed_slots // + self._block_size) - self._max_block_sliding_window + for idx in range(0, end_block_idx): + b = self._blocks[idx] + if b is not null_block: + self._allocator.free(b) + self._blocks[idx] = null_block + + # Ensure there are enough empty slots for the new tokens plus + # lookahead slots + self.ensure_num_empty_slots(num_empty_slots=len(token_ids) + + num_lookahead_slots) + + # Update the blocks with the new tokens + first_block_idx = self._num_full_slots // self._block_size + token_blocks = self._chunk_token_blocks_for_append(token_ids) + + for i, token_block in enumerate(token_blocks): + self._blocks.append_token_ids(first_block_idx + i, token_block) + + self._num_full_slots += len(token_ids) + + def ensure_num_empty_slots(self, num_empty_slots: int) -> None: + """Ensures that the BlockTable has at least the specified number of + empty slots available. + + This method checks if the BlockTable has enough empty slots (i.e., + available space) to accommodate the requested number of tokens. If not, + it allocates additional blocks on the GPU to ensure that the required + number of empty slots is available. + + Args: + num_empty_slots (int): The minimum number of empty slots required. + """ + # Currently the block table only supports + # appending tokens to GPU blocks. + device = Device.GPU + assert self._is_allocated + + if self._num_empty_slots >= num_empty_slots: + return + + slots_to_allocate = num_empty_slots - self._num_empty_slots + blocks_to_allocate = cdiv(slots_to_allocate, self._block_size) + + for _ in range(blocks_to_allocate): + assert len(self._blocks) > 0 + self._blocks.append( + self._allocator.allocate_mutable_block( + prev_block=self._blocks[-1], device=device)) + + def fork(self) -> "BlockTable": + """Creates a new BlockTable instance with a copy of the blocks from the + current instance. + + This method creates a new BlockTable instance with the same block size, + block allocator, and a copy of the blocks from the current instance. The + new BlockTable has its own independent set of blocks, but shares the + same underlying memory allocation with the original BlockTable. + + Returns: + BlockTable: A new BlockTable instance with a copy of the blocks from + the current instance. + """ + assert self._is_allocated + assert len(self._blocks) > 0 + forked_blocks = self._allocator.fork(self._blocks[-1]) + return BlockTable( + block_size=self._block_size, + block_allocator=self._allocator, + _blocks=forked_blocks, + max_block_sliding_window=self._max_block_sliding_window, + ) + + def free(self) -> None: + """Frees the memory occupied by the blocks in the BlockTable. + + This method iterates over all the blocks in the `_blocks` list and calls + the `free` method of the `_allocator` object to release the memory + occupied by each block. After freeing all the blocks, the `_blocks` list + is set to `None`. + """ + for block in self.blocks: + self._allocator.free(block) + self._blocks.reset() + + @property + def physical_block_ids(self) -> List[int]: + """Returns a list of physical block indices for the blocks in the + BlockTable. + + This property returns a list of integers, where each integer represents + the physical block index of a corresponding block in the `_blocks` list. + The physical block index is a unique identifier for the memory location + occupied by the block. + + Returns: + List[int]: A list of physical block indices for the blocks in the + BlockTable. + """ + return self._blocks.ids() + + def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]: + """Get the number of "unseen" tokens in the sequence. + + Unseen tokens are tokens in the sequence corresponding to this block + table, but are not yet appended to this block table. + + Args: + sequence_token_ids (List[int]): The list of token ids in the + sequence. + + Returns: + List[int]: The postfix of sequence_token_ids that has not yet been + appended to the block table. + """ + + # Since the block table is append-only, the unseen token ids are the + # ones after the appended ones. + return sequence_token_ids[self.num_full_slots:] + + def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block], + token_ids: List[int], + device: Device) -> List[Block]: + blocks: List[Block] = [] + + block_token_ids = [] + tail_token_ids = [] + for cur_token_ids in chunk_list(token_ids, self._block_size): + if len(cur_token_ids) == self._block_size: + block_token_ids.append(cur_token_ids) + else: + tail_token_ids.append(cur_token_ids) + + if block_token_ids: + blocks.extend( + self._allocator.allocate_immutable_blocks( + prev_block, block_token_ids=block_token_ids, + device=device)) + prev_block = blocks[-1] + + if tail_token_ids: + assert len(tail_token_ids) == 1 + cur_token_ids = tail_token_ids[0] + + block = self._allocator.allocate_mutable_block( + prev_block=prev_block, device=device) + block.append_token_ids(cur_token_ids) + + blocks.append(block) + + return blocks + + def _get_all_token_ids(self) -> List[int]: + # NOTE: This function is O(seq_len); use sparingly. + token_ids: List[int] = [] + + if not self._is_allocated: + return token_ids + + for block in self.blocks: + token_ids.extend(block.token_ids) + + return token_ids + + def _get_num_token_ids(self) -> int: + res = 0 + for block in self.blocks: + res += len(block.token_ids) + + return res + + @property + def _is_allocated(self) -> bool: + return len(self._blocks) > 0 + + @property + def blocks(self) -> List[Block]: + return self._blocks.list() + + @property + def _num_empty_slots(self) -> int: + assert self._is_allocated + return len(self._blocks) * self._block_size - self._num_full_slots + + @property + def num_full_slots(self) -> int: + """Returns the total number of tokens currently stored in the + BlockTable. + + Returns: + int: The total number of tokens currently stored in the BlockTable. + """ + return self._num_full_slots + + def get_num_blocks_touched_by_append_slots( + self, token_ids: List[int], num_lookahead_slots: int) -> int: + """Determine how many blocks will be "touched" by appending the token + ids. + + This is required for the scheduler to determine whether a sequence can + continue generation, or if it must be preempted. + """ + # Math below is equivalent to: + # all_token_ids = token_ids + [-1] * num_lookahead_slots + # token_blocks = self._chunk_token_blocks_for_append(all_token_ids) + # return len(token_blocks) + + num_token_ids = len(token_ids) + num_lookahead_slots + first_chunk_size = self._block_size - (self._num_full_slots % + self._block_size) + num_token_blocks = (1 + math.ceil( + (num_token_ids - first_chunk_size) / self._block_size)) + return num_token_blocks + + def _chunk_token_blocks_for_append( + self, token_ids: List[int]) -> List[List[int]]: + """Split the token ids into block-sized chunks so they can be easily + appended to blocks. The first such "token block" may have less token ids + than the block size, since the last allocated block may be partially + full. + + If no token ids are provided, then no chunks are returned. + """ + + if not token_ids: + return [] + + first_chunk_size = self._block_size - (self._num_full_slots % + self._block_size) + token_blocks = [token_ids[:first_chunk_size]] + token_blocks.extend( + chunk_list(token_ids[first_chunk_size:], self._block_size)) + return token_blocks diff --git a/vllm/core/block/common.py b/vllm/core/block/common.py new file mode 100644 index 00000000..eb190adf --- /dev/null +++ b/vllm/core/block/common.py @@ -0,0 +1,360 @@ +from collections import deque +from dataclasses import dataclass +from typing import Deque, Dict, Iterable, List, Optional, Protocol, Tuple + +from vllm.core.block.interfaces import Block, BlockAllocator + +BlockId = int +RefCount = int + + +class RefCounterProtocol(Protocol): + + def incr(self, block_id: BlockId) -> RefCount: + raise NotImplementedError + + def decr(self, block_id: BlockId) -> RefCount: + raise NotImplementedError + + def get(self, block_id: BlockId) -> RefCount: + raise NotImplementedError + + +class RefCounter(RefCounterProtocol): + """A class for managing reference counts for a set of block indices. + + The RefCounter class maintains a dictionary that maps block indices to their + corresponding reference counts. It provides methods to increment, decrement, + and retrieve the reference count for a given block index. + + Args: + all_block_indices (Iterable[BlockId]): An iterable of block indices + to initialize the reference counter with. + """ + + def __init__(self, all_block_indices: Iterable[BlockId]): + deduped = set(all_block_indices) + self._refcounts: Dict[BlockId, + RefCount] = {index: 0 + for index in deduped} + + def incr(self, block_id: BlockId) -> RefCount: + assert block_id in self._refcounts + pre_incr_refcount = self._refcounts[block_id] + + assert pre_incr_refcount >= 0 + + post_incr_refcount = pre_incr_refcount + 1 + self._refcounts[block_id] = post_incr_refcount + return post_incr_refcount + + def decr(self, block_id: BlockId) -> RefCount: + assert block_id in self._refcounts + refcount = self._refcounts[block_id] + + assert refcount > 0 + refcount -= 1 + + self._refcounts[block_id] = refcount + + return refcount + + def get(self, block_id: BlockId) -> RefCount: + assert block_id in self._refcounts + return self._refcounts[block_id] + + def as_readonly(self) -> "ReadOnlyRefCounter": + return ReadOnlyRefCounter(self) + + +class ReadOnlyRefCounter(RefCounterProtocol): + """A read-only view of the RefCounter class. + + The ReadOnlyRefCounter class provides a read-only interface to access the + reference counts maintained by a RefCounter instance. It does not allow + modifications to the reference counts. + + Args: + refcounter (RefCounter): The RefCounter instance to create a read-only + view for. + """ + + def __init__(self, refcounter: RefCounter): + self._refcounter = refcounter + + def incr(self, block_id: BlockId) -> RefCount: + raise ValueError("Incr not allowed") + + def decr(self, block_id: BlockId) -> RefCount: + raise ValueError("Decr not allowed") + + def get(self, block_id: BlockId) -> RefCount: + return self._refcounter.get(block_id) + + +class CopyOnWriteTracker: + """A class for tracking and managing copy-on-write operations for blocks. + + The CopyOnWriteTracker class maintains a mapping of source block indices to + their corresponding copy-on-write destination block indices. It works in + conjunction with a RefCounter. + + Args: + refcounter (RefCounter): The reference counter used to track block + reference counts. + """ + + def __init__(self, refcounter: RefCounterProtocol): + self._copy_on_writes: List[Tuple[BlockId, BlockId]] = [] + self._refcounter = refcounter + + def is_appendable(self, block: Block) -> bool: + """Checks if the block is shared or not. If shared, then it cannot + be appended and needs to be duplicated via copy-on-write + """ + block_id = block.block_id + if block_id is None: + return True + + refcount = self._refcounter.get(block_id) + return refcount <= 1 + + def record_cow(self, src_block_id: Optional[BlockId], + trg_block_id: Optional[BlockId]) -> None: + """Records a copy-on-write operation from source to target block id + Args: + src_block_id (BlockId): The source block id from which to copy + the data + trg_block_id (BlockId): The target block id to which the data + is copied + """ + assert src_block_id is not None + assert trg_block_id is not None + self._copy_on_writes.append((src_block_id, trg_block_id)) + + def clear_cows(self) -> List[Tuple[BlockId, BlockId]]: + """Clears the copy-on-write tracking information and returns the current + state. + + This method returns a list mapping source block indices to + destination block indices for the current copy-on-write operations. + It then clears the internal tracking information. + + Returns: + List[Tuple[BlockId, BlockId]]: A list mapping source + block indices to destination block indices for the + current copy-on-write operations. + """ + cows = self._copy_on_writes + self._copy_on_writes = [] + return cows + + +class BlockPool: + """Used to pre-allocate block objects, in order to avoid excessive python + object allocations/deallocations. + The pool starts from "pool_size" objects and will increase to more objects + if necessary + + Note that multiple block objects may point to the same physical block id, + which is why this pool is needed, so that it will be easier to support + prefix caching and more complicated sharing of physical blocks. + """ + + def __init__(self, block_size: int, create_block: Block.Factory, + allocator: BlockAllocator, pool_size: int): + self._block_size = block_size + self._create_block = create_block + self._allocator = allocator + self._pool_size = pool_size + assert self._pool_size >= 0 + + self._free_ids: Deque[int] = deque(range(self._pool_size)) + self._pool = [] + for i in range(self._pool_size): + self._pool.append( + self._create_block(prev_block=None, + token_ids=[], + block_size=self._block_size, + allocator=self._allocator, + block_id=None)) + + def increase_pool(self): + """Doubles the internal pool size + """ + cur_pool_size = self._pool_size + new_pool_size = cur_pool_size * 2 + self._pool_size = new_pool_size + + self._free_ids += deque(range(cur_pool_size, new_pool_size)) + + for i in range(cur_pool_size, new_pool_size): + self._pool.append( + self._create_block(prev_block=None, + token_ids=[], + block_size=self._block_size, + allocator=self._allocator, + block_id=None)) + + def init_block(self, prev_block: Optional[Block], token_ids: List[int], + block_size: int, physical_block_id: Optional[int]) -> Block: + if len(self._free_ids) == 0: + self.increase_pool() + assert len(self._free_ids) > 0 + + pool_id = self._free_ids.popleft() + + block = self._pool[pool_id] + block.__init__( # type: ignore[misc] + prev_block=prev_block, + token_ids=token_ids, + block_size=block_size, + allocator=block._allocator, # type: ignore[attr-defined] + block_id=physical_block_id) + block.pool_id = pool_id # type: ignore[attr-defined] + return block + + def free_block(self, block: Block) -> None: + self._free_ids.appendleft(block.pool_id) # type: ignore[attr-defined] + + +class BlockList: + """This class is an optimization to allow fast-access to physical + block ids. It maintains a block id list that is updated with the + block list and this avoids the need to reconstruct the block id + list on every iteration of the block manager + """ + + def __init__(self, blocks: List[Block]): + self._blocks: List[Block] = [] + self._block_ids: List[int] = [] + + self.update(blocks) + + def _add_block_id(self, block_id: Optional[BlockId]) -> None: + assert block_id is not None + self._block_ids.append(block_id) + + def _update_block_id(self, block_index: int, + new_block_id: Optional[BlockId]) -> None: + assert new_block_id is not None + self._block_ids[block_index] = new_block_id + + def update(self, blocks: List[Block]): + self._blocks = blocks + + # Cache block ids for fast query + self._block_ids = [] + for block in self._blocks: + self._add_block_id(block.block_id) + + def append_token_ids(self, block_index: int, token_ids: List[int]) -> None: + block = self._blocks[block_index] + prev_block_id = block.block_id + + block.append_token_ids(token_ids) + + # CoW or promotion may update the internal block_id + if prev_block_id != block.block_id: + self._update_block_id(block_index, block.block_id) + + def append(self, new_block: Block): + self._blocks.append(new_block) + self._add_block_id(new_block.block_id) + + def __len__(self) -> int: + return len(self._blocks) + + def __getitem__(self, block_index: int) -> Block: + return self._blocks[block_index] + + def __setitem__(self, block_index: int, new_block: Block) -> None: + self._blocks[block_index] = new_block + self._update_block_id(block_index, new_block.block_id) + + def reset(self): + self._blocks = [] + self._block_ids = [] + + def list(self) -> List[Block]: + return self._blocks + + def ids(self) -> List[int]: + return self._block_ids + + +@dataclass +class CacheMetricData: + """A utility dataclass to maintain cache metric. + To avoid overflow, we maintain the hit rate in block granularity, so that + we can maintain a single hit rate for n_completed_block x block_size, + and calculate the real time hit rate by the following: + BS = The number of queries per block. + nB = The number of completed blocks. + HR = hit rate of (nB x BS) queries. + Q = current number of queries (< BS). + H = current number of hits (< BS). + hit rate = ((HR x nB) + (H / Q) x (Q / BS)) / (nB + Q / BS) + """ + num_completed_blocks: int = 0 + completed_block_cache_hit_rate: float = 0.0 + num_incompleted_block_queries: int = 0 + num_incompleted_block_hit: int = 0 + block_size: int = 1000 + + def query(self, hit: bool): + self.num_incompleted_block_queries += 1 + self.num_incompleted_block_hit += 1 if hit else 0 + + # When a block is completed, update the cache hit rate + # and reset the incomplete numbers. + if self.num_incompleted_block_queries == self.block_size: + hit_rate = (self.num_incompleted_block_hit / + self.num_incompleted_block_queries) + self.completed_block_cache_hit_rate = ( + self.completed_block_cache_hit_rate * self.num_completed_blocks + + hit_rate) / (self.num_completed_blocks + 1) + self.num_incompleted_block_queries = 0 + self.num_incompleted_block_hit = 0 + self.num_completed_blocks += 1 + + def get_hit_rate(self): + incomplete_ratio = self.num_incompleted_block_queries / self.block_size + total_blocks = self.num_completed_blocks + incomplete_ratio + if total_blocks == 0: + return 0.0 + + completed_block_hit, incompleted_block_hit = 0.0, 0.0 + if self.num_completed_blocks > 0: + completed_block_hit = (self.completed_block_cache_hit_rate * + self.num_completed_blocks) + if self.num_incompleted_block_queries > 0: + incompleted_hit_rate = (self.num_incompleted_block_hit / + self.num_incompleted_block_queries) + incompleted_block_hit = (incompleted_hit_rate * incomplete_ratio) + return (completed_block_hit + incompleted_block_hit) / total_blocks + + +def get_all_blocks_recursively(last_block: Block) -> List[Block]: + """Retrieves all the blocks in a sequence starting from the last block. + + This function recursively traverses the sequence of blocks in reverse order, + starting from the given last block, and returns a list of all the blocks in + the sequence. + + Args: + last_block (Block): The last block in the sequence. + + Returns: + List[Block]: A list of all the blocks in the sequence, in the order they + appear. + """ + + def recurse(block: Block, lst: List[Block]) -> None: + if block.prev_block is not None: + recurse(block.prev_block, lst) + lst.append(block) + + all_blocks: List[Block] = [] + recurse(last_block, all_blocks) + return all_blocks diff --git a/vllm/core/block/cpu_gpu_block_allocator.py b/vllm/core/block/cpu_gpu_block_allocator.py new file mode 100644 index 00000000..6eda5f99 --- /dev/null +++ b/vllm/core/block/cpu_gpu_block_allocator.py @@ -0,0 +1,404 @@ +from typing import Dict, FrozenSet, List, Optional, Tuple + +from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId, + DeviceAwareBlockAllocator) +from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator +from vllm.core.block.prefix_caching_block import PrefixCachingBlockAllocator +from vllm.utils import Device + + +class CpuGpuBlockAllocator(DeviceAwareBlockAllocator): + """A block allocator that can allocate blocks on both CPU and GPU memory. + + This class implements the `DeviceAwareBlockAllocator` interface and provides + functionality for allocating and managing blocks of memory on both CPU and + GPU devices. + + The `CpuGpuBlockAllocator` maintains separate memory pools for CPU and GPU + blocks, and allows for allocation, deallocation, forking, and swapping of + blocks across these memory pools. + """ + + @staticmethod + def create( + allocator_type: str, + num_gpu_blocks: int, + num_cpu_blocks: int, + block_size: int, + ) -> DeviceAwareBlockAllocator: + """Creates a CpuGpuBlockAllocator instance with the specified + configuration. + + This static method creates and returns a CpuGpuBlockAllocator instance + based on the provided parameters. It initializes the CPU and GPU block + allocators with the specified number of blocks, block size, and + allocator type. + + Args: + allocator_type (str): The type of block allocator to use for CPU + and GPU blocks. Currently supported values are "naive" and + "prefix_caching". + num_gpu_blocks (int): The number of blocks to allocate for GPU + memory. + num_cpu_blocks (int): The number of blocks to allocate for CPU + memory. + block_size (int): The size of each block in number of tokens. + + Returns: + DeviceAwareBlockAllocator: A CpuGpuBlockAllocator instance with the + specified configuration. + + Notes: + - The block IDs are assigned contiguously, with GPU block IDs coming + before CPU block IDs. + """ + block_ids = list(range(num_gpu_blocks + num_cpu_blocks)) + gpu_block_ids = block_ids[:num_gpu_blocks] + cpu_block_ids = block_ids[num_gpu_blocks:] + + if allocator_type == "naive": + gpu_allocator: BlockAllocator = NaiveBlockAllocator( + create_block=NaiveBlock, # type: ignore + num_blocks=num_gpu_blocks, + block_size=block_size, + block_ids=gpu_block_ids, + ) + + cpu_allocator: BlockAllocator = NaiveBlockAllocator( + create_block=NaiveBlock, # type: ignore + num_blocks=num_cpu_blocks, + block_size=block_size, + block_ids=cpu_block_ids, + ) + elif allocator_type == "prefix_caching": + gpu_allocator = PrefixCachingBlockAllocator( + num_blocks=num_gpu_blocks, + block_size=block_size, + block_ids=gpu_block_ids, + ) + + cpu_allocator = PrefixCachingBlockAllocator( + num_blocks=num_cpu_blocks, + block_size=block_size, + block_ids=cpu_block_ids, + ) + else: + raise ValueError(f"Unknown allocator type {allocator_type=}") + + return CpuGpuBlockAllocator( + cpu_block_allocator=cpu_allocator, + gpu_block_allocator=gpu_allocator, + ) + + def __init__(self, cpu_block_allocator: BlockAllocator, + gpu_block_allocator: BlockAllocator): + assert not ( + cpu_block_allocator.all_block_ids + & gpu_block_allocator.all_block_ids + ), "cpu and gpu block allocators can't have intersection of block ids" + + self._allocators = { + Device.CPU: cpu_block_allocator, + Device.GPU: gpu_block_allocator, + } + + self._swap_mapping: Dict[int, int] = {} + self._null_block: Optional[Block] = None + + self._block_ids_to_allocator: Dict[int, BlockAllocator] = {} + for _, allocator in self._allocators.items(): + for block_id in allocator.all_block_ids: + self._block_ids_to_allocator[block_id] = allocator + + def allocate_or_get_null_block(self) -> Block: + if self._null_block is None: + self._null_block = NullBlock( + self.allocate_mutable_block(None, Device.GPU)) + return self._null_block + + def allocate_mutable_block(self, prev_block: Optional[Block], + device: Device) -> Block: + """Allocates a new mutable block on the specified device. + + Args: + prev_block (Optional[Block]): The previous block to in the sequence. + Used for prefix hashing. + device (Device): The device on which to allocate the new block. + + Returns: + Block: The newly allocated mutable block. + """ + return self._allocators[device].allocate_mutable_block(prev_block) + + def allocate_immutable_blocks(self, prev_block: Optional[Block], + block_token_ids: List[List[int]], + device: Device) -> List[Block]: + """Allocates a new group of immutable blocks with the provided block + token IDs on the specified device. + + Args: + prev_block (Optional[Block]): The previous block in the sequence. + Used for prefix hashing. + block_token_ids (List[int]): The list of block token IDs to be + stored in the new blocks. + device (Device): The device on which to allocate the new block. + + Returns: + List[Block]: The newly allocated list of immutable blocks + containing the provided block token IDs. + """ + return self._allocators[device].allocate_immutable_blocks( + prev_block, block_token_ids) + + def allocate_immutable_block(self, prev_block: Optional[Block], + token_ids: List[int], + device: Device) -> Block: + """Allocates a new immutable block with the provided token IDs on the + specified device. + + Args: + prev_block (Optional[Block]): The previous block in the sequence. + Used for prefix hashing. + token_ids (List[int]): The list of token IDs to be stored in the new + block. + device (Device): The device on which to allocate the new block. + + Returns: + Block: The newly allocated immutable block containing the provided + token IDs. + """ + return self._allocators[device].allocate_immutable_block( + prev_block, token_ids) + + def free(self, block: Block) -> None: + """Frees the memory occupied by the given block. + + Args: + block (Block): The block to be freed. + """ + # Null block should never be freed + if isinstance(block, NullBlock): + return + block_id = block.block_id + assert block_id is not None + allocator = self._block_ids_to_allocator[block_id] + allocator.free(block) + + def fork(self, last_block: Block) -> List[Block]: + """Creates a new sequence of blocks that shares the same underlying + memory as the original sequence. + + Args: + last_block (Block): The last block in the original sequence. + + Returns: + List[Block]: A new list of blocks that shares the same memory as the + original sequence. + """ + # do not attempt to fork the null block + assert not isinstance(last_block, NullBlock) + block_id = last_block.block_id + assert block_id is not None + allocator = self._block_ids_to_allocator[block_id] + return allocator.fork(last_block) + + def get_num_free_blocks(self, device: Device) -> int: + """Returns the number of free blocks available on the specified device. + + Args: + device (Device): The device for which to query the number of free + blocks. AssertionError is raised if None is passed. + + Returns: + int: The number of free blocks available on the specified device. + """ + return self._allocators[device].get_num_free_blocks() + + def get_num_total_blocks(self, device: Device) -> int: + return self._allocators[device].get_num_total_blocks() + + def get_physical_block_id(self, device: Device, absolute_id: int) -> int: + """Returns the zero-offset block id on certain device given the + absolute block id. + + Args: + device (Device): The device for which to query relative block id. + absolute_id (int): The absolute block id for the block in + whole allocator. + + Returns: + int: The zero-offset block id on certain device. + """ + return self._allocators[device].get_physical_block_id(absolute_id) + + def swap(self, blocks: List[Block], src_device: Device, + dst_device: Device) -> Dict[int, int]: + """Execute the swap for the given blocks from source_device + on to dest_device, save the current swap mapping and append + them to the accumulated `self._swap_mapping` for each + scheduling move. + + Args: + blocks: List of blocks to be swapped. + src_device (Device): Device to swap the 'blocks' from. + dst_device (Device): Device to swap the 'blocks' to. + + Returns: + Dict[int, int]: Swap mapping from source_device + on to dest_device. + """ + src_block_ids = [block.block_id for block in blocks] + self._allocators[src_device].swap_out(blocks) + self._allocators[dst_device].swap_in(blocks) + dst_block_ids = [block.block_id for block in blocks] + + current_swap_mapping: Dict[int, int] = {} + for src_block_id, dst_block_id in zip(src_block_ids, dst_block_ids): + if src_block_id is not None and dst_block_id is not None: + self._swap_mapping[src_block_id] = dst_block_id + current_swap_mapping[src_block_id] = dst_block_id + return current_swap_mapping + + def get_num_full_blocks_touched(self, blocks: List[Block], + device: Device) -> int: + """Returns the number of full blocks that will be touched by + swapping in/out the given blocks on to the 'device'. + + Args: + blocks: List of blocks to be swapped. + device (Device): Device to swap the 'blocks' on. + + Returns: + int: the number of full blocks that will be touched by + swapping in/out the given blocks on to the 'device'. + Non full blocks are ignored when deciding the number + of blocks to touch. + """ + return self._allocators[device].get_num_full_blocks_touched(blocks) + + def clear_copy_on_writes(self) -> List[Tuple[int, int]]: + """Clears the copy-on-write (CoW) state and returns the mapping of + source to destination block IDs. + + Returns: + List[Tuple[int, int]]: A list mapping source block IDs to + destination block IDs. + """ + # CoW only supported on GPU + device = Device.GPU + return self._allocators[device].clear_copy_on_writes() + + def mark_blocks_as_accessed(self, block_ids: List[int], + now: float) -> None: + """Mark blocks as accessed, only use for prefix caching.""" + # Prefix caching only supported on GPU. + device = Device.GPU + return self._allocators[device].mark_blocks_as_accessed(block_ids, now) + + def mark_blocks_as_computed(self, block_ids: List[int]) -> None: + """Mark blocks as accessed, only use for prefix caching.""" + # Prefix caching only supported on GPU. + device = Device.GPU + return self._allocators[device].mark_blocks_as_computed(block_ids) + + def get_computed_block_ids(self, prev_computed_block_ids: List[int], + block_ids: List[int], + skip_last_block_id: bool) -> List[int]: + # Prefix caching only supported on GPU. + device = Device.GPU + return self._allocators[device].get_computed_block_ids( + prev_computed_block_ids, block_ids, skip_last_block_id) + + def get_common_computed_block_ids( + self, computed_seq_block_ids: List[List[int]]) -> List[int]: + # Prefix caching only supported on GPU. + device = Device.GPU + return self._allocators[device].get_common_computed_block_ids( + computed_seq_block_ids) + + @property + def all_block_ids(self) -> FrozenSet[int]: + return frozenset(self._block_ids_to_allocator.keys()) + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + """Prefix cache hit rate. -1 means not supported or disabled.""" + assert device in self._allocators + return self._allocators[device].get_prefix_cache_hit_rate() + + def get_and_reset_swaps(self) -> List[Tuple[int, int]]: + """Returns and clears the mapping of source to destination block IDs. + Will be called after every swapping operations for now, and after every + schedule when BlockManagerV2 become default. Currently not useful. + + Returns: + List[Tuple[int, int]]: A mapping of source to destination block IDs. + """ + mapping = self._swap_mapping.copy() + self._swap_mapping.clear() + return list(mapping.items()) + + +class NullBlock(Block): + """ + Null blocks are used as a placeholders for KV cache blocks that have + been dropped due to sliding window. + This implementation just wraps an ordinary block and prevents it from + being modified. It also allows for testing if a block is NullBlock + via isinstance(). + """ + + def __init__(self, proxy: Block): + super().__init__() + self._proxy = proxy + + def append_token_ids(self, token_ids: List[BlockId]): + raise ValueError("null block should not be modified") + + @property + def block_id(self): + return self._proxy.block_id + + @block_id.setter + def block_id(self, value: Optional[BlockId]): + raise ValueError("null block should not be modified") + + @property + def token_ids(self) -> List[BlockId]: + return self._proxy.token_ids + + @property + def num_tokens_total(self) -> int: + raise NotImplementedError( + "num_tokens_total is not used for null block") + + @property + def num_empty_slots(self) -> BlockId: + return self._proxy.num_empty_slots + + @property + def is_full(self): + return self._proxy.is_full + + @property + def prev_block(self): + return self._proxy.prev_block + + @property + def computed(self): + return self._proxy.computed + + @computed.setter + def computed(self, value): + self._proxy.computed = value + + @property + def last_accessed(self) -> float: + return self._proxy.last_accessed + + @last_accessed.setter + def last_accessed(self, last_accessed_ts: float): + self._proxy.last_accessed = last_accessed_ts + + @property + def content_hash(self): + return self._proxy.content_hash diff --git a/vllm/core/block/interfaces.py b/vllm/core/block/interfaces.py new file mode 100644 index 00000000..72bbab1d --- /dev/null +++ b/vllm/core/block/interfaces.py @@ -0,0 +1,286 @@ +from abc import ABC, abstractmethod +from typing import Dict, FrozenSet, List, Optional, Protocol, Tuple + +from vllm.utils import Device + +BlockId = int + + +class Block(ABC): + + @abstractmethod + def append_token_ids(self, token_ids: List[int]) -> None: + pass + + @property + @abstractmethod + def block_id(self) -> Optional[int]: + pass + + @block_id.setter + @abstractmethod + def block_id(self, value: Optional[int]) -> None: + """NOTE: Do not use this API outside Block.""" + self._block_id = value + + @property + @abstractmethod + def token_ids(self) -> List[int]: + pass + + @property + @abstractmethod + def num_tokens_total(self) -> int: + """The number of tokens till the current block (inclusive) + """ + pass + + @property + @abstractmethod + def num_empty_slots(self) -> int: + pass + + @property + @abstractmethod + def is_full(self) -> bool: + pass + + @property + @abstractmethod + def prev_block(self) -> Optional["Block"]: + pass + + @property + @abstractmethod + def computed(self) -> bool: + raise NotImplementedError + + @computed.setter + @abstractmethod + def computed(self, value) -> bool: + """Should be only used by PrefixCacingAllocator""" + raise NotImplementedError + + @property + @abstractmethod + def last_accessed(self) -> float: + raise NotImplementedError + + @last_accessed.setter + @abstractmethod + def last_accessed(self, last_accessed_ts: float): + raise NotImplementedError + + class Factory(Protocol): + + @abstractmethod + def __call__( + self, + prev_block: Optional["Block"], + token_ids: List[int], + block_size: int, + allocator: "BlockAllocator", + block_id: Optional[int] = None, + ) -> "Block": + pass + + @property + @abstractmethod + def content_hash(self) -> Optional[int]: + """Return the content-based hash of the current block, or None if it is + not yet defined or not supported. + + For the content-based hash to be defined, the current block must be + full. + """ + return None + + +class BlockAllocator(ABC): + + @abstractmethod + def allocate_mutable_block(self, prev_block: Optional[Block]) -> Block: + pass + + @abstractmethod + def allocate_immutable_block(self, prev_block: Optional[Block], + token_ids: List[int]) -> Block: + pass + + @abstractmethod + def allocate_immutable_blocks( + self, prev_block: Optional[Block], + block_token_ids: List[List[int]]) -> List[Block]: + pass + + @abstractmethod + def free(self, block: Block) -> None: + pass + + @abstractmethod + def fork(self, last_block: Block) -> List[Block]: + pass + + @abstractmethod + def get_num_total_blocks(self) -> int: + pass + + @abstractmethod + def get_num_free_blocks(self) -> int: + pass + + @abstractmethod + def get_physical_block_id(self, absolute_id: int) -> int: + pass + + @abstractmethod + def swap_out(self, blocks: List[Block]) -> None: + pass + + @abstractmethod + def swap_in(self, blocks: List[Block]) -> None: + pass + + @property + @abstractmethod + def all_block_ids(self) -> FrozenSet[int]: + pass + + @abstractmethod + def clear_copy_on_writes(self) -> List[Tuple[int, int]]: + pass + + @abstractmethod + def mark_blocks_as_accessed(self, block_ids: List[int], + now: float) -> None: + pass + + @abstractmethod + def mark_blocks_as_computed(self, block_ids: List[int]) -> None: + pass + + @abstractmethod + def get_computed_block_ids(self, prev_computed_block_ids: List[int], + block_ids: List[int], + skip_last_block_id: bool) -> List[int]: + pass + + @abstractmethod + def get_common_computed_block_ids( + self, computed_seq_block_ids: List[List[int]]) -> List[int]: + pass + + @abstractmethod + def cow_block_if_not_appendable(self, block: Block) -> BlockId: + """NOTE: This should not be used besides Block""" + pass + + @abstractmethod + def promote_to_immutable_block(self, block: Block) -> BlockId: + """NOTE: This should not be used besides Block""" + pass + + @abstractmethod + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int: + pass + + @abstractmethod + def get_prefix_cache_hit_rate(self) -> float: + """Prefix cache hit rate. -1 means not supported or disabled.""" + pass + + class NoFreeBlocksError(ValueError): + pass + + +class DeviceAwareBlockAllocator(ABC): + + @abstractmethod + def allocate_mutable_block(self, prev_block: Optional[Block], + device: Device) -> Block: + pass + + @abstractmethod + def allocate_immutable_block(self, prev_block: Optional[Block], + token_ids: List[int], + device: Device) -> Block: + pass + + @abstractmethod + def allocate_immutable_blocks(self, prev_block: Optional[Block], + block_token_ids: List[List[int]], + device: Device) -> List[Block]: + pass + + @abstractmethod + def get_num_free_blocks(self, device: Device) -> int: + pass + + @abstractmethod + def get_num_total_blocks(self, device: Device) -> int: + pass + + @abstractmethod + def free(self, block: Block) -> None: + pass + + @abstractmethod + def fork(self, last_block: Block) -> List[Block]: + pass + + @property + @abstractmethod + def all_block_ids(self) -> FrozenSet[int]: + pass + + @abstractmethod + def clear_copy_on_writes(self) -> List[Tuple[int, int]]: + pass + + @abstractmethod + def mark_blocks_as_accessed(self, block_ids: List[int], + now: float) -> None: + pass + + @abstractmethod + def mark_blocks_as_computed(self, block_ids: List[int]) -> None: + pass + + @abstractmethod + def get_computed_block_ids(self, prev_computed_block_ids: List[int], + block_ids: List[int], + skip_last_block_id: bool) -> List[int]: + pass + + @abstractmethod + def get_common_computed_block_ids( + self, computed_seq_block_ids: List[List[int]]) -> List[int]: + pass + + @abstractmethod + def get_num_full_blocks_touched(self, blocks: List[Block], + device: Device) -> int: + pass + + @abstractmethod + def swap(self, blocks: List[Block], src_device: Device, + dst_device: Device) -> Dict[int, int]: + pass + + @abstractmethod + def get_physical_block_id(self, device: Device, absolute_id: int) -> int: + pass + + @abstractmethod + def allocate_or_get_null_block(self) -> Block: + """ + Null blocks are used as a placeholders for KV cache blocks that have + been dropped due to sliding window. + There is at most one null block per allocator. + """ + pass + + @abstractmethod + def get_prefix_cache_hit_rate(self, device: Device) -> float: + """Prefix cache hit rate. -1 means not supported or disabled.""" + pass diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py new file mode 100644 index 00000000..9341a518 --- /dev/null +++ b/vllm/core/block/naive_block.py @@ -0,0 +1,449 @@ +from collections import deque +from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple + +from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter, + get_all_blocks_recursively) +from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device + +Refcount = int + + +class NaiveBlockAllocator(BlockAllocator): + """A simple block allocator that manages blocks of memory without prefix + caching. + + Args: + create_block (Block.Factory): A factory function for creating new + blocks. This is used when a NaiveBlockAllocator is composed within + a prefix caching allocator -- the naive block allocator must + construct prefix caching blocks (but shouldn't know anything else + about them). + num_blocks (int): The total number of blocks to manage. + block_size (int): The size of each block in tokens. + block_ids (Optional[Iterable[int]], optional): An optional iterable of + block IDs. If not provided, block IDs will be assigned sequentially + from 0 to num_blocks - 1. + """ + + def __init__( + self, + create_block: Block.Factory, + num_blocks: int, + block_size: int, + block_ids: Optional[Iterable[int]] = None, + block_pool: Optional[BlockPool] = None, + ): + if block_ids is None: + block_ids = range(num_blocks) + + self._free_block_indices: Deque[BlockId] = deque(block_ids) + self._all_block_indices = frozenset(block_ids) + assert len(self._all_block_indices) == num_blocks + + self._refcounter = RefCounter( + all_block_indices=self._free_block_indices) + self._block_size = block_size + + self._cow_tracker = CopyOnWriteTracker( + refcounter=self._refcounter.as_readonly()) + + if block_pool is None: + extra_factor = 4 + # Pre-allocate "num_blocks * extra_factor" block objects. + # The "* extra_factor" is a buffer to allow more block objects + # than physical blocks + self._block_pool = BlockPool(self._block_size, create_block, self, + num_blocks * extra_factor) + else: + # In this case, the block pool is provided by the caller, + # which means that there is most likely a need to share + # a block pool between allocators + self._block_pool = block_pool + + def allocate_immutable_block(self, + prev_block: Optional[Block], + token_ids: List[int], + device: Optional[Device] = None) -> Block: + """Allocates a new immutable block with the given token IDs, linked to + the previous block. + + Args: + prev_block (Optional[Block]): The previous block in the sequence. If + None, then the block to be allocated is the first block in the + sequence. + token_ids (List[int]): The token IDs to be stored in the new block. + + Returns: + Block: The newly allocated immutable block. + """ + assert device is None + block = self.allocate_mutable_block(prev_block=prev_block) + block.append_token_ids(token_ids) + return block + + def allocate_immutable_blocks( + self, + prev_block: Optional[Block], + block_token_ids: List[List[int]], + device: Optional[Device] = None) -> List[Block]: + assert device is None + num_blocks = len(block_token_ids) + + block_ids = [] + for i in range(num_blocks): + block_ids.append(self._allocate_block_id()) + + blocks = [] + for i in range(num_blocks): + prev_block = self._block_pool.init_block( + prev_block=prev_block, + token_ids=block_token_ids[i], + block_size=self._block_size, + physical_block_id=block_ids[i]) + blocks.append(prev_block) + + return blocks + + def allocate_mutable_block(self, + prev_block: Optional[Block], + device: Optional[Device] = None) -> Block: + """Allocates a new mutable block, linked to the previous block. + + Args: + prev_block (Optional[Block]): The previous block in the sequence. If + None, then the block to be allocated is the first block in the + sequence. + + Returns: + Block: The newly allocated mutable block. + """ + assert device is None + block_id = self._allocate_block_id() + block = self._block_pool.init_block(prev_block=prev_block, + token_ids=[], + block_size=self._block_size, + physical_block_id=block_id) + return block + + def _allocate_block_id(self) -> BlockId: + if not self._free_block_indices: + raise BlockAllocator.NoFreeBlocksError() + + block_id = self._free_block_indices.popleft() + self._refcounter.incr(block_id) + return block_id + + def _free_block_id(self, block: Block) -> None: + block_id = block.block_id + assert block_id is not None + + refcount = self._refcounter.decr(block_id) + if refcount == 0: + self._free_block_indices.appendleft(block_id) + + block.block_id = None + + def free(self, block: Block, keep_block_object: bool = False) -> None: + # Release the physical block id + self._free_block_id(block) + + # Release the block object + if not keep_block_object: + self._block_pool.free_block(block) + + def fork(self, last_block: Block) -> List[Block]: + """Creates a new sequence of blocks that shares the same underlying + memory as the original sequence. + + Args: + last_block (Block): The last block in the original sequence. + + Returns: + List[Block]: The new sequence of blocks that shares the same memory + as the original sequence. + """ + source_blocks = get_all_blocks_recursively(last_block) + + forked_blocks: List[Block] = [] + prev_block = None + for block in source_blocks: + + # Increment refcount for each block. + assert block.block_id is not None + refcount = self._refcounter.incr(block.block_id) + assert refcount != 1, "can't fork free'd block" + + forked_block = self._block_pool.init_block( + prev_block=prev_block, + token_ids=block.token_ids, + block_size=self._block_size, + physical_block_id=block.block_id) + + forked_blocks.append(forked_block) + prev_block = forked_blocks[-1] + + return forked_blocks + + def get_num_free_blocks(self) -> int: + return len(self._free_block_indices) + + def get_num_total_blocks(self) -> int: + return len(self._all_block_indices) + + def get_physical_block_id(self, absolute_id: int) -> int: + """Returns the zero-offset block id on certain block allocator + given the absolute block id. + + Args: + absolute_id (int): The absolute block id for the block + in whole allocator. + + Returns: + int: The zero-offset block id on certain device. + """ + return sorted(self._all_block_indices).index(absolute_id) + + @property + def refcounter(self): + return self._refcounter + + @property + def all_block_ids(self) -> FrozenSet[int]: + return self._all_block_indices + + def cow_block_if_not_appendable(self, block: Block) -> BlockId: + """Performs a copy-on-write operation on the given block if it is not + appendable. + + Args: + block (Block): The block to check for copy-on-write. + + Returns: + BlockId: The block index of the new block if a copy-on-write + operation was performed, or the original block index if + no copy-on-write was necessary. + """ + src_block_id = block.block_id + assert src_block_id is not None + + if self._cow_tracker.is_appendable(block): + return src_block_id + + self._free_block_id(block) + trg_block_id = self._allocate_block_id() + + self._cow_tracker.record_cow(src_block_id, trg_block_id) + + return trg_block_id + + def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]: + """Returns the copy-on-write source->destination mapping and clears it. + + Returns: + List[Tuple[BlockId, BlockId]]: A list mapping source + block indices to destination block indices. + """ + return self._cow_tracker.clear_cows() + + def mark_blocks_as_accessed(self, block_ids: List[int], + now: float) -> None: + """Mark blocks as accessed, used in prefix caching. + + Since the naive allocator does not implement prefix caching, we do + nothing. + """ + pass + + def mark_blocks_as_computed(self, block_ids: List[int]) -> None: + """Mark blocks as computed, used in prefix caching. + + Since the naive allocator does not implement prefix caching, we do + nothing. + """ + pass + + def get_computed_block_ids(self, prev_computed_block_ids: List[int], + block_ids: List[int], + skip_last_block_id: bool) -> List[int]: + """No prefix caching here => return empty list + """ + return [] + + def get_common_computed_block_ids( + self, computed_seq_block_ids: List[List[int]]) -> List[int]: + """Determine blocks that can be skipped in prefill. + + Since the naive allocator does not support prefix caching, always return + an empty list. + """ + return [] + + def promote_to_immutable_block(self, block: Block) -> BlockId: + raise NotImplementedError("There is no promotion for naive blocks") + + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int: + """Returns the number of full blocks that will be touched by + swapping in/out. + + Args: + blocks: List of blocks to be swapped. + Returns: + int: the number of full blocks that will be touched by + swapping in/out the given blocks. Non full blocks are ignored + when deciding the number of blocks to touch. + """ + # NOTE: for naive block, we use set to eliminate common blocks among + # seqs, also we compare the empty slots in the mutable blocks with + # lookahead slots to get the number of unique new block that are + # needed. + old_block_set = set() + for block in blocks: + if block.is_full: + old_block_set.add(block) + return len(old_block_set) + + def swap_out(self, blocks: List[Block]) -> None: + for block in blocks: + self._free_block_id(block) + + def swap_in(self, blocks: List[Block]) -> None: + for block in blocks: + # Here we allocate either immutable or mutable block and then + # extract its block_id. Note that the block object is released + # and the block_id is assigned to "block" to allow reusing the + # existing "block" object + if block.is_full: + tmp_block = self.allocate_immutable_block( + prev_block=block.prev_block, token_ids=block.token_ids) + else: + tmp_block = self.allocate_mutable_block( + prev_block=block.prev_block) + tmp_block.append_token_ids(block.token_ids) + + block_id = tmp_block.block_id + tmp_block.block_id = None + self._block_pool.free_block(tmp_block) + + block.block_id = block_id # Assign block_id + + def get_prefix_cache_hit_rate(self) -> float: + return -1 + + +class NaiveBlock(Block): + """An implementation of the Block class that does not support prefix + caching. + + The NaiveBlock class represents a block of token IDs with a fixed size. It + provides methods for appending token IDs to the block and manages copy-on + -write operations when necessary. + + Args: + prev_block (Block): The previous block in the sequence. + token_ids (List[int]): The initial token IDs to be stored in the block. + block_size (int): The maximum number of token IDs that can be stored in + the block. + allocator (BlockAllocator): The block allocator associated with this + block. + block_id (Optional[int], optional): The physical block index + of this block. Defaults to None, which means no allocation has been + made. + _cow_target (Optional[Block], optional): The copy-on-write target block. + If not provided, it defaults to self. + """ + + def __init__(self, + prev_block: Optional[Block], + token_ids: List[int], + block_size: int, + allocator: BlockAllocator, + block_id: Optional[int] = None, + _cow_target: Optional[Block] = None): + self._token_ids: List[int] = [] + self._block_size = block_size + self._prev_block = prev_block + self._block_id = block_id + self._allocator = allocator + self._cow_target = _cow_target if _cow_target is not None else self + + self._append_token_ids_no_cow(token_ids) + + def append_token_ids(self, token_ids: List[int]) -> None: + """Appends the given token IDs to the block and performs a + copy-on-write if necessary. + + Args: + token_ids (Optional[List[int]]): The token IDs to be appended + to the block. + """ + self._append_token_ids_no_cow(token_ids) + + if self._block_id is not None: + self._block_id = (self._allocator.cow_block_if_not_appendable( + self._cow_target)) + + def _append_token_ids_no_cow(self, token_ids: List[int]) -> None: + """Appends the given token IDs to the block + + Args: + token_ids (List[int]): The token IDs to be appended to the block. + """ + if len(token_ids) == 0: + return + + assert len(token_ids) <= self.num_empty_slots + + self._token_ids.extend(token_ids) + + @property + def computed(self) -> bool: + raise NotImplementedError + + @computed.setter + def computed(self, value) -> None: + raise NotImplementedError + + @property + def last_accessed(self) -> float: + raise NotImplementedError + + @last_accessed.setter + def last_accessed(self, last_accessed_ts: float): + raise NotImplementedError + + @property + def block_id(self) -> Optional[int]: + return self._block_id + + @block_id.setter + def block_id(self, value: Optional[int]) -> None: + self._block_id = value + + @property + def is_full(self) -> bool: + return self.num_empty_slots == 0 + + @property + def num_empty_slots(self) -> int: + return self._block_size - len(self.token_ids) + + @property + def token_ids(self) -> List[int]: + return self._token_ids + + @property + def num_tokens_total(self) -> int: + raise NotImplementedError( + "num_tokens_total is not used for naive block") + + @property + def block_size(self) -> int: + return self._block_size + + @property + def prev_block(self) -> Optional["Block"]: + return self._prev_block + + @property + def content_hash(self) -> Optional[int]: + return None diff --git a/vllm/core/block/prefix_caching_block.py b/vllm/core/block/prefix_caching_block.py new file mode 100644 index 00000000..7c8a2bc4 --- /dev/null +++ b/vllm/core/block/prefix_caching_block.py @@ -0,0 +1,970 @@ +"""Token blocks.""" +from os.path import commonprefix +from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple + +from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker, + get_all_blocks_recursively) +from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device +from vllm.core.block.naive_block import (BlockPool, NaiveBlock, + NaiveBlockAllocator) +from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor + +PrefixHash = int + +# By default, we init our block access time as _DEFAULT_LAST_ACCESSED_TIME +# so that if we find one block is still hold _DEFAULT_LAST_ACCESSED_TIME, +# then we know this block hasn't been accessed yet. +_DEFAULT_LAST_ACCESSED_TIME = -1 + + +class BlockTracker: + """Used to track the status of a block inside the prefix caching allocator + """ + __slots__ = ("active", "last_accessed", "computed") + + def reset(self): + self.last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME + self.computed: bool = False + + def __init__(self): + self.active: bool = False + self.reset() + + def enable(self): + assert not self.active + self.active = True + self.reset() + + def disable(self): + assert self.active + self.active = False + self.reset() + + +class PrefixCachingBlockAllocator(BlockAllocator): + """A block allocator that implements prefix caching. + + The PrefixCachingBlockAllocator maintains a cache of blocks based on their + content hash. It reuses blocks with the same content hash to avoid redundant + memory allocation. The allocator also supports copy-on-write operations. + + Args: + num_blocks (int): The total number of blocks to manage. + block_size (int): The size of each block in tokens. + block_ids(Optional[Iterable[int]], optional): An optional iterable of + block IDs. If not provided, block IDs will be assigned sequentially + from 0 to num_blocks - 1. + """ + + def __init__( + self, + num_blocks: int, + block_size: int, + block_ids: Optional[Iterable[int]] = None, + eviction_policy: EvictionPolicy = EvictionPolicy.LRU, + ): + if block_ids is None: + block_ids = range(num_blocks) + + self._block_size = block_size + + # A mapping of prefix hash to block index. All blocks which have a + # prefix hash will be in this dict, even if they have refcount 0. + self._cached_blocks: Dict[PrefixHash, BlockId] = {} + + # A list of immutable block IDs that have been touched by scheduler + # and should be marked as computed after an entire batch of sequences + # are scheduled. + self._touched_blocks: Set[BlockId] = set() + + # Used to track status of each physical block id + self._block_tracker: Dict[BlockId, BlockTracker] = {} + for block_id in block_ids: + self._block_tracker[block_id] = BlockTracker() + + # Pre-allocate "num_blocks * extra_factor" block objects. + # The "* extra_factor" is a buffer to allow more block objects + # than physical blocks + extra_factor = 4 + self._block_pool = BlockPool(self._block_size, self._create_block, + self, num_blocks * extra_factor) + + # An allocator for blocks that do not have prefix hashes. + self._hashless_allocator = NaiveBlockAllocator( + create_block=self._create_block, # type: ignore + num_blocks=num_blocks, + block_size=block_size, + block_ids=block_ids, + block_pool=self._block_pool, # Share block pool here + ) + + # Evitor used to maintain how we want to handle those computed blocks + # if we find memory pressure is high. + self.evictor: Evictor = make_evictor(eviction_policy) + + # We share the refcounter between allocators. This allows us to promote + # blocks originally allocated in the hashless allocator to immutable + # blocks. + self._refcounter = self._hashless_allocator.refcounter + + self._cow_tracker = CopyOnWriteTracker( + refcounter=self._refcounter.as_readonly()) + + self.metric_data = CacheMetricData() + + # Implements Block.Factory. + def _create_block( + self, + prev_block: Optional[Block], + token_ids: List[int], + block_size: int, + allocator: BlockAllocator, + block_id: Optional[int] = None, + computed: bool = False, + ) -> Block: + # Bind block to self. + allocator = self + + return PrefixCachingBlock( + prev_block=prev_block, + token_ids=token_ids, + block_size=block_size, + block_id=block_id, + allocator=allocator, + computed=computed, + ) + + def allocate_immutable_block(self, + prev_block: Optional[Block], + token_ids: List[int], + device: Optional[Device] = None) -> Block: + """Allocates an immutable block with the given token IDs, reusing cached + blocks if possible. + + Args: + prev_block (Optional[Block]): The previous block in the sequence. + token_ids (List[int]): The token IDs to be stored in the block. + + Returns: + Block: The allocated immutable block. + """ + assert device is None + assert_prefix_caching_block_or_none(prev_block) + + # First, try to create a block that points to cached data + block = self._block_pool.init_block(prev_block=prev_block, + token_ids=token_ids, + block_size=self._block_size, + physical_block_id=None) + assert block.content_hash is not None + + cached_block_id = self._cached_blocks.get(block.content_hash, None) + if cached_block_id is not None: + self.metric_data.query(hit=True) + block.block_id = cached_block_id + self._incr_refcount_cached_block(block) + return block + self.metric_data.query(hit=False) + self._block_pool.free_block(block) + + # No cached block => Allocate a new block + block = self.allocate_mutable_block(prev_block) + block.append_token_ids(token_ids) + return block + + def allocate_immutable_blocks( + self, + prev_block: Optional[Block], + block_token_ids: List[List[int]], + device: Optional[Device] = None) -> List[Block]: + blocks = [] + for token_ids in block_token_ids: + prev_block = self.allocate_immutable_block(prev_block=prev_block, + token_ids=token_ids, + device=device) + blocks.append(prev_block) + return blocks + + def allocate_mutable_block(self, + prev_block: Optional[Block], + device: Optional[Device] = None) -> Block: + """Allocates a mutable block. If there are no free blocks, this will + evict unused cached blocks. + + Args: + prev_block (Block): The previous block in the sequence. + None is not allowed unlike it is super class. + + Returns: + Block: The allocated mutable block. + """ + assert device is None + assert_prefix_caching_block_or_none(prev_block) + + block_id = self._allocate_block_id() + block = self._block_pool.init_block(prev_block=prev_block, + token_ids=[], + block_size=self._block_size, + physical_block_id=block_id) + assert not block.computed + assert block.content_hash is None + return block + + def _incr_refcount_cached_block(self, block: Block) -> None: + # Set this block to be "computed" since it is pointing to a + # cached block id (which was already computed) + block.computed = True + + block_id = block.block_id + assert block_id is not None + + refcount = self._refcounter.incr(block_id) + if refcount == 1: + # In case a cached block was evicted, restore its tracking + if block_id in self.evictor: + self.evictor.remove(block_id) + + self._track_block_id(block_id, computed=True) + + def _decr_refcount_cached_block(self, block: Block) -> None: + # Ensure this is immutable/cached block + assert block.content_hash is not None + + block_id = block.block_id + assert block_id is not None + + refcount = self._refcounter.decr(block_id) + if refcount > 0: + block.block_id = None + return + else: + assert refcount == 0 + + # No longer used + assert block.content_hash in self._cached_blocks + + # Add the cached block to the evictor + # (This keeps the cached block around so it can be reused) + self.evictor.add(block_id, block.content_hash, block.num_tokens_total, + self._block_tracker[block_id].last_accessed) + + # Stop tracking the block + self._untrack_block_id(block_id) + + block.block_id = None + + def _decr_refcount_hashless_block(self, block: Block) -> None: + block_id = block.block_id + assert block_id is not None + + # We may have a fork case where block is shared, + # in which case, we cannot remove it from tracking + refcount = self._refcounter.get(block_id) + if refcount == 1: + self._untrack_block_id(block_id) + + # Decrement refcount of the block_id, but do not free the block object + # itself (will be handled by the caller) + self._hashless_allocator.free(block, keep_block_object=True) + + def _allocate_block_id(self) -> BlockId: + """First tries to allocate a block id from the hashless allocator, + and if there are no blocks, then tries to evict an unused cached block. + """ + hashless_block_id = self._maybe_allocate_hashless_block_id() + if hashless_block_id is not None: + return hashless_block_id + + evicted_block_id = self._maybe_allocate_evicted_block_id() + if evicted_block_id is not None: + return evicted_block_id + + # No block available in hashless allocator, nor in unused cache blocks. + raise BlockAllocator.NoFreeBlocksError() + + def _maybe_allocate_hashless_block_id(self) -> Optional[BlockId]: + try: + # Allocate mutable block and extract its block_id + block = self._hashless_allocator.allocate_mutable_block( + prev_block=None) + block_id = block.block_id + self._block_pool.free_block(block) + + self._track_block_id(block_id, computed=False) + return block_id + except BlockAllocator.NoFreeBlocksError: + return None + + def _maybe_allocate_evicted_block_id(self) -> Optional[BlockId]: + if self.evictor.num_blocks == 0: + return None + + # Here we get an evicted block, which is only added + # into evictor if its ref counter is 0 + # and since its content would be changed, we need + # to remove it from _cached_blocks's tracking list + block_id, content_hash_to_evict = self.evictor.evict() + + # Sanity checks + assert content_hash_to_evict in self._cached_blocks + _block_id = self._cached_blocks[content_hash_to_evict] + assert self._refcounter.get(_block_id) == 0 + assert _block_id == block_id + + self._cached_blocks.pop(content_hash_to_evict) + + self._refcounter.incr(block_id) + self._track_block_id(block_id, computed=False) + + return block_id + + def _free_block_id(self, block: Block) -> None: + """Decrements the refcount of the block. The block may be in two + possible states: (1) immutable/cached or (2) mutable/hashless. + In the first case, the refcount is decremented directly and the block + may be possibly added to the evictor. In other case, hashless + allocator free(..) with keep_block_object=True is called to only free + the block id (since the block object may be reused by the caller) + """ + block_id = block.block_id + assert block_id is not None, "Freeing unallocated block is undefined" + + if block.content_hash is not None: + # Immutable: This type of block is always cached, and we want to + # keep it in the evictor for future reuse + self._decr_refcount_cached_block(block) + else: + # Mutable: This type of block is not cached, so we release it + # directly to the hashless allocator + self._decr_refcount_hashless_block(block) + + assert block.block_id is None + + def free(self, block: Block, keep_block_object: bool = False) -> None: + """Release the block (look at free_block_id(..) docs) + """ + # Release the physical block index + self._free_block_id(block) + + # Release the block object to the pool + if not keep_block_object: + self._block_pool.free_block(block) + + def fork(self, last_block: Block) -> List[Block]: + """Creates a new sequence of blocks that shares the same underlying + memory as the original sequence. + + Args: + last_block (Block): The last block in the original sequence. + + Returns: + List[Block]: The new sequence of blocks that shares the same memory + as the original sequence. + """ + source_blocks = get_all_blocks_recursively(last_block) + + forked_blocks: List[Block] = [] + prev_block = None + for block in source_blocks: + block_id = block.block_id + assert block_id is not None + + refcount = self._refcounter.incr(block_id) + assert refcount != 1, "can't fork free'd block_id = {}".format( + block_id) + + forked_block = self._block_pool.init_block( + prev_block=prev_block, + token_ids=block.token_ids, + block_size=self._block_size, + physical_block_id=block_id) + + forked_blocks.append(forked_block) + prev_block = forked_blocks[-1] + + return forked_blocks + + def get_num_free_blocks(self, device: Optional[Device] = None) -> int: + assert device is None + # The number of free blocks is the number of hashless free blocks + # plus the number of blocks evictor could free from its list. + return self._hashless_allocator.get_num_free_blocks( + ) + self.evictor.num_blocks + + def get_num_total_blocks(self) -> int: + return self._hashless_allocator.get_num_total_blocks() + + def get_physical_block_id(self, absolute_id: int) -> int: + """Returns the zero-offset block id on certain block allocator + given the absolute block id. + + Args: + absolute_id (int): The absolute block id for the block + in whole allocator. + + Returns: + int: The rzero-offset block id on certain device. + """ + return sorted(self.all_block_ids).index(absolute_id) + + @property + def all_block_ids(self) -> FrozenSet[int]: + return self._hashless_allocator.all_block_ids + + def get_prefix_cache_hit_rate(self) -> float: + return self.metric_data.get_hit_rate() + + def is_block_cached(self, block: Block) -> bool: + assert block.content_hash is not None + return block.content_hash in self._cached_blocks + + def promote_to_immutable_block(self, block: Block) -> BlockId: + """Once a mutable block is full, it can be promoted to an immutable + block. This means that its content can be referenced by future blocks + having the same prefix. + + Note that if we already have a cached block with the same content, we + will replace the newly-promoted block's mapping with the existing cached + block id. + + Args: + block: The mutable block to be promoted. + + Returns: + BlockId: Either the original block index, or the block index of + the previously cached block matching the same content. + """ + # Ensure block can be promoted + assert block.content_hash is not None + assert block.block_id is not None + assert self._refcounter.get(block.block_id) > 0 + + if block.content_hash not in self._cached_blocks: + # No cached content hash => Set this block as cached. + # Note that this block cannot be marked as computed yet + # because other sequences in the same batch cannot reuse + # this block. + self._cached_blocks[block.content_hash] = block.block_id + # Mark this block as touched so that it can be marked as + # computed after the entire batch of sequences are scheduled. + self._touched_blocks.add(block.block_id) + return block.block_id + + # Reuse the cached content hash + self._decr_refcount_hashless_block(block) + block.block_id = self._cached_blocks[block.content_hash] + + # Increment refcount of the cached block and (possibly) restore + # it from the evictor. + # Note that in this case, the block is marked as computed + self._incr_refcount_cached_block(block) + + return block.block_id + + def cow_block_if_not_appendable(self, block: Block) -> BlockId: + """Performs a copy-on-write operation on the given block if it is not + appendable. + + Args: + block (Block): The block to check for copy-on-write. + + Returns: + BlockId: The block index of the new block if a copy-on-write + operation was performed, or the original block index if + no copy-on-write was necessary. + """ + src_block_id = block.block_id + assert src_block_id is not None + + if self._cow_tracker.is_appendable(block): + return src_block_id + + self._free_block_id(block) + trg_block_id = self._allocate_block_id() + + self._cow_tracker.record_cow(src_block_id, trg_block_id) + + return trg_block_id + + def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]: + """Returns the copy-on-write source->destination mapping and clears it. + + Returns: + List[Tuple[BlockId, BlockId]]: A list mapping source + block indices to destination block indices. + """ + return self._cow_tracker.clear_cows() + + def mark_blocks_as_accessed(self, block_ids: List[int], + now: float) -> None: + """Mark blocks as accessed, used in prefix caching. + + If the block is added into evictor, we need to update corresponding + info in evictor's metadata. + """ + + for block_id in block_ids: + if self._block_tracker[block_id].active: + self._block_tracker[block_id].last_accessed = now + elif block_id in self.evictor: + self.evictor.update(block_id, now) + else: + raise ValueError( + "Mark block as accessed which is not belonged to GPU") + + def mark_blocks_as_computed(self, block_ids: List[int]) -> None: + # Mark all touched blocks as computed. + for block_id in self._touched_blocks: + self._block_tracker[block_id].computed = True + self._touched_blocks.clear() + + def _track_block_id(self, block_id: Optional[BlockId], + computed: bool) -> None: + assert block_id is not None + self._block_tracker[block_id].enable() + self._block_tracker[block_id].computed = computed + + def _untrack_block_id(self, block_id: Optional[BlockId]) -> None: + assert block_id is not None + self._block_tracker[block_id].disable() + + def block_is_computed(self, block_id: int) -> bool: + if self._block_tracker[block_id].active: + return self._block_tracker[block_id].computed + else: + return block_id in self.evictor + + def get_computed_block_ids(self, + prev_computed_block_ids: List[int], + block_ids: List[int], + skip_last_block_id: bool = True) -> List[int]: + prev_prefix_size = len(prev_computed_block_ids) + cur_size = len(block_ids) + if skip_last_block_id: + cur_size -= 1 + + # Sanity checks + assert cur_size >= 0 + assert prev_prefix_size <= cur_size + + ret = prev_computed_block_ids + for i in range(prev_prefix_size, cur_size): + block_id = block_ids[i] + if self.block_is_computed(block_id): + ret.append(block_id) + return ret + + def get_common_computed_block_ids( + self, computed_seq_block_ids: List[List[int]]) -> List[int]: + """Return the block ids that are common for a given sequence group. + + Only those blocks that are immutable and already be marked + compyted would be taken consideration. + """ + + # NOTE We exclude the last block to avoid the case where the entire + # prompt is cached. This would cause erroneous behavior in model + # runner. + + # It returns a list of int although type annotation says list of string. + if len(computed_seq_block_ids) == 1: + return computed_seq_block_ids[0] + + return commonprefix([ + ids for ids in computed_seq_block_ids # type: ignore + if ids + ]) + + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int: + """Returns the number of full blocks that will be touched by + swapping in/out. + + Args: + blocks: List of blocks to be swapped. + Returns: + int: the number of full blocks that will be touched by + swapping in/out the given blocks. Non full blocks are ignored + when deciding the number of blocks to touch. + """ + num_touched_blocks: int = 0 + for block in blocks: + # If the block has a match in the cache and the cached + # block is not referenced, then we still count it as a + # touched block + if block.is_full and (not self.is_block_cached(block) or \ + (block.content_hash is not None and \ + self._cached_blocks[block.content_hash] in \ + self.evictor)): + num_touched_blocks += 1 + return num_touched_blocks + + def swap_out(self, blocks: List[Block]) -> None: + """Execute the swap out actions. Basically just free the + given blocks. + + Args: + blocks: List of blocks to be swapped out. + """ + for block in blocks: + self._free_block_id(block) + + def swap_in(self, blocks: List[Block]) -> None: + """Execute the swap in actions. Change the block id from + old allocator to current allocator for each block to finish + the block table update. + + Args: + blocks: List of blocks to be swapped in. + """ + for block in blocks: + # Here we allocate either immutable or mutable block and then + # extract its block_id. Note that the block object is released + # and the block_id is assigned to "block" to allow reusing the + # existing "block" object + if block.is_full: + tmp_block = self.allocate_immutable_block( + prev_block=block.prev_block, token_ids=block.token_ids) + else: + tmp_block = self.allocate_mutable_block( + prev_block=block.prev_block) + tmp_block.append_token_ids(block.token_ids) + + block_id = tmp_block.block_id + self._block_pool.free_block(tmp_block) + + block.block_id = block_id # Assign block_id + + +class PrefixCachingBlock(Block): + """A block implementation that supports prefix caching. + + The PrefixCachingBlock class represents a block of token IDs with prefix + caching capabilities. It wraps a NaiveBlock internally and provides + additional functionality for content hashing and promoting immutable blocks + with the prefix caching allocator. + + Args: + prev_block (Optional[PrefixCachingBlock]): The previous block in the + sequence. + token_ids (List[int]): The initial token IDs to be stored in the block. + block_size (int): The maximum number of token IDs that can be stored in + the block. + allocator (BlockAllocator): The prefix + caching block allocator associated with this block. + block_id (Optional[int], optional): The physical block index + of this block. Defaults to None. + """ + + def __init__( + self, + prev_block: Optional[Block], + token_ids: List[int], + block_size: int, + allocator: BlockAllocator, + block_id: Optional[int] = None, + computed: bool = False, + ): + assert isinstance(allocator, PrefixCachingBlockAllocator), ( + "Currently this class is only tested with " + "PrefixCachingBlockAllocator. Got instead allocator = {}".format( + allocator)) + assert_prefix_caching_block_or_none(prev_block) + + self._prev_block = prev_block + self._cached_content_hash: Optional[int] = None + self._cached_num_tokens_total: int = 0 + self._allocator = allocator + self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME + self._computed = computed + + # On the first time, we create the block object, and next we only + # reinitialize it + if hasattr(self, "_block"): + self._block.__init__( # type: ignore[has-type] + prev_block=prev_block, + token_ids=token_ids, + block_size=block_size, + block_id=block_id, + allocator=self._allocator) + else: + self._block = NaiveBlock(prev_block=prev_block, + token_ids=token_ids, + block_size=block_size, + block_id=block_id, + allocator=self._allocator) + + self._update_num_tokens_total() + + def _update_num_tokens_total(self): + """Incrementally computes the number of tokens that there is + till the current block (included) + """ + res = 0 + + # Add all previous blocks + if self._prev_block is not None: + res += self._prev_block.num_tokens_total + + # Add current block + res += len(self.token_ids) + + self._cached_num_tokens_total = res + + @property + def computed(self) -> bool: + return self._computed + + @computed.setter + def computed(self, value) -> None: + self._computed = value + + @property + def last_accessed(self) -> float: + return self._last_accessed + + @last_accessed.setter + def last_accessed(self, last_accessed_ts: float): + self._last_accessed = last_accessed_ts + + def append_token_ids(self, token_ids: List[int]) -> None: + """Appends the given token IDs to the block and registers the block as + immutable if the block becomes full. + + Args: + token_ids (List[int]): The token IDs to be appended to the block. + """ + # Ensure this is mutable block (not promoted) + assert self.content_hash is None + assert not self.computed + + if len(token_ids) == 0: + return + + # Ensure there are input tokens + assert token_ids, "Got token_ids = {}".format(token_ids) + + # Naive block handles CoW. + self._block.append_token_ids(token_ids) + self._update_num_tokens_total() + + # If the content hash is present, then the block can be made immutable. + # Register ourselves with the allocator, potentially replacing the + # physical block index. + if self.content_hash is not None: + self.block_id = self._allocator.promote_to_immutable_block(self) + + @property + def block_id(self) -> Optional[int]: + return self._block.block_id + + @block_id.setter + def block_id(self, value) -> None: + self._block.block_id = value + + @property + def is_full(self) -> bool: + return self._block.is_full + + @property + def num_empty_slots(self) -> int: + return self._block.num_empty_slots + + @property + def num_tokens_total(self) -> int: + return self._cached_num_tokens_total + + @property + def block_size(self) -> int: + return self._block.block_size + + @property + def token_ids(self) -> List[int]: + return self._block.token_ids + + @property + def prev_block(self) -> Optional[Block]: + return self._prev_block + + @property + def content_hash(self) -> Optional[int]: + """Return the content-based hash of the current block, or None if it is + not yet defined. + + For the content-based hash to be defined, the current block must be + full. + """ + # If the hash is already computed, return it. + if self._cached_content_hash is not None: + return self._cached_content_hash + + # We cannot compute a hash for the current block because it is not full. + if not self.is_full: + return None + + is_first_block = self._prev_block is None + prev_block_hash = ( + None if is_first_block else + self._prev_block.content_hash # type: ignore + ) + + # Previous block exists but does not yet have a hash. + # Return no hash in this case. + if prev_block_hash is None and not is_first_block: + return None + + self._cached_content_hash = PrefixCachingBlock.hash_block_tokens( + is_first_block, + prev_block_hash, + cur_block_token_ids=self.token_ids) + return self._cached_content_hash + + @staticmethod + def hash_block_tokens(is_first_block: bool, prev_block_hash: Optional[int], + cur_block_token_ids: List[int]) -> int: + """Computes a hash value corresponding to the contents of a block and + the contents of the preceding block(s). The hash value is used for + prefix caching. + + NOTE: Content-based hashing does not yet support LoRA. + + Parameters: + - is_first_block (bool): A flag indicating if the block is the first in + the sequence. + - prev_block_hash (Optional[int]): The hash of the previous block. None + if this is the first block. + - cur_block_token_ids (List[int]): A list of token ids in the current + block. The current block is assumed to be full. + + Returns: + - int: The computed hash value for the block. + """ + assert (prev_block_hash is None) == is_first_block + return hash((is_first_block, prev_block_hash, *cur_block_token_ids)) + + +class ComputedBlocksTracker: + """Handles caching of per-sequence computed block ids. + When a sequence appears for the first time, it traverses all of the + blocks and detects the prefix of blocks that is computed. On the + subsequent times, it only traverses the new blocks that were added + and updates the already recorded prefix of blocks with the newly + computed blocks. + + To avoid redundant traversals, the algorithm also detects when there + is a "gap" in the computed prefix. For example, if we have blocks = + [1,2,3,4,5], and we have detected [1,2,3] as the computed prefix, then + we won't try to add more computed blocks to [1,2,3] in this sequence + iteration, and will add more computed blocks only after the sequence is + freed and reused again. + + Note that currently, for a given sequence, we also skip the last + block id for caching purposes, to avoid caching of a full sequence + """ + + def __init__(self, allocator): + self._allocator = allocator + self._cached_computed_seq_blocks: Dict[int, Tuple[List[int], + bool]] = {} + + def add_seq(self, seq_id: int) -> None: + """Start tracking seq_id + """ + assert seq_id not in self._cached_computed_seq_blocks + self._cached_computed_seq_blocks[seq_id] = ([], False) + + def remove_seq(self, seq_id: int) -> None: + """Stop tracking seq_id + """ + assert seq_id in self._cached_computed_seq_blocks + del self._cached_computed_seq_blocks[seq_id] + + def get_cached_computed_blocks_and_update( + self, seq_id: int, block_ids: List[int]) -> List[int]: + """ Look at the class documentation for details + """ + # Ensure seq_id is already tracked + assert seq_id in self._cached_computed_seq_blocks + + # Get cached data (may be empty on the first time) + prev_computed_block_ids, has_gap = self._cached_computed_seq_blocks[ + seq_id] + + if has_gap: + # When gap is detected, we do not add more computed blocks at this + # sequence iteration + return prev_computed_block_ids + + # We do not consider the last block id for caching purposes. + num_cur_blocks = len(block_ids) - 1 + assert num_cur_blocks >= 0 + + if len(prev_computed_block_ids) >= num_cur_blocks: + # Cache HIT + assert len(prev_computed_block_ids) == num_cur_blocks + return prev_computed_block_ids + + # If here, then we may possibly add more computed blocks. As a result, + # traverse the additional blocks after prev_computed_block_ids to + # detect more computed blocks and add them. + + # Incremental init for seq_id => Look only at the new blocks + computed_block_ids = self._allocator.get_computed_block_ids( # noqa: E501 + prev_computed_block_ids, + block_ids, + skip_last_block_id= + True, # We skip last block id to avoid caching of full seq + ) + + # Detect if there is a "gap" + has_gap = len(computed_block_ids) < num_cur_blocks + + # Record + self._cached_computed_seq_blocks[seq_id] = (computed_block_ids, + has_gap) + + return computed_block_ids + + +class LastAccessBlocksTracker: + """Manages the last access time of the tracked sequences, in order to allow + an efficient update of allocator's block last access times + """ + + def __init__(self, allocator): + self._allocator = allocator + self._seq_last_access: Dict[int, Optional[float]] = {} + + def add_seq(self, seq_id: int) -> None: + """Start tracking seq_id + """ + assert seq_id not in self._seq_last_access + self._seq_last_access[seq_id] = None + + def remove_seq(self, seq_id: int) -> None: + """Stop tracking seq_id + """ + assert seq_id in self._seq_last_access + del self._seq_last_access[seq_id] + + def update_last_access(self, seq_id: int, time: float) -> None: + assert seq_id in self._seq_last_access + self._seq_last_access[seq_id] = time + + def update_seq_blocks_last_access(self, seq_id: int, + block_ids: List[int]) -> None: + assert seq_id in self._seq_last_access + + ts = self._seq_last_access[seq_id] + + if ts is None: + # No last access was recorded, no need to update. + return + + self._allocator.mark_blocks_as_accessed(block_ids, ts) + + +def assert_prefix_caching_block_or_none(block: Optional[Block]): + if block is None: + return + assert isinstance(block, + PrefixCachingBlock), "Got block = {}".format(block) diff --git a/vllm/core/block/utils.py b/vllm/core/block/utils.py new file mode 100644 index 00000000..28839437 --- /dev/null +++ b/vllm/core/block/utils.py @@ -0,0 +1,48 @@ +"""Block manager utils.""" +from vllm.sequence import SequenceGroup +from vllm.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE, + STR_NOT_IMPL_ENC_DEC_SWA) + + +def _get_block_mgr_sliding_window_attr(block_mgr): + ''' + BlockManagerV1 and BlockManagerV2 have slightly different + members related to sliding window attention (SWA). This + function extracts the appropriate member to use for determining + whether SWA is enabled. + + Arguments: + + * block_mgr: BlockManagerV1 or BlockManagerV2 instance + ''' + + if hasattr(block_mgr, 'block_sliding_window'): + return block_mgr.block_sliding_window + if hasattr(block_mgr, 'max_block_sliding_window'): + return block_mgr.max_block_sliding_window + + raise AttributeError("Block manager instance has neither " + \ + "block_sliding_window nor " + \ + "max_block_sliding_window attributes.") + + +def check_no_caching_or_swa_for_blockmgr_encdec( + block_mgr, seq_group: SequenceGroup) -> None: + ''' + Enforce that prefix caching & sliding-window attention (SWA) + are currently unsupported *specifically* for encoder/decoder models. + + Raises NotImplementedError if unsupported scenario is detected. + + Arguments: + + * block_mgr: BlockSpaceManager instance + * seq_group: SequenceGroup passed to block_mgr + ''' + + if seq_group.is_encoder_decoder(): + if _get_block_mgr_sliding_window_attr(block_mgr) is not None: + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_SWA) + + if block_mgr.enable_caching: + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE) diff --git a/vllm/core/block_manager_v1.py b/vllm/core/block_manager_v1.py new file mode 100644 index 00000000..8bc0ce2b --- /dev/null +++ b/vllm/core/block_manager_v1.py @@ -0,0 +1,743 @@ +"""A block manager that manages token blocks.""" +import math +from abc import ABC, abstractmethod +from itertools import count, takewhile +from os.path import commonprefix +from typing import Dict, List, Optional +from typing import Sequence as GenericSequence +from typing import Set, Tuple + +from vllm.block import BlockTable, PhysicalTokenBlock +from vllm.core.block.common import CacheMetricData +from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec +from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor +from vllm.core.interfaces import AllocStatus, BlockSpaceManager +from vllm.logger import init_logger +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus +from vllm.utils import Device + +logger = init_logger(__name__) + + +class BlockAllocatorBase(ABC): + """Manages free physical token blocks for a device. + + The allocator maintains a list of free blocks and allocates a block when + requested. When a block is freed, its reference count is decremented. If + the reference count becomes zero, the block is added back to the free list. + """ + + @abstractmethod + def __init__(self, + device: Device, + block_size: int, + num_blocks: int, + eviction_policy: EvictionPolicy = EvictionPolicy.LRU): + pass + + @abstractmethod + def allocate(self, + block_hash: Optional[int] = None, + num_hashed_tokens: int = 0) -> PhysicalTokenBlock: + pass + + @abstractmethod + def free(self, block: PhysicalTokenBlock) -> None: + pass + + @abstractmethod + def get_num_free_blocks(self) -> int: + pass + + @abstractmethod + def get_num_total_blocks(self) -> int: + pass + + @abstractmethod + def contains_block(self, block_hash: int) -> bool: + pass + + @abstractmethod + def update_hash(self, block_hash: int, block: PhysicalTokenBlock): + pass + + @abstractmethod + def get_prefix_cache_hit_rate(self) -> float: + """Prefix cache hit rate. -1 means not supported or disabled.""" + pass + + +class CachedBlockAllocator(BlockAllocatorBase): + """Manages free physical token blocks for a device. + + The allocator maintains a list of free blocks and allocates a block when + requested. When a block is freed, its reference count is decremented. If + the reference count becomes zero, the block is added back to the free list. + """ + + def __init__(self, + device: Device, + block_size: int, + num_blocks: int, + eviction_policy: EvictionPolicy = EvictionPolicy.LRU) -> None: + self.device = device + self.block_size = block_size + self.num_blocks = num_blocks + + self.current_num_blocks = 0 + self.cached_blocks: Dict[int, PhysicalTokenBlock] = {} + + self.evictor: Evictor = make_evictor(eviction_policy) + + self.default_hash_ctr = count() + + self.cache_metric_data = CacheMetricData() + + def allocate_block(self, block_hash: int, + num_hashed_tokens: int) -> PhysicalTokenBlock: + if self.current_num_blocks == self.num_blocks: + block = self.evictor.evict() + block.block_hash = block_hash + block.num_hashed_tokens = num_hashed_tokens + return block + block = PhysicalTokenBlock(device=self.device, + block_number=self.current_num_blocks, + block_size=self.block_size, + block_hash=block_hash, + num_hashed_tokens=num_hashed_tokens) + self.current_num_blocks += 1 + return block + + def allocate(self, + block_hash: Optional[int] = None, + num_hashed_tokens: int = 0) -> PhysicalTokenBlock: + if block_hash is None: + block_hash = next(self.default_hash_ctr) + + if block_hash in self.evictor: + assert block_hash not in self.cached_blocks + block = self.evictor.remove(block_hash) + assert block.ref_count == 0 + self.cached_blocks[block_hash] = block + + if block_hash in self.cached_blocks: + self.cache_metric_data.query(hit=True) + else: + self.cache_metric_data.query(hit=False) + self.cached_blocks[block_hash] = self.allocate_block( + block_hash, num_hashed_tokens) + block = self.cached_blocks[block_hash] + assert block.block_hash == block_hash + block.ref_count += 1 + return block + + def free(self, block: PhysicalTokenBlock) -> None: + if block.ref_count == 0: + raise ValueError(f"Double free! {block} is already freed.") + block.ref_count -= 1 + if block.ref_count == 0: + assert block.block_hash not in self.evictor + self.evictor.add(block) + + # Remove the block from the cached_blocks + del self.cached_blocks[block.block_hash] + + def get_num_free_blocks(self) -> int: + return (self.num_blocks - self.current_num_blocks + + self.evictor.num_blocks) + + def get_num_total_blocks(self) -> int: + return self.num_blocks + + def contains_block(self, block_hash: int) -> bool: + return block_hash in self.cached_blocks or block_hash in self.evictor + + def update_hash(self, block_hash: int, block: PhysicalTokenBlock): + # Update the hash of block and the cached_blocks dictionary. + assert not self.contains_block(block_hash) + old_hash = block.block_hash + block.block_hash = block_hash + del self.cached_blocks[old_hash] + self.cached_blocks[block_hash] = block + + def get_prefix_cache_hit_rate(self) -> float: + return self.cache_metric_data.get_hit_rate() + + +class UncachedBlockAllocator(BlockAllocatorBase): + """Manages free physical token blocks for a device. + + The allocator maintains a list of free blocks and allocates a block when + requested. When a block is freed, its reference count is decremented. If + the reference count becomes zero, the block is added back to the free list. + """ + + def __init__( + self, + device: Device, + block_size: int, + num_blocks: int, + ) -> None: + self.device = device + self.block_size = block_size + self.num_blocks = num_blocks + + # Initialize the free blocks. + self.free_blocks: List[PhysicalTokenBlock] = [] + for i in range(num_blocks): + block = PhysicalTokenBlock(device=device, + block_number=i, + block_size=block_size, + block_hash=-1, + num_hashed_tokens=0) + self.free_blocks.append(block) + + def allocate(self, + block_hash: Optional[int] = None, + num_hashed_tokens: int = 0) -> PhysicalTokenBlock: + if not self.free_blocks: + raise ValueError("Out of memory! No free blocks are available.") + block = self.free_blocks.pop() + block.ref_count = 1 + return block + + def free(self, block: PhysicalTokenBlock) -> None: + if block.ref_count == 0: + raise ValueError(f"Double free! {block} is already freed.") + block.ref_count -= 1 + if block.ref_count == 0: + self.free_blocks.append(block) + + def get_num_free_blocks(self) -> int: + return len(self.free_blocks) + + def get_num_total_blocks(self) -> int: + return self.num_blocks + + def contains_block(self, block_hash: int) -> bool: + raise NotImplementedError( + "Invalid codepath for uncached block allocator.") + + def update_hash(self, block_hash: int, block: PhysicalTokenBlock): + raise NotImplementedError( + "Invalid codepath for uncached block allocator.") + + def get_prefix_cache_hit_rate(self) -> float: + return -1 + + +class BlockSpaceManagerV1(BlockSpaceManager): + """Manages the mapping between logical and physical token blocks.""" + + def __init__( + self, + block_size: int, + num_gpu_blocks: int, + num_cpu_blocks: int, + watermark: float = 0.01, + sliding_window: Optional[int] = None, + enable_caching: bool = False, + ) -> None: + self.block_size = block_size + self.num_total_gpu_blocks = num_gpu_blocks + self.num_total_cpu_blocks = num_cpu_blocks + + if enable_caching and sliding_window is not None: + raise NotImplementedError( + "Sliding window is not allowed with prefix caching enabled!") + + self.block_sliding_window = None + if sliding_window is not None: + # Round up to nearest block size to regularize sliding window + # allocation sizes. + self.block_sliding_window = math.ceil(sliding_window / block_size) + + self.watermark = watermark + assert watermark >= 0.0 + + self.enable_caching = enable_caching + + self.watermark_blocks = int(watermark * num_gpu_blocks) + + if self.enable_caching: + logger.info("Automatic prefix caching is enabled.") + self.gpu_allocator: BlockAllocatorBase = CachedBlockAllocator( + Device.GPU, block_size, num_gpu_blocks) + self.cpu_allocator: BlockAllocatorBase = CachedBlockAllocator( + Device.CPU, block_size, num_cpu_blocks) + else: + self.gpu_allocator = UncachedBlockAllocator( + Device.GPU, block_size, num_gpu_blocks) + self.cpu_allocator = UncachedBlockAllocator( + Device.CPU, block_size, num_cpu_blocks) + # Mapping: seq_id -> BlockTable. + self.block_tables: Dict[int, BlockTable] = {} + + # Mapping: req_id -> BlockTable + # Note that each SequenceGroup has a unique + # request ID + self.cross_block_tables: Dict[str, BlockTable] = {} + + def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int: + return 0 if seq is None else seq.n_blocks + + def can_allocate(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> AllocStatus: + # FIXME(woosuk): Here we assume that all sequences in the group share + # the same prompt. This may not be true for preempted sequences. + + assert (num_lookahead_slots == 0 + ), "lookahead allocation not supported in BlockSpaceManagerV1" + + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group) + + self_num_required_blocks = self._get_seq_num_required_blocks( + seq_group.get_seqs(status=SequenceStatus.WAITING)[0]) + cross_num_required_blocks = self._get_seq_num_required_blocks( + seq_group.get_encoder_seq()) + num_required_blocks = self_num_required_blocks + \ + cross_num_required_blocks + + if self.block_sliding_window is not None: + + num_required_blocks = min(num_required_blocks, + self.block_sliding_window) + num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks() + + # Use watermark to avoid frequent cache eviction. + if (self.num_total_gpu_blocks - num_required_blocks < + self.watermark_blocks): + return AllocStatus.NEVER + if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks: + return AllocStatus.OK + else: + return AllocStatus.LATER + + def _allocate_sequence(self, \ + seq: Optional[Sequence], \ + ref_count: int, \ + is_encoder_decoder: bool = True) -> BlockTable: + # Allocate new physical token blocks that will store the prompt tokens. + num_prompt_blocks = self._get_seq_num_required_blocks(seq) + + block_table: BlockTable = BlockTable() + assert seq is not None + for logical_idx in range(num_prompt_blocks): + if (self.block_sliding_window is not None + and logical_idx >= self.block_sliding_window): + block = block_table[logical_idx % self.block_sliding_window] + # Set the reference counts of the token blocks. + block.ref_count = ref_count + elif not is_encoder_decoder and self.enable_caching: + block = self.gpu_allocator.allocate( + seq.hash_of_block(logical_idx), + seq.num_hashed_tokens_of_block(logical_idx)) + else: + block = self.gpu_allocator.allocate() + # Set the reference counts of the token blocks. + block.ref_count = ref_count + block_table.append(block) + + return block_table + + def allocate(self, seq_group: SequenceGroup) -> None: + is_encoder_decoder = seq_group.is_encoder_decoder() + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group) + + # Allocate decoder sequences + # + # NOTE: Here we assume that all sequences in the group have the same + # decoder prompt. + wait_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING) + seq = wait_seqs[0] + block_table: BlockTable = \ + self._allocate_sequence(seq, + seq_group.num_seqs(), + is_encoder_decoder) + + # Assign the self-attention block tables for each sequence. + if len(wait_seqs) == 1: + self.block_tables[seq.seq_id] = block_table + else: + for seq in wait_seqs: + self.block_tables[seq.seq_id] = block_table.copy() + + # Allocate encoder sequence + if is_encoder_decoder: + # A SequenceGroup has only a single encoder sequence (at most), + # thus allocate with a ref count of 1 + block_table = self._allocate_sequence(seq_group.get_encoder_seq(), + 1, is_encoder_decoder) + # Assign the cross-attention block table for the SequenceGroup. + self.cross_block_tables[seq_group.request_id] = block_table + + def can_append_slots(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> bool: + assert (num_lookahead_slots == 0 + ), "lookahead allocation not supported in BlockSpaceManagerV1" + + # Simple heuristic: If there is at least one free block + # for each sequence, we can append. + num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks() + num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING) + return num_seqs <= num_free_gpu_blocks + + def _promote_last_block( + self, + seq: Sequence, + last_block: PhysicalTokenBlock, + ) -> PhysicalTokenBlock: + assert self.enable_caching + + # Compute a new hash for the block so that it can be shared by other + # Sequences + new_hash = seq.hash_of_block(seq.n_blocks - 1) + + # if new_hash is already in the cached table, then free last_block + # and return the cached version + if self.gpu_allocator.contains_block(new_hash): + self.gpu_allocator.free(last_block) + return self.gpu_allocator.allocate(new_hash) + else: + self.gpu_allocator.update_hash(new_hash, last_block) + return last_block + + def _is_last_block_full( + self, + seq: Sequence, + ) -> bool: + token_ids_len = seq.data.get_len() + return token_ids_len > 0 and token_ids_len % seq.block_size == 0 + + def _maybe_promote_last_block( + self, + seq: Sequence, + last_block: PhysicalTokenBlock, + ) -> PhysicalTokenBlock: + if self._is_last_block_full(seq): + return self._promote_last_block(seq, last_block) + else: + return last_block + + def _allocate_last_physical_block( + self, + seq: Sequence, + ) -> PhysicalTokenBlock: + # Called before a new block is appended. + # This is in charge of allocating a new physical block (to be appended). + + # None if the last block is not full. Otherwise, we set it to the + # content hash. + if not self.enable_caching: + return self.gpu_allocator.allocate() + block_hash: Optional[int] = None + n_blocks = seq.n_blocks + if (self._is_last_block_full(seq)): + block_hash = seq.hash_of_block(n_blocks - 1) + num_hashed_tokens = seq.num_hashed_tokens_of_block(n_blocks - 1) + + # num_hashed_tokens is used to compute future hashes + # (e.g. in the hashing function, it is used to ask the sequence for + # prefix tokens) + new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens) + + # If the block_hash is None, then the block is not full. + # If the block is not full, then we expect it to have a refcount of 1. + if block_hash is None: + assert new_block.ref_count == 1 + return new_block + + def append_slots( + self, + seq: Sequence, + num_lookahead_slots: int = 0, + ) -> List[Tuple[int, int]]: + """Allocate a physical slot for a new token.""" + n_blocks = seq.n_blocks + block_table = self.block_tables[seq.seq_id] + # If we need to allocate a new physical block + if len(block_table) < n_blocks: + # Currently this code only supports adding one physical block + assert len(block_table) == n_blocks - 1 + + if (self.block_sliding_window + and len(block_table) >= self.block_sliding_window): + # reuse a block + block_table.append(block_table[len(block_table) % + self.block_sliding_window]) + else: + # The sequence hash a new logical block. + # Allocate a new physical block. + new_block = self._allocate_last_physical_block(seq) + block_table.append(new_block) + return [] + + # We want to append the token to the last physical block. + last_block = block_table[-1] + assert last_block.device == Device.GPU + if last_block.ref_count == 1: + # Not shared with other sequences. Appendable. + if self.enable_caching: + # If the last block is now complete, we may reuse an old block + # to save memory. + maybe_new_block = self._maybe_promote_last_block( + seq, last_block) + block_table[-1] = maybe_new_block + return [] + else: + # The last block is shared with other sequences. + # Copy on Write: Allocate a new block and copy the tokens. + new_block = self._allocate_last_physical_block(seq) + + block_table[-1] = new_block + self.gpu_allocator.free(last_block) + return [(last_block.block_number, new_block.block_number)] + + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None: + # NOTE: fork does not allocate a new physical block. + # Thus, it is always safe from OOM. + if parent_seq.seq_id not in self.block_tables: + # Parent sequence has either been freed or never existed. + return + src_block_table = self.block_tables[parent_seq.seq_id] + self.block_tables[child_seq.seq_id] = src_block_table.copy() + + # When using a sliding window, blocks will be eventually reused. + # In this case the block tables will contain repeated blocks. + # When forking, we must make sure that each block's `ref_count` + # is only incremented by one, so we deduplicate them by wrapping + # them in a set. + for block in set(src_block_table): + block.ref_count += 1 + + def _get_physical_blocks( + self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]: + + # NOTE: Here, we assume that the physical blocks are only shared by + # the sequences in the same group. + request_id = seq_group.request_id + blocks: Set[PhysicalTokenBlock] = set() + for seq in seq_group.get_seqs(): + if seq.is_finished(): + continue + blocks.update(self.block_tables[seq.seq_id]) + # Cross-attention blocks + if seq_group.is_encoder_decoder(): + blocks.update(self.cross_block_tables[request_id]) + return list(blocks) + + def can_swap_in(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> AllocStatus: + assert (num_lookahead_slots == 0 + ), "BlockSpaceManagerV1 does not support lookahead allocation" + + blocks = self._get_physical_blocks(seq_group) + num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED) + if seq_group.is_encoder_decoder(): + num_swapped_seqs += 1 + num_free_blocks = self.gpu_allocator.get_num_free_blocks() + # NOTE: Conservatively, we assume that every sequence will allocate + # at least one free block right after the swap-in. + # NOTE: This should match the logic in can_append_slot(). + num_required_blocks = len(blocks) + num_swapped_seqs + if self.gpu_allocator.get_num_total_blocks() < num_required_blocks: + return AllocStatus.NEVER + elif num_free_blocks - num_required_blocks >= self.watermark_blocks: + return AllocStatus.OK + else: + return AllocStatus.LATER + + def _swap_block_table( + self, block_table: BlockTable, src_allocator: BlockAllocatorBase, + dest_allocator: BlockAllocatorBase, + mapping: Dict[PhysicalTokenBlock, + PhysicalTokenBlock]) -> BlockTable: + new_block_table: BlockTable = BlockTable() + + for from_block in block_table: + if from_block in mapping: + to_block = mapping[from_block] + to_block.ref_count += 1 + else: + to_block = dest_allocator.allocate( + from_block.block_hash, from_block.num_hashed_tokens) + mapping[from_block] = to_block + new_block_table.append(to_block) + # Free the source block swapped in to destination. + src_allocator.free(from_block) + + return new_block_table + + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + + request_id = seq_group.request_id + + # CPU block -> GPU block. + # dict is efficient in lookup `if cpu_block in mapping` + mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {} + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED): + self.block_tables[seq.seq_id] = \ + self._swap_block_table(self.block_tables[seq.seq_id], + self.cpu_allocator, self.gpu_allocator, + mapping) + + if seq_group.is_encoder_decoder(): + self.cross_block_tables[request_id] = \ + self._swap_block_table(self.cross_block_tables[request_id], + self.cpu_allocator, + self.gpu_allocator, + mapping) + + return [(cpu_block.block_number, gpu_block.block_number) + for cpu_block, gpu_block in mapping.items()] + + def can_swap_out(self, seq_group: SequenceGroup) -> bool: + blocks = self._get_physical_blocks(seq_group) + return len(blocks) <= self.cpu_allocator.get_num_free_blocks() + + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + request_id = seq_group.request_id + + # GPU block -> CPU block. + # dict is efficient in lookup `if gpu_block in mapping` + mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {} + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + self.block_tables[seq.seq_id] = \ + self._swap_block_table(self.block_tables[seq.seq_id], + self.gpu_allocator, self.cpu_allocator, + mapping) + + if seq_group.is_encoder_decoder(): + self.cross_block_tables[request_id] = \ + self._swap_block_table(self.cross_block_tables[request_id], + self.gpu_allocator, + self.cpu_allocator, + mapping) + + return [(cpu_block.block_number, gpu_block.block_number) + for cpu_block, gpu_block in mapping.items()] + + def _free_block_table(self, block_table: BlockTable) -> None: + # when using a sliding window, each seq will only use up + # to `self.block_sliding_window` blocks. When freeing + # the block table, we must make sure to not free blocks more + # than once. If no sliding window is used, there is no block + # reuse in the block table, so we must free all blocks. + blocks_to_free = (block_table[-self.block_sliding_window:] + if self.block_sliding_window is not None else + block_table) + for block in set(blocks_to_free): + if block.device == Device.GPU: + self.gpu_allocator.free(block) + else: + self.cpu_allocator.free(block) + + def free(self, seq: Sequence) -> None: + if seq.seq_id not in self.block_tables: + # Already freed or haven't been scheduled yet. + return + block_table = self.block_tables[seq.seq_id] + self._free_block_table(block_table) + del self.block_tables[seq.seq_id] + + def free_cross(self, seq_group: SequenceGroup) -> None: + if seq_group.request_id not in self.cross_block_tables: + # Already freed or hasn't ben scheduled yet. + return + block_table = self.cross_block_tables[seq_group.request_id] + self._free_block_table(block_table) + del self.cross_block_tables[seq_group.request_id] + + def reset(self) -> None: + # Free decoder block tables + for block_table in self.block_tables.values(): + self._free_block_table(block_table) + self.block_tables.clear() + # Free cross-attention block tables + for block_table in self.cross_block_tables.values(): + self._free_block_table(block_table) + self.cross_block_tables.clear() + + def get_block_table(self, seq: Sequence) -> List[int]: + return self.block_tables[seq.seq_id].ids() + + def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]: + block_table = self.cross_block_tables[seq_group.request_id] + return [block.block_number for block in block_table] + + def get_num_free_gpu_blocks(self) -> int: + return self.gpu_allocator.get_num_free_blocks() + + def get_num_free_cpu_blocks(self) -> int: + return self.cpu_allocator.get_num_free_blocks() + + def access_all_blocks_in_seq( + self, + seq: Sequence, + access_time: float, + ) -> None: + if self.enable_caching: + # Update the last accessed time of all the blocks accessed + # in this step. + block_table = self.block_tables[seq.seq_id] + for block in block_table: + block.last_accessed = access_time + + def compute_full_blocks_in_seq(self, seq: Sequence, token_chunk_size: int): + if seq.seq_id not in self.block_tables: + return + + # When chunked prefill is enabled, the computed full blocks + # should be calculated based on the number of computed tokens. + max_computed_tokens = (seq.data.get_num_computed_tokens() + + token_chunk_size) + computed_full_blocks = max_computed_tokens // self.block_size + + block_table = self.block_tables[seq.seq_id] + if computed_full_blocks == 0: + return + for i in reversed(range(computed_full_blocks)): + if block_table[i].computed: + break + block_table[i].computed = True + + def get_all_computed_blocks(self, seq: Sequence) -> List[int]: + if seq.seq_id not in self.block_tables: + return [] + block_table = self.block_tables[seq.seq_id] + # NOTE We exclude the last block to avoid the case where the entire + # prompt is cached. This would cause erroneous behavior in model + # runner. + return [ + b.block_number + for b in takewhile(lambda b: b.computed, block_table[:-1]) + ] + + def get_common_computed_block_ids( + self, seqs: List[Sequence]) -> GenericSequence[int]: + """Return the block ids that are common for a given sequence group. + + Used in prefill (can skip prefill of some blocks). + """ + # Can return non-empty result only with prefix caching enabled. + if not self.enable_caching: + return [] + + ids_list = [self.get_all_computed_blocks(seq) for seq in seqs] + return commonprefix([ids for ids in ids_list if ids != []]) + + def mark_blocks_as_computed(self, seq_group: SequenceGroup, + token_chunk_size: int): + if self.enable_caching: + for seq in seq_group.get_seqs(): + self.compute_full_blocks_in_seq(seq, token_chunk_size) + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + if device == Device.GPU: + return self.gpu_allocator.get_prefix_cache_hit_rate() + if device == Device.CPU: + return self.cpu_allocator.get_prefix_cache_hit_rate() + raise ValueError(f"Invalid device: {device}") diff --git a/vllm/core/block_manager_v2.py b/vllm/core/block_manager_v2.py new file mode 100644 index 00000000..cb047c83 --- /dev/null +++ b/vllm/core/block_manager_v2.py @@ -0,0 +1,505 @@ +"""A block manager that manages token blocks.""" +from typing import Dict, List, Optional +from typing import Sequence as GenericSequence +from typing import Tuple + +from vllm.core.block.block_table import BlockTable +from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator +from vllm.core.block.interfaces import Block +from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker, + LastAccessBlocksTracker) +from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec +from vllm.core.interfaces import AllocStatus, BlockSpaceManager +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus +from vllm.utils import Device + +SeqId = int +EncoderSeqId = str + + +class BlockSpaceManagerV2(BlockSpaceManager): + """BlockSpaceManager which manages the allocation of KV cache. + + It owns responsibility for allocation, swapping, allocating memory for + autoregressively-generated tokens, and other advanced features such as + prefix caching, forking/copy-on-write, and sliding-window memory allocation. + + This class implements the design described in + https://github.com/vllm-project/vllm/pull/3492. + + Lookahead slots + The block manager has the notion of a "lookahead slot". These are slots + in the KV cache that are allocated for a sequence. Unlike the other + allocated slots, the content of these slots is undefined -- the worker + may use the memory allocations in any way. + + In practice, a worker could use these lookahead slots to run multiple + forward passes for a single scheduler invocation. Each successive + forward pass would write KV activations to the corresponding lookahead + slot. This allows low inter-token latency use-cases, where the overhead + of continuous batching scheduling is amortized over >1 generated tokens. + + Speculative decoding uses lookahead slots to store KV activations of + proposal tokens. + + See https://github.com/vllm-project/vllm/pull/3250 for more information + on lookahead scheduling. + + Args: + block_size (int): The size of each memory block. + num_gpu_blocks (int): The number of memory blocks allocated on GPU. + num_cpu_blocks (int): The number of memory blocks allocated on CPU. + watermark (float, optional): The threshold used for memory swapping. + Defaults to 0.01. + sliding_window (Optional[int], optional): The size of the sliding + window. Defaults to None. + enable_caching (bool, optional): Flag indicating whether caching is + enabled. Defaults to False. + """ + + def __init__( + self, + block_size: int, + num_gpu_blocks: int, + num_cpu_blocks: int, + watermark: float = 0.01, + sliding_window: Optional[int] = None, + enable_caching: bool = False, + ) -> None: + self.block_size = block_size + self.num_total_gpu_blocks = num_gpu_blocks + self.num_total_cpu_blocks = num_cpu_blocks + + self.sliding_window = sliding_window + # max_block_sliding_window is the max number of blocks that need to be + # allocated + self.max_block_sliding_window = None + if sliding_window is not None: + # +1 here because // rounds down + num_blocks = sliding_window // block_size + 1 + # +1 here because the last block may not be full, + # and so the sequence stretches one more block at the beginning + # For example, if sliding_window is 3 and block_size is 4, + # we may need 2 blocks when the second block only holds 1 token. + self.max_block_sliding_window = num_blocks + 1 + + self.watermark = watermark + assert watermark >= 0.0 + + self.enable_caching = enable_caching + + self.watermark_blocks = int(watermark * num_gpu_blocks) + + self.block_allocator = CpuGpuBlockAllocator.create( + allocator_type="prefix_caching" if enable_caching else "naive", + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks, + block_size=block_size, + ) + + self.block_tables: Dict[SeqId, BlockTable] = {} + self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {} + + self._computed_blocks_tracker = ComputedBlocksTracker( + self.block_allocator) + self._last_access_blocks_tracker = LastAccessBlocksTracker( + self.block_allocator) + + def can_allocate(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> AllocStatus: + # FIXME(woosuk): Here we assume that all sequences in the group share + # the same prompt. This may not be true for preempted sequences. + + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group) + + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0] + num_required_blocks = BlockTable.get_num_required_blocks( + seq.get_token_ids(), + block_size=self.block_size, + num_lookahead_slots=num_lookahead_slots, + ) + + if seq_group.is_encoder_decoder(): + encoder_seq = seq_group.get_encoder_seq() + assert encoder_seq is not None + num_required_blocks += BlockTable.get_num_required_blocks( + encoder_seq.get_token_ids(), + block_size=self.block_size, + ) + + if self.max_block_sliding_window is not None: + num_required_blocks = min(num_required_blocks, + self.max_block_sliding_window) + + num_free_gpu_blocks = self.block_allocator.get_num_free_blocks( + device=Device.GPU) + + # Use watermark to avoid frequent cache eviction. + if (self.num_total_gpu_blocks - num_required_blocks < + self.watermark_blocks): + return AllocStatus.NEVER + if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks: + return AllocStatus.OK + else: + return AllocStatus.LATER + + def _allocate_sequence(self, seq: Sequence) -> BlockTable: + block_table = BlockTable( + block_size=self.block_size, + block_allocator=self.block_allocator, + max_block_sliding_window=self.max_block_sliding_window, + ) + if seq.get_token_ids(): + # Add blocks to the block table only if the sequence is non empty. + block_table.allocate(seq.get_token_ids()) + + return block_table + + def allocate(self, seq_group: SequenceGroup) -> None: + + # Allocate self-attention block tables for decoder sequences + waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING) + assert not (set(seq.seq_id for seq in waiting_seqs) + & self.block_tables.keys()), "block table already exists" + + # NOTE: Here we assume that all sequences in the group have the same + # prompt. + seq = waiting_seqs[0] + block_table: BlockTable = self._allocate_sequence(seq) + self.block_tables[seq.seq_id] = block_table + + # Track seq + self._computed_blocks_tracker.add_seq(seq.seq_id) + self._last_access_blocks_tracker.add_seq(seq.seq_id) + + # Assign the block table for each sequence. + for seq in waiting_seqs[1:]: + self.block_tables[seq.seq_id] = block_table.fork() + + # Track seq + self._computed_blocks_tracker.add_seq(seq.seq_id) + self._last_access_blocks_tracker.add_seq(seq.seq_id) + + # Allocate cross-attention block table for encoder sequence + # + # NOTE: Here we assume that all sequences in the group have the same + # encoder prompt. + request_id = seq_group.request_id + + assert (request_id + not in self.cross_block_tables), \ + "block table already exists" + + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group) + + if seq_group.is_encoder_decoder(): + encoder_seq = seq_group.get_encoder_seq() + assert encoder_seq is not None + block_table = self._allocate_sequence(encoder_seq) + self.cross_block_tables[request_id] = block_table + + def can_append_slots(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> bool: + """Determine if there is enough space in the GPU KV cache to continue + generation of the specified sequence group. + + We use a worst-case heuristic: assume each touched block will require a + new allocation (either via CoW or new block). We can append slots if the + number of touched blocks is less than the number of free blocks. + + "Lookahead slots" are slots that are allocated in addition to the slots + for known tokens. The contents of the lookahead slots are not defined. + This is used by speculative decoding when speculating future tokens. + """ + + num_touched_blocks = 0 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + block_table = self.block_tables[seq.seq_id] + + num_touched_blocks += ( + block_table.get_num_blocks_touched_by_append_slots( + token_ids=block_table.get_unseen_token_ids( + seq.get_token_ids()), + num_lookahead_slots=num_lookahead_slots, + )) + + num_free_gpu_blocks = self.block_allocator.get_num_free_blocks( + Device.GPU) + return num_touched_blocks <= num_free_gpu_blocks + + def append_slots( + self, + seq: Sequence, + num_lookahead_slots: int, + ) -> List[Tuple[int, int]]: + + block_table = self.block_tables[seq.seq_id] + + block_table.append_token_ids( + token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()), + num_lookahead_slots=num_lookahead_slots, + num_computed_slots=seq.data.get_num_computed_tokens(), + ) + # Return any new copy-on-writes. + new_cows = self.block_allocator.clear_copy_on_writes() + return new_cows + + def free(self, seq: Sequence) -> None: + seq_id = seq.seq_id + + if seq_id not in self.block_tables: + # Already freed or haven't been scheduled yet. + return + + # Update seq block ids with the latest access time + self._last_access_blocks_tracker.update_seq_blocks_last_access( + seq_id, self.block_tables[seq.seq_id].physical_block_ids) + + # Untrack seq + self._last_access_blocks_tracker.remove_seq(seq_id) + self._computed_blocks_tracker.remove_seq(seq_id) + + # Free table/blocks + self.block_tables[seq_id].free() + del self.block_tables[seq_id] + + def free_cross(self, seq_group: SequenceGroup) -> None: + request_id = seq_group.request_id + if request_id not in self.cross_block_tables: + # Already freed or hasn't been scheduled yet. + return + self.cross_block_tables[request_id].free() + del self.cross_block_tables[request_id] + + def get_block_table(self, seq: Sequence) -> List[int]: + block_ids = self.block_tables[seq.seq_id].physical_block_ids + return block_ids # type: ignore + + def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]: + request_id = seq_group.request_id + assert request_id in self.cross_block_tables + block_ids = self.cross_block_tables[request_id].physical_block_ids + assert all(b is not None for b in block_ids) + return block_ids # type: ignore + + def access_all_blocks_in_seq(self, seq: Sequence, now: float): + if self.enable_caching: + # Record the latest access time for the sequence. The actual update + # of the block ids is deferred to the sequence free(..) call, since + # only during freeing of block ids, the blocks are actually added to + # the evictor (which is when the most updated time is required) + # (This avoids expensive calls to mark_blocks_as_accessed(..)) + self._last_access_blocks_tracker.update_last_access( + seq.seq_id, now) + + def mark_blocks_as_computed(self, seq_group: SequenceGroup, + token_chunk_size: int): + # If prefix caching is enabled, mark immutable blocks as computed + # right after they have been scheduled (for prefill). This assumes + # the scheduler is synchronous so blocks are actually computed when + # scheduling the next batch. + self.block_allocator.mark_blocks_as_computed([]) + + def get_common_computed_block_ids( + self, seqs: List[Sequence]) -> GenericSequence[int]: + """Determine which blocks for which we skip prefill. + + With prefix caching we can skip prefill for previously-generated blocks. + Currently, the attention implementation only supports skipping cached + blocks if they are a contiguous prefix of cached blocks. + + This method determines which blocks can be safely skipped for all + sequences in the sequence group. + """ + computed_seq_block_ids = [] + for seq in seqs: + computed_seq_block_ids.append( + self._computed_blocks_tracker. + get_cached_computed_blocks_and_update( + seq.seq_id, + self.block_tables[seq.seq_id].physical_block_ids)) + + # NOTE(sang): This assumes seq_block_ids doesn't contain any None. + return self.block_allocator.get_common_computed_block_ids( + computed_seq_block_ids) # type: ignore + + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None: + if parent_seq.seq_id not in self.block_tables: + # Parent sequence has either been freed or never existed. + return + src_block_table = self.block_tables[parent_seq.seq_id] + self.block_tables[child_seq.seq_id] = src_block_table.fork() + + # Track child seq + self._computed_blocks_tracker.add_seq(child_seq.seq_id) + self._last_access_blocks_tracker.add_seq(child_seq.seq_id) + + def can_swap_in(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> AllocStatus: + """Returns the AllocStatus for the given sequence_group + with num_lookahead_slots. + + Args: + sequence_group (SequenceGroup): The sequence group to swap in. + num_lookahead_slots (int): Number of lookahead slots used in + speculative decoding, default to 0. + + Returns: + AllocStatus: The AllocStatus for the given sequence group. + """ + return self._can_swap(seq_group, Device.GPU, SequenceStatus.SWAPPED, + num_lookahead_slots) + + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + """Returns the block id mapping (from CPU to GPU) generated by + swapping in the given seq_group with num_lookahead_slots. + + Args: + seq_group (SequenceGroup): The sequence group to swap in. + + Returns: + List[Tuple[int, int]]: The mapping of swapping block from CPU + to GPU. + """ + physical_block_id_mapping = [] + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED): + blocks = self.block_tables[seq.seq_id].blocks + if len(blocks) == 0: + continue + + seq_swap_mapping = self.block_allocator.swap(blocks=blocks, + src_device=Device.CPU, + dst_device=Device.GPU) + + # Refresh the block ids of the table (post-swap) + self.block_tables[seq.seq_id].update(blocks) + + seq_physical_block_id_mapping = { + self.block_allocator.get_physical_block_id( + Device.CPU, cpu_block_id): + self.block_allocator.get_physical_block_id( + Device.GPU, gpu_block_id) + for cpu_block_id, gpu_block_id in seq_swap_mapping.items() + } + + physical_block_id_mapping.extend( + list(seq_physical_block_id_mapping.items())) + + return physical_block_id_mapping + + def can_swap_out(self, seq_group: SequenceGroup) -> bool: + """Returns whether we can swap out the given sequence_group + with num_lookahead_slots. + + Args: + seq_group (SequenceGroup): The sequence group to swap in. + num_lookahead_slots (int): Number of lookahead slots used in + speculative decoding, default to 0. + + Returns: + bool: Whether it's possible to swap out current sequence group. + """ + alloc_status = self._can_swap(seq_group, Device.CPU, + SequenceStatus.RUNNING) + return alloc_status == AllocStatus.OK + + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + """Returns the block id mapping (from GPU to CPU) generated by + swapping out the given sequence_group with num_lookahead_slots. + + Args: + sequence_group (SequenceGroup): The sequence group to swap in. + + Returns: + List[Tuple[int, int]]: The mapping of swapping block from + GPU to CPU. + """ + physical_block_id_mapping = [] + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + blocks = self.block_tables[seq.seq_id].blocks + if len(blocks) == 0: + continue + + seq_swap_mapping = self.block_allocator.swap(blocks=blocks, + src_device=Device.GPU, + dst_device=Device.CPU) + + # Refresh the block ids of the table (post-swap) + self.block_tables[seq.seq_id].update(blocks) + + seq_physical_block_id_mapping = { + self.block_allocator.get_physical_block_id( + Device.GPU, gpu_block_id): + self.block_allocator.get_physical_block_id( + Device.CPU, cpu_block_id) + for gpu_block_id, cpu_block_id in seq_swap_mapping.items() + } + + physical_block_id_mapping.extend( + list(seq_physical_block_id_mapping.items())) + + return physical_block_id_mapping + + def get_num_free_gpu_blocks(self) -> int: + return self.block_allocator.get_num_free_blocks(Device.GPU) + + def get_num_free_cpu_blocks(self) -> int: + return self.block_allocator.get_num_free_blocks(Device.CPU) + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + return self.block_allocator.get_prefix_cache_hit_rate(device) + + def _can_swap(self, + seq_group: SequenceGroup, + device: Device, + status: SequenceStatus, + num_lookahead_slots: int = 0) -> AllocStatus: + """Returns the AllocStatus for swapping in/out the given sequence_group + on to the 'device'. + + Args: + sequence_group (SequenceGroup): The sequence group to swap in. + device (Device): device to swap the 'seq_group' on. + status (SequenceStatus): The status of sequence which is needed + for action. RUNNING for swap out and SWAPPED for swap in + num_lookahead_slots (int): Number of lookahead slots used in + speculative decoding, default to 0. + + Returns: + AllocStatus: The AllocStatus for swapping in/out the given + sequence_group on to the 'device'. + """ + # First determine the number of blocks that will be touched by this + # swap. Then verify if there are available blocks in the device + # to perform the swap. + num_blocks_touched = 0 + blocks: List[Block] = [] + for seq in seq_group.get_seqs(status=status): + block_table = self.block_tables[seq.seq_id] + if block_table.blocks is not None: + # Compute the number blocks to touch for the tokens to be + # appended. This does NOT include the full blocks that need + # to be touched for the swap. + num_blocks_touched += \ + block_table.get_num_blocks_touched_by_append_slots( + block_table.get_unseen_token_ids(seq.get_token_ids()), + num_lookahead_slots=num_lookahead_slots) + blocks.extend(block_table.blocks) + # Compute the number of full blocks to touch and add it to the + # existing count of blocks to touch. + num_blocks_touched += self.block_allocator.get_num_full_blocks_touched( + blocks, device=device) + + watermark_blocks = 0 + if device == Device.GPU: + watermark_blocks = self.watermark_blocks + + if self.block_allocator.get_num_total_blocks( + device) < num_blocks_touched: + return AllocStatus.NEVER + elif self.block_allocator.get_num_free_blocks( + device) - num_blocks_touched >= watermark_blocks: + return AllocStatus.OK + else: + return AllocStatus.LATER diff --git a/vllm/core/evictor_v1.py b/vllm/core/evictor_v1.py new file mode 100644 index 00000000..5db5a08a --- /dev/null +++ b/vllm/core/evictor_v1.py @@ -0,0 +1,106 @@ +import enum +from abc import ABC, abstractmethod +from typing import OrderedDict + +from vllm.block import PhysicalTokenBlock + + +class EvictionPolicy(enum.Enum): + """Enum for eviction policy used by make_evictor to instantiate the correct + Evictor subclass. + """ + LRU = enum.auto() + + +class Evictor(ABC): + """The Evictor subclasses should be used by the BlockAllocator class to + handle eviction of freed PhysicalTokenBlocks. + """ + + @abstractmethod + def __init__(self): + pass + + @abstractmethod + def __contains__(self, block_hash: int) -> bool: + pass + + @abstractmethod + def evict(self) -> PhysicalTokenBlock: + """Runs the eviction algorithm and returns the evicted block""" + pass + + @abstractmethod + def add(self, block: PhysicalTokenBlock): + """Adds block to the evictor, making it a candidate for eviction""" + pass + + @abstractmethod + def remove(self, block_hash: int) -> PhysicalTokenBlock: + """Simply removes the block with the hash value block_hash from the + evictor. Caller is responsible for making sure that block_hash is + contained in the evictor before calling remove. Should be used to + "bring back" blocks that have been freed but not evicted yet. + """ + pass + + @property + @abstractmethod + def num_blocks(self) -> int: + pass + + +class LRUEvictor(Evictor): + """Evicts in a least-recently-used order using the last_accessed timestamp + that's recorded in the PhysicalTokenBlock. If there are multiple blocks with + the same last_accessed time, then the one with the largest num_hashed_tokens + will be evicted. If two blocks each have the lowest last_accessed time and + highest num_hashed_tokens value, then one will be chose arbitrarily + """ + + def __init__(self): + self.free_table: OrderedDict[int, PhysicalTokenBlock] = OrderedDict() + + def __contains__(self, block_hash: int) -> bool: + return block_hash in self.free_table + + def evict(self) -> PhysicalTokenBlock: + if len(self.free_table) == 0: + raise ValueError("No usable cache memory left") + + evicted_block = next(iter(self.free_table.values())) + # The blocks with the lowest timestamps should be placed consecutively + # at the start of OrderedDict. Loop through all these blocks to + # find the one with maximum number of hashed tokens. + for _, block in self.free_table.items(): + if evicted_block.last_accessed < block.last_accessed: + break + if evicted_block.num_hashed_tokens < block.num_hashed_tokens: + evicted_block = block + + self.free_table.pop(evicted_block.block_hash) + + evicted_block.computed = False + return evicted_block + + def add(self, block: PhysicalTokenBlock): + self.free_table[block.block_hash] = block + + def remove(self, block_hash: int) -> PhysicalTokenBlock: + if block_hash not in self.free_table: + raise ValueError( + "Attempting to remove block that's not in the evictor") + block: PhysicalTokenBlock = self.free_table[block_hash] + self.free_table.pop(block_hash) + return block + + @property + def num_blocks(self) -> int: + return len(self.free_table) + + +def make_evictor(eviction_policy: EvictionPolicy) -> Evictor: + if eviction_policy == EvictionPolicy.LRU: + return LRUEvictor() + else: + raise ValueError(f"Unknown cache eviction policy: {eviction_policy}") diff --git a/vllm/core/evictor_v2.py b/vllm/core/evictor_v2.py new file mode 100644 index 00000000..0b943e6e --- /dev/null +++ b/vllm/core/evictor_v2.py @@ -0,0 +1,131 @@ +import enum +from abc import ABC, abstractmethod +from typing import OrderedDict, Tuple + + +class EvictionPolicy(enum.Enum): + """Enum for eviction policy used by make_evictor to instantiate the correct + Evictor subclass. + """ + LRU = enum.auto() + + +class Evictor(ABC): + """The Evictor subclasses should be used by the BlockAllocator class to + handle eviction of freed PhysicalTokenBlocks. + """ + + @abstractmethod + def __init__(self): + pass + + @abstractmethod + def __contains__(self, block_id: int) -> bool: + pass + + @abstractmethod + def evict(self) -> Tuple[int, int]: + """Runs the eviction algorithm and returns the evicted block's + content hash along with physical block id along with physical block id + """ + pass + + @abstractmethod + def add(self, block_id: int, content_hash: int, num_hashed_tokens: int, + last_accessed: float): + """Adds block to the evictor, making it a candidate for eviction""" + pass + + @abstractmethod + def update(self, block_id: int, last_accessed: float): + """Update corresponding block's access time in metadata""" + pass + + @abstractmethod + def remove(self, block_id: int): + """Remove a given block id from the cache.""" + pass + + @property + @abstractmethod + def num_blocks(self) -> int: + pass + + +class BlockMetaData(): + """Data structure for storing key data describe cached block, so that + evitor could use to make its decision which one to choose for eviction + + Here we use physical block id as the dict key, as there maybe several + blocks with the same content hash, but their physical id is unique. + """ + + def __init__(self, content_hash: int, num_hashed_tokens: int, + last_accessed: float): + self.content_hash = content_hash + self.num_hashed_tokens = num_hashed_tokens + self.last_accessed = last_accessed + + +class LRUEvictor(Evictor): + """Evicts in a least-recently-used order using the last_accessed timestamp + that's recorded in the PhysicalTokenBlock. If there are multiple blocks with + the same last_accessed time, then the one with the largest num_hashed_tokens + will be evicted. If two blocks each have the lowest last_accessed time and + highest num_hashed_tokens value, then one will be chose arbitrarily + """ + + def __init__(self): + self.free_table: OrderedDict[int, BlockMetaData] = OrderedDict() + + def __contains__(self, block_id: int) -> bool: + return block_id in self.free_table + + def evict(self) -> Tuple[int, int]: + if len(self.free_table) == 0: + raise ValueError("No usable cache memory left") + + evicted_block, evicted_block_id = None, None + # The blocks with the lowest timestamps should be placed consecutively + # at the start of OrderedDict. Loop through all these blocks to + # find the one with maximum number of hashed tokens. + for _id, block in self.free_table.items(): + if evicted_block is None: + evicted_block, evicted_block_id = block, _id + continue + if evicted_block.last_accessed < block.last_accessed: + break + if evicted_block.num_hashed_tokens < block.num_hashed_tokens: + evicted_block, evicted_block_id = block, _id + + assert evicted_block is not None + assert evicted_block_id is not None + self.free_table.pop(evicted_block_id) + + return evicted_block_id, evicted_block.content_hash + + def add(self, block_id: int, content_hash: int, num_hashed_tokens: int, + last_accessed: float): + self.free_table[block_id] = BlockMetaData(content_hash, + num_hashed_tokens, + last_accessed) + + def update(self, block_id: int, last_accessed: float): + self.free_table[block_id].last_accessed = last_accessed + + def remove(self, block_id: int): + if block_id not in self.free_table: + raise ValueError( + "Attempting to remove block that's not in the evictor") + self.free_table.pop(block_id) + + @property + def num_blocks(self) -> int: + return len(self.free_table) + + +def make_evictor(eviction_policy: EvictionPolicy) -> Evictor: + if eviction_policy == EvictionPolicy.LRU: + return LRUEvictor() + else: + raise ValueError(f"Unknown cache eviction policy: {eviction_policy}") diff --git a/vllm/core/interfaces.py b/vllm/core/interfaces.py new file mode 100644 index 00000000..9e1d1b02 --- /dev/null +++ b/vllm/core/interfaces.py @@ -0,0 +1,127 @@ +import enum +from abc import ABC, abstractmethod +from typing import List +from typing import Sequence as GenericSequence +from typing import Tuple + +from vllm.sequence import Sequence, SequenceGroup +from vllm.utils import Device + + +class AllocStatus(enum.Enum): + """Result for BlockSpaceManager.can_allocate + + 1. Ok: seq_group can be allocated now. + 2. Later: seq_group cannot be allocated. + The capacity of allocator is larger than seq_group required. + 3. Never: seq_group can never be allocated. + The seq_group is too large to allocated in GPU. + """ + OK = enum.auto() + LATER = enum.auto() + NEVER = enum.auto() + + +class BlockSpaceManager(ABC): + + @staticmethod + def get_block_space_manager_class(version: str): + version = version.lower() + + if version == "v1": + from vllm.core.block_manager_v1 import BlockSpaceManagerV1 + return BlockSpaceManagerV1 + + if version == "v2": + from vllm.core.block_manager_v2 import BlockSpaceManagerV2 + return BlockSpaceManagerV2 + + if version == "placeholder": + from vllm.core.placeholder_block_space_manager import ( + PlaceholderBlockSpaceManager) + return PlaceholderBlockSpaceManager + + raise ValueError(f"Unknown version {version=}") + + @abstractmethod + def can_allocate(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> AllocStatus: + pass + + @abstractmethod + def allocate(self, seq_group: SequenceGroup) -> None: + pass + + @abstractmethod + def can_append_slots(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> bool: + pass + + @abstractmethod + def append_slots( + self, + seq: Sequence, + num_lookahead_slots: int, + ) -> List[Tuple[int, int]]: + pass + + @abstractmethod + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None: + pass + + @abstractmethod + def can_swap_in(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> AllocStatus: + pass + + @abstractmethod + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + pass + + @abstractmethod + def can_swap_out(self, seq_group: SequenceGroup) -> bool: + pass + + @abstractmethod + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + pass + + @abstractmethod + def free(self, seq: Sequence) -> None: + pass + + @abstractmethod + def get_block_table(self, seq: Sequence) -> List[int]: + pass + + @abstractmethod + def get_num_free_gpu_blocks(self) -> int: + pass + + @abstractmethod + def get_num_free_cpu_blocks(self) -> int: + pass + + @abstractmethod + def access_all_blocks_in_seq( + self, + seq: Sequence, + access_time: float, + ) -> None: + pass + + @abstractmethod + def get_common_computed_block_ids( + self, seqs: List[Sequence]) -> GenericSequence[int]: + pass + + @abstractmethod + def mark_blocks_as_computed(self, seq_group: SequenceGroup, + token_chunk_size: int): + pass + + @abstractmethod + def get_prefix_cache_hit_rate(self, device: Device) -> float: + """Prefix cache hit rate. -1 means not supported or disabled.""" + pass diff --git a/vllm/core/placeholder_block_space_manager.py b/vllm/core/placeholder_block_space_manager.py new file mode 100644 index 00000000..a337392b --- /dev/null +++ b/vllm/core/placeholder_block_space_manager.py @@ -0,0 +1,91 @@ +from typing import List, Tuple + +from vllm.core.interfaces import AllocStatus, BlockSpaceManager +from vllm.sequence import Sequence, SequenceGroup +from vllm.utils import Device + + +class PlaceholderBlockSpaceManager(BlockSpaceManager): + """A version of BlockSpaceManager for use in environments + where block management is not required. + For example: embedding models or attention-free models like Mamba. + + This class provides the same interface as BlockSpaceManager, but its + methods perform no actions or return simple values like True in specific + actions. It's designed to be used in scenarios where the overhead of + block management is unnecessary, such as in an embedding environment. + """ + + def __init__( + self, + **kwargs, + ) -> None: + pass + + def can_allocate(self, + seq_group: SequenceGroup, + num_lookahead_slots: int = 0) -> AllocStatus: + # Always return OK for dummy purposes + return AllocStatus.OK + + def allocate(self, seq_group: SequenceGroup) -> None: + # No actual allocation logic needed + pass + + def can_append_slots(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> bool: + return True + + def append_slots( + self, + seq: Sequence, + num_lookahead_slots: int, + ) -> List[Tuple[int, int]]: + return [] + + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None: + pass + + def can_swap_in(self, seq_group: SequenceGroup, + num_lookahead_slots: int) -> AllocStatus: + return AllocStatus.OK + + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + return None # type: ignore + + def can_swap_out(self, seq_group: SequenceGroup) -> bool: + return True + + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]: + return None # type: ignore + + def free(self, seq: Sequence) -> None: + # No operation on free + return + + def get_block_table(self, seq: Sequence) -> List[int]: + return None # type: ignore + + def get_num_free_gpu_blocks(self) -> int: + return 1 + + def get_num_free_cpu_blocks(self) -> int: + return 1 + + def access_all_blocks_in_seq( + self, + seq: Sequence, + access_time: float, + ) -> None: + pass + + def get_common_computed_block_ids(self, + seq_group: List[Sequence]) -> List[int]: + return [] + + def mark_blocks_as_computed(self, seq_group: SequenceGroup, + token_chunk_size: int): + pass + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + return -1 diff --git a/vllm/core/scheduler.py b/vllm/core/scheduler.py new file mode 100644 index 00000000..1f0a1217 --- /dev/null +++ b/vllm/core/scheduler.py @@ -0,0 +1,1650 @@ +import enum +import os +import random +import time +from collections import deque +from dataclasses import dataclass, field +from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set, + Tuple, Union) + +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.core.interfaces import AllocStatus, BlockSpaceManager +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import (Sequence, SequenceData, SequenceGroup, + SequenceGroupMetadata, SequenceGroupMetadataDelta, + SequenceStatus) +from vllm.utils import Device, PyObjectCache + +logger = init_logger(__name__) + +# Test-only. If configured, decode is preempted with +# ARTIFICIAL_PREEMPTION_PROB% probability. +ENABLE_ARTIFICIAL_PREEMPT = bool( + os.getenv("VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT", False)) # noqa +ARTIFICIAL_PREEMPTION_PROB = 0.5 +ARTIFICIAL_PREEMPTION_MAX_CNT = 500 + + +class PreemptionMode(enum.Enum): + """Preemption modes. + + 1. Swapping: Swap out the blocks of the preempted sequences to CPU memory + and swap them back in when the sequences are resumed. + 2. Recomputation: Discard the blocks of the preempted sequences and + recompute them when the sequences are resumed, treating the sequences as + new prompts. + """ + SWAP = enum.auto() + RECOMPUTE = enum.auto() + + +@dataclass +class SchedulingBudget: + """The available slots for scheduling. + + TODO(sang): Right now, the budget is request_id-aware meaning it can ignore + budget update from the same request_id. It is because in normal scheduling + path, we update RUNNING num_seqs ahead of time, meaning it could be + updated more than once when scheduling RUNNING requests. Since this won't + happen if we only have chunked prefill scheduling, we can remove this + feature from the API when chunked prefill is enabled by default. + """ + token_budget: int + max_num_seqs: int + _request_ids_num_batched_tokens: Set[str] = field(default_factory=set) + _request_ids_num_curr_seqs: Set[str] = field(default_factory=set) + _num_batched_tokens: int = 0 + _num_curr_seqs: int = 0 + + def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int): + assert num_new_tokens != 0 + assert num_new_seqs != 0 + return (self.num_batched_tokens + num_new_tokens <= self.token_budget + and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs) + + def remaining_token_budget(self): + return self.token_budget - self.num_batched_tokens + + def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int): + if req_id in self._request_ids_num_batched_tokens: + return + + self._request_ids_num_batched_tokens.add(req_id) + self._num_batched_tokens += num_batched_tokens + + def subtract_num_batched_tokens(self, req_id: str, + num_batched_tokens: int): + if req_id in self._request_ids_num_batched_tokens: + self._request_ids_num_batched_tokens.remove(req_id) + self._num_batched_tokens -= num_batched_tokens + + def add_num_seqs(self, req_id: str, num_curr_seqs: int): + if req_id in self._request_ids_num_curr_seqs: + return + + self._request_ids_num_curr_seqs.add(req_id) + self._num_curr_seqs += num_curr_seqs + + def subtract_num_seqs(self, req_id: str, num_curr_seqs: int): + if req_id in self._request_ids_num_curr_seqs: + self._request_ids_num_curr_seqs.remove(req_id) + self._num_curr_seqs -= num_curr_seqs + + @property + def num_batched_tokens(self): + return self._num_batched_tokens + + @property + def num_curr_seqs(self): + return self._num_curr_seqs + + +@dataclass +class ScheduledSequenceGroup: + # A sequence group that's scheduled. + seq_group: SequenceGroup + # The total chunk size (number of tokens) to process for next iteration. + # 1 for decoding. Same as prompt tokens for prefill, but if prefill is + # chunked, it can be smaller than that. + token_chunk_size: int + + +@dataclass +class SchedulerOutputs: + """The scheduling decision made from a scheduler.""" + # Scheduled sequence groups. + scheduled_seq_groups: Iterable[ScheduledSequenceGroup] + # Number of prefill groups scheduled. + num_prefill_groups: int + # Total number of batched tokens. + num_batched_tokens: int + # Blocks to swap in. List of CPU -> GPU block number. + blocks_to_swap_in: List[Tuple[int, int]] + # Blocks to swap out. List of GPU -> CPU block number. + blocks_to_swap_out: List[Tuple[int, int]] + # Blocks to copy. Source to dest block. + blocks_to_copy: List[Tuple[int, int]] + # Sequence groups that are going to be ignored. + ignored_seq_groups: List[SequenceGroup] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + # The number of requests in the running queue + running_queue_size: int + preempted: int + + def __post_init__(self): + # Swap in and swap out should never happen at the same time. + assert not (self.blocks_to_swap_in and self.blocks_to_swap_out) + + self.num_loras: int = len(self.lora_requests) + if self.num_loras > 0: + self._sort_by_lora_ids() + + self.num_prompt_adapters: int = len(self.prompt_adapter_requests) + + def is_empty(self) -> bool: + # NOTE: We do not consider the ignored sequence groups. + return (not self.scheduled_seq_groups and not self.blocks_to_swap_in + and not self.blocks_to_swap_out and not self.blocks_to_copy) + + def _sort_by_lora_ids(self): + self.scheduled_seq_groups = sorted( + self.scheduled_seq_groups, + key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id)) + + @property + def lora_requests(self) -> Set[LoRARequest]: + return { + g.seq_group.lora_request + for g in self.scheduled_seq_groups + if g.seq_group.lora_request is not None + } + + @property + def prompt_adapter_requests(self) -> Set[PromptAdapterRequest]: + return { + g.seq_group.prompt_adapter_request + for g in self.scheduled_seq_groups + if g.seq_group.prompt_adapter_request is not None + } + + +@dataclass +class SchedulerRunningOutputs: + """The requests that are scheduled from a running queue. + + Could contain prefill (prefill that's chunked) or decodes. If there's not + enough memory, it can be preempted (for recompute) or swapped out. + """ + # Selected sequences that are running and in a decoding phase. + decode_seq_groups: List[ScheduledSequenceGroup] + # Selected sequences that are running and in a prefill phase. + # I.e., it means the prefill has been chunked. + prefill_seq_groups: List[ScheduledSequenceGroup] + # The preempted sequences. + preempted: List[SequenceGroup] + # Sequences that are swapped out. + swapped_out: List[SequenceGroup] + # The blocks to swap out. + blocks_to_swap_out: List[Tuple[int, int]] + # The blocks to copy. + blocks_to_copy: List[Tuple[int, int]] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + + # Optimization for fast-access to seq_group lists + decode_seq_groups_list: List[SequenceGroup] + prefill_seq_groups_list: List[SequenceGroup] + + @classmethod + def create_empty(cls) -> "SchedulerRunningOutputs": + return SchedulerRunningOutputs( + decode_seq_groups=[], + prefill_seq_groups=[], + preempted=[], + swapped_out=[], + blocks_to_swap_out=[], + blocks_to_copy=[], + num_lookahead_slots=0, + decode_seq_groups_list=[], + prefill_seq_groups_list=[], + ) + + +@dataclass +class SchedulerSwappedInOutputs: + """The requests that are scheduled from a swap queue. + + Could contain prefill (prefill that's chunked) or decodes. + """ + # Selected sequences that are going to be swapped in and is in a + # decoding phase. + decode_seq_groups: List[ScheduledSequenceGroup] + # Selected sequences that are going to be swapped in and in a prefill + # phase. I.e., it means the prefill has been chunked. + prefill_seq_groups: List[ScheduledSequenceGroup] + # The blocks to swap in. + blocks_to_swap_in: List[Tuple[int, int]] + # The blocks to copy. + blocks_to_copy: List[Tuple[int, int]] + # The number of slots for lookahead decoding. + num_lookahead_slots: int + # Infeasible sequence groups. + infeasible_seq_groups: List[SequenceGroup] + + @classmethod + def create_empty(cls) -> "SchedulerSwappedInOutputs": + return SchedulerSwappedInOutputs( + decode_seq_groups=[], + prefill_seq_groups=[], + blocks_to_swap_in=[], + blocks_to_copy=[], + num_lookahead_slots=0, + infeasible_seq_groups=[], + ) + + +@dataclass +class SchedulerPrefillOutputs: + """The requests that are scheduled from a waiting queue. + + Could contain a fresh prefill requests or preempted requests that need + to be recomputed from scratch. + """ + # Selected sequences for prefill. + seq_groups: List[ScheduledSequenceGroup] + # Ignored sequence groups. + ignored_seq_groups: List[SequenceGroup] + num_lookahead_slots: int + + @classmethod + def create_empty(cls) -> "SchedulerPrefillOutputs": + return SchedulerPrefillOutputs( + seq_groups=[], + ignored_seq_groups=[], + num_lookahead_slots=0, + ) + + +def seq_group_metadata_builder(): + return SequenceGroupMetadata(request_id="", + is_prompt=False, + seq_data={}, + sampling_params=None, + block_tables={}) + + +def scheduler_running_outputs_builder(): + return SchedulerRunningOutputs(decode_seq_groups=[], + prefill_seq_groups=[], + preempted=[], + swapped_out=[], + blocks_to_swap_out=[], + blocks_to_copy=[], + num_lookahead_slots=0, + prefill_seq_groups_list=[], + decode_seq_groups_list=[]) + + +def scheduled_seq_group_builder(): + return ScheduledSequenceGroup(SequenceGroup("", [], -1), + token_chunk_size=0) + # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0) + + +class Scheduler: + + def __init__( + self, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig, + lora_config: Optional[LoRAConfig], + pipeline_parallel_size: int = 1, + output_proc_callback: Optional[Callable] = None, + ) -> None: + self.scheduler_config = scheduler_config + self.cache_config = cache_config + # Note for LoRA scheduling: the current policy is extremely + # simple and NOT fair. It can lead to starvation of some + # LoRAs. This should be improved in the future. + self.lora_config = lora_config + + version = "v1" + if self.scheduler_config.use_v2_block_manager: + version = "v2" + if (self.scheduler_config.embedding_mode + or self.cache_config.is_attention_free): + version = "placeholder" + + BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class( + version) + + num_gpu_blocks = cache_config.num_gpu_blocks + if num_gpu_blocks: + num_gpu_blocks //= pipeline_parallel_size + + num_cpu_blocks = cache_config.num_cpu_blocks + if num_cpu_blocks: + num_cpu_blocks //= pipeline_parallel_size + + # Create the block space manager. + self.block_manager = BlockSpaceManagerImpl( + block_size=self.cache_config.block_size, + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks, + sliding_window=self.cache_config.sliding_window, + enable_caching=self.cache_config.enable_prefix_caching) + + # Sequence groups in the WAITING state. + # Contain new prefill or preempted requests. + self.waiting: Deque[SequenceGroup] = deque() + # Sequence groups in the RUNNING state. + # Contain decode requests. + self.running: Deque[SequenceGroup] = deque() + # Sequence groups in the SWAPPED state. + # Contain decode requests that are swapped out. + self.swapped: Deque[SequenceGroup] = deque() + # Sequence groups finished requests ids since last step iteration. + # It lets the model know that any state associated with these requests + # can and must be released after the current step. + # This is used to evict the finished requests from the Mamba cache. + self._finished_requests_ids: List[str] = list() + # Time at previous scheduling step + self.prev_time = 0.0 + # Did we schedule a prompt at previous step? + self.prev_prompt = False + # Latency of the last prompt step + self.last_prompt_latency = 0.0 + # preemption mode, RECOMPUTE or SWAP + self.user_specified_preemption_mode = scheduler_config.preemption_mode + + # The following field is test-only. It is used to inject artificial + # preemption. + self.enable_artificial_preemption = ENABLE_ARTIFICIAL_PREEMPT + self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT + if self.enable_artificial_preemption + else 0) + self.num_cumulative_preemption: int = 0 + + # Used to cache python objects + self._seq_group_metadata_cache: List[PyObjectCache] = [] + self._scheduler_running_outputs_cache: List[PyObjectCache] = [] + self._scheduled_seq_group_cache: List[PyObjectCache] = [] + + # For async output processing, we need to swap cache buffers between + # iterations. I.e. since the output processing is lagged one step, + # we cannot reuse the cached objects immediately when the schedule() + # is called again, but only when schedule() is called the second time. + self.output_proc_callback = output_proc_callback + self.use_async_output_proc = self.output_proc_callback is not None + self.num_cache_iters = 2 if self.use_async_output_proc else 1 + + self.cache_id = 0 + for i in range(self.num_cache_iters): + self._seq_group_metadata_cache.append( + PyObjectCache(seq_group_metadata_builder)) + self._scheduler_running_outputs_cache.append( + PyObjectCache(scheduler_running_outputs_builder)) + self._scheduled_seq_group_cache.append( + PyObjectCache(scheduled_seq_group_builder)) + + # For async postprocessor, the extra decode run cannot be done + # when the request reaches max_model_len. In this case, the request + # will be stopped during schedule() call and added to this stop list + # for processing and deallocation by the free_finished_seq_groups() + self._async_stopped: List[SequenceGroup] = [] + + @property + def next_cache_id(self): + return (self.cache_id + 1) % self.num_cache_iters + + @property + def lora_enabled(self) -> bool: + return bool(self.lora_config) + + @property + def num_decoding_tokens_per_seq(self) -> int: + """The number of new tokens.""" + return 1 + + def add_seq_group(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the waiting queue. + self.waiting.append(seq_group) + + def _add_seq_group_to_running(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the running queue. + # Only for testing purposes. + self.running.append(seq_group) + + def _add_seq_group_to_swapped(self, seq_group: SequenceGroup) -> None: + # Add sequence groups to the swapped queue. + # Only for testing purposes. + self.swapped.append(seq_group) + + def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None: + """Aborts a sequence group with the given ID. + + Check if the sequence group with the given ID + is present in any of the state queue. + If present, remove the sequence group from the state queue. + Also, if any of the sequences in the sequence group is not finished, + free the sequence with status `FINISHED_ABORTED`. + Otherwise, do nothing. + + Args: + request_id: The ID(s) of the sequence group to abort. + """ + if isinstance(request_id, str): + request_id = (request_id, ) + request_ids = set(request_id) + for state_queue in [self.waiting, self.running, self.swapped]: + aborted_groups: List[SequenceGroup] = [] + for seq_group in state_queue: + if not request_ids: + # Using 'break' here may add two extra iterations, + # but is acceptable to reduce complexity. + break + if seq_group.request_id in request_ids: + # Appending aborted group into pending list. + aborted_groups.append(seq_group) + request_ids.remove(seq_group.request_id) + for aborted_group in aborted_groups: + # Remove the sequence group from the state queue. + state_queue.remove(aborted_group) + # Remove the aborted request from the Mamba cache. + self._finished_requests_ids.append(aborted_group.request_id) + for seq in aborted_group.get_seqs(): + if seq.is_finished(): + continue + seq.status = SequenceStatus.FINISHED_ABORTED + self.free_seq(seq) + + self._free_seq_group_cross_attn_blocks(aborted_group) + + def _free_seq_group_cross_attn_blocks( + self, + seq_group: SequenceGroup, + ) -> None: + """ + Free a sequence group from a cross-attention block table. + Has no effect on decoder-only models. + """ + if seq_group.is_encoder_decoder(): + self.block_manager.free_cross(seq_group) + + def has_unfinished_seqs(self) -> bool: + return len(self.waiting) != 0 or len(self.running) != 0 or len( + self.swapped) != 0 + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + return self.block_manager.get_prefix_cache_hit_rate(device) + + def get_num_unfinished_seq_groups(self) -> int: + return len(self.waiting) + len(self.running) + len(self.swapped) + + def get_and_reset_finished_requests_ids(self) -> List[str]: + """Flushes the list of request ids of previously finished seq_groups.""" + finished_requests_ids = self._finished_requests_ids + self._finished_requests_ids = list() + return finished_requests_ids + + def _schedule_running( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerRunningOutputs: + """Schedule sequence groups that are running. + + Running queue should include decode and chunked prefill requests. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any decodes are preempted. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any decodes are preempted. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerRunningOutputs. + """ + ret: SchedulerRunningOutputs = \ + self._scheduler_running_outputs_cache[self.cache_id].get_object() + ret.blocks_to_swap_out.clear() + ret.blocks_to_copy.clear() + ret.decode_seq_groups.clear() + ret.prefill_seq_groups.clear() + ret.preempted.clear() + ret.swapped_out.clear() + + ret.num_lookahead_slots = self._get_num_lookahead_slots( + is_prefill=False, enable_chunking=enable_chunking) + + ret.decode_seq_groups_list.clear() + ret.prefill_seq_groups_list.clear() + + # Blocks that need to be swapped or copied before model execution. + blocks_to_swap_out: List[Tuple[int, int]] = ret.blocks_to_swap_out + blocks_to_copy: List[Tuple[int, int]] = ret.blocks_to_copy + + decode_seq_groups: List[ScheduledSequenceGroup] = ret.decode_seq_groups + prefill_seq_groups: List[ + ScheduledSequenceGroup] = ret.prefill_seq_groups + preempted: List[SequenceGroup] = ret.preempted + swapped_out: List[SequenceGroup] = ret.swapped_out + + running_queue = self.running + assert len(self._async_stopped) == 0 + while running_queue: + seq_group = running_queue[0] + num_running_tokens = self._get_num_new_tokens( + seq_group, SequenceStatus.RUNNING, enable_chunking, budget) + + if num_running_tokens == 0: + # No budget => Stop + break + + running_queue.popleft() + + # With async postprocessor, an extra decode run is done + # to process the final tokens. The check below avoids this extra + # decode run when the model max len is reached, in order to avoid + # a memory overflow. + if self.use_async_output_proc and seq_group.seqs[0].get_len( + ) > self.scheduler_config.max_model_len: + self._async_stopped.append(seq_group) + continue + + # NOTE(woosuk): Preemption happens only when there is no available + # slot to keep all the sequence groups in the RUNNING state. + while not self._can_append_slots(seq_group, enable_chunking): + budget.subtract_num_batched_tokens(seq_group.request_id, + num_running_tokens) + num_running_seqs = seq_group.get_max_num_running_seqs() + budget.subtract_num_seqs(seq_group.request_id, + num_running_seqs) + + if (curr_loras is not None and seq_group.lora_int_id > 0 + and seq_group.lora_int_id in curr_loras): + curr_loras.remove(seq_group.lora_int_id) + + # Determine victim sequence + cont_loop = True + if running_queue: + # Preempt the lowest-priority sequence group. + victim_seq_group = running_queue.pop() + else: + # No other sequence group can be preempted. + # Preempt the current sequence group. + # Note: This is also where we stop this loop + # (since there is nothing else to preempt) + victim_seq_group = seq_group + cont_loop = False + + # With async postprocessor, before preempting a sequence + # we need to ensure it has no pending async postprocessor + do_preempt = True + if self.use_async_output_proc: + assert self.output_proc_callback is not None + self.output_proc_callback( + request_id=victim_seq_group.request_id) + + # It may be that the async pending "victim_seq_group" + # becomes finished, in which case we simply free it. + if victim_seq_group.is_finished(): + self._free_finished_seq_group(victim_seq_group) + do_preempt = False + + # Do preemption + if do_preempt: + preempted_mode = self._preempt(victim_seq_group, + blocks_to_swap_out) + if preempted_mode == PreemptionMode.RECOMPUTE: + preempted.append(victim_seq_group) + else: + swapped_out.append(victim_seq_group) + + if not cont_loop: + break + else: + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + is_prefill = seq_group.is_prefill() + + scheduled_seq_group: ScheduledSequenceGroup = \ + self._scheduled_seq_group_cache[self.cache_id].get_object() + scheduled_seq_group.seq_group = seq_group + if is_prefill: + scheduled_seq_group.token_chunk_size = num_running_tokens + prefill_seq_groups.append(scheduled_seq_group) + ret.prefill_seq_groups_list.append(seq_group) + else: + scheduled_seq_group.token_chunk_size = 1 + decode_seq_groups.append(scheduled_seq_group) + ret.decode_seq_groups_list.append(seq_group) + + budget.add_num_batched_tokens(seq_group.request_id, + num_running_tokens) + # OPTIMIZATION: Note that get_max_num_running_seqs is + # expensive. For the default scheduling chase where + # enable_chunking is False, num_seqs are updated before running + # this method, so we don't have to update it again here. + if enable_chunking: + num_running_seqs = seq_group.get_max_num_running_seqs() + budget.add_num_seqs(seq_group.request_id, num_running_seqs) + if curr_loras is not None and seq_group.lora_int_id > 0: + curr_loras.add(seq_group.lora_int_id) + + self._scheduler_running_outputs_cache[self.next_cache_id].reset() + self._scheduled_seq_group_cache[self.next_cache_id].reset() + + return ret + + def _schedule_swapped( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerSwappedInOutputs: + """Schedule sequence groups that are swapped out. + + It schedules swapped requests as long as it fits `budget` and + curr_loras <= max_lora from the scheduling config. The input arguments + `budget` and `curr_loras` are updated based on scheduled seq_groups. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are swapped in. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any requests are swapped in. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerSwappedInOutputs. + """ + # Blocks that need to be swapped or copied before model execution. + blocks_to_swap_in: List[Tuple[int, int]] = [] + blocks_to_copy: List[Tuple[int, int]] = [] + decode_seq_groups: List[ScheduledSequenceGroup] = [] + prefill_seq_groups: List[ScheduledSequenceGroup] = [] + infeasible_seq_groups: List[SequenceGroup] = [] + + swapped_queue = self.swapped + + leftover_swapped: Deque[SequenceGroup] = deque() + while swapped_queue: + seq_group = swapped_queue[0] + + # If the sequence group cannot be swapped in, stop. + is_prefill = seq_group.is_prefill() + alloc_status = self.block_manager.can_swap_in( + seq_group, + self._get_num_lookahead_slots(is_prefill, enable_chunking)) + if alloc_status == AllocStatus.LATER: + break + elif alloc_status == AllocStatus.NEVER: + logger.warning( + "Failing the request %s because there's not enough kv " + "cache blocks to run the entire sequence.", + seq_group.request_id) + for seq in seq_group.get_seqs(): + seq.status = SequenceStatus.FINISHED_IGNORED + infeasible_seq_groups.append(seq_group) + swapped_queue.popleft() + continue + + lora_int_id = 0 + if self.lora_enabled: + lora_int_id = seq_group.lora_int_id + assert curr_loras is not None + assert self.lora_config is not None + if (lora_int_id > 0 and (lora_int_id not in curr_loras) + and len(curr_loras) >= self.lora_config.max_loras): + # We don't have a space for another LoRA, so + # we ignore this request for now. + leftover_swapped.appendleft(seq_group) + swapped_queue.popleft() + continue + + # The total number of sequences in the RUNNING state should not + # exceed the maximum number of sequences. + num_new_seqs = seq_group.get_max_num_running_seqs() + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.SWAPPED, + enable_chunking, budget) + + if (num_new_tokens == 0 + or not budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + if lora_int_id > 0 and curr_loras is not None: + curr_loras.add(lora_int_id) + swapped_queue.popleft() + self._swap_in(seq_group, blocks_to_swap_in) + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + is_prefill = seq_group.is_prefill() + if is_prefill: + prefill_seq_groups.append( + ScheduledSequenceGroup(seq_group, + token_chunk_size=num_new_tokens)) + else: + decode_seq_groups.append( + ScheduledSequenceGroup(seq_group, token_chunk_size=1)) + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens) + budget.add_num_seqs(seq_group.request_id, num_new_seqs) + + swapped_queue.extendleft(leftover_swapped) + + return SchedulerSwappedInOutputs( + decode_seq_groups=decode_seq_groups, + prefill_seq_groups=prefill_seq_groups, + blocks_to_swap_in=blocks_to_swap_in, + blocks_to_copy=blocks_to_copy, + num_lookahead_slots=self._get_num_lookahead_slots( + is_prefill=False, enable_chunking=enable_chunking), + infeasible_seq_groups=infeasible_seq_groups, + ) + + def _get_prompt_limit(self, seq_group: SequenceGroup) -> int: + if self.scheduler_config.chunked_prefill_enabled and \ + not self.scheduler_config.is_multi_step: + prompt_limit = self.scheduler_config.max_model_len + else: + prompt_limit = min(self.scheduler_config.max_model_len, + self.scheduler_config.max_num_batched_tokens) + + # Model is fine tuned with long context. Return the fine tuned max_len. + if (seq_group.lora_request + and seq_group.lora_request.long_lora_max_len): + assert prompt_limit <= seq_group.lora_request.long_lora_max_len + return seq_group.lora_request.long_lora_max_len + else: + return prompt_limit + + def _get_priority(self, + seq_group: SequenceGroup) -> Tuple[Optional[int], float]: + """ Get the priority of the sequence group. + Highest preference to user-defined priority, followed by arrival time. + Args: + seq_group: The sequence group input. + Returns: + The priority of the sequence group. + """ + return seq_group.priority, seq_group.arrival_time + + def _schedule_priority_preemption( + self, + budget: SchedulingBudget, + ) -> int: + """Sorts waiting and running queue. Also, force preempt requests + from the running queue if their priority is lower. + Priority-based preemption is used with the priority policy. + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are scheduled. + Returns: + A count of priority-based preemptions. + """ + + waiting_queue = self.waiting + + running_queue = deque(sorted(self.running, key=self._get_priority)) + + blocks_to_swap_out: List[Tuple[int, int]] = [] + force_preemption_count = 0 + + if waiting_queue: + seq_group = waiting_queue.popleft() + num_new_seqs = seq_group.get_max_num_running_seqs() + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.WAITING, + False, budget) + + #Only preempt if priority inversion exists + while running_queue and self._get_priority( + running_queue[-1]) > self._get_priority(seq_group): + #Only preempt if waiting sequence cannot be allocated + can_allocate = self.block_manager.can_allocate(seq_group) + if (num_new_tokens and can_allocate == AllocStatus.OK + and budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + #Adjust budget to remove the victim sequence group + vseq_group = running_queue.pop() + num_running_tokens = self._get_num_new_tokens( + vseq_group, SequenceStatus.RUNNING, False, budget) + budget.subtract_num_batched_tokens(vseq_group.request_id, + num_running_tokens) + num_running_seqs = vseq_group.get_max_num_running_seqs() + budget.subtract_num_seqs(vseq_group.request_id, + num_running_seqs) + + #Preempt out the victim sequence group + self._preempt(vseq_group, blocks_to_swap_out, + PreemptionMode.RECOMPUTE) + waiting_queue.appendleft(vseq_group) + force_preemption_count += 1 + #Put the sequence back into the waiting queue + waiting_queue.appendleft(seq_group) + + waiting_queue = deque(sorted(waiting_queue, key=self._get_priority)) + + self.waiting = waiting_queue + self.running = running_queue + return force_preemption_count + + def _schedule_prefills( + self, + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, + ) -> SchedulerPrefillOutputs: + """Schedule sequence groups that are in prefill stage. + + Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE + as a new prefill (that starts from beginning -> most recently generated + tokens). + + It schedules waiting requests as long as it fits `budget` and + curr_loras <= max_lora from the scheduling config. The input arguments + `budget` and `curr_loras` are updated based on scheduled seq_groups. + + Args: + budget: The scheduling budget. The argument is in-place updated + when any requests are scheduled. + curr_loras: Currently batched lora request ids. The argument is + in-place updated when any requests are scheduled. + enable_chunking: If True, seq group can be chunked and only a + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. + + Returns: + SchedulerPrefillOutputs. + """ + ignored_seq_groups: List[SequenceGroup] = [] + seq_groups: List[ScheduledSequenceGroup] = [] + + waiting_queue = self.waiting + + leftover_waiting_sequences: Deque[SequenceGroup] = deque() + while self._passed_delay(time.time()) and waiting_queue: + seq_group = waiting_queue[0] + + waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING) + assert len(waiting_seqs) == 1, ( + "Waiting sequence group should have only one prompt " + "sequence.") + num_new_tokens = self._get_num_new_tokens(seq_group, + SequenceStatus.WAITING, + enable_chunking, budget) + if not enable_chunking: + num_prompt_tokens = waiting_seqs[0].get_len() + assert num_new_tokens == num_prompt_tokens + + prompt_limit = self._get_prompt_limit(seq_group) + if num_new_tokens > prompt_limit: + logger.warning( + "Input prompt (%d tokens) is too long" + " and exceeds limit of %d", num_new_tokens, prompt_limit) + for seq in waiting_seqs: + seq.status = SequenceStatus.FINISHED_IGNORED + ignored_seq_groups.append(seq_group) + waiting_queue.popleft() + continue + + num_lookahead_slots: int = 0 + if self.scheduler_config.is_multi_step and enable_chunking: + num_lookahead_slots = self._get_num_lookahead_slots( + True, enable_chunking) + + # If the sequence group cannot be allocated, stop. + can_allocate = self.block_manager.can_allocate( + seq_group, num_lookahead_slots=num_lookahead_slots) + if can_allocate == AllocStatus.LATER: + break + elif can_allocate == AllocStatus.NEVER: + logger.warning( + "Input prompt (%d tokens) + lookahead slots (%d) is " + "too long and exceeds the capacity of block_manager", + num_new_tokens, num_lookahead_slots) + for seq in waiting_seqs: + seq.status = SequenceStatus.FINISHED_IGNORED + ignored_seq_groups.append(seq_group) + waiting_queue.popleft() + continue + + lora_int_id = 0 + if self.lora_enabled: + lora_int_id = seq_group.lora_int_id + assert curr_loras is not None + assert self.lora_config is not None + if (self.lora_enabled and lora_int_id > 0 + and lora_int_id not in curr_loras + and len(curr_loras) >= self.lora_config.max_loras): + # We don't have a space for another LoRA, so + # we ignore this request for now. + leftover_waiting_sequences.appendleft(seq_group) + waiting_queue.popleft() + continue + + num_new_seqs = seq_group.get_max_num_running_seqs() + if (num_new_tokens == 0 + or not budget.can_schedule(num_new_tokens=num_new_tokens, + num_new_seqs=num_new_seqs)): + break + + # Can schedule this request. + if curr_loras is not None and lora_int_id > 0: + curr_loras.add(lora_int_id) + waiting_queue.popleft() + self._allocate_and_set_running(seq_group) + + if enable_chunking and self.scheduler_config.is_multi_step: + blocks_to_copy: List[Tuple[int, int]] = [] + # init_multi_step_from_lookahead_slots happens in append_slots + self._append_slots(seq_group, blocks_to_copy, enable_chunking) + # This assert will trip when a copy-on-write happens. This is + # not a concern as the very first sequence-group block + # allocation happens above. Still, we have the assert to + # catch any edge-cases. + assert not blocks_to_copy + else: + seq_group.init_multi_step_from_lookahead_slots( + num_lookahead_slots, + num_scheduler_steps=self.scheduler_config. + num_scheduler_steps, + is_multi_step=self.scheduler_config.is_multi_step, + enable_chunking=enable_chunking) + + seq_groups.append( + ScheduledSequenceGroup(seq_group=seq_group, + token_chunk_size=num_new_tokens)) + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens) + budget.add_num_seqs(seq_group.request_id, num_new_seqs) + + # Queue requests that couldn't be scheduled. + waiting_queue.extendleft(leftover_waiting_sequences) + if len(seq_groups) > 0: + self.prev_prompt = True + + return SchedulerPrefillOutputs( + seq_groups=seq_groups, + ignored_seq_groups=ignored_seq_groups, + num_lookahead_slots=self._get_num_lookahead_slots( + is_prefill=True, enable_chunking=enable_chunking)) + + def _schedule_default(self) -> SchedulerOutputs: + """Schedule queued requests. + + The current policy is designed to optimize the throughput. First, + it batches as many prefill requests as possible. And it schedules + decodes. If there's a pressure on GPU memory, decode requests can + be swapped or preempted. + """ + # Include running requests to the budget. + budget = SchedulingBudget( + token_budget=self.scheduler_config.max_num_batched_tokens, + max_num_seqs=self.scheduler_config.max_num_seqs, + ) + # Make sure we include num running seqs before scheduling prefill, + # so that we don't schedule beyond max_num_seqs for prefill. + for seq_group in self.running: + budget.add_num_seqs(seq_group.request_id, + seq_group.get_max_num_running_seqs()) + curr_loras = set( + seq_group.lora_int_id for seq_group in self.running + if seq_group.lora_int_id > 0) if self.lora_enabled else None + + prefills = SchedulerPrefillOutputs.create_empty() + running_scheduled = SchedulerRunningOutputs.create_empty() + swapped_in = SchedulerSwappedInOutputs.create_empty() + + # If any requests are swapped, prioritized swapped requests. + if not self.swapped: + prefills = self._schedule_prefills(budget, + curr_loras, + enable_chunking=False) + + if len(prefills.seq_groups + ) == 0 and self.scheduler_config.policy == "priority": + self._schedule_priority_preemption(budget) + + # Don't schedule decodes if prefills are scheduled. + # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running + # only contains decode requests, not chunked prefills. + if len(prefills.seq_groups) == 0: + running_scheduled = self._schedule_running(budget, + curr_loras, + enable_chunking=False) + + # If any sequence group is preempted, do not swap in any sequence + # group. because it means there's no slot for new running requests. + if len(running_scheduled.preempted) + len( + running_scheduled.swapped_out) == 0: + swapped_in = self._schedule_swapped(budget, curr_loras) + + assert (budget.num_batched_tokens <= + self.scheduler_config.max_num_batched_tokens) + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs + + # Update waiting requests. + self.waiting.extendleft(running_scheduled.preempted) + # Update new running requests. + if len(prefills.seq_groups) > 0: + self.running.extend([s.seq_group for s in prefills.seq_groups]) + + self.running.extend(running_scheduled.decode_seq_groups_list) + + if len(swapped_in.decode_seq_groups) > 0: + self.running.extend( + [s.seq_group for s in swapped_in.decode_seq_groups]) + + # Update swapped requests. + self.swapped.extend(running_scheduled.swapped_out) + preempted = (len(running_scheduled.preempted) + + len(running_scheduled.swapped_out)) + + # There should be no prefill from running queue because this policy + # doesn't allow chunked prefills. + assert len(running_scheduled.prefill_seq_groups) == 0 + assert len(swapped_in.prefill_seq_groups) == 0 + + # Merge lists + num_prefill_groups = len(prefills.seq_groups) + if num_prefill_groups > 0: + scheduled_seq_groups = prefills.seq_groups + scheduled_seq_groups.extend(running_scheduled.decode_seq_groups) + else: + scheduled_seq_groups = running_scheduled.decode_seq_groups + scheduled_seq_groups.extend(swapped_in.decode_seq_groups) + + blocks_to_copy = running_scheduled.blocks_to_copy + blocks_to_copy.extend(swapped_in.blocks_to_copy) + + ignored_seq_groups = prefills.ignored_seq_groups + ignored_seq_groups.extend(swapped_in.infeasible_seq_groups) + + return SchedulerOutputs( + scheduled_seq_groups=scheduled_seq_groups, + num_prefill_groups=num_prefill_groups, + num_batched_tokens=budget.num_batched_tokens, + blocks_to_swap_in=swapped_in.blocks_to_swap_in, + blocks_to_swap_out=running_scheduled.blocks_to_swap_out, + blocks_to_copy=blocks_to_copy, + ignored_seq_groups=ignored_seq_groups, + num_lookahead_slots=running_scheduled.num_lookahead_slots, + running_queue_size=len(self.running), + preempted=preempted, + ) + + def _schedule_chunked_prefill(self) -> SchedulerOutputs: + """Schedule queued requests. + + Chunked prefill allows to chunk prefill requests, batch them together + with decode requests. This policy 1. schedule as many decoding requests + as possible. 2. schedule chunked prefill requests that are not + finished. 3. schedule swapped request. 4. schedule new prefill + requests. + + The policy can sustain the high GPU utilization because it can put + prefill and decodes requests to the same batch, while it improves + inter token latency because decodes requests don't need to be blocked + by prefill requests. + """ + budget = SchedulingBudget( + token_budget=self.scheduler_config.max_num_batched_tokens, + max_num_seqs=self.scheduler_config.max_num_seqs, + ) + curr_loras: Set[int] = set() + + prefills = SchedulerPrefillOutputs.create_empty() + swapped_in = SchedulerSwappedInOutputs.create_empty() + + # Decoding should be always scheduled first by fcfs. + running_scheduled = self._schedule_running(budget, + curr_loras, + enable_chunking=True) + + # Schedule swapped out requests. + # If preemption happens, it means we don't have space for swap-in. + if len(running_scheduled.preempted) + len( + running_scheduled.swapped_out) == 0: + swapped_in = self._schedule_swapped(budget, curr_loras) + + # Schedule new prefills. + prefills = self._schedule_prefills(budget, + curr_loras, + enable_chunking=True) + + assert (budget.num_batched_tokens <= + self.scheduler_config.max_num_batched_tokens) + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs + + # Update waiting requests. + self.waiting.extendleft(running_scheduled.preempted) + + # Update new running requests. + # By default, vLLM scheduler prioritizes prefills. + # Once chunked prefill is enabled, + # the policy is changed to prioritize decode requests. + self.running.extend( + [s.seq_group for s in swapped_in.decode_seq_groups]) + self.running.extend( + [s.seq_group for s in swapped_in.prefill_seq_groups]) + self.running.extend( + [s.seq_group for s in running_scheduled.decode_seq_groups]) + self.running.extend( + [s.seq_group for s in running_scheduled.prefill_seq_groups]) + self.running.extend([s.seq_group for s in prefills.seq_groups]) + + # Update swapped requests. + self.swapped.extend(running_scheduled.swapped_out) + return SchedulerOutputs( + scheduled_seq_groups=(prefills.seq_groups + + running_scheduled.prefill_seq_groups + + swapped_in.prefill_seq_groups + + running_scheduled.decode_seq_groups + + swapped_in.decode_seq_groups), + num_prefill_groups=(len(prefills.seq_groups) + + len(swapped_in.prefill_seq_groups) + + len(running_scheduled.prefill_seq_groups)), + num_batched_tokens=budget.num_batched_tokens, + blocks_to_swap_in=swapped_in.blocks_to_swap_in, + blocks_to_swap_out=running_scheduled.blocks_to_swap_out, + blocks_to_copy=running_scheduled.blocks_to_copy + + swapped_in.blocks_to_copy, + ignored_seq_groups=prefills.ignored_seq_groups + + swapped_in.infeasible_seq_groups, + num_lookahead_slots=running_scheduled.num_lookahead_slots, + running_queue_size=len(self.running), + preempted=(len(running_scheduled.preempted) + + len(running_scheduled.swapped_out)), + ) + + def _schedule(self) -> SchedulerOutputs: + """Schedule queued requests.""" + if self.scheduler_config.chunked_prefill_enabled: + return self._schedule_chunked_prefill() + else: + return self._schedule_default() + + def _can_append_slots(self, seq_group: SequenceGroup, + enable_chunking: bool) -> bool: + """Determine whether or not we have enough space in the KV cache to + continue generation of the sequence group. + """ + # It is True only for testing case to trigger artificial preemption. + if (self.enable_artificial_preemption + and random.uniform(0, 1) < ARTIFICIAL_PREEMPTION_PROB + and self.artificial_preempt_cnt > 0): + self.artificial_preempt_cnt -= 1 + return False + + is_prefill = seq_group.is_prefill() + num_lookahead_slots = self._get_num_lookahead_slots( + is_prefill, enable_chunking) + + if is_prefill and num_lookahead_slots > 0: + # Appending prefill slots only happens multi-step and + # chunked-prefill are enabled together. + assert self.scheduler_config.is_multi_step and enable_chunking + + return self.block_manager.can_append_slots( + seq_group=seq_group, num_lookahead_slots=num_lookahead_slots) + + def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool: + # async_output_proc is allowed only when we have a single sequence + # in the sequence group + no_single_seq = seq_group.sampling_params is None or ( + seq_group.sampling_params.n == 1) + return no_single_seq + + def schedule( + self + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, bool]: + # Schedule sequence groups. + # This function call changes the internal states of the scheduler + # such as self.running, self.swapped, and self.waiting. + scheduler_start_time = time.perf_counter() + + scheduler_outputs: SchedulerOutputs = self._schedule() + now = time.time() + + if not self.cache_config.enable_prefix_caching: + common_computed_block_nums = [] + + allow_async_output_proc: bool = self.use_async_output_proc + + # Create input data structures. + seq_group_metadata_list: List[SequenceGroupMetadata] = [] + for i, scheduled_seq_group in enumerate( + scheduler_outputs.scheduled_seq_groups): + seq_group = scheduled_seq_group.seq_group + token_chunk_size = scheduled_seq_group.token_chunk_size + seq_group.maybe_set_first_scheduled_time(now) + + seq_group_metadata = self._seq_group_metadata_cache[ + self.cache_id].get_object() + seq_group_metadata.seq_data.clear() + seq_group_metadata.block_tables.clear() + + # seq_id -> SequenceData + seq_data: Dict[int, SequenceData] = {} + # seq_id -> physical block numbers + block_tables: Dict[int, List[int]] = {} + + if seq_group.is_encoder_decoder(): + # Encoder associated with SequenceGroup + encoder_seq = seq_group.get_encoder_seq() + assert encoder_seq is not None + encoder_seq_data = encoder_seq.data + # Block table for cross-attention + # Also managed at SequenceGroup level + cross_block_table = self.block_manager.get_cross_block_table( + seq_group) + else: + encoder_seq_data = None + cross_block_table = None + + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + seq_id = seq.seq_id + seq_data[seq_id] = seq.data + block_tables[seq_id] = self.block_manager.get_block_table(seq) + self.block_manager.access_all_blocks_in_seq(seq, now) + + if self.cache_config.enable_prefix_caching: + common_computed_block_nums = ( + self.block_manager.get_common_computed_block_ids( + seq_group.get_seqs(status=SequenceStatus.RUNNING))) + + do_sample = True + is_prompt = seq_group.is_prefill() + # We should send the metadata to workers when the first prefill + # is sent. Subsequent requests could be chunked prefill or decode. + is_first_prefill = False + if is_prompt: + seqs = seq_group.get_seqs() + # Prefill has only 1 sequence. + assert len(seqs) == 1 + num_computed_tokens = seqs[0].data.get_num_computed_tokens() + is_first_prefill = num_computed_tokens == 0 + # In the next iteration, all prompt tokens are not computed. + # It means the prefill is chunked, and we don't need sampling. + # NOTE: We use get_len instead of get_prompt_len because when + # a sequence is preempted, prefill includes previous generated + # output tokens. + if (token_chunk_size + num_computed_tokens < + seqs[0].data.get_len()): + do_sample = False + + # It assumes the scheduled_seq_groups is ordered by + # prefill < decoding. + if is_first_prefill or not self.scheduler_config.send_delta_data: + seq_group_metadata = SequenceGroupMetadata( + request_id=seq_group.request_id, + is_prompt=is_prompt, + seq_data=seq_data, + sampling_params=seq_group.sampling_params, + block_tables=block_tables, + do_sample=do_sample, + pooling_params=seq_group.pooling_params, + token_chunk_size=token_chunk_size, + lora_request=seq_group.lora_request, + computed_block_nums=common_computed_block_nums, + encoder_seq_data=encoder_seq_data, + cross_block_table=cross_block_table, + state=seq_group.state, + # `multi_modal_data` will only be present for the 1st comm + # between engine and worker. + # the subsequent comms can still use delta, but + # `multi_modal_data` will be None. + multi_modal_data=seq_group.multi_modal_data + if scheduler_outputs.num_prefill_groups > 0 else None, + mm_processor_kwargs=seq_group.mm_processor_kwargs, + prompt_adapter_request=seq_group.prompt_adapter_request, + ) + else: + # When SPMD mode is enabled, we only send delta data except for + # the first request to reduce serialization cost. + seq_data_delta = {} + for id, data in seq_data.items(): + seq_data_delta[id] = data.get_delta_and_reset() + seq_group_metadata = SequenceGroupMetadataDelta( + seq_data_delta, + seq_group.request_id, + block_tables, + is_prompt, + do_sample=do_sample, + token_chunk_size=token_chunk_size, + computed_block_nums=common_computed_block_nums, + ) + seq_group_metadata_list.append(seq_group_metadata) + + if allow_async_output_proc: + allow_async_output_proc = self._allow_async_output_proc( + seq_group) + + # Now that the batch has been created, we can assume all blocks in the + # batch will have been computed before the next scheduling invocation. + # This is because the engine assumes that a failure in model execution + # will crash the vLLM instance / will not retry. + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups: + self.block_manager.mark_blocks_as_computed( + scheduled_seq_group.seq_group, + scheduled_seq_group.token_chunk_size) + + self._seq_group_metadata_cache[self.next_cache_id].reset() + + scheduler_time = time.perf_counter() - scheduler_start_time + # Add this to scheduler time to all the sequences that are currently + # running. This will help estimate if the scheduler is a significant + # component in the e2e latency. + for seq_group in self.running: + if seq_group is not None and seq_group.metrics is not None: + if seq_group.metrics.scheduler_time is not None: + seq_group.metrics.scheduler_time += scheduler_time + else: + seq_group.metrics.scheduler_time = scheduler_time + + # Move to next cache (if exists) + self.cache_id = self.next_cache_id + + # Return results + return (seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc) + + def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None: + self.block_manager.fork(parent_seq, child_seq) + + def free_seq(self, seq: Sequence) -> None: + """Free a sequence from a block table.""" + self.block_manager.free(seq) + + def _free_finished_seqs(self, seq_group: SequenceGroup) -> None: + """Free finished seqs in a sequence group.""" + for seq in seq_group.get_seqs(): + if seq.is_finished(): + self.free_seq(seq) + + def _free_finished_seq_group(self, seq_group: SequenceGroup) -> None: + if seq_group.is_finished(): + # Free cross-attention block table, if it exists + self._free_seq_group_cross_attn_blocks(seq_group) + + # Add the finished requests to the finished requests list. + # This list will be used to update the Mamba cache in the + # next step. + self._finished_requests_ids.append(seq_group.request_id) + + # Free finished seqs + self._free_finished_seqs(seq_group) + + def free_finished_seq_groups(self) -> None: + remaining: Deque[SequenceGroup] = deque() + for seq_group in self.running: + self._free_finished_seq_group(seq_group) + if not seq_group.is_finished(): + remaining.append(seq_group) + + self.running = remaining + + # Handle async stopped sequence groups + # (ones that reached max model len) + if self._async_stopped: + for seq_group in self._async_stopped: + self._free_seq_group_cross_attn_blocks(seq_group) + self._finished_requests_ids.append(seq_group.request_id) + + # Free finished seqs + self._free_finished_seqs(seq_group) + + self._async_stopped.clear() + + def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None: + self.block_manager.allocate(seq_group) + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING): + seq.status = SequenceStatus.RUNNING + + def _append_slots(self, + seq_group: SequenceGroup, + blocks_to_copy: List[Tuple[int, int]], + enable_chunking: bool = False) -> None: + """Appends new slots to the sequences in the given sequence group. + + Args: + seq_group (SequenceGroup): The sequence group containing the + sequences to append slots to. + blocks_to_copy (List[Tuple[int, int]]): A list of tuple of two + ints, the first int is the source block index, and the second + int is the destination block index. This list is updated with + the new source and destination block indices for the appended + slots. + enable_chunking (bool): True if chunked prefill is enabled. + """ + is_prefill: bool = seq_group.is_prefill() + num_lookahead_slots: int = self._get_num_lookahead_slots( + is_prefill, enable_chunking) + + seq_group.init_multi_step_from_lookahead_slots( + num_lookahead_slots, + num_scheduler_steps=self.scheduler_config.num_scheduler_steps, + is_multi_step=self.scheduler_config.is_multi_step, + enable_chunking=enable_chunking) + + seq_status: Optional[SequenceStatus] = SequenceStatus.RUNNING + if self.scheduler_config.is_multi_step and enable_chunking: + # In multi-step chunked-prefill any sequence type can have + # slots appended. + seq_status = None + + for seq in seq_group.get_seqs(status=seq_status): + cows = self.block_manager.append_slots(seq, num_lookahead_slots) + if len(cows) > 0: + blocks_to_copy.extend(cows) + + def _preempt( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + preemption_mode: Optional[PreemptionMode] = None, + ) -> PreemptionMode: + # If preemption mode is not specified, we determine the mode as follows: + # We use recomputation by default since it incurs lower overhead than + # swapping. However, when the sequence group has multiple sequences + # (e.g., beam search), recomputation is not currently supported. In + # such a case, we use swapping instead. + # FIXME(woosuk): This makes our scheduling policy a bit bizarre. + # As swapped sequences are prioritized over waiting sequences, + # sequence groups with multiple sequences are implicitly prioritized + # over sequence groups with a single sequence. + # TODO(woosuk): Support recomputation for sequence groups with multiple + # sequences. This may require a more sophisticated CUDA kernel. + if self.user_specified_preemption_mode is None: + if seq_group.get_max_num_running_seqs() == 1: + preemption_mode = PreemptionMode.RECOMPUTE + else: + preemption_mode = PreemptionMode.SWAP + + elif self.user_specified_preemption_mode == "swap": + preemption_mode = PreemptionMode.SWAP + else: + preemption_mode = PreemptionMode.RECOMPUTE + + if self.num_cumulative_preemption % 50 == 0: + logger.warning( + "Sequence group %s is preempted by %s mode because there is " + "not enough KV cache space. This can affect the end-to-end " + "performance. Increase gpu_memory_utilization or " + "tensor_parallel_size to provide more KV cache memory. " + "total_num_cumulative_preemption=%d", seq_group.request_id, + preemption_mode, self.num_cumulative_preemption + 1) + self.num_cumulative_preemption += 1 + + if preemption_mode == PreemptionMode.RECOMPUTE: + self._preempt_by_recompute(seq_group) + elif preemption_mode == PreemptionMode.SWAP: + self._preempt_by_swap(seq_group, blocks_to_swap_out) + else: + raise AssertionError("Invalid preemption mode.") + return preemption_mode + + def _preempt_by_recompute( + self, + seq_group: SequenceGroup, + ) -> None: + seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING) + assert len(seqs) == 1 + for seq in seqs: + seq.status = SequenceStatus.WAITING + self.free_seq(seq) + seq.reset_state_for_recompute() + + def _preempt_by_swap( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + ) -> None: + self._swap_out(seq_group, blocks_to_swap_out) + + def _swap_in( + self, + seq_group: SequenceGroup, + blocks_to_swap_in: List[Tuple[int, int]], + ) -> None: + mapping = self.block_manager.swap_in(seq_group) + blocks_to_swap_in.extend(mapping) + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED): + seq.status = SequenceStatus.RUNNING + + def _swap_out( + self, + seq_group: SequenceGroup, + blocks_to_swap_out: List[Tuple[int, int]], + ) -> None: + if not self.block_manager.can_swap_out(seq_group): + # FIXME(woosuk): Abort the sequence group instead of aborting the + # entire engine. + raise RuntimeError( + "Aborted due to the lack of CPU swap space. Please increase " + "the swap space to avoid this error.") + mapping = self.block_manager.swap_out(seq_group) + blocks_to_swap_out.extend(mapping) + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): + seq.status = SequenceStatus.SWAPPED + + def _passed_delay(self, now: float) -> bool: + if self.prev_prompt: + self.last_prompt_latency = now - self.prev_time + self.prev_time, self.prev_prompt = now, False + # Delay scheduling prompts to let waiting queue fill up + if self.scheduler_config.delay_factor > 0 and self.waiting: + earliest_arrival_time = min( + [e.metrics.arrival_time for e in self.waiting]) + passed_delay = ( + (now - earliest_arrival_time) > + (self.scheduler_config.delay_factor * self.last_prompt_latency) + or not self.running) + else: + passed_delay = True + return passed_delay + + def _get_num_lookahead_slots(self, is_prefill: bool, + enable_chunking: bool) -> int: + """The number of slots to allocate per sequence per step, beyond known + token ids. Speculative decoding uses these slots to store KV activations + of tokens which may or may not be accepted. + + Speculative decoding does not yet support prefill, so we do not perform + lookahead allocation for prefill. + + When chunking is enabled with multi-step, we allocate lookahead slots + for the prefills for when the prefills turn into decodes in the first + step. + """ + if is_prefill: + if self.scheduler_config.is_multi_step and enable_chunking: + # num_lookahead_slots was introduced in the context of decodes, + # in Speculative Decoding. + # When the num_scheduler_steps is 8, say, then the + # num_lookahead_slots is 7. Meaning, we are doing a 1-step of + # decode anyways and we wish to do 7 more. + # + # "lookaheads" for prefills, is introduced in support for + # Chunked-Prefill in Multi-Step. + return self.scheduler_config.num_lookahead_slots + 1 + else: + return 0 + + return self.scheduler_config.num_lookahead_slots + + def _get_num_new_tokens(self, seq_group: SequenceGroup, + status: SequenceStatus, enable_chunking: bool, + budget: SchedulingBudget) -> int: + """Get the next new tokens to compute for a given sequence group + that's in a given `status`. + + The API could chunk the number of tokens to compute based on `budget` + if `enable_chunking` is True. If a sequence group has multiple + sequences (e.g., running beam search), it means it is in decoding + phase, so chunking doesn't happen. + + Returns 0 if the new token cannot be computed due to token budget. + """ + num_new_tokens = 0 + seqs = seq_group.get_seqs(status=status) + for seq in seqs: + num_new_tokens += seq.get_num_new_tokens() + assert num_new_tokens > 0 + # Chunk if a running request cannot fit in the given budget. + # If number of seq > 1, it means it is doing beam search + # in a decode phase. Do not chunk. + if enable_chunking and len(seqs) == 1: + remaining_token_budget = budget.remaining_token_budget() + if self.scheduler_config.is_multi_step: + # The current multi-step + chunked prefill capability does + # not actually support chunking prompts. + # + # Therefore, `num_new_tokens` is computed in the same fashion + # for both multi-step+chunked-prefill & + # multi-step+chunked-prefill+APC + # + # Prompts with more tokens than the current remaining budget + # are postponed to future scheduler steps + if num_new_tokens > self._get_prompt_limit(seq_group): + # If the seq_group is in prompt-stage, pass the + # num_new_tokens as-is so the caller can ignore + # the sequence. + pass + else: + num_new_tokens = 0 \ + if num_new_tokens > remaining_token_budget \ + else num_new_tokens + elif self.cache_config.enable_prefix_caching: + # When prefix caching is enabled, we always allocate + # the number of new tokens that is dividable by the block + # size to avoid partial block matching. + block_size = self.cache_config.block_size + remainder = budget.token_budget % block_size + if remainder != 0: + raise ValueError("When enabling chunked prefill and " + "prefix caching, max_num_batched_tokens " + "(chunk size) must be dividable by " + "block size, but got chunk_size " + f"({budget.token_budget}) % block_size " + f"({block_size}) = {remainder}") + if remaining_token_budget < num_new_tokens: + num_new_tokens = (remaining_token_budget // + block_size) * block_size + else: + num_new_tokens = min(num_new_tokens, remaining_token_budget) + return num_new_tokens diff --git a/vllm/distributed/__init__.py b/vllm/distributed/__init__.py new file mode 100644 index 00000000..db325cfa --- /dev/null +++ b/vllm/distributed/__init__.py @@ -0,0 +1,3 @@ +from .communication_op import * +from .parallel_state import * +from .utils import * diff --git a/vllm/distributed/__pycache__/__init__.cpython-310.pyc b/vllm/distributed/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..912dfd764ba8d5733e37f2c77b202fe99a775f82 GIT binary patch literal 233 zcmd1j<>g`k0>icYGJ=8hV-N=!FabFZKwK;XBvKes7;_jxAT%SCW&+Yo!3>(rFBySK zG#PI(YWZn0M+qe7=jN8?WhN(ch*r(_nF u6lErrmZYZW$H!;pWtPOp>lIYq;;_lhPbtkwwF5b(m<>p9FmN#PFaiMOpFE}j literal 0 HcmV?d00001 diff --git a/vllm/distributed/__pycache__/communication_op.cpython-310.pyc b/vllm/distributed/__pycache__/communication_op.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52ccfdc211b24fef223dda2f64ac0c7a34f9c911 GIT binary patch literal 1409 zcmbtTJ8u**5cXqtZ||AQ6CzSvdqr}Jgn$AeB9Nd8MWEScwOKoN8|_P8FX*D{$PdYt z6!{Aj{ETg>@)sx)X1sSKAfSmQkH<4(d%l^^S-0CFF!nFMPA)A%zT@WZK<`VK#WxUw z2pW@wdNg4kOSs1q%d;rjS!{c@Sshrpu;NDIdM@~EX`eQQgYv`@4dI?!UaQ#@8w*cM zwijMUv__EI3;sHW;~m@N#1%cU0&(3YYZ9>~`jSf~skG29=?Q}vnA=3G z$~L^YCe|UkC)g1gKDYuLWN20VAe~m$>nPNf`!3f}mIiTUe@H<&yrLLGtIk+zU+4Z< zWo2GDQJR<9uN)6hR3LRo6mjiVhrs-@Slv^+}Hs=P;<2Uz!;w)hB z8U{Y`S1da)7tsO<&I=X(yF)t$)G57Sb5b;BYwRxH%kJ7P1wkhnALe44dmBMm(t0v>U z-xB^s>Ou!v)?dqcZ_+f3u6UGc!vs@Clg6ENP2z@G?sNDhWh`G{YQ+-}4~ffM+F@Iq gIjtK;*^NbEZ7|xp#vNnq+GU`-jP}9O0ZZ@tCmjA*bN~PV literal 0 HcmV?d00001 diff --git a/vllm/distributed/__pycache__/parallel_state.cpython-310.pyc b/vllm/distributed/__pycache__/parallel_state.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f696dc6c037f430b9170ffa1a5a545a7b380d44d GIT binary patch literal 29555 zcmcJ236xw{dRD!q)~;UEYPA;2@{?@YYFlbaUL;$VG?um5W4Wb~WP7G&Jf-TouUpk! zOZ(PqNnNSVI3tHl7ULvv2r!w@%E{(HfFy8$goFWd2q7deha`kzJ>fubNCF%~GVzSu zeBXcHtF5~=P6*YfZr%6ZeRuu$|G&3ugM*nEK8Jte=dVBi3$fUL=0*2s6E2>^x1Mlg zu~%X$mWwG%*<~wd$*-NW<=4qM_;t!|C7z30EaR4~N+Or2By&l*j+axFbS^FVL^)F# z$PGw7S+9;Wz%n5Q$Y8dF=|a&retvD~D*e~@qaht&A-nA++; zhW8&sscmXIN*zY2$EDN|+T9VgfztP(^o+V+?Lw(1{N1->-)-sp(aw`|PpJo#H*2Zg zZ`;aa%W4m6RrX9mJ*Xaf%f(2J_($fBu6epwJuFY34(oXYZT8i|`cTh4H6bOR@t^6f zCw%^BR1;?KF|`9TxC1kGOxcBJ{Z0Nc8Sj3KV6%U0##RT^Bt~&u9mGr>#0(zt530j5 z6USy!>T&f1o}NhC=_m2E)vS67<)ma+*(0d;B(9J8Co$el{$t2LjbGNsRz}n_>eyTH z+{e&|=WyTj;n{W{j;j-B<&=La_dH6RZkIUOF7Yw-97>$=&*aYf33W<6|5gk$dCtGz zKabxR)ak;D>WqI9<2kF&q2$NY=kfOfzAvil=*vO%aeQCGxWe(xWYh)qGWzsVdJ2EG zDyfSoaY;sRM!;1|sZXfOD0KlP<~AsCMO{USm(?rk)wg1~DfMLGBKr86% z+OgP;3m0DYRH+{5()42BE3Y0D0)H~|V&DaZdB5(}Zur^@uKQlGRt|HBW zOF_P?vV2!%*Sw(Su|TO%E;UdSYU?WMbrqfLsnC?qS8B>H=NAeZE&1hZUQK(~{Azts z`+bk{oko;2n(1#=s`8?$yjfjaT&PcS7=`K*=0>4sK6+W5^=_7e>ub7N4;E);I-^!T zX1uo4*9~bpd?Yn&&}uYyQ7v6_vS#|atLrM)J9n8EXS}7_qF1Yym%OSEFu=qw*6%sv zGS1p!;7KPBE|eC04ByYRC#h7g?3Y@Lg{oJdFDN!D$yAs}%*|qv*BOAL z&v|~2+-z#XYGta0im!sjg|a_kw}uFm{;i-=s1{~@-5RJamdoKi6i%x`P$-rQ^*SyF z>(>j~SNV!xsp+MOWXm~KU23IH12ziNWxwT~DHVg3djYu6N?lxF^M!INew9tN;;&S3 zjauRhg`jvHO(&l(6zBb_YGp6!TG6l9&l4n2Vn86eT$@FQa5++3)EbD9Unm!XnVPQP z-e`SsVWFmjdcL?=4{DWsjW1@fMx=IHo6qaQ!u8XIg#ciF+DuO?ff@A+l_`ummCthu^Z8aLPw-nTGoQ`p-&ib^ z!zb~8*xPc8i>lB{T#;_iT4eUARwyhs+x}48_;j z#BMt?_8rTLHEld|nR_E;$~9f{gx5`bhKpl0e(K6Xxdh5eswiCrITJuH=p|4bAYE{? zMg$5#B0YQ#PkN`kGC-1Kci+6fwBNf?C@=aGC{)t*Ne_t2+Ta68u&~B%qj}yBunddU(i@Au z$A&zhy{wY-#7yhiTZIY;i&^D7FeYzakbdsl6D;hR@D7~xkU4f}k1WK!3mrCvG@<)3nzov`#F{@D63esamGf0NzmpV^WAnlGbtoeD)Nk+McHv#zC|#ARzlXx!;qO{-G1 z03y*!n%S+l;xe|LDP8NWN#NXWE$jUgf|C@kP*v8pD`b;!ScH96s!naR9L z3UF3y%>P)|pdiGM$rSx}9eomA)E{HA-o%ZroskmC%9?L;GFA#}K4j@LxYsiaoDr@n zSM+^O2S=cvMn;zAUAwE@Hg7QKYvnbH$?CYaK9783yICEG5dF*3(`e7svrGT)Q;$wR zhl;`p)u)heZRws#^vO)6=1}|LZu2@_zJYQ*^Ov#oi@4V_6wV)q!%C%agc0E^VggB6 za~Z#~5G;X6&inctZuE6PrtfpCgB51%BGR4@k?lEr>t(dokrtGdb9@(SuYEk`Cwv!* zTU-GT<(?b<#>3wPev^2f41ZJjP2oBn{$}u-_A^q)KvzE7l^;AFOAn=o)5GeMY6LAx zn;@8R-&LFZ0Y7V$YL)g=DueGRzMJtKL(N%~bItF}sM?~&p|}kB!)mMACaT&9^4qD5 zLS5U0{C#Std0Qc^PT4riEh%uU466%mTjNb1`Kc1dc zpAMftf#;@7nr)#MDfOg!3gsSvN@wag;=8E%sCpWAJ@vYJ#>}`f_m8P(aep^z9ftCE z+)r>d_)MKZxuht3C)LMLW)I%XD7N?r`kGMBsZ;3dgHrYpKZ7=&SEq63A*>Z@Lw$KD zziEFjR%n2|Lz&PZ(%XVMYg)ocHz*CY_?$YAK0GY-UPHa>)QY~P36JUpKds18H?A+G zUQ{1PeUF6o9r?&_Ce%yn0^Z!GURG03ktYBF7iG1h*3Il(QlCJnM+HJI)HbPx!Tt;SDd1h<=RhTeX@`oi*R3OXHzA=fK&z%$P%x?_ z#k&QgNBE)OweG865YuE1)X!>F->Wb9#nMcv2o1DS0mm#FHEP=ECi|ril;Zr!*-3Be z^yv%xz4PT-ZNK;Or5E>meo!>hNIzc>b^jtXrdgkT1M3tCrni|IMq$(euTZH~XWPwD zByA4|f^C=dg-!cw(ZEVss(57oYQ&sziLX`XY@N)`(yGgNsaKdoy8 zRV+}-IXBfBF!G7%)^<_TdWmFp~I z$3!!8Bc?xtGO_${Fw)FO4V#*ol~F#u!}>Ni2iSHK<@4z`4&grAx9aNSvCrfA9D0ST zd0qu$D_icw^itrgjH`IlhIkL94)2Xu6LQbgINo$|Z!6TlSbkeGfpoh{s#F<<>g7R5 zb4it^p?Z0Uk{zDzYP$MI(b6WBnP+Z@eZY5I4ZO|S*(IeOXvX!=v`hU=bFi<}1E^OF z-OIyay?dGo{gw6`Uuh2W4K?^Sd%OpwhsqCD9$Fp=OYdza_3ySz|9*31bUD^?0O_J?pv3TLv0%4CJl?#zu-H@PxVfKw*?{OL2%!*6pNqXm#eL!t(L(gI z$cub$w9tVri$W_!O^Vo|Ut*~#CZAw(2}#SHuGPx=K4fzdf{I3%8!XlHmBn&U%2#2@ z%cY?y8PwGm_?}bttNLZ;;-#wcZ|RH3>rvLjH#sMe9x1JHWSaVuzQW{FEOL}`?qpXl zCqv@Ujhpq!4owJ+K>BsQz5k=VZRm&oEL*+GUt@1xWpB0@VT20&JWa&;kUf}bp4bEnj$fY9ud%SK+9#Rook^YJJ<)2o7_v5< z)s^Wo6Tt49cKfst7{A0im;Ygu5Cf2#O<7cUvUbAapJ%1)gzZ|1)sbWd*BK~6yl=lb zWDnul7)wgotb?30WZ6)UT`TorB91)XWIh{Dp!ATG9pW7;vzl6MZ13l@lTm-xD9)7Z zNp{~SJqE&p^lk^DPS}m7FNDV zN~)_U(%7-4pY1{$cJuu#-sA-1Yt0)o-LqkrZ(jEyGKB6AFl%HXfKiIrc$Puqg!EK4 z>NqJzXZqk}HPf7R8oSnXzo+PiL(p%aEm#+ieZy{VIN4FJ7dGm37F}siW`;9fN72Up z>vg$ZuBXRPMLWtb@sg9iOjc~s{yJwjVJO)OOX@fj0D<%q_lBI1f?z!gG?|p)S~816 z$C7V24oo3JtK0MfHw5rIcl$^LX#Xt(SKzm?_sMt?P6B4Ib)c(V^p8K#ZQTPs`NtUO zg7#s~g=3Cj*fmnUh@NViA=Y3DUb!KdGR}3og(B;WrCq4CMWo?;(YO#GL5ho|$B>ai z9VAReeCwmA^;=+-5%F#~z1Lc}GI{1UA(FlG5-jcRJBN_Q<}%BPW}<$I^(-f=nPy^c zfVNt+j(U_0(J|RfzB7Qb)~g^9uR+Lh@ps#?V%5~)SpCyCQ8)iS)&Fz^PvQo1 zE?%Sj=ex`E-h?xC6>g%0U!`|Rk7K4LvLdmF0HSx|hB!Cmk+5q`lNzOe@!jNxBF~X= z-sA>u7GVE27sBInStw{#i1Ok-nMfJPZm?oDXGDOJN?~al8V6MhS;*V~>+Pv$%#>;C zAX2U$j0Sta0?`<;`r~NDelz7{z$U>IZLrQ&u*%g;oJ@7hPK>&+!@D+~uHvb+bJc$H z(r>Rr=X$DcC19R}0Z#8l^*vxmcEmXnedIfkAqi2S+B!+&dtlv;)i=SF15!f3;XRaa zl)Y$akGVOwF25^q=G-w)6Y}&KJWb%K{H~z3JC=SXNX@0^GIIlP;Kb&#)IQLLxd&|} zx@X#%a0|4?wRO%{>b^cOQz+6Q403cJ)KF&@tK#qRrhRDtzIPK!Xfc524SrW5GqJi| zN7s98O&cy}FZ{d)?}E5$uhS~v7YoqtyfaHM@z*?61BIX|IrKEbT~vqn27Q5#RZo`! z4Ubd1rrPwJUkto`*C>NsGeQIEgw_d16D$n{&j_bpZDG=DH%muVUFcA|qR^NjPLBee zQGiyox?ePH46W)xiJ12mjVpMT9#dE@+U^7i9_Um?3TMi&!u@ppi}zJX-+Gh7zP4ZL z)YH{M*Yv%JnNSL`1-0t_C}5#8&N`;BX9C4dfU*oC&?13hVC^;vSc;pGPcO80)am-+ zj9sa5og=sUOlcOLSr6t9uVkEewFQV3?W#)E17Z(nLnw5hHWOYKuKNX;G4M{Qf(VTW z3`DbnYg`25i6PNb^kyK3exAuJl2!^+M>3=|O~WB!s9&!w!m1JJ9onci)V!eZ}VEz$gfPImV#j2-8r?=-UJ!Eo*0)))Qs&ALOPH1I^*`Lah zrUgB>DH?ve-3hmAoGjRKunhepet|V`#1tPB@I8_SW-;c*!)rvMbv+9`gpE>0+t5N$ zwtIMbQxii2VWZ=yL)ff2M?Z)Nx>(ueKawW z85LLn35Qy|u$a3KPHlwXK%|1Z8`zUag|LK);Yv@lIJNWKvr$WvWKsx4nv@cg?e`|5 z0(tYGOPki8M-A{WcYu@nZ_Y(L$}M@8+nay!ljkm8e);U>{MEBlS1w-8zkKn`*$eqg zr!JqmZ~>X~moL6@segruJq@n#He?b|<^a*z#Q&j3)JnYnE)Zbe(%TSTfIIWC`rayd9Q=>) zG{XM~A&X)cY*(Sc3{96-F8vfk80tvL_aI1tkHI_X&O@1oolfdIN^CNMTRce2 zC7aF+fCS>Wg-{0z_9Xzzasp4t4-+BxO9%iZRt6QtGn)e0$`C*cwSO!81x<&--1gX5 zY=(k8@)Yn@jL){h1IqFcl#)|I!af*$u?qKp2?0Q*ins5BT@zv;7g->ptP)yf+*N>#BhOnHwhwGR0sl2x?)( zfdSNH1@Um`Un{A7>BmH}3-F{>sv5aeWIM@aj4&g%<3SJ-N;by%+z)Wyju*d^0CxI4c`tzIM1W}S^Z=qmK*arSKy(Dq4mNooH2~0H zcASH501$uf+W;CWt`cvh0=ea=dD*Brs8&?N zXHhDH5{^nk;unCzy9Tf_jUR7w$*GH1&mN=f0+D4Ad|8x6$_hR31)(Q&6A1G*b~;FC zx>oNnXuyX^8DfO*@L|)!HB2oeD5IWG`vE*_WsWoI z1{TPLlV2vxvG(y;Nd6j!LK!5H?w}75 zCu47e9+GlwnBssocQXT^-j2Hp3chuNA-9~mF7}XXVh6EC$n}0?=;W9qDINqF24U@l zf$+X^0O4bXh=a_zjn6(G`Emq)x*@Lv2O$`t?R$Vxy|$=}pn*LQ^ucIEK=HDHx_uv5 z6UOqMFzeD;fppuk)Ul&NGU{;AzkmVulc>Lhd%wXsP9d|yr5xvG)-fiu>xS$!J=Zgd zoIrxHOyUYMA9)g?+A9~_>=E#h_`F@;jY)?lQR)+~jhWJo?z zDj5kq%AK{Aov;*1Qhymo#l;YnT7Th;()6dXaQgc&4f-!Kc>;-%#To}fFNi0OIj0xrse(Z2$5Ozd5+BHtQmznC{YHU_Z{AZ+O`Ai;SCDCjkb<7)UL`R3HpTP%#m26rW!QP0id%DCYl%CD&iX03gH4vbC2+ ze+hTHurpLl{tEB57rTEQ_>K)~ULQebN0KO-q7eqNg6piKob=DROm`2)5?k9CODdc0 z%r*3_KEg+mXw5tIt$1Q0;xOBRNS=)RgL>kEQ9H;%#-yokmUkgvDoB$s1ll5DQd!A{ zt21bDHhe-%?R%n9Ag}4h6IT#<+C^ErmMk(1tYN<$nhCxg(oDg`#!C?~UGt82rjLn7 z>|SIBm^~;~gtj|FqQM;ZVBYj+n0$fB_aK>yly$L1bJcShy2hrlQvfll3GAAfXXv?F ztSlhFauLGl(-L6UidP}Rlv{2YF&BoP!h43OTj8dHvvEVn1!u`fTZ3jcLeaA|KzVe1 zITX)OS4b?z2uTm40^8mJHVT<@bv#ZP6f)i-&-9RT>VF(P}?7Bnj z8>b==>wXSvQJc4aL%=dZV+X8AMjFqBy@IXC*!8#s;2Pp3Y#RnE+{YlGDxol4WhN&= zC8`aVnzmN`BTS~5`~Z_Su(e#qhwC5W%^yV4ikA_x0*D2aE-bZ@L2Y^o0k)BFCzR`5 zl*^^rBZ;cfKgR0;xi+2F0=m6srq%!%PY9Wh6EY9swGL#Cx+y1w#5gn}!sy)yHZ#@f-eQ2nBf)n zC?w$Bh>0o=VTkq~yqRhyC4>f{HVmv$3H0RB8|RzpIRq-Pmk6t20LrisR+qsmZ@bHx z>foAZPV}sqT0wY?>}xpROriF~@<4N7Wz!w_Oy0N=j&eYyDUwnDDw+Khsb&IYUqU~# z%YzutU@(fE6#okCY;F$Djp4fm-*J@z$?xv%Uq%0{CPGNx_%7-7-(SZ8ft z)Z9Og7RNC$&?v3P_>HdKwNK+uh5b{`>rjFZcAmY3Kt5=OFwl23wcneC2_M`;a0Z|j zy9yD#!}@RI&Xh2eiLG7A)ewwt?R{0S`HECQJDWto82RrJT((^9KG73=a*zpSIQ_#+ z-eN*&-=H~Fs+LRtllFL1=)r@`?P0PT37DDa4ix{ox=6}yT6~N-YE8L8V>g8FN|pV( zVQC3-mCJTN(+Bv@B$GZ$&?U5*1(H2KS4k5-kK$tk;72e%??6h5V9Ot@CARxv)){gc zkT2CXb9d6v4HN}62aZZ=K#l{+!iVl+oKlwZl=vQ7#-5GHJ-;d@_CrK0yT8OXY* zkEZW?N?KiAZjE%An+&m`WE*`3kV)Vx#0GxPjfoAtT!=|Rbe#=jFl~S4&V9;}J1&R} zNfF}KKyaYzNO{J^!N>#a9LUqSSk##Vp*lh0BPwm&5GoRwHfbVnOl&}!Qb1{vCYWp> zxzQAz1~GVMIfM8-k|nCe{u`f>{#km-+- zX~earnwcOCHu%>8v4rXYe6#q9ej{uZ(xqU>3}RPPP3*J)^JWjN+8bcHc-jPxvcz(> znbn_b4$KWN58}xPaEbVW2zu5tw@JSvSTV{NSg>45!y#5wvumDUPlsr~S^X{?O=Hag z<$y6G@>XgdgqA%$22#)xSLdOONw|=bv~PL z8pp}IJG+`Uvg0sh1tA!<>VawaDds6h-z+Q{zap%m!divNUPXt+8>oeE?%?G#6VYbL zeIs)x2W}X*ywi((^7d&?AeoO1Od&HHLb}swY*{mzWM9S`xd+92%$Q_c?djxFVLCZs z?dcS@!|9~SjAlz>47Wm1#52q)E)%@3iK(n3MfehgKTdGd#RS`PDY?VV2acFUGMr#O zgHPfSkZ+nWy5@LYvxS*UqiYOwO{mPE(9YfHd{-F25Ad>YYC|XegM7r9FnWT_^m>!b zDlT)9NiA#`nMSNpi0pKRx`}OfVa{AT)g(b1lbrw+36#<+K1@sS3KjHlH!gNV@Tu{c z?`l7Fss74d4ViH9s6Q;%A}7f8q+C5{^YT}^Y$`=?Xp|>N!uJF976>+SD1M;cw=lu+m5hjWW z^%|XIGKd7Owm|D)UY2;7!hC?h21~ladkai76Rw}W$z+KM#vR5`E;A<+yN@{njDCU1 z^GwL6#Xo3NT2XVwwk`s-2m)yq`8!O0mdVdC`FSR!6D{`&%%uio1+Kr$C%bwRaY8Fbaz@RAGvG2k@^>sULe+^NTAJ|Fzk02R; zHIcUeFpiFdN!(*wPo9qWs=Us~6c33ZhGw+T+x@`>R zTRdSn7M~<|cXV`@{g4sQ-LGv6%O}u}1p2Wpk@*0kr{#WqOv>AL`D9x>^ELZ9sVTg> z&CPs0b$(ac9yRSk;79E?kTY{PJJw3%U%Pnu!kHqbq~~Mf>NLLZbEN`c7xrSMpi{!Q z#Xa;OP4-k{mu$O>7@vt}?6PY>vHC^4z1}*shPe;O?AjB)c?+N-JJ#hegJ0rnLh-4+ z=uac*;c|d=XN`KN|;xAXEWlm{PEdT~l#s4LJT9%5S`dukuxOuor>{3%V>R-k?=S)Ygu#FrOHd-Xg zMIYvghce2CkpbgjXm%nXv%&bxc*%~yP~y-a?6=e8$_Q*ue}iF4i(O-FZO@;bI`#a8 zv-#7nT)BGj=!{_3S7GCiLNIQ2ipZ35zCsMM@?920Hy47BY9%wNTuUELK@ zAuE|BUVL+f{{jg*bLk^==1_Ffk~w$@LAd64t9#+ezr1I67Q*iQBxfy! zq985c891lmE4b*H9&!VcO5UuK8h;CLqtgqS)5g+eNqdRa>t@UQkqQQ>S_o<_ZrQ+$?i z1_*Ts6Mam9?b`XZXFcJRGvoC!gJwJ>I^7A=u0r0!24I{w#FLaH#ID_3)RRf?v^lh@ zjPr(|2T8oanwr3~ai&yzr@PzW8; z8L7I4E@>|+0Z^<%Z(Z`r1?(-jZZ^7e(=PcQ?4DsTdT(Eh`Zxu?L&Bp)dLpi4Q*JVx z?k?&@!X;&X7FZ_%&-R-m!;Cph*_S?)Cs*}1q0kz^o8;m&zCu-+u>u%<6ohT*e~t&> z;e4^LvDOwWX*)2z9hv=WXr4xnSPH%*zEwl=0e0BjICk(L&UC_-W7&Jx+S(N!Y&Lwt z@^ZbVdi;3Qyh!Y%e?No_^*|cwc_51)pvvSQ%GT0gYm@S$sA4s zqdo?cpCeI;jCHj2L6k*+l0Lvd+GU#G8EBLe=6WVb2N|h#0H=pR+(DdUmJVk`KRN}J zfyS1a8;nZ84Sk8Gc8W!^t0+0uc+toKH0GYztHiC#-_E%NyxL#}PVB{25py^MvW?3d z6!R+B(@ly6LHtHZ0jLCv+B}X}-SuksVS><~$-i^v7(6rY#=!j&Zp)JCUq`O-tPBHG zgPTAKIKrWzcm`T@*dvO5ruyaU1|+=q(-(Pa+DF)!q6NU!oTJ?+Jd8z z+&a2|aQU6meAf(hAItb}0LHR)%1zRj1RoQmf5UPiV~65Of_^1>v!Mv^2(D3joL2R} zXWLJ(uRB7^U!VBHez$f;B0}H!lHzOck#xz++eokj=t`p;!*}aR?wOD)2ix+15DQ!6 z$_?VTVI7B67=9*#{#Xp#^&Sn9Kplqv0v#9xEGJ&MUrG{iS#Ae6`K45RF(z(p0cd!X z;_x9!*zfCOaD2l)&T|#UXsu4tjx-kIQN8e1#a9NJNpV9TC%Q-<27zWNxSnaPP83pT z#X>8oU`Sdi0c{4@de{mgIpBpFZnxs#^a{i$_0(PKsXMC%QTGt)MvahrhYP=il@mJr zVjE1?6~cHhsE5xfJ&Qn{YIK+%Z*9D4$2`PKOoOsxhY!q#unqFm5}py$EqzM8tcagT z^utM;9fRP#)_&tQqPNU=Hf}|pT=0WOz{3km+}CuyfB={R_*sCR=;-^?ruWK+REz=O zxWU;&-sI$@H+xk0^-UuqVRYW(rYKlpdnDm0uSU`~1K%&NJ-J?4=?yt_D4B<6kdmR3 zc0n@WiL}w4JBnfQhOy3iPxj30fv69LBtgCc_k7&zotXYY=KMac#l0)*lw;XX&mMwn z|M2Vkk$W7uCnWbIa!*O_2y#bXZ!f|}??k6^R?7;HNQH-ydjj95@C_@T!s-~=jUeXF z;1HpbUoAr5<9wQp#gWKd!AZcBo3YMn4hPI(Qt3G~#&aC1+7mQ?=KLpdM$xTF?;Ptj z#~gat=>||?(M8#Y__&8+2M)8KZ>kBYeH)ApACU#$Z^D)iQC>OM2R=GjTJ2_IE( zsbsdKSAakiIQ&#!Y1rm#SW$LO$~LkcKI-jjOioa+qbBF`Bvfg`oF=*sg++Os&v>g1 z9P80pZ=d*pry7rU<8K?%L&k?T*WrCa;;**sU;!dpuM*sNvKJyhT7~CY_5$k?b3jKg zAMWFu`rnbY43G_mQi2v+3GJ4x>}4FDiLHln@}L$Cmt0f-TPBY(p-OEyBNa{<3H}3b z{6{8~351pANan)G!~SU#S)eYKfRLPBeb~1;5M#$bcptfh(e+ zJ?4ZuF&DT7?-y{l&(2{q8e(~<0}H-%NF)hpv~r!hlu!WlsIHA@P=pZu6dqD5N5Dqy zZqO;d-RXJg4V!Arg_zb$$=5?LsM^?n3vRWDGaO9g9y@l-6Z_1%h~FU>`zB5_Jnsfv zdZxjCuw+dq8dsvR_oL`K2)}WEV4UeRe4?>OnvH5+x8aFQ7gqlk@veiz_xBOAzDO~` zYwMwpG6iB9GP1GThvvD#o<8^McDoI<^mn~$IWG}QC()Hmnz)&Oi4%XsG2$9>Fna?_ zQzA4I`*;oE_`l(mBECJJ$e1Vb@US$)7g8?M{u4suUpt2Xe@v|8Z%4KD9bU_k0YURO+yejW^rW~ zw%k(y(C3S^mDpZn+sbqi+#sn==ktfyd%#VByd-cA3Hg zbHa0l$#)bwbV>QM+6OD=e+`g5d8pAGAs(VWfFGugs6&vb!;`To>O`~X?K|J+zn(ZI z5<+{-T}J>I=09;ocE1ZrlYU-OAt$!O2-bP3D zudxKLHtdrAT|7ls5FgJY*uBbW&|f#CL0`f-u1bQpWPAFW**lP8LQ-`nou@2%Ix zHaU0*Pk~)N{o-$p(m+eD}V#98pS(2L5)6;K1m zP=hbC^Ba+b)_COh?Ip(FM<~eRO<8wYeH>!``?#@YqT8d`kEcDOfQ>D(S?Mv!#btCo zj_8DOoP8NDcd?3}=H)+Nvfi{m5OsP3tHuwtdk;|xM?Y|T{Hp#Rs1Bm%EH;(Mwgx$d zsAnGtJy305+fEG3NJDZ?I78NPW(#%`4M#+fPhob01cVqn9LJ6;$VqbL;3}yc`01Q6 z){>H)M&@WCM5IEpiYvswDn!3FzQ6HJL1twj$ih=mMlb|SbTHUm3kK(g82T=oiIcEJ zB<#e!F!MdP^a2m1w4$g~YyEq?nm~n&ffF#=f5*}3a9o_zq-X1Zo5B>gusP#4P z{#f_Buf?j{+*mNSgk1~EIK!)v4YuIKO9uGEUHw|@Q)qcP^UawmcGU3qrgal{n@Rnv z&E(2ff>9L_jrupDtDCW#W=6J|6~H*6IUQ}!>F?tWd{?&f{WbmjYIpx+&URoPxt{2e z+N4I`LZHgZef-x9w~36K8($In9U1Hk@ZjLEFNDs6`*eb`gp)*m5=gLq0sbFS2Xe8{ zA}+hNc-*5@7;f9@j_swd1^ZUu_CXDyDKV5>3jzMG2yN}5Cd#^`8uQOZ5VwXXEgWKi zz&&wO6zOwe;qXTIKS)AF^jRDe0s4_=Eph;59=!>@Rj|Rb546A+`nRK#wM#z@u;^`p+WYvTpT_L4P-H|4!8Xu~_4gZ_xc{)BMEFE{gkGY-bor z&gGJ7POzxp5`Ue;VWYtlkLR|M;q{oZg^J=>;?Pxtf3ea^aXT0)|70Eii-{P=DL&*jud^6Nn;-0f&76ayLd|Y6cBN&l`QH?q*O}Iz z;Iz~C%%E|5HHDZ|DgZ+`31i%Wl1_C2>Uhc;g5^95C7;o&83z%qVLs)!E{`D^vUh$s zj=T1oJKuBP9NMlsS)-Us2SM@T=S)P7V)9W+p3$ zo)m}>D9vqJd)f}cRRXzkX*ms+P8Hq4&D?-o8@G(!#5+TB2d7!dKR&9P6LT|ia}jz+ zU5=^BjYVhsbf1xHLb#?`i;L$fHzKb^PC*`0HPCgT-$Y4soSw(rFEH814#kp%|3y|Hs`W23rsZESsGX}1#^$>E54Trf) zBH0+t#+;2U9Ye0|i~(zQ+L;f>wvReH2QvM-iUoJ z`Wv}1d3Ys$4!jbNRzgM(*`!X z$kT`y*6=m_wrjAZ2E?&VNbPr{XN+1yl-<|u%kG+cshw_|<4bMBXyU0sFa3W3qzcp8 literal 0 HcmV?d00001 diff --git a/vllm/distributed/__pycache__/utils.cpython-310.pyc b/vllm/distributed/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4fac5b03978519367c446f8895585a7379a041b GIT binary patch literal 2840 zcmbVO&2Jn@6|buPn4S+iw%5CfvjIvG5{#4`vqA!7k_Zd3A}C~|*b30ns<%B=p6R$h zCRNoAGwDIgIvhD5aYIP)aSz-%@rU#gPJ83Zie2+wjXm}_%O19+uC9;QuipFB`@L7i z(o&1S^X}jNG5lwnkbmLe;)AO{fmi<-8bJh2Nk&7Ov5-+5v(yP4`*kxf^z59I`dK4v zP$C;LIHF-wxWao$!j|ww1HSEMAl5}wI!`;IC6`27E{l$&INl)pfmjmDFTL=JxB_b{ z@UFt!kxVYhrF~{4*2Ehx-Ec*)9nyXCQ?QJ5Ssm=iXQj;hvi5e%Q7RvHY28lp#6)Q^ z7)aG;D0uo{--Gw3@M;cCMJoCh_;vbQ-Jn_-b}9;RuwDH3f| zvR4`@HcOMFx;GlvZ%FGxbZT&tCT4sE^lBfPHRjURiGna7uVLlDq{V6tI>usi1QCF! zT_=DJo&5T1p~K@`a3xKt@)n23f@jeza()=6rR;U7HA$g~7KB-Yky-obh>&kWU%Sw> zoPeqhbl$6d1HKljUIsv;QLId2k|Njj)qatiWKb5Rj`Rosi2B1aKhU*5Goz2Lo;|qd zJWPHLgNnQ&#}Fph$kZ_uM(mhW&IZ76#HOxsp1J^uTRDRL(V2P`Q#Xw#9O1qsmG?7p zM33kW^Z+m~ehhKJchCYJ^O>&LV`KUCf>Us+ji#Tjy1n)`3i+z5M4mSn2pxA?$8MUC~|A4o}@9*#v7X3KYcYE`nyS_qZ+1RaJolGE$LGtJiYtJfz z5#}@yTSBpAY>>ENaHcd|I_pLFd)PkR8bP{j^xI!y+zrsGkwTr-HM+uFwnkm1-iC>K z6+>v42*^v|%UCL1H%6mK#rc8l;Jyp=!IXtOwVC7J&@tpGojO1dT2l4lq$8McfEvy# zCfs9+1evd)hhV@oO7NrD4!yll=fE|7Xe^qpYTqV;6^c(w7 zZGhbDI++F)t%7x8WdgenHi-5$a=?_SF35K~lfUmOs0js^hcZvcK64avm=Z~Qk_@5%jhpJLi{7xRc}G#`vGOt zrvcmpmK`7Z?g^vpB>0G(be!w712gL|x57F!Xa@fYK(Yh3hJdLq{NAPSbyuPAjz@3| z3KOovjH@u)hE2Sc^bjibZ5+4lcy={Y7+G}?MR}abD60L_X}pTqlo{X zq&6klZ7+C-2}jwRL2x@OM48IFSOfGeXja^Sb torch.Tensor: + """All-reduce the input tensor across model parallel group.""" + return get_tp_group().all_reduce(input_) + + +def tensor_model_parallel_all_gather(input_: torch.Tensor, + dim: int = -1) -> torch.Tensor: + """All-gather the input tensor across model parallel group.""" + return get_tp_group().all_gather(input_, dim) + + +def tensor_model_parallel_gather(input_: torch.Tensor, + dst: int = 0, + dim: int = -1) -> Optional[torch.Tensor]: + """Gather the input tensor across model parallel group.""" + return get_tp_group().gather(input_, dst, dim) + + +def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor, + Any]]] = None, + src: int = 0): + if not torch.distributed.is_initialized(): + return tensor_dict + return get_tp_group().broadcast_tensor_dict(tensor_dict, src) diff --git a/vllm/distributed/device_communicators/__init__.py b/vllm/distributed/device_communicators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/distributed/device_communicators/__pycache__/__init__.cpython-310.pyc b/vllm/distributed/device_communicators/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2619583b2e1c7c4eb825d60fadb4846d5e0fd30d GIT binary patch literal 178 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;x_$eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_eoAI>Nl|7}X-R5|eoAUtW^!tL va(-@ZXVo82cv3`7fW?p7Ve7s&kHtU_CxC*&s zW|wkAtbnv~QJ_Fkpg<2rP*mj9_NR>{`Cn zwSBu=@he@&cNARjRQ)REt&ZEB@n@JOW<+h9`Llwp*ZnzF&wrqCn^zuayn^weZNpy> z24D`a0#@DDfq$60!V!zhT1)3My!OEKmw++L>%gdQ>;4h0-_{y)!++js$EnwfJHkU7 zdi{YE-p$Rd-AlatGVJ$-^jbaXWo_X_10KrE>%{kDC^uK!tC<&W^kSZ>*+dBLWj!zI zb^Amv^it6eWta&sZHE$*qlKrvFyYwEje5z3NMey>YKuWCy!&l|eTBC^NHcFjG3<0e zt|jh!tw9oHaW6?%+{X~Gw%TAtm4{gvb;2}ln1yj6*(}UUag-J2^*GH6=SH6vgdJec z#7Ue5o!;7-kP$84`!g^&kCGloB{XPE^L5UIv2JpG5prB(zQqkl%`6s36j%Gv$D;co z4$~1q<~wnc35ks!e+2@o4ZCoHAPKu72nsg{x;;JsUQpD6;EO@n8O_*Hw%He{Z%4ts z%}n@C(27Ner@BUa7YppB(- zR}+RXIXkBb3$2dU7S^(czHv_DCS+w5&XtkQ3X{%URHR7h%!PxasKc)|Dw4cP&QMiD zRTwx$F&6}3lJv4rX*>udsYqH>O+-`n<_I>A>I6zkM{+esci22r(887?8^|PLm>K^n z=n&>bl=SbYINM^G#^F7GT-wre{gU=8cgx7N+}P0MVQy?OrX}o)>L)mvshQd~JUMaB zYU#=Qn4$ltC^<`Cp_Avv4!rmw+cI+_H@W#+eOcSlA8K1xZtWN}!|i2l%g*f`lUFcu zFmL7BLzdq72wt4qUu)dWwXe0mn%nF*EVm%dnXrK)xE_8NqNpc%oUD=0g?k6>EbFK5 zHJjZqd1s|3*I*;b;7b~w>4$3~rQX}kzU)OoKjuLAfprD6aP6=`}JejG<_FHWM)fD47LNKXZV7!#dd z$PpRtZSGyZ;(dZ(6DA%-glrO@aL!hY$U*$|!>m2pR1prxlqSK6WOQXXx9fmIi?|H*n6_rK=J@~WP>x-nYz=Qa*u*lE)X!@Nv zD_n_Cm&IhLg`2`jv-EzPwHp=R0Ra^T3aix{B)nj)!h}WyQ2<_H(pd_JBu*ReF>SKy z;xL^VF^l9BjF@?mOqMFM;xY$+6My}C{j1~Oo0h4&#yqRDBPiw*QxGu~!SUP&}^ihxID;Sj+@O)wIIDl0jGSu#xuJCO|r#iv{ zK4G9BLIB55IL-u0Gp?9fGZnl#qFGZEt0Ri;2wLt`!-`p=v*!BsSVL}{7l*_`3w|aJ z^Oyp2RDPV}^A9X!#6yT%3w*X^^22-)8S@#wL=bTZkR$vV zK#uaGVtItK%#Q(bjK9U7<;T%`7QGYvBznjBbA$!{s#xjq=K(vxU!d`5?J52uU?)eg zmw^59h_1t5;jd!uIo?qCuc>->_H}+5v)-7Rs9}dQdx*Mb!9F zF?$hF|JI%BW#%qsDGA&b*(H@AiX|GB{o9*K)Rw&@9*Saa7ko>kA}h1r2O_&HWl!FQ z`6p|IOGqChLHDA1X*uv z^Fb6wZBZ(9tEmY84LUh;`T&UnQJ!rh+N&{3 z=Mira^$}xP#@6)>CZEfU9W!T$-pB(4%aP;Cc3I1{z*3ktFt5ehmVucHvc-`tGjnon z-O9~%+`(z7G7tNej*(17W72O)wpf6+9ufo$wEHWFeOuBSK zLpmIXx;RmxFd9fnSEy0!ZOkf%moMYI@+?)es0#DqrR&!Vg96B#G+sc&>q$hsU~;7@ zD!5sMStjKnTDCydVZyZtC8YpJsX)F(6K|vPZM8TpOhS+z!FLn0ie9(^sXJHD#-s{a z6s}yRBQ&|Xi(ZYnx{0=~0vSd~+QTCkarTM5u8fIL9#03IOdbKXDOJ49RB-Xb%NMWZz*V;qIx?d1_rqh;2iuEWSc>S>vB;Vy$6CYy z)&<*zt-yY9ztA5tWiN`~p*6@_XWj+(?*g;1xrlmPV5s>18DRUERjYmtcuM;$vY|K4 z9U0(0cL!0rQyyS)>}KB{1tleOZ5|G$hwDBn1 z2Y`P|>>XvpXAiPB31U-MJId-4j>uN_aFPC+^!>rnn21LE#9S|zu!+0e56;v=L# z5&ROW{h=g+O{)}TZG@cxE@2TaWm!Z!jepdi$d4X_d#xvDxt`tQ-xBdPJ`nCYos!o;*WTv$BWYM%jn?YV&*2&4aGY0J6wTE zK67i^oN#KaKBl9|6{=`)!!FsqL4cc75xf2&k`WRy680t03K4G$^K)E6_n(94`~^iR z*hCON`E&&jR>vQRofrfrw@$-qAz-LOwez)!KyCa@cpdZX@Ir7~A}jiyFXnlw<@ zZS^+a&o`0y8%j!VPFtsxW8NfYH~hU?*Q^Kug2>xi2|NId@V%u-IX|8zv@ z8_Pxo=m-Gy=q!MC~l~KZqkmB*3B?=;Z z<_Y?K<>rm6t9LHnk{^MN4>(1rAS>#$w6v>hWKIrEgKqo z+jyTTm|h{6$?8wku8n7kN)T{7#e-6sGn0>5B9#;a3me}^aK|mBp@d@rtfb*v_@ooH z{Tcf7vr>N7k`x-0>I!=V`2|7lQuRwzeic{uL4@xiqI4trZX%VyU!oofV~>E8s7OkM mbFTb`^*&jw`buVMj#XnvnPb#+N4IU8Ef~5v&lc>3WB&*1MF*+? literal 0 HcmV?d00001 diff --git a/vllm/distributed/device_communicators/__pycache__/custom_all_reduce.cpython-310.pyc b/vllm/distributed/device_communicators/__pycache__/custom_all_reduce.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..045bef61d98d9ffc68580be52a526434929423e3 GIT binary patch literal 8276 zcmb_h+ix3JdY?P5q9}>F*^WKF#0ht8D#>P>G)>YvvYjgNDyk(X*g?|S(VU?~YB)om z8QKv$Dt1?X%|oAy7Sdv0jG{pMQuL|lV}Sz2J`eg(v_Js^{Rl4MC@5|e$YBU`deZN*nQs-<=` zOY7*C-Z3mgz*RqI6UGz(~+l%6`V~5=c-P%sKXLsM~+Kx-JY>y;G?a=g6D`>^GAMViEa$0qJoO?lRcYP;rhP(sVdAG+Ln|Y5~u4lVp52I1+#Ga4s zx9+AIcY?jN_&DS~v!hnuOAYSDJs!9UJ#p}-;#WsmMTyo?#Zn?4%D1Jj%mXD+5@}E2 zrH5K9w-tUNR*y6$AA>hyy=@%j5-CxR6$KR7l;S+~4wdK{=%H*%4>f^LZN2~Q=3cAY zZ3R2###^U%#1nkLts4LWW4prV;ItGj_aG^MXf_HTU_w zdq1%^KV93fv6sF2$9GmgO^pmosn!abVXDU=cXv~bkYg>lPB6sK0PkJTU4? z51&Sr%B`moUqb7*f4UNPx+~pQ_h#(y9WTE5PN%oqd9(9)#cyq|bob-kFj#ry`<)dA zPT;NWUhJ`z5fwU}UeIzKTq0T-(BJWWn|rM1dX4UWnz!jrV1)BHEutgKa$Z(sL)K7h zFJ_e@+R6(};pgaPYAfivkj&8^wKWt$vA5Akq$BBA>Mr9~K9-J@V}hun){do`v_yUV zq`wF1JeAk$hBeg!AA-$WJH4rdyf}QlB z*YW(i%;_Zpd0DlLZaRytAA8Q;sr_FEJR)5*D_8KqilRN|Cb}-l*xyJa{RhCFWXVjj zlpScMU&)qQP}miwdK%L_-7~zLmv0xE>axUiWrk%&GcDnB!* zB8)&W{ai}*9a1l;kue;vVqo2+iNn7cG-$eDk1w$glM~5?NTQZwB`T6wCInMs?MP1` z&au`u+POV7I`@ghv_tuqa-5F~iA*T{MmdzDdR#mzu^b6Lt;YOvBDGbP|3*I0(3=|d z^h9R`sEiN9DSBLKPd}3Rx>#RK^teipL-|-8>@<@o@$8|(zl`UOYOItfe<}Tyax5Px zK{2MWD#n^L21Ns?fX%gQP&2_N(l2FFPF(Iza!&!mQAtBH|jxwO|$9~DL$9zqy#a)i1%Gg zib-McM!NY7%}I)4ycAzLGLNO`ALu^7*sOoKbLF566?y4kDw#UEnv`(!3h*+wr}5t; zmFPDD8?_NOuO%8zQ=)$t@cAKJ(2ef)YJC0ZHHfdeD53W{?op$=;^lSpe~_pX&>s?% zElfafB(<&XVj#qN84cT6hDy)sN*bm4oSnYEl98O`P@?K~Zib(@B+9wBI~7F!m#? zcpJSR4)0iBbp8?gsAvxUZt3=Ztfgs5TcrVwb*P$;v9_n!k%$2>V`qo+HU!+c%SalarQhj z>T%EX9Cz1r!xNh^x&d~{t{j?<35)Wu2NNkQ{S9F|vHRG<$ZQ2T7s~{~NBcN9MU3D? zI2a~57$?`gyS3hbXA&dPZrJmgi8niOj0<`|9Ni)agPZHCtM{@K8l3DcN4%kU&+s8KBp6g_k1$2yrX4G|p$aGvDMv=H=)OafM*Kou$I;a2i zdpBml4&5;Ts-j+(oCh2yJRg%x)qCcvtYL1xWF*2+)G)U~%=5j#CAP!m9y>gMXL1Ag z^4wN)fB1%>ONaLjO!@~~9EW>fcm0;ziv9gYUl$DWTlq$+y4~JW#nM^4-}N||zcE8j zIzET8i4vKpc3`N#Dj&$P)P~7}UXt1hlbQ03`WMPW7;ShggbaRzs+*_)n8#0Q#9k1E zoP342uB1xija0TLEpqXF9}Ww+t@~jQiy|@-hNLL7FN_!c8xvg6ux;Uh+Ku-Gr$-;6 zFZ(v?3#}ilNZvos#i3l|{^1{oj zkhYD1&&#jS_^-d06^1<4is4HJVGKbV@iK$c{O5=}q5p_aeGW<|sQ|XP`7fE+BBpVzcRhQNhi>Lzd;&f!F$v(F| zh;SgKe1*0q-WOuUXVJ21s}FA9vOizjT>I$W9sBm3&(~J5R3o-c)sE9m^$1cLbMlb* z6slB*Rq(#%h>9hvNHV?Sz(#OhCP0pyV+tbX$lGrWsT4H`$bU>zDwxvjVO{XZZv}h8 zVYl?Y2Q8MW2>a4P$BP}|En9jI!uf8h5hT^UPB-39jqP5u>2a$-7_pgyk$lJbS}Zsf zoJ`q-t5(;w#p2SbBXI?a6G!w(HM=Jt&-S>Tolu`iwaD|Ef_xduKEVS{&ai6q zD*F*o3{^b{nIfy0*&MsYB5St0yB{I6@rMCbi)ENAWmvNb(bnAASy6_pDRIIOjr<$XE5woLDMsC zG#$opsmP1S(PKjgg~S)m1B7q|1z+JannU4qR*4`q=Q>?v zJiNaLScKZj&y}1(qyjsD)UbR8h_1ZoFHVweuu{afV+j+obt=j#?MaF=vvU-BQ^JJF z%SeMT1!13Z05TmZiFz!P?Ox|U!0czVwys*!NY}tCKxQV3xvjDt?ZR6VD#uQ%164GV z`ns0AkYa7aA)k%++Bcr8=kmtwU|v4YMt#D!|29vQ^6aV}9sLaB-9(GP^HHE>fd04#7h@KN~U5OBg(!b|yAz#{_{JSe&NqW|_; z#0pa~qTkE>sLY$I(-Gv-)m7Lz?w-=o#4zN6T6ylQ8h1ZQ=6^vlXhnCnEb9d5Th7pX;I7UA=zhSTjrj_di1!+uKJ=gArpmsym5 z+lw+ph!=jm*} zYnfQ)v@Rbpr$C3GU#g4DpYYl^dGq`$sz_$)y5Nu0$ZU3MWG0(`C+29;f1`*>)Iw@fiGo4Ayxu zS85||xTi!{v0mq|3*BYx0hKgb*Vpept77v^EGgEg=Y-Yamk4^1Dw8T9isZj=A}H+B zh>ff~vbCA}HZB}#H4CdxJhlJ9(ugojiQlSRhI2x0NmVVPJ}ViM%i&-4VwaFI{G;}W z)2^cfGbHsSUJx7yF+uuFIf6-ok_N947bQ@WnSkh7AXyZCDRjf0CwL~UkRHfFdHn1M zMtOSb!JOfsNkz~E>{}E;_&j-E^rVZEgdgAf%v(mHPFnDL-vVOdJrMYl_ZV4l;L+{w zph~N!PU{83r|>nJutU{2z4$I**zx2IwU?4*a#so|x*!Z0o8jNk@Cfex2vFkQ2;vMy z0J0z>3l0QKd|+uKCwnaMMT{CWN*s(2EusZjK5Jz}D`ed1V#~}$5&_Wyy4HCSykwt@ z5IcEJ2cI3t7lNMPL?13tL$8FX6a3pE=r|VA8Fvu)jko~c`lN{LKoty2&J+H%qS3!R zN!>Fm$J8P@z{IcuCxXC)lNn$^=5qr!#hanSDN_v-4H+F2sJ$%rq-D=1Rh}uh9KzCW z7MsAxX>d|B7(w`}1cmPV>%OmWU`WTSfzfKx*T(fsy$ho%ynZb2b7d z-?p#P(X>rjKI?7{=HO!rNp42oWr}_R`YdRtMceL#tcTFdPRq9au;=*M2>+1glcUB- zV6wpY52hnB}lWMR&5aXxTHU?4e>C7yJ{o-rxX^D?pek8snT9?l3(RDbcg7D&-tjL8}B{3`e zT**sxc11vjL2;C29*P3k6T4HMg5Ra;T~uk|&eyIdzCH5}0q3Y%p%HC641JM-;9UZm vR9&X(B?L?J?`Q1vAqk?OT3LcirjS^aPuH>n2QCjNX!<;_EvVJ0S>yizjhR3Y literal 0 HcmV?d00001 diff --git a/vllm/distributed/device_communicators/__pycache__/custom_all_reduce_utils.cpython-310.pyc b/vllm/distributed/device_communicators/__pycache__/custom_all_reduce_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44034d54cb45ad081f002fab76e49fe98a4b3cbd GIT binary patch literal 6685 zcmbtZOKcoRdhXZs^gQ_xNr{$ZwPnc?*WyspI%~r#DE8`ME086vMB8w$cU#S=BBzJb z-J|LnQ6zc-XX&f6dtVqB9(~9qxd+KDx8#%neGLLcz+Q4)ti^nP^^8PHUMzwP(e&^5luPwwj8>lK|HV`bjprxrEl z@H%%FwRMYCSoQ0Y_lD}Nz0zA}4fLLVK5sIjiIHz&zRjlC^w*YmhUvjs(5!NozqxKK zYKt0wi_Kio*epNAj^Jye-DY!a9;45(`3>DW&rh<2O+)+#H%Cjn)csNS?a_rjZBPHw z^p?Grx5CfxW!~B~yo>x2Utvdgjo-|8m-*$kqU` zTKaEWXs5p~@KGCmON&qGHt#IydATnVHfYNwE4QwPZJArQ!&K(Z-M$QyIOyfh1O8~h z<2J|O#|N|kPv8FL!et73L7tAeVcDeKJG)v<%- z(3TCf+l$(+&Pto6_!YB<6r}>jYtkMbO)(AERKAnXUHkOtiT2V%lH;9K1xV*0J!mCpClc3j2+Idmt zQOadLPwmB+2itMGgL?^gxOaq_Z}rF4?AQO?Uv-scmmTo+& znpNHUy*Nil9czNzkcVWR*igZzWd8Rk{27^Nhh$zKnUG&d<EN@Go^ zDNU#;O|YN|A?-?O!Y`l+lT}Nj!OU;yDSh9=l;K`g9sUK;0iW_C-` z&LCo0DYG+Uy~K=f3>ZvmFJ~~ZhuWR2Oy&`S;EeO~s4}h>c^L-Mgnzr4IYlY`_^-5I z>l-ksvW9i^T?0n)^q0o4I;>^(sE%FL$9W^GXAOu+Pc}wPX2FQoln_=%QxGi}^hb9y zl(YJz48v&$58xlFE`Fb;18D5_1-ISd?aeUWaKm-?-sO94piG7vrY?`k2QjxD#O@k* zSrT)1ZClNL2+ePIT4cyt=$`ntVy@rkT%3o?y5K@vmmaplzdy>xmdd zX$!`Z_h5!a3!a^YU=%1x*i!*2%NzZH`p`JGrN9--EaB-g_P3mJ9xcmcc`Ff{v@*!_ zwpT8{_v6byd1vL)MYY~)hYOCy_yuB@Ji9CE}J+KO1j|* z7ZkY{oKV#}iR5k)yZy_3bs2Qm?yY;*-dDZP#fO9f`y)5w4pN?C3U@!c_h~T<1s8(g zE-zno6#*!@wM1Lm_r6>?@6>4(rPblh@{Guok0Homn6mb z^S>;Gcjqiq8>ACL%3nR|d)8 zKd6`+0Xw;5w3Z7GMYggZh~n1JKiP!3EpQE#6E}FAgs_}+ff#@AUa&*xCz&Q3wI77S zP1fmFQ>DdVj;kO)$vjVKa?6Dh6r$kDtwVP_rZpbu!k{aPP{M*F-G-t@rv=YM*xnoq z8?YoV7(&-o06RBW2ZnTc(C(EKfFa2CW=<>Z{Sp^{rL{P#40`Ivl5OePE zC!uU749I*Z>?@Fz+r_&rFK-1R2LE$A=mRgYrKW=E1>zxyjN50KcWt0iC@{Mm$?bz|QKcAk_RplJF# zhMrkfDo_&8^Rv2HH!NNJC5YtnKHLq_#Uf;>{Wh+ci=U!TTtRXO9uWYf<0*Lb7{G~M zk9b8kq?T#FLuew?f2VzGF5)F7^^pNcVn`EjJX2cT5wZy*yH^yo)!hoI`Csb24WaFVDZ#0I4d&s$3Wy z-P3;ai(yS38!cuP0#V)9)=liQE{}Il$dldI<*5xLt7AWnZ_Ht1$Icofmp~le2PZ8} zzOntCt__>IwlgD7vu0MoiKenD(&?aifk6U?K`*`9dOpire0dKQS39?0a zr6-%k3r^3c$e;lL_L#pRl0kpxENz2aGdvI-{{UQ93QczTY>I=F!kD5G&34eo>xcWv zK%&0$Q}x0FTp)V}2eZ`yJXZc;;%k5&xggw0`Pikp>mrHV9gAG=j`4mqHw0o1TTh7% z*tQ+U4BmR@3~V&}lRHy1&ZfC8mVP8Yz|7pDz=GEdQ=fumVbDW_g$a7fixo=fT`MSL zpx&Ihk)*lR4`j!i`t0`YPyA~iUi(H6YOfx`n-C{|Z5zPDbBYaf zmg_v%*TwrF;u(I3uPg2l*_R&rb1y3cY0W=iV45LuY&bH*Hf-hAY`8FlfSxUwbvQVb z_Um}|*YyRXT2npyb;Cedz}bp(?PRkQXDjC2g^nXzvmRN#DNJZsu@{lvcU)rtRE zn3)5GS^f2Wn?eWb@AZAlnAJsu4bY85K^WskjIA~CGc+yzMG*$sKv~?Q5Gq9K)zriS0}1+vhF4WBDyUcJ zC&g2VO(f#JU#N_U_@f}mZ~$sqYv6`LAce|L`F;=sNvg+1ikXVvWwau8=;tL8KOsW{d|xCI^IwxNwV*W)2|bfH-pF59uqXhy#cmkfO~lzgO+~Vh151Z1wZ2SJm&m z`c=IT=QA^&fzPEs{JHrI(=MNzye^U<($-x7S(6>khX^|HBkXN1~) zg#}nzz;Au9I9P%;n=kUEk83!PGu*@xoGlmL++LfaGRop^q;k1d);2Z94i~rME>KJ= zY{Oxs`du5_JNVGWqkZ4NQ~bj;3}x(@`$plsW_)CVXA^TDzXNMvZ&}6SYq)&7=11nl zoETvZ^A62}uY-4|d}Cq^8k}u0`5d<}=W!df$!nl3URyWDbu}}wcImzLdq%cs8)|m@ zmTA!U!8hJF=$$)2Tb`^+@J*LfW_`&zoV4@tO`+3L)$=G5}u zS@jU+mImi`;q#6Sj~^zFcd(DaBNKa4mnMq?G z2?48Hd(Z9z$DcOd18%lq8@t#|=3X+2zumzK`o53n;7o9bn$h2J@SZ>YT$7u{<--vP z^<}#3tG@7uRR~uiU*-OtTyFV!w&LIH`D=M5F8HzX@5D*sj|xGroX|wtxadWrM4eQ% za$;F^Z7YivPo_5FjB6(zyz!Vm^5BFjvUuRphM4Ne{oWIlST6r0$-7Y!$|&2~dE%rG z$3teicLWdlK{!yt@~2CVEFV~Uz=J6khkKvC=~Fx&nZ2p}xX|k1R3hXvg)c=h%(Eii zh?7`tll3&(5`Hm~!WU7u?{|?6590pNEVNW$<$mO27U>VBDHR+U8(%&J#-g8(67B=2 zNGYfn0GxDWd3I56Szm>f&v9RjAHKadUcP zrfnMI3MM=(pds*%__7h*j^ZTRNJQzv8Zo?d3SCQN9d491`CB$#`LHW=wJM#eGL&|l z_41%i)2|Bku8;*vSTH*sAJmhwQ9&EVT+Wcb8gg&9A9&Zdm3UiZMJ`J_p?$hB%A-Uz z49=3n(;x~pn0h-!+1QYI#Jf?UN=N1NR@o4!YC=XzlwJ&c3l(KZ<`$G`7y++>1wuB^ z)sQOVO~CmU0D>H&(y3}^P#X)G7o~@L6-r%rgJ!Xvb^9{U09?l_?Ls6yjj~oQ7idS0 z(%T4Xgmw`$Hn6i$J7>nFO(UEnYVc=S~6^uX2IK1oz5tyL3Z+Ny2auCW;M^*8GymO#8mW zTC8PymSfV_hOBM2OlLY~UplU1E-;U^nQJYWp1H(aa}H}flSZC7#~ib5JM_-|7HeB1 zv&~2U>9xm?JQy7-2Y$$lgbmAwCrT-mRVt^tlzs#T$=HzS#nw?&1}Js|bBh&cse}^$ zDS9`OqhQ#&q*~k<8S*jAu^lA*fwgC=+Qi&*_KjWhh=ZiQZzy-Kfww&r=O#E57rfD; zvN~v^yw#E5VfQm@ZG2Jn5ub$RiCjTJ+eKRi{3*nzY1|*5zu@CgbP|fQA#wT!(DFR< zJF^lEs!`pLhlzUxB$y%f*Ai;J(9%h2C=heF7vT^$v_{uBkxi0%J(;6>qc|yRoLW5D zA~hW%4O-Ep`y5tb+Og#YnjpO8^F*{wx@!8q$>)gsCP*Jo@fe6>+c;7>M*F^f-^Ep! z!`H(RyQX`0eEyVUJ&K9a;)QzPP{}Pi)Gs6mVE(@y>c1Z)bpwN@YT*!PaEMNIi1OPo z+i9N)yi35ZBU@_B<#(`BzCeV)mft1vJtBwj%F7s@fc5({e+sOW5Hdc&>=!gk;YERS zgzGXe1*-SPXHUVD-jz#u??0j`uaM#Ywzs!L_xAt3H*(W0?QwXpQyQv<{Bbnz5xAU~(h(L?}Ke`xHWv(uwljL;2&RzZ!vQ}=x6pj*_*Vb#0n z#wOI^G4!BD2rcNz*!b3AHNS?y)qk&0`}O}T3fOKx$~JX_<)bbr;pe@ob2!!y92NVu zjwK(0mlYTk!ktdqj1AtGHn{OF#8)w3ewDe=GGnN zBB46^aVaG;&m2E~%I!F=p=08|;o4efneu!Vr6LSVFAUS1j}qcrVfeu)N~#%2xJ$~k z@@XQ^5b=pnUm@v)Q%$xQ-2QrV%-P0O6 z50&2ZD5(L|E;z`6Xq_t8^~QXd28Rf$8VWNK-GrdC|FMY2r`t`j*=gg`kBtj)LF&zxvJkmt{ryGaz=L>O|Kbs(=+Rq zXVr6Fu5Np_q^q^Om&dqKbLs`JAZcc;SRe2PWZ$Zl>Vw{(s_+3mw4-{%T-g}$jsT8& zV>cC+WA+ycvw@HA=-z~+PfBt|;Y^XOLA{DyvDJ1#uA8f+#8*i9XxdN{snx7$?SJ zHHs5^xfNHNji82^_Fc}}AzyBxU#K>!v0rPht#JW#xKeE}zY&HtzgArpf!GRZ{obDr zWCkrd0l*c~tfw)R>l+5sMwN=Trg|pR!JCm3X{Gy(>Z3MaV4huV#9V-fXP@EKrLvXS zzTXJy-1ig5_v=m8u2H|}`;Xc|EgP}Ic&o)D&kFt3t(beZU#apMi~O$@F@$v_`1hyN zalJL&syH+KxG!W_+_6azAX= z>+ME04C1DUFj>m~SOl#W7gMb*L1y<2w1_OsRz_7xc}X)SMrUXm131u(r!&=qZb&Ph zg_dK6XEPIaW_b?Fc?Gm0v%LYd63csoXhY2LhFQTI;YV1Jk8X^y0X|+aScwgOVZbIP z*bp>4oD31K^Vs=&P}22AyDn6G{FwwA^=P}E5#P^#s#-wLGux7om)1#L&jh@5F_9EZxpYxdu3X-yT)FJ!Dz#=1zkeku z(0eyhB%|Wp11jE5=i>bXDps&pyGuyN__&KX4}w~oUl*b&%37j_wMd+Vnh(rd3sF#_;Sw1~c7D~9^t|C)A8mG6X@e(1OF#UFxv z$cr_2u?{ab;Ke4q*z#v_OEGEOGC&b$2)*z8_=s|LuKoq2HT^dcCRHf-4mT#SBkhxqmy6cCgZ}d$~Jzm*?-# z-Ag^|j%cz(zdid&I^lNk#M5stEM-`=;pwvvZl>7m=9+jH>yJT*KmZXQ+8*RP=zfianheWTq7 zDTOBnCB`HtcqAjXjeJL)XsMj*6FAY4>XJd<4;qbT9MId5?+ao=keT=BTxwq`o}6eC zEusYBD7I#+L#iO^OVZFwGicGDkZ*dWK{;S-nA}=ZcXX~Z4e}VfIneAqHKXYnEx%7O zGK#ZDZ^N|+t{HL#ti=oX8yiIwUM24E!L^(mvE)cfr?+Utk|Vj-j@TG+h>tbf8>%WO z;*e(fXhnn0$5{TF0@}Ew?bEX{2m3LNggn8I97U-k%QVxQWP{vdL;MJWz%U#6!tjo! z6h>j;2pa|Em>k#G7#jz699SOjPq0Z~C)iPbk{v@ZIgaNLcPchJ!A@f26g$NW>@=E- zkuzv#(cVDFd6T0g#QH<*E%r8Mo|ZGav)<{ORbub5GG@7l)_U)yS>IsiFzbx`0f50e3!GFziaiK;XWCM&$b(%xMjZlm@0P+X;e=Ag}y0sj`DqqdA#*-@UV+Y^Yf>V~$dim_PVH9G2^ z`c&&^pqU-GY5F{>bQItEfzp^TlsLC_NmZWWY_VO4?cF>@)Y#c|_LM!)I{N3RlAr27 z(I2aiDSoGGdsXGBfqr3L`O)}OGcKaKu{!3)z>fBmo~4fEm+&0K6Rl&=*fr(RM;&@g z%Q(SPax%me1PW~_xCLjt5kOq##!VMx3Ug`K+^nE?Df_!`Mefu>>{cU}KWXuh%BE_? z-D#XTa8`TIb_%?y-1O@>TV{U$NQOr|U2QNwf`je2$16E?CXQ{;G zRL&)KSDa)7ik9`MU&$hQl;m(y3F26YE41_n1g_GemYfrnbBPggR8co*h!vo~x~-BEW{rtQ%=RbLXU? zi}A3@IM6#76!!_pOeBF!NA%Y3(IO3CRNX$+%Uu>+dIHcP7pFu=O}aQ0rX7{34|O`G z?4nOj9fQG+);&r{7XqtveA+HbuaVn`3P_XuC&jz%MqI6D#kzjLx0jbQeZ;3e5fVol z-`uV1bKn017M6+CmRdKSZ;$uNb1*R^7P>z=v{93C2y|j)SKULPf*cyujRWRUqRBwD z1G!WsMxffTc4XLM$ZNYfrbAx6qcMY-Umj2lP)w%uP|S}0mBJ8iBs8HL6C0RY65qj> zrD_uOLR!R9{hFv#Wjl2XGlcQQ;uD%D_t#wA61*b5MbpepI^@7VprLB3Y)RE6xy>w5 z6tWH}Ffr@F2As#NR~z(zD~9WRT9w@V5er9jYO&R0q)=@{9Z_K<$9k2T8A+Cr4k?yw zYG1KrAZh%ETA|tv+-g9rS^Q7c+96xkmc&PxTQ=o2z=60@3F3z|lp1uB$E^bHKKv@{ z=bje$D_TS;$b#$Hs`bLqwvWGp^ZnfEJZ_2_^P}oE7Yy$uhCl)h4RpIdD(fh-NmHRB zQ%YpKB$2U)4As#(`VM-?P*h|>uJ7Uv6r1uuNBP$8+07pD%+&5VW=lpy1--tmlTleWG8~Lsfwtp#)fMS1O5(*t{uwX4r)HB{to3QOTPg9MIC|t*9>U?LX$o}Vq1v5&hzb~{gNM4KL$gE6VeWLf7GWOL_7NTvj8%9 zy+Vo|t5=3u%oGg(oC*$@Kj6Db$s$dUMEF{MMT;1K zU42Yn>*c0Ti;rmhwcOC#PM4ce4H@ed+)$x-DEgCglWRy?u*p>QecQzVnzoMWvzA4*d|5wZMoy3DC9~u>7HXFl@Sqot zo%RUJnx;KRo%6g@+^1x7OJ+Z1+?4ek97C3Mbbgno^#gT_$3%+(%4gEBw}^gH*uP@u zks$L*EbIr0 literal 0 HcmV?d00001 diff --git a/vllm/distributed/device_communicators/__pycache__/shm_broadcast.cpython-310.pyc b/vllm/distributed/device_communicators/__pycache__/shm_broadcast.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2b96161d60091d3950d69ad2d28b312c26b9848 GIT binary patch literal 13087 zcmb7LTW}o5b)EOlKCuMA2l$XilqGR33M3_2vT24QDS;9l3X(xevd3mdi|qlh;O;Eg zGYb&NB9R5AtR%%KPk+R51qr|7 zqiOo4PI+d-YTCZtbbN=?RwL7NeV5~QBiqdRxn|zabJ}SXnnl0JalKLUOWgNFv+VEV zyi8-FIq6Sw+-*!Xr~PS;XA$4;?{6OP4>S+@2b+idL(Lg~rrXcq<`Mr0$8(LN&13#C zj^`VXG#~XJZF;`fe9V7L*TN&K$Nl4?;6EN73%|0X`%k>8iJ~ao(L`y-@ShBAq$flf z>GF<&@~4pACnk}eMEMD!U(+g6pU^tZSM;=0Yeh+TJ84#<>T)P6Mw%7Xq*`lK<2bdK z>S05mRxw_w%1{K&u-THEsAOHO$4TnGv6j?ZQMG|8_F6Tmtsqi(Q?_bh9KR}C?KPyW zD{sF2Ld8zaYj0gjvk0!eyl~~xTxxyiO=J`(4Bnnwxb()gxis^#q6W?K^{Ac%jn*PN zwlzs(gPQzZnw86=?juTW8u7-rwDin_di_P&jR!{DgMvRAr=U`(t`{^d z=XIZpp5ks#aknS2TizmXmsfADHNs{XB~@Z{Z>c4{McJx~S~X6*TC3S?NA(()&G!47 zi(@pXR~wDZlU_aX>akb#ZneX9=xwZoGW1$eV-o?A^+v;cH;lOOU9TDmui0)Sb&T2X zP1O;hsx#|dUkN?%i5J(m(9@C^R%|L=_ije5jfnEAV!ayG!oe)BP+d$7D-oXbHbPIdqNfrs z3eiu}^5V^?wjx^*QAw0kYlI0X=@T+E)S%aXsS=momNcrHt#)#@U)tL`&m*uJq#hD$ zhDntw?$O@!IuB_ymfd+s;}f;El-Tc}E^b%G+VqnhZ=~9uU9I?3-}vcgzWDjF z@_y^5`(1WC#-r0%6poxhWOjD8m&9*xDXB^?KB(2p@dm5)WmJ4w`JPk`fYQ57Og;>w zw4?p3809C-4A%XmG5UqU!W$@=PzOctF*e?`HP<8Xu&;V;tGDbUbsg57)6&G0>K=t-mP*p)t_6 z(P(YNX9u-U5c+s$D&FarUU~^cB0Q^brO~S0q%DhDNpGs&W|*8L#G^c#3n#dA0Jo~g zXDi;B)2C0PM}%WNuC}p}-Mv;l^!w@!Hs0NH=O32|ujnY|tf zIt3SbTlnO`!AOS>Pr1A39!}TiDtu;A(IzRML%T_bZw<8KfP;)s!+~tPK}TwJDWNiO zP`BC=bjWxmjJYzM>Qu%wC>d5{Gab#Uyg4W)ZW|q7LCsK|-FsK|PAth*6T%DX&R`le z7%ewwq6T#0c1Q1ZzuB8ex9J%c{>tot!S`Hk)#;EJ@L~@Q#}(+X?Js#(P-umpS1YXW z3#b$(in1POFdz^Jf%96?dMFcGxezJ^CPMPoX9ZMJ^2aa+H9>9Z7w;R>HtSt{=w4BU zJ@$&*4b&LLJB*iD)$T?rAXT)pGWEcw_9i*9RRcDff8+X`5}1|*Llb~N1B9VbjS`P^ zx07Am=XK)^9U1yW+(IVwD(FLm^>IQP9@M8UPBBwN%nqa6(4|YA2MYg&M2z4sX*KJ$ zS?|h{(zRHzRO*@xu}OeICYhy&b;e~P1vc37iW>bnh8XN3G`=dpvA28Ap6p`}y78Lz z~6a zV4l?8-jUeTv7tm#)lkfO@6aHTw+8EzhVOUTt>Cp5S1}gR-eP`SQ?@2V9I&}ouib1A z>(mpqkmGe|UHV-@SLPWKZKOz}zcE9A=p<7U{hb6RqXu>s)i#;No<=)FWlUX-8WU=c z0}lHHhMOiSn7)DY}F|duGc}%zy@l`_*A^o4Epxb z?Am7Po@-Q_i=ukIMHCTvmbP_b#3Y^#0yg!7*>gA@s*U(OBC1hEmxs`#G9~xpCoT5$ zfq+&^9zdQvh``^Uw32E=SNAYrAU%^fGdA9q{Udf631F3)k4m z@jClBV2tzcdOh4oO>~&ru#svjevUw14dB~th{xA`xJep~K)OWF5d`VJ%k@S$-%2jG z+L4%(vL!1Hwm57o@k9oc^Yn&DQ~eJ1Mtanp+gl+@#jhZ6UdZVV{w(|%_buJI?-(WB zH5>zR-MxRnH1tFI@y{LGvQY27fzks;LANa9=f?fgr$+Ah|7;!DdmC!RK)r|Jv^X#0 zjgKOLNlq_}bapV1qrqCWm#ih@nkK0NW||EG_8$eo_;jf&p7Y1L{3^x!T&QN6Lz7Un z!B@c+ieqxuSfJaXS$@8CXq@6+0^UM)&T4h$MaWzR4hop9`(TYR)1o}*!b_A@sDQA%wpGGDKMA!&|AD}FTt3k`@ zv|lCN`ph!6_K(kk>!$f3L$XGa(tBvph_^`6fBJm{KTx|sd!SEi2eoa3TtDRO*)|i3 znX49ZdhVLSw$-uT#vI`sf%tXZGng;jGf#~;N06uX(3X#59L1+kQj9n5Da1au)y6%K zcxvOMf)maRBO$49d>veJ){zkY5QRUbg)rVL8#(=e?&`azBSJI%7Cx-EgLLu20-B@|Z|nz#9{b^jw>7X`vZfLO-X4cIXJ>uI^_-xG!fk zMhW8aN+lj8p8Z1?7 z0L#sEk^#Nz?+jov+>GZ_r;E$fgN<@rB;KEDS}B!K2!M;1#=Y89`WMI%M^Q+g1z?RA_{(lSIMsC8VB^U zQP#2YRPw;e_npdrjndxAXZ5U+HQ`$J$fc|t$}<{V%eHXvW{~Hwi``%C91EwqeLf-{y5x;{>Nwo;tDTI{Z(8z1V((~?E{ts_adxzAw_|YrjU$5USGU`7 zce6?EBXS#)n>xQ+NQ%Pz$k4T>`qXZhhRDXaih?OY^Ex032oEw1hd4fy9PT*e-d}Y)*-l;*@4}fM9Ko!PiV|{< z@OZ%R%EUZtm|NL1&%QW&8h>Z5-zz7zwX>&A^>fZ{6)(XmXG^Q)JrRF$ACH{DF5c6> zcTc}@&ruET>1QujY~FXp3oJ=}`Pq7U{p!`%;RAJZGN+CJTR~ReKVvNc$c?AQc zHlScROwC3ZrKJd$JW%`kw<6@JV+c@|mE4CGa)JVaNfzYPiQ(P}6IrC*3SA)KtTaak zEuyWpM3$(*B-L@$MN(>uaIw9dT8o&vB-NdI5jPGf82jIt8;tzfe*NdYM!e z6Zz!ejV0ii}2uZYEJ<&eWp)_e7BLJizG!qL@L2#t4 z+UuHJz%PN39v^0~LJGBTej)@^t=hY8M+f8>K^DSl1Uc>{-!TakIKR{}R!h4RNm*Dd zmUQp7k73A4CU>VG)|}6^rV%F?B6{|Bph$BJSC2Frhgt4F$72yl7(r=hEVNnOKNu?^ zwe}#OXq)p7B{RZh8nToRFe2bN%n&qA%l_RiEks2nKYy8V7$`Sc3{G?EVqi_Rm5y;ud83MN7uvW4R>#`rV*sA`E2!yMC941n z2xx24Z`rg+y@zrMOeO>UGt1mMHo_O?T#(4Lc=0UGIP_ESJ8``WC^5H ze;xghhED^GX$^YXD+zQiVM7Nay$XEsBKRXaVr||kK?_6Q=M)9SE(Q5*r{j!UrjZM~ zG#s`|lM;QpK(q+sAU+rX?>gHV>ahcp+5B%xuNfsZdDl>7!#l6jjPWffPk zZI12sK+zQiy~uaUZA&^Vj=FHcqz&4<{<#tFA@M^ z%3Y}REP4-T{O=F5CNXx8Di)mqdl$cey?<@E55?Zcn7wVp# zLT{s~Zkt%@0%{Gj6dk&*p3CjQQlFF$6GN5k5oY>VFw@Kkv<{CkqBWAnNYKiM?Y|&` zUC`pp094`p1hlRKJnsTnQ3`JgpzuawlgUea2H>>iH#^#{`;qoxVH*GikTTDWz#>E4 zso~v1%j_D6 z0qE;~kp!2Hi?cXs;izWlV?L=A6|6aY-4ep2YB$&7wEWtYS6`dEc0G7=;f?EWy!^&h zb(&ts;wkQ76Q@H-I$>o*LH}s2SPGd;T=4iM_*n7z6ny+5KzdL9$aHD^Q(}!r&=>3- zXjk;+XzLAN3H*#lKhK$EAkNnG$i{aYkKhOC1pPh#eI&46ntTC!pkoJTpj_!fhX)60 z52-)#Ok(W9V(1;8uqe!72P^OG<0Cs^$mh|gwQECxvbeQHtvX75a^xIJp?E!5nz#ba zllS$Cr3`!Chwi=&%XIfEu}W5n?=~&*uOmq92EJK~)xxvFQl+9BR8&|XJ*63{Yl^OS zmn1TETew$T!WS3veX6~eqLb;4_!qPb9%?De($Xg_e z>>23B#LKcr&R*UJ*etu)0g{UjJXT1-jo))un@We!%2mV61$Z~LcWp{Lr=YOaz=^}%g za0GF!LdW_*yJ^VliQCC_@|^-X)`WZ4-Y#~EyIF8lDbZGQozk-2aX!Ea*MUpwt_h4e zD70mLdxA?RM$0-A#P4Y7?&dosvRApc0$-mLq0Z+NHv%T6>QR7;(SR;MgQz9Kpc>;g z6Fv*Rvm`@nMAm4PX!S@li4P%I21PgA?;G1>mm3o3`g_JLx>D-z9=-uKW=~7fKT>_@ z`cit7lRHOFU-+HY&i@1U^QXi_hqSG;BT_$XVRdbcV41#B9=>_(b;B|?LNxVBL{H%z zBAS2!i*N2pv>|^RwVCRV@lCVRCD~o1X68}pT?GFGswLe_z&d@HZwPm|W-M!|%|us< zsYMsEKHsv?68j~(!PscYhTt1g{~)DDW~!X<{-boAJ3DlpJG)ku^l424<~45OP{i3x0JVGfK?yT;2wf>=AqmF_k;lFQ72~; zP8DE>O+vNckTq5Ed(ofsshc57A6~G}9lBVhJZQ1-qyag$p8UnQjZplTiL9##=G8@$ z&Ztt^KYqJQuDWdma}kyY#W~<=bn?v{j9WLnA5sjF&@k5#Lgzo(LWsE+X5z0-O}@YaTrSi26c-wG4=%nUqPl0K93-~ zh8&4}o#Pkr1vYGyIGJt2gR3s#PoUJ_$M^NaOpc-GW=MU&Apd~6pk?sQt>!J|9?DZ* zfr26h>^~;gv!7*w38VGcw>ZpmhVm*)0)IcpIc?bdvGQ3@QWpdkN2nMAJu3E7&^cJNdeiL9!} zx~NOgmy=p+nHyfJ*z6ZN1DjVKX#;yt$0xitsj3pZK0iWG9>A=gz#<8z#T-GUyYC1` zxaS`8geU!@K=`=sdlr(%La}&d@5$)uSs?mi;oN6EF#xE;c4?=qH;OE)^E8P~p8APv`X{)`P#dQoU4c$JrG^aFWMHfbt|J=r>NRc=Fd}IYze7-vzlIz+z)%MmM7Jg`)I4kZv4ToUQ74l1zkl>_cjFP?-u-ss z=q0$N5uzXGBUH3Rd@5MC)zoJ87;uIjQZv>AizPuK5_p|gx3?D|1$4yB$> zs9u8$ki{+7x+J?t6L&5#Jo9FDZ-ea0W9H_eU4@Nk|Dp9lP$$XUT1q!srcVw~gU literal 0 HcmV?d00001 diff --git a/vllm/distributed/device_communicators/cuda_wrapper.py b/vllm/distributed/device_communicators/cuda_wrapper.py new file mode 100644 index 00000000..d5a53381 --- /dev/null +++ b/vllm/distributed/device_communicators/cuda_wrapper.py @@ -0,0 +1,172 @@ +"""This file is a pure Python wrapper for the cudart library. +It avoids the need to compile a separate shared library, and is +convenient for use when we just need to call a few functions. +""" + +import ctypes +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +# this line makes it possible to directly load `libcudart.so` using `ctypes` +import torch # noqa + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# === export types and functions from cudart to Python === +# for the original cudart definition, please check +# https://docs.nvidia.com/cuda/cuda-runtime-api/index.html + +cudaError_t = ctypes.c_int +cudaMemcpyKind = ctypes.c_int + + +class cudaIpcMemHandle_t(ctypes.Structure): + _fields_ = [("internal", ctypes.c_byte * 128)] + + +@dataclass +class Function: + name: str + restype: Any + argtypes: List[Any] + + +def find_loaded_library(lib_name) -> Optional[str]: + """ + According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html, + the file `/proc/self/maps` contains the memory maps of the process, which includes the + shared libraries loaded by the process. We can use this file to find the path of the + a loaded library. + """ # noqa + found = False + with open("/proc/self/maps") as f: + for line in f: + if lib_name in line: + found = True + break + if not found: + # the library is not loaded in the current process + return None + # if lib_name is libcudart, we need to match a line with: + # address /path/to/libcudart-hash.so.11.0 + start = line.index("/") + path = line[start:].strip() + filename = path.split("/")[-1] + assert filename.rpartition(".so")[0].startswith(lib_name), \ + f"Unexpected filename: {filename} for library {lib_name}" + return path + + +class CudaRTLibrary: + exported_functions = [ + # ​cudaError_t cudaSetDevice ( int device ) + Function("cudaSetDevice", cudaError_t, [ctypes.c_int]), + # cudaError_t cudaDeviceSynchronize ( void ) + Function("cudaDeviceSynchronize", cudaError_t, []), + # ​cudaError_t cudaDeviceReset ( void ) + Function("cudaDeviceReset", cudaError_t, []), + + # const char* cudaGetErrorString ( cudaError_t error ) + Function("cudaGetErrorString", ctypes.c_char_p, [cudaError_t]), + + # ​cudaError_t cudaMalloc ( void** devPtr, size_t size ) + Function("cudaMalloc", cudaError_t, + [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t]), + # ​cudaError_t cudaFree ( void* devPtr ) + Function("cudaFree", cudaError_t, [ctypes.c_void_p]), + # ​cudaError_t cudaMemset ( void* devPtr, int value, size_t count ) + Function("cudaMemset", cudaError_t, + [ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t]), + # ​cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind ) # noqa + Function("cudaMemcpy", cudaError_t, [ + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, cudaMemcpyKind + ]), + + # cudaError_t cudaIpcGetMemHandle ( cudaIpcMemHandle_t* handle, void* devPtr ) # noqa + Function("cudaIpcGetMemHandle", cudaError_t, + [ctypes.POINTER(cudaIpcMemHandle_t), ctypes.c_void_p]), + # ​cudaError_t cudaIpcOpenMemHandle ( void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags ) # noqa + Function("cudaIpcOpenMemHandle", cudaError_t, [ + ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint + ]), + ] + + # class attribute to store the mapping from the path to the library + # to avoid loading the same library multiple times + path_to_library_cache: Dict[str, Any] = {} + + # class attribute to store the mapping from library path + # to the corresponding dictionary + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {} + + def __init__(self, so_file: Optional[str] = None): + if so_file is None: + so_file = find_loaded_library("libcudart") + assert so_file is not None, \ + "libcudart is not loaded in the current process" + if so_file not in CudaRTLibrary.path_to_library_cache: + lib = ctypes.CDLL(so_file) + CudaRTLibrary.path_to_library_cache[so_file] = lib + self.lib = CudaRTLibrary.path_to_library_cache[so_file] + + if so_file not in CudaRTLibrary.path_to_dict_mapping: + _funcs = {} + for func in CudaRTLibrary.exported_functions: + f = getattr(self.lib, func.name) + f.restype = func.restype + f.argtypes = func.argtypes + _funcs[func.name] = f + CudaRTLibrary.path_to_dict_mapping[so_file] = _funcs + self.funcs = CudaRTLibrary.path_to_dict_mapping[so_file] + + def CUDART_CHECK(self, result: cudaError_t) -> None: + if result != 0: + error_str = self.cudaGetErrorString(result) + raise RuntimeError(f"CUDART error: {error_str}") + + def cudaGetErrorString(self, error: cudaError_t) -> str: + return self.funcs["cudaGetErrorString"](error).decode("utf-8") + + def cudaSetDevice(self, device: int) -> None: + self.CUDART_CHECK(self.funcs["cudaSetDevice"](device)) + + def cudaDeviceSynchronize(self) -> None: + self.CUDART_CHECK(self.funcs["cudaDeviceSynchronize"]()) + + def cudaDeviceReset(self) -> None: + self.CUDART_CHECK(self.funcs["cudaDeviceReset"]()) + + def cudaMalloc(self, size: int) -> ctypes.c_void_p: + devPtr = ctypes.c_void_p() + self.CUDART_CHECK(self.funcs["cudaMalloc"](ctypes.byref(devPtr), size)) + return devPtr + + def cudaFree(self, devPtr: ctypes.c_void_p) -> None: + self.CUDART_CHECK(self.funcs["cudaFree"](devPtr)) + + def cudaMemset(self, devPtr: ctypes.c_void_p, value: int, + count: int) -> None: + self.CUDART_CHECK(self.funcs["cudaMemset"](devPtr, value, count)) + + def cudaMemcpy(self, dst: ctypes.c_void_p, src: ctypes.c_void_p, + count: int) -> None: + cudaMemcpyDefault = 4 + kind = cudaMemcpyDefault + self.CUDART_CHECK(self.funcs["cudaMemcpy"](dst, src, count, kind)) + + def cudaIpcGetMemHandle(self, + devPtr: ctypes.c_void_p) -> cudaIpcMemHandle_t: + handle = cudaIpcMemHandle_t() + self.CUDART_CHECK(self.funcs["cudaIpcGetMemHandle"]( + ctypes.byref(handle), devPtr)) + return handle + + def cudaIpcOpenMemHandle(self, + handle: cudaIpcMemHandle_t) -> ctypes.c_void_p: + cudaIpcMemLazyEnablePeerAccess = 1 + devPtr = ctypes.c_void_p() + self.CUDART_CHECK(self.funcs["cudaIpcOpenMemHandle"]( + ctypes.byref(devPtr), handle, cudaIpcMemLazyEnablePeerAccess)) + return devPtr diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py new file mode 100644 index 00000000..7de5b05a --- /dev/null +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -0,0 +1,292 @@ +from contextlib import contextmanager +from typing import Any, List, Optional, Union + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.distributed.device_communicators.custom_all_reduce_utils import ( + gpu_p2p_access_check) +from vllm.distributed.parallel_state import in_the_same_node_as +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils import cuda_device_count_stateless + +try: + ops.meta_size() + custom_ar = True +except Exception: + # For AMD GPUs and CPUs + custom_ar = False + +logger = init_logger(__name__) + + +def _can_p2p(rank: int, world_size: int) -> bool: + for i in range(world_size): + if i == rank: + continue + if envs.VLLM_SKIP_P2P_CHECK: + logger.info( + "Skipping P2P check and trusting the driver's P2P report.") + return torch.cuda.can_device_access_peer(rank, i) + if not gpu_p2p_access_check(rank, i): + return False + return True + + +def is_weak_contiguous(inp: torch.Tensor): + return inp.is_contiguous() or (inp.storage().nbytes() - + inp.storage_offset() * inp.element_size() + == inp.numel() * inp.element_size()) + + +class CustomAllreduce: + + _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8] + + # max_size: max supported allreduce size + def __init__(self, + group: ProcessGroup, + device: Union[int, str, torch.device], + max_size=8192 * 1024) -> None: + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the CustomAllreduce to. If None, + it will be bind to f"cuda:{local_rank}". + It is the caller's responsibility to make sure each communicator + is bind to a unique device, and all communicators in this group + are in the same node. + """ + self._IS_CAPTURING = False + self.disabled = True + + if not custom_ar: + # disable because of missing custom allreduce library + # e.g. in a non-cuda environment + return + + self.group = group + + assert dist.get_backend(group) != dist.Backend.NCCL, ( + "CustomAllreduce should be attached to a non-NCCL group.") + + if not all(in_the_same_node_as(group, source_rank=0)): + # No need to initialize custom allreduce for multi-node case. + logger.warning( + "Custom allreduce is disabled because this process group" + " spans across nodes.") + return + + rank = dist.get_rank(group=self.group) + world_size = dist.get_world_size(group=self.group) + if world_size == 1: + # No need to initialize custom allreduce for single GPU case. + return + + if world_size not in CustomAllreduce._SUPPORTED_WORLD_SIZES: + logger.warning( + "Custom allreduce is disabled due to an unsupported world" + " size: %d. Supported world sizes: %s. To silence this " + "warning, specify disable_custom_all_reduce=True explicitly.", + world_size, str(CustomAllreduce._SUPPORTED_WORLD_SIZES)) + return + + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES + if cuda_visible_devices: + device_ids = list(map(int, cuda_visible_devices.split(","))) + else: + device_ids = list(range(cuda_device_count_stateless())) + + physical_device_id = device_ids[device.index] + tensor = torch.tensor([physical_device_id], + dtype=torch.int, + device="cpu") + gather_list = [ + torch.tensor([0], dtype=torch.int, device="cpu") + for _ in range(world_size) + ] + dist.all_gather(gather_list, tensor, group=self.group) + physical_device_ids = [t.item() for t in gather_list] + + # test nvlink first, this will filter out most of the cases + # where custom allreduce is not supported + # this checks hardware and driver support for NVLink + assert current_platform.is_cuda() + from vllm.platforms.cuda import CudaPlatform + cuda_platform: CudaPlatform = current_platform + full_nvlink = cuda_platform.is_full_nvlink(physical_device_ids) + if world_size > 2 and not full_nvlink: + logger.warning( + "Custom allreduce is disabled because it's not supported on" + " more than two PCIe-only GPUs. To silence this warning, " + "specify disable_custom_all_reduce=True explicitly.") + return + # test P2P capability, this checks software/cudaruntime support + # this is expensive to compute at the first time + # then we cache the result + if not _can_p2p(rank, world_size): + logger.warning( + "Custom allreduce is disabled because your platform lacks " + "GPU P2P capability or P2P test failed. To silence this " + "warning, specify disable_custom_all_reduce=True explicitly.") + return + + self.disabled = False + # buffers memory are owned by this Python class and passed to C++ + # meta data composes of two parts: meta data for synchronization + # (256 bytes) and a temporary buffer for storing intermediate + # allreduce results. + self.meta = torch.zeros(ops.meta_size() + max_size, + dtype=torch.uint8, + device=self.device) + # This is a pre-registered IPC buffer. In eager mode, input tensors + # are first copied into this buffer before allreduce is performed + self.buffer = torch.empty(max_size, + dtype=torch.uint8, + device=self.device) + # This is a buffer for storing the tuples of pointers pointing to + # IPC buffers from all ranks. Each registered tuple has size of + # 8*world_size bytes where world_size is at most 8. Allocating 8MB + # is enough for 131072 such tuples. The largest model I've seen only + # needs less than 10000 of registered tuples. + self.rank_data = torch.empty(8 * 1024 * 1024, + dtype=torch.uint8, + device=self.device) + self.max_size = max_size + self.rank = rank + self.world_size = world_size + handles, offsets = self._get_ipc_meta(self.meta) + self.full_nvlink = full_nvlink + self._ptr = ops.init_custom_ar(self.meta, self.rank_data, handles, + offsets, rank, self.full_nvlink) + self.register_buffer(self.buffer) + + @contextmanager + def capture(self): + """ + The main responsibility of this context manager is the + `register_graph_buffers` call at the end of the context. + It records all the buffer addresses used in the CUDA graph. + """ + try: + self._IS_CAPTURING = True + yield + finally: + self._IS_CAPTURING = False + if not self.disabled: + self.register_graph_buffers() + + def _get_ipc_meta(self, inp: torch.Tensor): + data = inp.untyped_storage()._share_cuda_() + shard_data = ( + data[1], # ipc handle to base ptr + data[3], # offset of base ptr + ) + return self._gather_ipc_meta(shard_data) + + def _gather_ipc_meta(self, shard_data): + # Note: don't use `[[None]] * self.world_size` here + # because it will create a list of the same reference + all_data: List[Optional[Any]] = [[None] + for i in range(self.world_size)] + all_data[self.rank][0] = shard_data + + ranks = dist.get_process_group_ranks(group=self.group) + ranks.sort() + for i, rank in enumerate(ranks): + dist.broadcast_object_list(all_data[i], + src=rank, + group=self.group, + device="cpu") + + # we cannot directly use `dist.all_gather_object` here + # because it is incompatible with `gloo` backend under inference mode. + # see https://github.com/pytorch/pytorch/issues/126032 for details. + + handles = [] + offsets = [] + for i in range(len(all_data)): + handles.append(all_data[i][0][0]) # type: ignore + offsets.append(all_data[i][0][1]) # type: ignore + return handles, offsets + + def register_buffer(self, inp: torch.Tensor): + handles, offsets = self._get_ipc_meta(inp) + ops.register_buffer(self._ptr, inp, handles, offsets) + + def register_graph_buffers(self): + handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr) + handles, offsets = self._gather_ipc_meta((bytes(handle), offset)) + logger.info("Registering %d cuda graph addresses", len(offset)) + ops.register_graph_buffers(self._ptr, handles, offsets) + + def should_custom_ar(self, inp: torch.Tensor): + if self.disabled: + return False + inp_size = inp.numel() * inp.element_size() + # custom allreduce requires input byte size to be multiples of 16 + if inp_size % 16 != 0: + return False + if not is_weak_contiguous(inp): + return False + # for 4 or more non NVLink-capable GPUs, custom allreduce provides + # little performance improvement over NCCL. + if self.world_size == 2 or self.full_nvlink: + return inp_size < self.max_size + return False + + # all reduce, assuming inp tensor is IPC registered with register_buffer, + # or, in the context of cuda graphs, register_graph_buffers + def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None): + if out is None: + out = torch.empty_like(inp) + ops.all_reduce_reg(self._ptr, inp, out) + return out + + # all reduce, assuming inp tensor is NOT IPC registered + def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None): + if out is None: + out = torch.empty_like(inp) + ops.all_reduce_unreg(self._ptr, inp, self.buffer, out) + return out + + def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]: + # when custom allreduce is disabled, this will be None + if self.disabled or not self.should_custom_ar(input): + return None + if self._IS_CAPTURING: + if torch.cuda.is_current_stream_capturing(): + return self.all_reduce_reg(input) + else: + # if warm up, mimic the allocation pattern + # since custom allreduce is out-of-place + return torch.empty_like(input) + else: + # note: outside of cuda graph context, + # custom allreduce incurs a cost of cudaMemcpy, which should + # be small(<=1% of overall latency) compared to the performance + # gains of using custom kernels + return self.all_reduce_unreg(input) + + return None + + def close(self): + if not self.disabled and self._ptr: + ops.dispose(self._ptr) + self._ptr = 0 + + def __del__(self): + self.close() diff --git a/vllm/distributed/device_communicators/custom_all_reduce_utils.py b/vllm/distributed/device_communicators/custom_all_reduce_utils.py new file mode 100644 index 00000000..983e772a --- /dev/null +++ b/vllm/distributed/device_communicators/custom_all_reduce_utils.py @@ -0,0 +1,255 @@ +import ctypes +import json +import os +import pickle +import subprocess +import sys +import tempfile +from itertools import product +from typing import Dict, List, Optional, Sequence + +import torch.distributed as dist +import torch.multiprocessing as mp + +import vllm.envs as envs +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary +from vllm.logger import init_logger +from vllm.utils import (cuda_device_count_stateless, + update_environment_variables) + +logger = init_logger(__name__) + + +def producer(batch_src: Sequence[int], + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices: Optional[str] = None): + if cuda_visible_devices is not None: + update_environment_variables( + {"CUDA_VISIBLE_DEVICES": cuda_visible_devices}) + + lib = CudaRTLibrary() + for i in batch_src: + lib.cudaSetDevice(i) + pointer = lib.cudaMalloc(1024) + lib.cudaMemset(pointer, 1, 1024) + lib.cudaDeviceSynchronize() + handle = lib.cudaIpcGetMemHandle(pointer) + producer_queue.put(handle) + open_success = consumer_queue.get() + if open_success: + # use two queues to simulate barrier + producer_queue.put(0) + consumer_queue.get() + # check if the memory is modified + host_data = (ctypes.c_char * 1024)() + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore + for i in range(1024): + if ord(host_data[i]) != 2: + open_success = False + break + result_queue.put(open_success) + lib.cudaDeviceReset() + + +def consumer(batch_tgt: Sequence[int], + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices: Optional[str] = None): + if cuda_visible_devices is not None: + update_environment_variables( + {"CUDA_VISIBLE_DEVICES": cuda_visible_devices}) + + lib = CudaRTLibrary() + for j in batch_tgt: + lib.cudaSetDevice(j) + handle = producer_queue.get() + open_success = False + try: + pointer = lib.cudaIpcOpenMemHandle(handle) # type: ignore + open_success = True + except RuntimeError: + # cannot error out here, because the producer process + # is still waiting for the response. + pass + consumer_queue.put(open_success) + if open_success: + # modify the memory + lib.cudaMemset(pointer, 2, 1024) + lib.cudaDeviceSynchronize() + # use two queues to simulate barrier + producer_queue.get() + consumer_queue.put(0) + # check if the memory is modified + host_data = (ctypes.c_char * 1024)() + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore + for i in range(1024): + if ord(host_data[i]) != 2: + open_success = False + break + result_queue.put(open_success) + lib.cudaDeviceReset() + + +def can_actually_p2p( + batch_src: Sequence[int], + batch_tgt: Sequence[int], +) -> Sequence[bool]: + """ + Usually, checking if P2P access is enabled can be done by + `torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes + the driver might be broken, and `torch.cuda.can_device_access_peer(src, tgt)` + returns `True` even if P2P access is not actually possible. + See https://github.com/vllm-project/vllm/issues/2728 and + https://forums.developer.nvidia.com/t/direct-gpu-gpu-communication-does-not-seem-to-work-properly/283264/10 + Therefore, we have to perform a real P2P access to check if it is actually + possible. + + Note on p2p and cuda IPC: + Usually, one process uses one GPU: + GPU src --> cuda context src --> tensor src --> process src + + We need to combine p2p and cuda IPC, so that: + GPU src --> cuda context src --> tensor src --> process src + |shared| + GPU tgt --> cuda context tgt --> tensor tgt --> process tgt + That is to say, process src creates a tensor in GPU src, passes IPC handle to + process tgt, and process tgt accesses the tensor in GPU tgt. Any operation on the + tensor in process tgt will be reflected in the tensor in process src, because + they are the same memory segment. + It is important to note that process tgt accesses the tensor in GPU tgt, not + GPU src. That's why we need p2p access. + + The most time-consuming part is the process creation. To avoid creating + processes for every pair of GPUs, we use batched testing. We create two + processes for testing all pairs of GPUs in batch. The trick is to reset + the device after each test (which is not available in PyTorch). + """ # noqa + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES + # pass the CUDA_VISIBLE_DEVICES to the child process + # to make sure they see the same set of GPUs + + # make sure the processes are spawned + smp = mp.get_context("spawn") + producer_queue = smp.Queue() + consumer_queue = smp.Queue() + result_queue = smp.Queue() + p_src = smp.Process(target=producer, + args=(batch_src, producer_queue, consumer_queue, + result_queue, cuda_visible_devices)) + p_tgt = smp.Process(target=consumer, + args=(batch_tgt, producer_queue, consumer_queue, + result_queue, cuda_visible_devices)) + p_src.start() + p_tgt.start() + p_src.join() + p_tgt.join() + assert p_src.exitcode == 0 and p_tgt.exitcode == 0 + result: List[bool] = [] + for src, tgt in zip(batch_src, batch_tgt): + a = result_queue.get() + b = result_queue.get() + if a != b: + logger.warning( + "Two processes do not agree on the P2P access" + " status on %d -> %d, treat as disabled.", src, tgt) + result.append(False) + else: + result.append(a) + return result + + +# why do we need this cache? +# we are testing peer-to-peer (p2p) access between GPUs,across processes. +# if we test it every time, it will be very slow, because we need to create +# N * N * 2 processes, where N is the world size. This is very slow. +# to reduce the time, we use a cache file to store the p2p access status. +# the cache file is generated by the master process if it does not exist. +# then all the processes can read the cache file to check the p2p access status. +# Note that the cache file is suffixed by the CUDA_VISIBLE_DEVICES, so that we +# can have different cache files for different CUDA_VISIBLE_DEVICES settings, +# e.g. used by different vllm engines. The device id in the cache file is a +# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number +# of visible devices in the vllm engine. +_gpu_p2p_access_cache: Optional[Dict[str, bool]] = None + + +def gpu_p2p_access_check(src: int, tgt: int) -> bool: + """Check if GPU src can access GPU tgt.""" + + # if the cache variable is already calculated, + # read from the cache instead of checking it again + global _gpu_p2p_access_cache + if _gpu_p2p_access_cache is not None: + return _gpu_p2p_access_cache[f"{src}->{tgt}"] + + is_distributed = dist.is_initialized() + + num_dev = cuda_device_count_stateless() + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES + if cuda_visible_devices is None: + cuda_visible_devices = ",".join(str(i) for i in range(num_dev)) + + path = os.path.join( + envs.VLLM_CACHE_ROOT, + f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json") + os.makedirs(os.path.dirname(path), exist_ok=True) + from vllm.distributed.parallel_state import get_world_group + if ((not is_distributed or get_world_group().local_rank == 0) + and (not os.path.exists(path))): + # only the local master process (with local_rank == 0) can + # enter this block to calculate the cache + logger.info("generating GPU P2P access cache in %s", path) + cache: Dict[str, bool] = {} + ids = list(range(num_dev)) + # batch of all pairs of GPUs + batch_src, batch_tgt = zip(*list(product(ids, ids))) + # NOTE: we use `subprocess` rather than `multiprocessing` here + # because the caller might not have `if __name__ == "__main__":`, + # in that case we cannot use spawn method in multiprocessing. + # However, `can_actually_p2p` requires spawn method. + # The fix is, we use `subprocess` to call the function, + # where we have `if __name__ == "__main__":` in this file. + + # use a temporary file to store the result + # we don't use the output of the subprocess directly, + # because the subprocess might produce logging output + with tempfile.NamedTemporaryFile() as output_file: + input_bytes = pickle.dumps( + (batch_src, batch_tgt, output_file.name)) + returned = subprocess.run([sys.executable, __file__], + input=input_bytes, + capture_output=True) + # check if the subprocess is successful + try: + returned.check_returncode() + except Exception as e: + # wrap raised exception to provide more information + raise RuntimeError( + f"Error happened when batch testing " + f"peer-to-peer access from {batch_src} to {batch_tgt}:\n" + f"{returned.stderr.decode()}") from e + with open(output_file.name, "rb") as f: + result = pickle.load(f) + for _i, _j, r in zip(batch_src, batch_tgt, result): + cache[f"{_i}->{_j}"] = r + with open(path, "w") as f: + json.dump(cache, f, indent=4) + if is_distributed: + get_world_group().barrier() + logger.info("reading GPU P2P access cache from %s", path) + with open(path, "r") as f: + cache = json.load(f) + _gpu_p2p_access_cache = cache + return _gpu_p2p_access_cache[f"{src}->{tgt}"] + + +__all__ = ["gpu_p2p_access_check"] + +if __name__ == "__main__": + batch_src, batch_tgt, output_file = pickle.loads(sys.stdin.buffer.read()) + result = can_actually_p2p(batch_src, batch_tgt) + with open(output_file, "wb") as f: + f.write(pickle.dumps(result)) diff --git a/vllm/distributed/device_communicators/pynccl.py b/vllm/distributed/device_communicators/pynccl.py new file mode 100644 index 00000000..73195665 --- /dev/null +++ b/vllm/distributed/device_communicators/pynccl.py @@ -0,0 +1,170 @@ +from contextlib import contextmanager +from typing import Optional, Union + +# ===================== import region ===================== +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup, ReduceOp + +from vllm.distributed.device_communicators.pynccl_wrapper import ( + NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum, + ncclRedOpTypeEnum, ncclUniqueId) +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class PyNcclCommunicator: + + def __init__( + self, + group: ProcessGroup, + device: Union[int, str, torch.device], + library_path: Optional[str] = None, + ): + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the PyNcclCommunicator to. If None, + it will be bind to f"cuda:{local_rank}". + library_path: the path to the NCCL library. If None, it will + use the default library path. + It is the caller's responsibility to make sure each communicator + is bind to a unique device. + """ + assert dist.is_initialized() + assert dist.get_backend(group) != dist.Backend.NCCL, ( + "PyNcclCommunicator should be attached to a non-NCCL group.") + self.group = group + # note: this rank is the rank in the group + self.rank = dist.get_rank(group) + self.world_size = dist.get_world_size(group) + + # if world_size == 1, no need to create communicator + if self.world_size == 1: + self.available = False + self.disabled = True + self.stream = None + return + try: + self.nccl = NCCLLibrary(library_path) + except Exception: + # disable because of missing NCCL library + # e.g. in a non-GPU environment + self.available = False + self.disabled = True + self.stream = None + return + + self.available = True + self.disabled = False + + logger.info("vLLM is using nccl==%s", self.nccl.ncclGetVersion()) + + if self.rank == 0: + # get the unique id from NCCL + self.unique_id = self.nccl.ncclGetUniqueId() + else: + # construct an empty unique id + self.unique_id = ncclUniqueId() + tensor = torch.ByteTensor(list(self.unique_id.internal)) + ranks = dist.get_process_group_ranks(group) + # arg `src` in `broadcast` is the global rank + dist.broadcast(tensor, src=ranks[0], group=group) + byte_list = tensor.tolist() + for i, byte in enumerate(byte_list): + self.unique_id.internal[i] = byte + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + # nccl communicator and stream will use this device + # `torch.cuda.device` is a context manager that changes the + # current cuda device to the specified one + with torch.cuda.device(device): + self.comm: ncclComm_t = self.nccl.ncclCommInitRank( + self.world_size, self.unique_id, self.rank) + self.stream = torch.cuda.Stream() + + # A small all_reduce for warmup. + data = torch.zeros(1, device=device) + self.all_reduce(data) + self.stream.synchronize() + del data + + # by default it is disabled, e.g. in profiling models and prefill phase. + # to use it, use under `with obj.change_state(enable=True)`, usually + # when we are using CUDA graph. + self.disabled = True + + def all_reduce(self, + tensor: torch.Tensor, + op: ReduceOp = ReduceOp.SUM, + stream=None): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}") + if stream is None: + stream = self.stream + self.nccl.ncclAllReduce(buffer_type(tensor.data_ptr()), + buffer_type(tensor.data_ptr()), tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), self.comm, + cudaStream_t(stream.cuda_stream)) + + def send(self, tensor: torch.Tensor, dst: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}") + if stream is None: + stream = self.stream + self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), dst, + self.comm, cudaStream_t(stream.cuda_stream)) + + def recv(self, tensor: torch.Tensor, src: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}") + if stream is None: + stream = self.stream + self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), src, + self.comm, cudaStream_t(stream.cuda_stream)) + + @contextmanager + def change_state(self, + enable: Optional[bool] = None, + stream: Optional[torch.cuda.Stream] = None): + """ + A context manager to change the state of the communicator. + """ + if enable is None: + # guess a default value when not specified + enable = self.available + + if stream is None: + stream = self.stream + + old_disable = self.disabled + old_stream = self.stream + + self.stream = stream + self.disabled = not enable + yield + + self.disabled = old_disable + self.stream = old_stream diff --git a/vllm/distributed/device_communicators/pynccl_wrapper.py b/vllm/distributed/device_communicators/pynccl_wrapper.py new file mode 100644 index 00000000..7619c98f --- /dev/null +++ b/vllm/distributed/device_communicators/pynccl_wrapper.py @@ -0,0 +1,278 @@ +# This file is a pure Python wrapper for the NCCL library. +# The main purpose is to use NCCL combined with CUDA graph. +# Before writing this script, we tried the following approach: +# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself +# often gets stuck when initializing the NCCL communicator. +# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce` +# contains many other potential cuda APIs, that are not allowed during +# capturing the CUDA graph. For further details, please check +# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ . +# +# Another rejected idea is to write a C/C++ binding for NCCL. It is usually +# doable, but we often encounter issues related with nccl versions, and need +# to switch between different versions of NCCL. See +# https://github.com/NVIDIA/nccl/issues/1234 for more details. +# A C/C++ binding is not flexible enough to handle this. It requires +# recompilation of the code every time we want to switch between different +# versions. This current implementation, with a **pure** Python wrapper, is +# more flexible. We can easily switch between different versions of NCCL by +# changing the environment variable `VLLM_NCCL_SO_PATH`, or the `so_file` +# variable in the code. + +import ctypes +import platform +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import torch +from torch.distributed import ReduceOp + +from vllm.logger import init_logger +from vllm.utils import find_nccl_library + +logger = init_logger(__name__) + +# === export types and functions from nccl to Python === +# for the original nccl definition, please check +# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in + +ncclResult_t = ctypes.c_int +ncclComm_t = ctypes.c_void_p + + +class ncclUniqueId(ctypes.Structure): + _fields_ = [("internal", ctypes.c_byte * 128)] + + +cudaStream_t = ctypes.c_void_p +buffer_type = ctypes.c_void_p + +ncclDataType_t = ctypes.c_int + + +class ncclDataTypeEnum: + ncclInt8 = 0 + ncclChar = 0 + ncclUint8 = 1 + ncclInt32 = 2 + ncclInt = 2 + ncclUint32 = 3 + ncclInt64 = 4 + ncclUint64 = 5 + ncclFloat16 = 6 + ncclHalf = 6 + ncclFloat32 = 7 + ncclFloat = 7 + ncclFloat64 = 8 + ncclDouble = 8 + ncclBfloat16 = 9 + ncclNumTypes = 10 + + @classmethod + def from_torch(cls, dtype: torch.dtype) -> int: + if dtype == torch.int8: + return cls.ncclInt8 + if dtype == torch.uint8: + return cls.ncclUint8 + if dtype == torch.int32: + return cls.ncclInt32 + if dtype == torch.int64: + return cls.ncclInt64 + if dtype == torch.float16: + return cls.ncclFloat16 + if dtype == torch.float32: + return cls.ncclFloat32 + if dtype == torch.float64: + return cls.ncclFloat64 + if dtype == torch.bfloat16: + return cls.ncclBfloat16 + raise ValueError(f"Unsupported dtype: {dtype}") + + +ncclRedOp_t = ctypes.c_int + + +class ncclRedOpTypeEnum: + ncclSum = 0 + ncclProd = 1 + ncclMax = 2 + ncclMin = 3 + ncclAvg = 4 + ncclNumOps = 5 + + @classmethod + def from_torch(cls, op: ReduceOp) -> int: + if op == ReduceOp.SUM: + return cls.ncclSum + if op == ReduceOp.PRODUCT: + return cls.ncclProd + if op == ReduceOp.MAX: + return cls.ncclMax + if op == ReduceOp.MIN: + return cls.ncclMin + if op == ReduceOp.AVG: + return cls.ncclAvg + raise ValueError(f"Unsupported op: {op}") + + +@dataclass +class Function: + name: str + restype: Any + argtypes: List[Any] + + +class NCCLLibrary: + exported_functions = [ + # const char* ncclGetErrorString(ncclResult_t result) + Function("ncclGetErrorString", ctypes.c_char_p, [ncclResult_t]), + # ncclResult_t ncclGetVersion(int *version); + Function("ncclGetVersion", ncclResult_t, + [ctypes.POINTER(ctypes.c_int)]), + # ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId); + Function("ncclGetUniqueId", ncclResult_t, + [ctypes.POINTER(ncclUniqueId)]), + # ncclResult_t ncclCommInitRank( + # ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank); + # note that ncclComm_t is a pointer type, so the first argument + # is a pointer to a pointer + Function("ncclCommInitRank", ncclResult_t, [ + ctypes.POINTER(ncclComm_t), ctypes.c_int, ncclUniqueId, + ctypes.c_int + ]), + # ncclResult_t ncclAllReduce( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function("ncclAllReduce", ncclResult_t, [ + buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t, + ncclRedOp_t, ncclComm_t, cudaStream_t + ]), + + # ncclResult_t ncclSend( + # const void* sendbuff, size_t count, ncclDataType_t datatype, + # int dest, ncclComm_t comm, cudaStream_t stream); + Function("ncclSend", ncclResult_t, [ + buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int, + ncclComm_t, cudaStream_t + ]), + + # ncclResult_t ncclRecv( + # void* recvbuff, size_t count, ncclDataType_t datatype, + # int src, ncclComm_t comm, cudaStream_t stream); + Function("ncclRecv", ncclResult_t, [ + buffer_type, ctypes.c_size_t, ncclDataType_t, ctypes.c_int, + ncclComm_t, cudaStream_t + ]), + + # be cautious! this is a collective call, it will block until all + # processes in the communicator have called this function. + # because Python object destruction can happen in random order, + # it is better not to call it at all. + # ncclResult_t ncclCommDestroy(ncclComm_t comm); + Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]), + ] + + # class attribute to store the mapping from the path to the library + # to avoid loading the same library multiple times + path_to_library_cache: Dict[str, Any] = {} + + # class attribute to store the mapping from library path + # to the corresponding dictionary + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {} + + def __init__(self, so_file: Optional[str] = None): + + so_file = so_file or find_nccl_library() + + try: + if so_file not in NCCLLibrary.path_to_dict_mapping: + lib = ctypes.CDLL(so_file) + NCCLLibrary.path_to_library_cache[so_file] = lib + self.lib = NCCLLibrary.path_to_library_cache[so_file] + except Exception as e: + logger.error( + "Failed to load NCCL library from %s ." + "It is expected if you are not running on NVIDIA/AMD GPUs." + "Otherwise, the nccl library might not exist, be corrupted " + "or it does not support the current platform %s." + "If you already have the library, please set the " + "environment variable VLLM_NCCL_SO_PATH" + " to point to the correct nccl library path.", so_file, + platform.platform()) + raise e + + if so_file not in NCCLLibrary.path_to_dict_mapping: + _funcs: Dict[str, Any] = {} + for func in NCCLLibrary.exported_functions: + f = getattr(self.lib, func.name) + f.restype = func.restype + f.argtypes = func.argtypes + _funcs[func.name] = f + NCCLLibrary.path_to_dict_mapping[so_file] = _funcs + self._funcs = NCCLLibrary.path_to_dict_mapping[so_file] + + def ncclGetErrorString(self, result: ncclResult_t) -> str: + return self._funcs["ncclGetErrorString"](result).decode("utf-8") + + def NCCL_CHECK(self, result: ncclResult_t) -> None: + if result != 0: + error_str = self.ncclGetErrorString(result) + raise RuntimeError(f"NCCL error: {error_str}") + + def ncclGetVersion(self) -> str: + version = ctypes.c_int() + self.NCCL_CHECK(self._funcs["ncclGetVersion"](ctypes.byref(version))) + version_str = str(version.value) + # something like 21903 --> "2.19.3" + major = version_str[0].lstrip("0") + minor = version_str[1:3].lstrip("0") + patch = version_str[3:].lstrip("0") + return f"{major}.{minor}.{patch}" + + def ncclGetUniqueId(self) -> ncclUniqueId: + unique_id = ncclUniqueId() + self.NCCL_CHECK(self._funcs["ncclGetUniqueId"]( + ctypes.byref(unique_id))) + return unique_id + + def ncclCommInitRank(self, world_size: int, unique_id: ncclUniqueId, + rank: int) -> ncclComm_t: + comm = ncclComm_t() + self.NCCL_CHECK(self._funcs["ncclCommInitRank"](ctypes.byref(comm), + world_size, unique_id, + rank)) + return comm + + def ncclAllReduce(self, sendbuff: buffer_type, recvbuff: buffer_type, + count: int, datatype: int, op: int, comm: ncclComm_t, + stream: cudaStream_t) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK(self._funcs["ncclAllReduce"](sendbuff, recvbuff, count, + datatype, op, comm, + stream)) + + def ncclSend(self, sendbuff: buffer_type, count: int, datatype: int, + dest: int, comm: ncclComm_t, stream: cudaStream_t) -> None: + self.NCCL_CHECK(self._funcs["ncclSend"](sendbuff, count, datatype, + dest, comm, stream)) + + def ncclRecv(self, recvbuff: buffer_type, count: int, datatype: int, + src: int, comm: ncclComm_t, stream: cudaStream_t) -> None: + self.NCCL_CHECK(self._funcs["ncclRecv"](recvbuff, count, datatype, src, + comm, stream)) + + def ncclCommDestroy(self, comm: ncclComm_t) -> None: + self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm)) + + +__all__ = [ + "NCCLLibrary", "ncclDataTypeEnum", "ncclRedOpTypeEnum", "ncclUniqueId", + "ncclComm_t", "cudaStream_t", "buffer_type" +] diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py new file mode 100644 index 00000000..c41a505e --- /dev/null +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -0,0 +1,492 @@ +import pickle +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from multiprocessing import shared_memory +from typing import List, Optional +from unittest.mock import patch + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup +from zmq import IPV6 # type: ignore +from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore + +import vllm.envs as envs +from vllm.logger import init_logger +from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address + +VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL + +# time to wait if the queue is full or empty +# if we sleep for too short, it will consume too much CPU +# if we sleep for too long, it will slow down the writer/reader +# 0.1 us is a good balance +RINGBUFFER_SLEEP_INTERVAL = 1e-7 + +logger = init_logger(__name__) + + +class ShmRingBuffer: + + def __init__(self, + n_reader: int, + max_chunk_bytes: int, + max_chunks: int, + name: Optional[str] = None): + """ + A shared memory ring buffer implementation for broadcast communication. + Essentially, it is a queue where only one will `enqueue` and multiple + will `dequeue`. The max size of each item, together with the max number + of items that can be stored in the buffer are known in advance. + In this case, we don't need to synchronize the access to + the buffer. + + Buffer memory layout: + data metadata + | | + | (current_idx) | (current_idx) + v v + +-------------------------------+----------------------------------------+ + | chunk0 | chunk1 | ... | chunk | metadata0 | metadata1 | ... | metadata | + +-------------------------------+----------------------------------------+ + | max_chunks x max_chunk_bytes | max_chunks x (1 + n_reader) bytes | + + metadata memory layout: each byte is a flag, the first byte is the written + flag, and the rest are reader flags. The flags are set to 0 by default. + +--------------+--------------+--------------+-----+--------------+ + | written_flag | reader0_flag | reader1_flag | ... | readerN_flag | + +--------------+--------------+--------------+-----+--------------+ + + The state of metadata is as follows: + + (case 1) 0???...???: the block is not written yet, cannot read, can write + (case 2) 1000...000: the block is just written, can read, cannot write + (case 3) 1???...???: the block is written and read by some readers, can read if not read, cannot write + (case 4) 1111...111: the block is written and read by all readers, cannot read, can write + + State transition for readers: + + When a reader finds a block that it can read (case 2 or 3), it can yield the block for caller to read. + Only after the caller finishes reading the block, the reader can mark the block as read. + Readers only mark the block as read (from 0 to 1), the writer marks the block as ready to read (from 1 to 0). + + State transition for writer: + + When the writer writes to a block (case 1 or 4), it first resets the written flag to 0, converting either case + to case 1. Then it can yield the block for caller to write. After the caller finishes writing the block, the writer + can reset the reader flags to 0, and mark the block as written (from 0 to 1). + NOTE: the order is important here, first reset the reader flags (so that we are still in case 1), then mark the block as written. The state transition is atomic. If we do it in the reverse order, it will go through case 3 and then back to case 2, and readers might read the intermediate case 3, which is not correct. + + During creation, `name` is None and the buffer is created. We can pass the + created object to other processes by pickling it. The other processes will + get the name of the shared memory and open it, so that they can access the + same shared memory buffer. + """# noqa + self.n_reader = n_reader + self.metadata_size = 1 + n_reader + self.max_chunk_bytes = max_chunk_bytes + self.max_chunks = max_chunks + self.total_bytes_of_buffer = (self.max_chunk_bytes + + self.metadata_size) * self.max_chunks + self.data_offset = 0 + self.metadata_offset = self.max_chunk_bytes * self.max_chunks + + if name is None: + # we are creating a buffer + self.is_creator = True + self.shared_memory = shared_memory.SharedMemory( + create=True, size=self.total_bytes_of_buffer) + # initialize the metadata section to 0 + with memoryview(self.shared_memory.buf[self.metadata_offset:] + ) as metadata_buffer: + torch.frombuffer(metadata_buffer, dtype=torch.uint8).fill_(0) + else: + # we are opening an existing buffer + self.is_creator = False + # fix to https://stackoverflow.com/q/62748654/9191338 + # Python incorrectly tracks shared memory even if it is not + # created by the process. The following patch is a workaround. + with patch("multiprocessing.resource_tracker.register", + lambda *args, **kwargs: None): + try: + self.shared_memory = shared_memory.SharedMemory(name=name) + assert ( + self.shared_memory.size == self.total_bytes_of_buffer) + except FileNotFoundError: + # we might deserialize the object in a different node + # in this case, this object is not used, + # and we should suppress the error + pass + + def __reduce__(self): + return ( + self.__class__, + (self.n_reader, self.max_chunk_bytes, self.max_chunks, + self.shared_memory.name), + ) + + def __del__(self): + if hasattr(self, "shared_memory"): + self.shared_memory.close() + if self.is_creator: + self.shared_memory.unlink() + + @contextmanager + def get_data(self, current_idx: int): + start = self.data_offset + current_idx * self.max_chunk_bytes + end = start + self.max_chunk_bytes + with memoryview(self.shared_memory.buf[start:end]) as buf: + yield buf + + @contextmanager + def get_metadata(self, current_idx: int): + start = self.metadata_offset + current_idx * self.metadata_size + end = start + self.metadata_size + with memoryview(self.shared_memory.buf[start:end]) as buf: + yield buf + + +@dataclass +class Handle: + connect_ip: str + local_reader_ranks: List[int] = field(default_factory=list) + + buffer: Optional[ShmRingBuffer] = None + local_subscribe_port: Optional[int] = None + remote_subscribe_port: Optional[int] = None + + +class MessageQueue: + + def __init__( + self, + n_reader, # number of all readers + n_local_reader, # number of local readers through shared memory + local_reader_ranks: Optional[List[int]] = None, + max_chunk_bytes: int = 1024 * 1024 * 10, + max_chunks: int = 10, + connect_ip: Optional[str] = None, + ): + if local_reader_ranks is None: + local_reader_ranks = list(range(n_local_reader)) + else: + assert len(local_reader_ranks) == n_local_reader + self.n_local_reader = n_local_reader + n_remote_reader = n_reader - n_local_reader + self.n_remote_reader = n_remote_reader + + if connect_ip is None: + connect_ip = get_ip() if n_remote_reader > 0 else "127.0.0.1" + + context = Context() + + if n_local_reader > 0: + # for local readers, we will: + # 1. create a shared memory ring buffer to communicate small data + # 2. create a publish-subscribe socket to communicate large data + self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes, + max_chunks) + + # XPUB is very similar to PUB, + # except that it can receive subscription messages + # to confirm the number of subscribers + self.local_socket = context.socket(XPUB) + # set the verbose option so that we can receive every subscription + # message. otherwise, we will only receive the first subscription + # see http://api.zeromq.org/3-3:zmq-setsockopt for more details + self.local_socket.setsockopt(XPUB_VERBOSE, True) + local_subscribe_port = get_open_port() + socket_addr = f"tcp://127.0.0.1:{local_subscribe_port}" + logger.debug("Binding to %s", socket_addr) + self.local_socket.bind(socket_addr) + + self.current_idx = 0 + + else: + self.buffer = None # type: ignore + local_subscribe_port = None + self.local_socket = None + self.current_idx = -1 + + if n_remote_reader > 0: + # for remote readers, we will: + # create a publish-subscribe socket to communicate large data + self.remote_socket = context.socket(XPUB) + self.remote_socket.setsockopt(XPUB_VERBOSE, True) + remote_subscribe_port = get_open_port() + if is_valid_ipv6_address(connect_ip): + self.remote_socket.setsockopt(IPV6, 1) + connect_ip = f"[{connect_ip}]" + socket_addr = f"tcp://{connect_ip}:{remote_subscribe_port}" + self.remote_socket.bind(socket_addr) + + else: + remote_subscribe_port = None + self.remote_socket = None + + self._is_writer = True + self._is_local_reader = False + self.local_reader_rank = -1 + # rank does not matter for remote readers + self._is_remote_reader = False + + self.handle = Handle( + connect_ip=connect_ip, + local_reader_ranks=local_reader_ranks, + buffer=self.buffer, + local_subscribe_port=local_subscribe_port, + remote_subscribe_port=remote_subscribe_port, + ) + + logger.info("vLLM message queue communication handle: %s", self.handle) + + def export_handle(self) -> Handle: + return self.handle + + @staticmethod + def create_from_handle(handle: Handle, rank) -> "MessageQueue": + self = MessageQueue.__new__(MessageQueue) + self.handle = handle + self._is_writer = False + + context = Context() + + if rank in handle.local_reader_ranks: + assert handle.buffer is not None + self.buffer = handle.buffer + self.current_idx = 0 + self.local_reader_rank = handle.local_reader_ranks.index(rank) + self._is_local_reader = True + self._is_remote_reader = False + + self.local_socket = context.socket(SUB) + self.local_socket.setsockopt_string(SUBSCRIBE, "") + socket_addr = f"tcp://127.0.0.1:{handle.local_subscribe_port}" + logger.debug("Connecting to %s", socket_addr) + self.local_socket.connect(socket_addr) + + self.remote_socket = None + else: + self.buffer = None # type: ignore + self.current_idx = -1 + self.local_reader_rank = -1 + self._is_local_reader = False + self._is_remote_reader = True + + self.local_socket = None + + self.remote_socket = context.socket(SUB) + self.remote_socket.setsockopt_string(SUBSCRIBE, "") + if is_valid_ipv6_address(handle.connect_ip): + self.remote_socket.setsockopt(IPV6, 1) + socket_addr = f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}" + logger.debug("Connecting to %s", socket_addr) + self.remote_socket.connect(socket_addr) + + return self + + def wait_until_ready(self): + """This is a collective operation. All processes (including the + readers and the writer) should call this function. + """ + if self._is_writer: + # wait for all readers to connect + + # local readers + for i in range(self.n_local_reader): + # wait for subscription messages from all local readers + self.local_socket.recv() + if self.n_local_reader > 0: + # send a message to all local readers + # to make sure the publish channel is working + self.local_socket.send(b"READY") + + # remote readers + for i in range(self.n_remote_reader): + # wait for subscription messages from all remote readers + self.remote_socket.recv() + if self.n_remote_reader > 0: + # send a message to all remote readers + # to make sure the publish channel is working + self.remote_socket.send(b"READY") + elif self._is_local_reader: + # wait for the writer to send a message + recv = self.local_socket.recv() + assert recv == b"READY" + elif self._is_remote_reader: + # wait for the writer to send a message + recv = self.remote_socket.recv() + assert recv == b"READY" + + @contextmanager + def acquire_write(self): + assert self._is_writer, "Only writers can acquire write" + start_time = time.monotonic() + n_warning = 1 + while True: + with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + read_count = sum(metadata_buffer[1:]) + written_flag = metadata_buffer[0] + if written_flag and read_count != self.buffer.n_reader: + # this block is written and not read by all readers + # for writers, `self.current_idx` is the next block to write + # if this block is not ready to write, + # we need to wait until it is read by all readers + + # wait for a while + time.sleep(RINGBUFFER_SLEEP_INTERVAL) + + # if we wait for a long time, we should warn the user + if (time.monotonic() - start_time > + VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning): + logger.warning( + "No available block found in %s second. ", + VLLM_RINGBUFFER_WARNING_INTERVAL) + n_warning += 1 + + continue + # found a block that is either + # (1) not written + # (2) read by all readers + + # mark the block as not written + metadata_buffer[0] = 0 + # let caller write to the buffer + with self.buffer.get_data(self.current_idx) as buf: + yield buf + + # caller has written to the buffer + # NOTE: order is important here + # first set the read flags to 0 + # then set the written flag to 1 + # otherwise, the readers may think they already read the block + for i in range(1, self.buffer.n_reader + 1): + # set read flag to 0, meaning it is not read yet + metadata_buffer[i] = 0 + # mark the block as written + metadata_buffer[0] = 1 + self.current_idx = (self.current_idx + + 1) % self.buffer.max_chunks + break + + @contextmanager + def acquire_read(self): + assert self._is_local_reader, "Only readers can acquire read" + start_time = time.monotonic() + n_warning = 1 + while True: + with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + read_flag = metadata_buffer[self.local_reader_rank + 1] + written_flag = metadata_buffer[0] + if not written_flag or read_flag: + # this block is either + # (1) not written + # (2) already read by this reader + + # for readers, `self.current_idx` is the next block to read + # if this block is not ready, + # we need to wait until it is written + + # wait for a while + time.sleep(RINGBUFFER_SLEEP_INTERVAL) + + # if we wait for a long time, we should warn the user + if (time.monotonic() - start_time > + VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning): + logger.warning( + "No available block found in %s second. ", + VLLM_RINGBUFFER_WARNING_INTERVAL) + n_warning += 1 + + continue + # found a block that is not read by this reader + # let caller read from the buffer + with self.buffer.get_data(self.current_idx) as buf: + yield buf + + # caller has read from the buffer + # set the read flag + metadata_buffer[self.local_reader_rank + 1] = 1 + self.current_idx = (self.current_idx + + 1) % self.buffer.max_chunks + break + + def enqueue(self, obj): + assert self._is_writer, "Only writers can enqueue" + serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + if self.n_local_reader > 0: + if len(serialized_obj) >= self.buffer.max_chunk_bytes: + with self.acquire_write() as buf: + buf[0] = 1 # overflow + self.local_socket.send(serialized_obj) + else: + with self.acquire_write() as buf: + buf[0] = 0 # not overflow + buf[1:len(serialized_obj) + 1] = serialized_obj + if self.n_remote_reader > 0: + self.remote_socket.send(serialized_obj) + + def dequeue(self): + if self._is_local_reader: + with self.acquire_read() as buf: + overflow = buf[0] == 1 + if not overflow: + # no need to know the size of serialized object + # pickle format contains the size information internally + # see https://docs.python.org/3/library/pickle.html + obj = pickle.loads(buf[1:]) + if overflow: + recv = self.local_socket.recv() + obj = pickle.loads(recv) + elif self._is_remote_reader: + recv = self.remote_socket.recv() + obj = pickle.loads(recv) + else: + raise RuntimeError("Only readers can dequeue") + return obj + + def broadcast_object(self, obj=None): + if self._is_writer: + self.enqueue(obj) + return obj + else: + return self.dequeue() + + @staticmethod + def create_from_process_group(pg: ProcessGroup, + max_chunk_bytes, + max_chunks, + writer_rank=0) -> "MessageQueue": + group_rank = dist.get_rank(pg) + group_world_size = dist.get_world_size(pg) + global_ranks = dist.get_process_group_ranks(pg) + + from vllm.distributed.parallel_state import in_the_same_node_as + status = in_the_same_node_as(pg, source_rank=writer_rank) + same_node_ranks = [i for i, s in enumerate(status) if s] + n_reader = group_world_size - 1 + n_local_reader = len(same_node_ranks) - 1 + local_reader_ranks = [i for i in same_node_ranks if i != writer_rank] + buffer_io: MessageQueue + if group_rank == writer_rank: + buffer_io = MessageQueue( + n_reader=n_reader, + n_local_reader=n_local_reader, + local_reader_ranks=local_reader_ranks, + max_chunk_bytes=max_chunk_bytes, + max_chunks=max_chunks, + ) + handle = buffer_io.export_handle() + dist.broadcast_object_list([handle], + src=global_ranks[writer_rank], + group=pg) + else: + recv = [None] + dist.broadcast_object_list(recv, + src=global_ranks[writer_rank], + group=pg) + handle = recv[0] # type: ignore + buffer_io = MessageQueue.create_from_handle(handle, group_rank) + buffer_io.wait_until_ready() + return buffer_io diff --git a/vllm/distributed/device_communicators/tpu_communicator.py b/vllm/distributed/device_communicators/tpu_communicator.py new file mode 100644 index 00000000..765a0f9c --- /dev/null +++ b/vllm/distributed/device_communicators/tpu_communicator.py @@ -0,0 +1,61 @@ +import os + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from vllm.platforms import current_platform + +if current_platform.is_tpu(): + import torch_xla.core.xla_model as xm + import torch_xla.runtime as xr + from torch_xla._internal import pjrt + + from vllm.executor import ray_utils + + +class TpuCommunicator: + + def __init__(self, group: ProcessGroup): + if not current_platform.is_tpu(): + self.disabled = True + return + self.disabled = False + + # NOTE(woosuk): When using TP > 1 on TPUs, every TPU on the same node + # must be used together. Therefore, the local rank and world size can + # be simply calculated as follows. + global_rank = dist.get_rank(group) + global_world_size = dist.get_world_size(group) + + # Calculate how many TPU nodes are in the current deployment. This + # is the Ray placement group if it is deployed with Ray. Default + # to the number of TPU nodes in the Ray cluster. The number of TPU + # nodes is computed by the total number of TPUs divided by the + # number of TPU accelerators per node, to account for clusters + # with both CPUs and TPUs. + num_nodes = ray_utils.get_num_tpu_nodes() + num_nodes_in_pg = ray_utils.get_num_nodes_in_placement_group() + if num_nodes_in_pg > 0: + num_nodes = num_nodes_in_pg + + local_world_size = global_world_size // num_nodes + local_rank = global_rank % local_world_size + + # Ensure environment variables are set for multihost deployments. + # On GKE, this is needed for libtpu and TPU driver to know which TPU + # chip is actually visible. Otherwise the TPU driver will fail to + # initialize because the number of devices would be different from + # the number of visible worker addresses. + os.environ["CLOUD_TPU_TASK_ID"] = str(global_rank) + os.environ["TPU_VISIBLE_CHIPS"] = str(local_rank) + + pjrt.initialize_multiprocess(local_rank, local_world_size) + xr._init_world_size_ordinal() + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + return xm.all_reduce(xm.REDUCE_SUM, x) + + def all_gather(self, x: torch.Tensor, dim: int = -1) -> torch.Tensor: + assert dim == -1, "TPUs only support dim=-1 for all-gather." + return xm.all_gather(x, dim=dim) diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py new file mode 100644 index 00000000..da1ad93e --- /dev/null +++ b/vllm/distributed/parallel_state.py @@ -0,0 +1,1258 @@ +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +"""vLLM distributed state. +It takes over the control of the distributed environment from PyTorch. +The typical workflow is: + +- call `init_distributed_environment` to initialize the distributed environment. +- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to + initialize the model parallel groups. + +- any code dealing with the distributed stuff + +- call `destroy_model_parallel` to destroy the model parallel groups. +- call `destroy_distributed_environment` to destroy the distributed environment. + +If you only need to use the distributed environment without model/pipeline + parallelism, you can skip the model parallel initialization and destruction + steps. +""" +import contextlib +import pickle +import weakref +from collections import namedtuple +from contextlib import contextmanager, nullcontext +from dataclasses import dataclass +from multiprocessing import shared_memory +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from unittest.mock import patch + +import torch +import torch.distributed +from torch.distributed import Backend, ProcessGroup + +import vllm.envs as envs +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils import supports_custom_op +from ixformer.distributed import all_reduce +# from ixformer.distributed import all_reduce, send, recv, gather +import ixformer.distributed as ixfd +import vllm._custom_ops as ops + + +@dataclass +class GraphCaptureContext: + stream: torch.cuda.Stream + + +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) + + +def _split_tensor_dict( + tensor_dict: Dict[str, Union[torch.Tensor, Any]] +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]: + """Split the tensor dictionary into two parts: + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced + by its metadata. + 2. A list of tensors. + """ + metadata_list: List[Tuple[str, Any]] = [] + tensor_list: List[torch.Tensor] = [] + for key, value in tensor_dict.items(): + if isinstance(value, torch.Tensor): + # Note: we cannot use `value.device` here, + # because it contains not only the device type but also the device + # index (e.g. "cuda:0"). We only need the device type. + # receiving side will set the device index. + device = value.device.type + metadata_list.append( + (key, TensorMetadata(device, value.dtype, value.size()))) + tensor_list.append(value) + else: + metadata_list.append((key, value)) + return metadata_list, tensor_list + + +_group_name_counter: Dict[str, int] = {} + + +def _get_unique_name(name: str) -> str: + """Get a unique name for the group. + Example: + _get_unique_name("tp") -> "tp:0" + _get_unique_name("tp") -> "tp:1" + """ + if name not in _group_name_counter: + _group_name_counter[name] = 0 + newname = f"{name}:{_group_name_counter[name]}" + _group_name_counter[name] += 1 + return newname + + +_groups: Dict[str, Callable[[], "GroupCoordinator"]] = {} + + +def _register_group(group: "GroupCoordinator") -> None: + # looks like Python 3.8 does not understand `ReferenceType` + _groups[group.unique_name] = weakref.ref(group) # type: ignore + + +if supports_custom_op(): + + @torch.library.custom_op("vllm::inplace_all_reduce", + mutates_args=["tensor"]) + def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + group._all_reduce_in_place(tensor) + + @inplace_all_reduce.register_fake + def _(tensor: torch.Tensor, group_name: str) -> None: + return + + @torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[]) + def outplace_all_reduce(tensor: torch.Tensor, + group_name: str) -> torch.Tensor: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + return group._all_reduce_out_place(tensor) + + @outplace_all_reduce.register_fake + def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor: + return torch.empty_like(tensor) + + +class GroupCoordinator: + """ + PyTorch ProcessGroup wrapper for a group of processes. + PyTorch ProcessGroup is bound to one specific communication backend, + e.g. NCCL, Gloo, MPI, etc. + GroupCoordinator takes charge of all the communication operations among + the processes in the group. It can route the communication to + a specific implementation (e.g. switch allreduce implementation + based on the tensor size and cuda graph mode). + """ + + # available attributes: + rank: int # global rank + ranks: List[int] # global ranks in the group + world_size: int # size of the group + # difference between `local_rank` and `rank_in_group`: + # if we have a group of size 4 across two nodes: + # Process | Node | Rank | Local Rank | Rank in Group + # 0 | 0 | 0 | 0 | 0 + # 1 | 0 | 1 | 1 | 1 + # 2 | 1 | 2 | 0 | 2 + # 3 | 1 | 3 | 1 | 3 + local_rank: int # local rank used to assign devices + rank_in_group: int # rank inside the group + cpu_group: ProcessGroup # group for CPU communication + device_group: ProcessGroup # group for device communication + use_pynccl: bool # a hint of whether to use PyNccl + use_custom_allreduce: bool # a hint of whether to use CustomAllreduce + # communicators are only created for world size > 1 + pynccl_comm: Optional[Any] # PyNccl communicator + ca_comm: Optional[Any] # Custom allreduce communicator + mq_broadcaster: Optional[Any] # shared memory broadcaster + + def __init__( + self, + group_ranks: List[List[int]], + local_rank: int, + torch_distributed_backend: Union[str, Backend], + use_pynccl: bool, + use_custom_allreduce: bool, + use_tpu_communicator: bool, + use_message_queue_broadcaster: bool = False, + group_name: Optional[str] = None, + world_size: Optional[str] = None, + ): + use_pynccl = False # we do not use pynccl + group_name = group_name or "anonymous" + self.unique_name = _get_unique_name(group_name) + _register_group(self) + + self.rank = torch.distributed.get_rank() + self.local_rank = local_rank + self.device_group = None + self.cpu_group = None + self.is_multi_node = False if world_size is None else bool(world_size > torch.cuda.device_count()) + + for ranks in group_ranks: + # if self.is_multi_node: + device_group = torch.distributed.new_group( + ranks, backend=torch_distributed_backend) + # else: + # device_group = ixfd.new_group( + # ranks, shmsize=1024 if world_size<=1 else None, backend=torch_distributed_backend,) + # a group with `gloo` backend, to allow direct coordination between + # processes through the CPU. + cpu_group = torch.distributed.new_group(ranks, backend="gloo") + if self.rank in ranks: + self.ranks = ranks + self.world_size = len(ranks) + self.rank_in_group = ranks.index(self.rank) + self.device_group = device_group + self.cpu_group = cpu_group + + assert self.cpu_group is not None + assert self.device_group is not None + + if current_platform.is_cuda_alike(): + self.device = torch.device(f"cuda:{local_rank}") + else: + self.device = torch.device("cpu") + + self.use_pynccl = use_pynccl + self.use_custom_allreduce = use_custom_allreduce + self.use_tpu_communicator = use_tpu_communicator + + # lazy import to avoid documentation build error + from vllm.distributed.device_communicators.custom_all_reduce import ( + CustomAllreduce) + from vllm.distributed.device_communicators.pynccl import ( + PyNcclCommunicator) + + self.pynccl_comm: Optional[PyNcclCommunicator] = None + if use_pynccl and self.world_size > 1: + self.pynccl_comm = PyNcclCommunicator( + group=self.cpu_group, + device=self.device, + ) + + self.ca_comm: Optional[CustomAllreduce] = None + if use_custom_allreduce and self.world_size > 1: + # Initialize a custom fast all-reduce implementation. + self.ca_comm = CustomAllreduce( + group=self.cpu_group, + device=self.device, + ) + + from vllm.distributed.device_communicators.tpu_communicator import ( + TpuCommunicator) + self.tpu_communicator: Optional[TpuCommunicator] = None + if use_tpu_communicator and self.world_size > 1: + self.tpu_communicator = TpuCommunicator(group=self.cpu_group) + + from vllm.distributed.device_communicators.shm_broadcast import ( + MessageQueue) + self.mq_broadcaster: Optional[MessageQueue] = None + if use_message_queue_broadcaster and self.world_size > 1: + self.mq_broadcaster = MessageQueue.create_from_process_group( + self.cpu_group, 1 << 22, 6) + + @property + def first_rank(self): + """Return the global rank of the first process in the group""" + return self.ranks[0] + + @property + def last_rank(self): + """Return the global rank of the last process in the group""" + return self.ranks[-1] + + @property + def is_first_rank(self): + """Return whether the caller is the first process in the group""" + return self.rank == self.first_rank + + @property + def is_last_rank(self): + """Return whether the caller is the last process in the group""" + return self.rank == self.last_rank + + @property + def next_rank(self): + """Return the global rank of the process that follows the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group + 1) % world_size] + + @property + def prev_rank(self): + """Return the global rank of the process that precedes the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group - 1) % world_size] + + @contextmanager + def graph_capture( + self, graph_capture_context: Optional[GraphCaptureContext] = None): + if graph_capture_context is None: + stream = torch.cuda.Stream() + graph_capture_context = GraphCaptureContext(stream) + else: + stream = graph_capture_context.stream + + ca_comm = self.ca_comm + maybe_ca_context = nullcontext( + ) if ca_comm is None else ca_comm.capture() + + # ensure all initialization operations complete before attempting to + # capture the graph on another stream + curr_stream = torch.cuda.current_stream() + if curr_stream != stream: + stream.wait_stream(curr_stream) + + with torch.cuda.stream(stream), maybe_ca_context: + # In graph mode, we have to be very careful about the collective + # operations. The current status is: + # allreduce \ Mode | Eager | Graph | + # -------------------------------------------- + # custom allreduce | enabled | enabled | + # PyNccl | disabled| enabled | + # torch.distributed | enabled | disabled| + # + # Note that custom allreduce will have a runtime check, if the + # tensor size is too large, it will fallback to the next + # available option. + # In summary: When using CUDA graph, we use + # either custom all-reduce kernel or pynccl. When not using + # CUDA graph, we use either custom all-reduce kernel or + # PyTorch NCCL. We always prioritize using custom all-reduce + # kernel but fall back to PyTorch or pynccl if it is + # disabled or not supported. + pynccl_comm = self.pynccl_comm + maybe_pynccl_context: Any + if not pynccl_comm: + maybe_pynccl_context = nullcontext() + else: + maybe_pynccl_context = pynccl_comm.change_state( + enable=True, stream=torch.cuda.current_stream()) + with maybe_pynccl_context: + yield graph_capture_context + + def all_reduce(self, input_: torch.Tensor) -> torch.Tensor: + """ + User-facing all-reduce function before we actually call the + all-reduce operation. + + We need this because Dynamo does not support passing an arbitrary + object (`self` in this case) to a custom op. We need to pass the + group name as a string, and then look up the group coordinator from + the group name, dispatch the all-reduce operation to the group + coordinator. + + In addition, PyTorch custom ops do not support mutation or returning + a new tensor in the same op. So we need to figure out if the op is + in-place or out-of-place ahead of time. + """ + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + + if not supports_custom_op(): + self._all_reduce_in_place(input_) + return input_ + + if self.tpu_communicator is not None and \ + not self.tpu_communicator.disabled: + # TPU handles Dynamo with its own logic. + return self.tpu_communicator.all_reduce(input_) + + if self.ca_comm is not None and \ + not self.ca_comm.disabled and \ + self.ca_comm.should_custom_ar(input_): + return torch.ops.vllm.outplace_all_reduce( + input_, group_name=self.unique_name) + else: + torch.ops.vllm.inplace_all_reduce(input_, + group_name=self.unique_name) + return input_ + + def _all_reduce_out_place(self, input_: torch.Tensor) -> torch.Tensor: + ca_comm = self.ca_comm + assert ca_comm is not None + assert not ca_comm.disabled + out = ca_comm.custom_all_reduce(input_) + assert out is not None + return out + + def _all_reduce_in_place(self, input_: torch.Tensor) -> None: + pynccl_comm = self.pynccl_comm + if (pynccl_comm is not None and not pynccl_comm.disabled): + pynccl_comm.all_reduce(input_) + elif input_.is_cpu: + import intel_extension_for_pytorch as ipex + ipex.distributed.all_reduce(input_, group=self.device_group) + else: + # if self.is_multi_node: + # torch.distributed.all_reduce(input_, group=self.device_group) + # else: + # all_reduce(input_, group=self.device_group, async_op=True) + from ixformer.contrib.torch.extension.ixformer_torch.distributed import create_ixformer_group_from_pg + _IXFORMER_TENSOR_MODEL_PARALLEL_GROUP = create_ixformer_group_from_pg(self.device_group) + all_reduce(input_, group=_IXFORMER_TENSOR_MODEL_PARALLEL_GROUP, async_op=True) + return input_ + + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + assert -input_.dim() <= dim < input_.dim(), ( + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}") + + # For TPUs, use TPU communicator. + tpu_comm = self.tpu_communicator + if tpu_comm is not None and not tpu_comm.disabled: + return tpu_comm.all_gather(input_, dim) + + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + input_size = input_.size() + # Allocate output tensor. + output_tensor = torch.empty((world_size, ) + input_size, + dtype=input_.dtype, + device=input_.device) + # All-gather. + torch.distributed.all_gather_into_tensor(output_tensor, + input_, + group=self.device_group) + # Reshape + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape(input_size[:dim] + + (world_size * + input_size[dim], ) + + input_size[dim + 1:]) + return output_tensor + + def gather(self, + input_: torch.Tensor, + dst: int = 0, + dim: int = -1) -> Optional[torch.Tensor]: + """ + NOTE: We assume that the input tensor is on the same device across + all the ranks. + NOTE: `dst` is the local rank of the destination rank. + """ + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + assert -input_.dim() <= dim < input_.dim(), ( + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}") + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + # Allocate output tensor. + if self.rank_in_group == dst: + gather_list = [torch.empty_like(input_) for _ in range(world_size)] + else: + gather_list = None + # # Gather. + # if self.is_multi_node: + torch.distributed.gather(input_, + gather_list, + dst=self.ranks[dst], + group=self.device_group, + ) + # else: + # gather(input_, + # gather_list, + # dst=self.ranks[dst], + # group=self.device_group, + # async_op=True) + if self.rank_in_group == dst: + output_tensor = torch.cat(gather_list, dim=dim) + else: + output_tensor = None + return output_tensor + + def broadcast(self, input_: torch.Tensor, src: int = 0): + """Broadcast the input tensor. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + # Broadcast. + # if self.is_multi_node: + torch.distributed.broadcast(input_, + src=self.ranks[src], + group=self.device_group) + # else: + # ops.broadcast(input_, + # src=self.ranks[src], + # group=self.device_group) + return input_ + + def broadcast_object(self, obj: Optional[Any] = None, src: int = 0): + """Broadcast the input object. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj + if self.mq_broadcaster is not None: + assert src == 0, "Message queue broadcaster only supports src=0" + return self.mq_broadcaster.broadcast_object(obj) + if self.rank_in_group == src: + torch.distributed.broadcast_object_list([obj], + src=self.ranks[src], + group=self.cpu_group) + return obj + else: + recv = [None] + torch.distributed.broadcast_object_list(recv, + src=self.ranks[src], + group=self.cpu_group) + return recv[0] + + def broadcast_object_list(self, + obj_list: List[Any], + src: int = 0, + group: Optional[ProcessGroup] = None): + """Broadcast the input object list. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj_list + # Broadcast. + torch.distributed.broadcast_object_list(obj_list, + src=self.ranks[src], + group=self.device_group) + return obj_list + + def send_object(self, obj: Any, dst: int) -> None: + """Send the input object list to the destination rank.""" + """NOTE: `dst` is the local rank of the destination rank.""" + + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + assert dst != self.rank_in_group, ( + "Invalid destination rank. Destination rank is the same " + "as the current rank.") + + # Serialize object to tensor and get the size as well + object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8) + + size_tensor = torch.tensor([object_tensor.numel()], + dtype=torch.long, + device="cpu") + + # Send object size + + torch.distributed.send(size_tensor, + dst=self.ranks[dst], + group=self.cpu_group) + + # Send object + torch.distributed.send(object_tensor, + dst=self.ranks[dst], + group=self.cpu_group) + + return None + + def recv_object(self, src: int) -> Any: + """Receive the input object list from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + + assert src < self.world_size, f"Invalid src rank ({src})" + + assert src != self.rank_in_group, ( + "Invalid source rank. Source rank is the same as the current rank." + ) + + size_tensor = torch.empty(1, dtype=torch.long, device="cpu") + + # Receive object size + rank_size = torch.distributed.recv(size_tensor, + src=self.ranks[src], + group=self.cpu_group) + + # Tensor to receive serialized objects into. + object_tensor = torch.empty( # type: ignore[call-overload] + size_tensor.item(), # type: ignore[arg-type] + dtype=torch.uint8, + device="cpu") + + rank_object = torch.distributed.recv(object_tensor, + src=self.ranks[src], + group=self.cpu_group) + + assert rank_object == rank_size, ( + "Received object sender rank does not match the size sender rank.") + + obj = pickle.loads(object_tensor.numpy().tobytes()) + + return obj + + def broadcast_tensor_dict( + self, + tensor_dict: Optional[Dict[str, Union[torch.Tensor, Any]]] = None, + src: int = 0, + group: Optional[ProcessGroup] = None, + metadata_group: Optional[ProcessGroup] = None + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Broadcast the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if (not torch.distributed.is_initialized() or self.world_size == 1): + return tensor_dict + + group = self.device_group + metadata_group = self.cpu_group + assert src < self.world_size, f"Invalid src rank ({src})" + + rank_in_group = self.rank_in_group + if rank_in_group == src: + metadata_list: List[Tuple[Any, Any]] = [] + assert isinstance( + tensor_dict, + dict), (f"Expecting a dictionary, got {type(tensor_dict)}") + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # `metadata_list` lives in CPU memory. + # `broadcast_object_list` has serialization & deserialization, + # all happening on CPU. Therefore, we can use the CPU group. + self.broadcast_object(metadata_list, src=src) + async_handles = [] + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast(tensor, + src=self.ranks[src], + group=metadata_group, + async_op=True) + else: + # use group for GPU tensors + # if self.is_multi_node: + handle = torch.distributed.broadcast(tensor, + src=self.ranks[src], + group=group, + async_op=True) + # else: + # handle = ops.broadcast(tensor, + # src=self.ranks[src], + # group=group, + # async_op=True) + async_handles.append(handle) + for async_handle in async_handles: + async_handle.wait() + + else: + metadata_list = self.broadcast_object(None, src=src) + tensor_dict = {} + async_handles = [] + for key, value in metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty(value.size, + dtype=value.dtype, + device=value.device) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + tensor_dict[key] = tensor + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast( + tensor, + src=self.ranks[src], + group=metadata_group, + async_op=True) + else: + # use group for GPU tensors + # if self.is_multi_node: + handle = torch.distributed.broadcast(tensor, + src=self.ranks[src], + group=group, + async_op=True) + # else: + # handle = ops.broadcast( + # tensor, + # src=self.ranks[src], + # group=group, + # async_op=True) + async_handles.append(handle) + tensor_dict[key] = tensor + else: + tensor_dict[key] = value + for async_handle in async_handles: + async_handle.wait() + return tensor_dict + + def send_tensor_dict( + self, + tensor_dict: Dict[str, Union[torch.Tensor, Any]], + dst: Optional[int] = None, + all_gather_group: Optional["GroupCoordinator"] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Send the input tensor dictionary. + NOTE: `dst` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return tensor_dict + + all_gather_size = (1 if all_gather_group is None else + all_gather_group.world_size) + all_gather_rank = (0 if all_gather_group is None else + all_gather_group.rank_in_group) + + group = self.device_group + metadata_group = self.cpu_group + + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + metadata_list: List[Tuple[Any, Any]] = [] + assert isinstance( + tensor_dict, + dict), f"Expecting a dictionary, got {type(tensor_dict)}" + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # `metadata_list` lives in CPU memory. + # `send_object_list` has serialization & deserialization, + # all happening on CPU. Therefore, we can use the CPU group. + self.send_object(metadata_list, dst=dst) + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip sending empty tensors. + continue + + # send-allgather: send only a slice, then do allgather. + if (all_gather_group is not None + and tensor.numel() % all_gather_size == 0): + tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank] + + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.send(tensor, + dst=self.ranks[dst], + group=metadata_group) + else: + # use group for GPU tensors + # if self.is_multi_node: + torch.distributed.send( + tensor, + dst=self.ranks[dst], + group=group) + # else: + # send(tensor, + # dst=self.ranks[dst], + # group=group) + return None + + def recv_tensor_dict( + self, + src: Optional[int] = None, + all_gather_group: Optional["GroupCoordinator"] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Recv the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return None + + all_gather_size = (1 if all_gather_group is None else + all_gather_group.world_size) + all_gather_rank = (0 if all_gather_group is None else + all_gather_group.rank_in_group) + + group = self.device_group + metadata_group = self.cpu_group + + if src is None: + src = (self.rank_in_group - 1) % self.world_size + assert src < self.world_size, f"Invalid src rank ({src})" + + recv_metadata_list = self.recv_object(src=src) + tensor_dict: Dict[str, Any] = {} + for key, value in recv_metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty(value.size, + dtype=value.dtype, + device=value.device) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + tensor_dict[key] = tensor + continue + + # send-allgather: send only a slice, then do allgather. + use_all_gather = (all_gather_group is not None + and tensor.numel() % all_gather_size == 0) + + if use_all_gather: + orig_shape = tensor.shape + tensor = tensor.reshape(all_gather_size, + -1)[all_gather_rank] + + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.recv(tensor, + src=self.ranks[src], + group=metadata_group) + else: + # if self.is_multi_node: + # use group for GPU tensors + torch.distributed.recv(tensor, + src=self.ranks[src], + group=group) + # else: + # recv(tensor, + # src=self.ranks[src], + # group=group) + if use_all_gather: + # do the allgather + tensor = all_gather_group.all_gather( # type: ignore + tensor, dim=0) + tensor = tensor.reshape(orig_shape) + + tensor_dict[key] = tensor + else: + tensor_dict[key] = value + return tensor_dict + + def barrier(self): + """Barrier synchronization among the group. + NOTE: don't use `device_group` here! `barrier` in NCCL is + terrible because it is internally a broadcast operation with + secretly created GPU tensors. It is easy to mess up the current + device. Use the CPU group instead. + """ + torch.distributed.barrier(group=self.cpu_group) + + def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None: + """Sends a tensor to the destination rank in a non-blocking way""" + """NOTE: `dst` is the local rank of the destination rank.""" + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.send(tensor, dst) + else: + # if self.is_multi_node: + torch.distributed.send(tensor, self.ranks[dst], self.device_group) + # else: + # send(tensor, self.ranks[dst], self.device_group) + + def recv(self, + size: torch.Size, + dtype: torch.dtype, + src: Optional[int] = None) -> torch.Tensor: + """Receives a tensor from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + if src is None: + src = (self.rank_in_group - 1) % self.world_size + + tensor = torch.empty(size, dtype=dtype, device=self.device) + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.recv(tensor, src) + else: + # if self.is_multi_node: + torch.distributed.recv(tensor, self.ranks[src], self.device_group) + # else: + # recv(tensor, self.ranks[src], self.device_group) + return tensor + + def destroy(self): + if self.device_group is not None: + torch.distributed.destroy_process_group(self.device_group) + self.device_group = None + if self.cpu_group is not None: + torch.distributed.destroy_process_group(self.cpu_group) + self.cpu_group = None + if self.pynccl_comm is not None: + self.pynccl_comm = None + if self.ca_comm is not None: + self.ca_comm = None + if self.mq_broadcaster is not None: + self.mq_broadcaster = None + + +_WORLD: Optional[GroupCoordinator] = None + + +def get_world_group() -> GroupCoordinator: + assert _WORLD is not None, ("world group is not initialized") + return _WORLD + + +def init_world_group(ranks: List[int], local_rank: int, + backend: str) -> GroupCoordinator: + return GroupCoordinator( + group_ranks=[ranks], + local_rank=local_rank, + torch_distributed_backend=backend, + use_pynccl=False, + use_custom_allreduce=False, + use_tpu_communicator=False, + group_name="world", + world_size=len(ranks), + ) + + +def init_model_parallel_group( + group_ranks: List[List[int]], + local_rank: int, + backend: str, + use_custom_allreduce: Optional[bool] = None, + use_message_queue_broadcaster: bool = False, + group_name: Optional[str] = None, + world_size: Optional[int] = None, +) -> GroupCoordinator: + if use_custom_allreduce is None: + use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE + return GroupCoordinator( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=backend, + use_pynccl=True, + use_custom_allreduce=use_custom_allreduce, + use_tpu_communicator=True, + use_message_queue_broadcaster=use_message_queue_broadcaster, + group_name=group_name, + world_size=world_size, + ) + + +_TP: Optional[GroupCoordinator] = None + + +def get_tp_group() -> GroupCoordinator: + assert _TP is not None, ("tensor model parallel group is not initialized") + return _TP + + +# kept for backward compatibility +get_tensor_model_parallel_group = get_tp_group + +_PP: Optional[GroupCoordinator] = None + + +def get_pp_group() -> GroupCoordinator: + assert _PP is not None, ( + "pipeline model parallel group is not initialized") + return _PP + + +# kept for backward compatibility +get_pipeline_model_parallel_group = get_pp_group + + +@contextmanager +def graph_capture(): + """ + `graph_capture` is a context manager which should surround the code that + is capturing the CUDA graph. Its main purpose is to ensure that the + some operations will be run after the graph is captured, before the graph + is replayed. It returns a `GraphCaptureContext` object which contains the + necessary data for the graph capture. Currently, it only contains the + stream that the graph capture is running on. This stream is set to the + current CUDA stream when the context manager is entered and reset to the + default stream when the context manager is exited. This is to ensure that + the graph capture is running on a separate stream from the default stream, + in order to explicitly distinguish the kernels to capture + from other kernels possibly launched on background in the default stream. + """ + with get_tp_group().graph_capture() as context, get_pp_group( + ).graph_capture(context): + yield context + + +logger = init_logger(__name__) + +_ENABLE_CUSTOM_ALL_REDUCE = True + + +def set_custom_all_reduce(enable: bool): + global _ENABLE_CUSTOM_ALL_REDUCE + _ENABLE_CUSTOM_ALL_REDUCE = enable + + +def init_distributed_environment( + world_size: int = -1, + rank: int = -1, + distributed_init_method: str = "env://", + local_rank: int = -1, + backend: str = "nccl", +): + logger.debug( + "world_size=%d rank=%d local_rank=%d " + "distributed_init_method=%s backend=%s", world_size, rank, local_rank, + distributed_init_method, backend) + if not torch.distributed.is_initialized(): + assert distributed_init_method is not None, ( + "distributed_init_method must be provided when initializing " + "distributed environment") + # this backend is used for WORLD + torch.distributed.init_process_group( + backend=backend, + init_method=distributed_init_method, + world_size=world_size, + rank=rank) + # set the local rank + # local_rank is not available in torch ProcessGroup, + # see https://github.com/pytorch/pytorch/issues/122816 + if local_rank == -1: + # local rank not set, this usually happens in single-node + # setting, where we can use rank as local rank + if distributed_init_method == "env://": + local_rank = envs.LOCAL_RANK + else: + local_rank = rank + global _WORLD + if _WORLD is None: + ranks = list(range(torch.distributed.get_world_size())) + _WORLD = init_world_group(ranks, local_rank, backend) + else: + assert _WORLD.world_size == torch.distributed.get_world_size(), ( + "world group already initialized with a different world size") + + +def initialize_model_parallel( + tensor_model_parallel_size: int = 1, + pipeline_model_parallel_size: int = 1, + backend: Optional[str] = None, +) -> None: + """ + Initialize model parallel groups. + + Arguments: + tensor_model_parallel_size: number of GPUs used for tensor model + parallelism. + pipeline_model_parallel_size: number of GPUs used for pipeline model + parallelism. + + Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we + use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize + the model pipeline. The present function will + create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: + 4 tensor model-parallel groups: + [g0, g1], [g2, g3], [g4, g5], [g6, g7] + 2 pipeline model-parallel groups: + [g0, g2, g4, g6], [g1, g3, g5, g7] + Note that for efficiency, the caller should make sure adjacent ranks + are on the same DGX box. For example if we are using 2 DGX-1 boxes + with a total of 16 GPUs, rank 0 to 7 belong to the first box and + ranks 8 to 15 belong to the second box. + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend( + get_world_group().device_group) + + if (world_size != + tensor_model_parallel_size * pipeline_model_parallel_size): + raise RuntimeError( + f"world_size ({world_size}) is not equal to " + f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " + f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") + + # Build the tensor model-parallel groups. + num_tensor_model_parallel_groups: int = (world_size // + tensor_model_parallel_size) + global _TP + assert _TP is None, ("tensor model parallel group is already initialized") + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + ranks = list( + range(i * tensor_model_parallel_size, + (i + 1) * tensor_model_parallel_size)) + group_ranks.append(ranks) + + # message queue broadcaster is only used in tensor model parallel group + _TP = init_model_parallel_group(group_ranks, + get_world_group().local_rank, + backend, + use_message_queue_broadcaster=True, + group_name="tp", + world_size=world_size) + + # Build the pipeline model-parallel groups. + num_pipeline_model_parallel_groups: int = (world_size // + pipeline_model_parallel_size) + global _PP + assert _PP is None, ( + "pipeline model parallel group is already initialized") + group_ranks = [] + for i in range(num_pipeline_model_parallel_groups): + ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) + group_ranks.append(ranks) + # pipeline parallel does not need custom allreduce + _PP = init_model_parallel_group(group_ranks, + get_world_group().local_rank, + backend, + use_custom_allreduce=False, + group_name="pp", + world_size=world_size) + + +def ensure_model_parallel_initialized( + tensor_model_parallel_size: int, + pipeline_model_parallel_size: int, + backend: Optional[str] = None, +) -> None: + """Helper to initialize model parallel groups if they are not initialized, + or ensure tensor-parallel and pipeline-parallel sizes are equal to expected + values if the model parallel groups are initialized. + """ + backend = backend or torch.distributed.get_backend( + get_world_group().device_group) + if not model_parallel_is_initialized(): + initialize_model_parallel(tensor_model_parallel_size, + pipeline_model_parallel_size, backend) + return + + assert ( + get_tensor_model_parallel_world_size() == tensor_model_parallel_size + ), ("tensor parallel group already initialized, but of unexpected size: " + f"{get_tensor_model_parallel_world_size()=} vs. " + f"{tensor_model_parallel_size=}") + pp_world_size = get_pp_group().world_size + assert (pp_world_size == pipeline_model_parallel_size), ( + "pipeline parallel group already initialized, but of unexpected size: " + f"{pp_world_size=} vs. " + f"{pipeline_model_parallel_size=}") + + +def model_parallel_is_initialized(): + """Check if tensor and pipeline parallel groups are initialized.""" + return (_TP is not None and _PP is not None) + + +_TP_STATE_PATCHED = False + + +@contextmanager +def patch_tensor_parallel_group(tp_group: GroupCoordinator): + """Patch the tp group temporarily until this function ends. + + This method is for draft workers of speculative decoding to run draft model + with different tp degree from that of target model workers. + + Args: + tp_group (GroupCoordinator): the tp group coordinator + """ + global _TP_STATE_PATCHED + assert not _TP_STATE_PATCHED, "Should not call when it's already patched" + + _TP_STATE_PATCHED = True + old_tp_group = get_tp_group() + global _TP + _TP = tp_group + try: + yield + finally: + # restore the original state + _TP_STATE_PATCHED = False + _TP = old_tp_group + + +def get_tensor_model_parallel_world_size(): + """Return world size for the tensor model parallel group.""" + return get_tp_group().world_size + + +def get_tensor_model_parallel_rank(): + """Return my rank for the tensor model parallel group.""" + return get_tp_group().rank_in_group + + +def destroy_model_parallel(): + """Set the groups to none and destroy them.""" + global _TP + if _TP: + _TP.destroy() + _TP = None + + global _PP + if _PP: + _PP.destroy() + _PP = None + + +def destroy_distributed_environment(): + global _WORLD + if _WORLD: + _WORLD.destroy() + _WORLD = None + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + +def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]: + """ + This is a collective operation that returns if each rank is in the same node + as the source rank. It tests if processes are attached to the same + memory system (shared access to shared memory). + """ + assert torch.distributed.get_backend( + pg) != torch.distributed.Backend.NCCL, ( + "in_the_same_node_as should be tested with a non-NCCL group.") + # local rank inside the group + rank = torch.distributed.get_rank(group=pg) + world_size = torch.distributed.get_world_size(group=pg) + + # local tensor in each process to store the result + is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32) + + # global ranks of the processes in the group + ranks = torch.distributed.get_process_group_ranks(pg) + + magic_message = b"magic_message" + shm = None + + try: + with contextlib.suppress(OSError): + if rank == source_rank: + # create a shared memory segment + shm = shared_memory.SharedMemory(create=True, size=128) + shm.buf[:len(magic_message)] = magic_message + torch.distributed.broadcast_object_list([shm.name], + src=ranks[source_rank], + group=pg) + is_in_the_same_node[rank] = 1 + else: + # try to open the shared memory segment + recv = [None] + torch.distributed.broadcast_object_list(recv, + src=ranks[source_rank], + group=pg) + name = recv[0] + # fix to https://stackoverflow.com/q/62748654/9191338 + # Python incorrectly tracks shared memory even if it is not + # created by the process. The following patch is a workaround. + with patch("multiprocessing.resource_tracker.register", + lambda *args, **kwargs: None): + shm = shared_memory.SharedMemory(name=name) + if shm.buf[:len(magic_message)] == magic_message: + is_in_the_same_node[rank] = 1 + except Exception as e: + logger.error("Error ignored in is_in_the_same_node: %s", e) + finally: + if shm: + shm.close() + + torch.distributed.barrier(group=pg) + + # clean up the shared memory segment + with contextlib.suppress(OSError): + if rank == source_rank and shm: + shm.unlink() + torch.distributed.all_reduce(is_in_the_same_node, group=pg) + + return [x == 1 for x in is_in_the_same_node.tolist()] diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py new file mode 100644 index 00000000..8c94ef8c --- /dev/null +++ b/vllm/distributed/utils.py @@ -0,0 +1,86 @@ +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +from typing import Sequence, Tuple + +import torch + +import vllm.envs as envs +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def ensure_divisibility(numerator, denominator): + """Ensure that numerator is divisible by the denominator.""" + assert numerator % denominator == 0, "{} is not divisible by {}".format( + numerator, denominator) + + +def divide(numerator, denominator): + """Ensure that numerator is divisible by the denominator and return + the division value.""" + ensure_divisibility(numerator, denominator) + return numerator // denominator + + +def split_tensor_along_last_dim( + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, +) -> Sequence[torch.Tensor]: + """ Split a tensor along its last dimension. + + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + + Returns: + A list of Tensors + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = divide(tensor.size()[last_dim], num_partitions) + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # NOTE: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + +def get_pp_indices(num_hidden_layers: int, pp_rank: int, + pp_size: int) -> Tuple[int, int]: + """Try to evenly distribute layers across partitions. + If the number of layers is not divisible by the number of partitions, + the last partition will have the remaining layers. + """ + partition_list_str = envs.VLLM_PP_LAYER_PARTITION + if partition_list_str is not None: + try: + partitions = [ + int(layer) for layer in partition_list_str.split(",") + ] + except ValueError as err: + raise ValueError("Invalid partition string: {}".format( + partition_list_str)) from err + if len(partitions) != pp_size: + raise ValueError(f"{len(partitions)=} does not match {pp_size=}.") + if sum(partitions) != num_hidden_layers: + raise ValueError( + f"{sum(partitions)=} does not match {num_hidden_layers=}.") + start_layer = sum(partitions[:pp_rank]) + end_layer = start_layer + partitions[pp_rank] + else: + layers_per_partition = num_hidden_layers // pp_size + start_layer = pp_rank * layers_per_partition + end_layer = start_layer + layers_per_partition + + if pp_rank == pp_size - 1: + end_layer = num_hidden_layers + + return (start_layer, end_layer) diff --git a/vllm/engine/__init__.py b/vllm/engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/engine/__pycache__/__init__.cpython-310.pyc b/vllm/engine/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20b1de9c8d70fbce854d116d6af17a57816f53bf GIT binary patch literal 152 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hueo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_erjHNW?rg(e0*kJW=VX!UP0w8 V4x8Nkl+v73JCLEpOhAH#0RVnbBMkrm literal 0 HcmV?d00001 diff --git a/vllm/engine/__pycache__/arg_utils.cpython-310.pyc b/vllm/engine/__pycache__/arg_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93db9cb553d0f489a09a2ad3de6d8269cf4de801 GIT binary patch literal 32837 zcmdsgd3YShb?00doCHV!ybrZNQ6xyr@X|?aqMJmSr+&AdsW@j z1A?-??6>)2fp4m-s_S@F_3G6-s;XV*AfyeN3za^PS zSP3gxPMGSMG?NlenJGL|#X&NS#LjdBaOI+k!FLn5iuL>#Faxg(&i)9CTsJH3G=YE z#oCJJ5i2>K7~b|8$Ls8HGU&95UU8~iblu^uVAaI4=f?`ipBOv-)Ty(N2kFu3LXbI8 zn(~6o>5}UO?Waqg?G(#F`;mc+!u5o zFHX(c$7|K;(oE2!o+oQgrRW8{C+w-3g+ju&C+zvsRFrnCI#a5Kp{~=lqNUS`UzY*@ zg;5>feWqsFWgQqeH|g5Wd~vc=E_n+&viE!uz?Yke&O5bA-5a%vbu>#S>Kg~ZR-+t6 ztQoJ{Q;l-bE6s=C3{2GK>}tuko%6Mtt`dFOGj<&*y_$2Z=-LQ(m#QVNP_E6)*bah& zPhT88J8{Y!oj7&wY~jq<#1rREj3a(mrMNI@7aV&AeQZ00De0{O;4c-+sH8BtP?#^4 z8!WO9z)j5+W@Z}Gh3Qh6QR_~Y?dzq9Wrxs+%$iuWjMUWMZj3necn9&c)2f6IOPy?c<40WR^OZdl9)8+uxtE zQcLacZoihimRQUzCg&5*3yaCc%*MoG7WKR!^>i`!CsI;dmTwOyuJ$c*es}wy7A$a$ zqJbX4Br||jUJ(c>-bw}Wb{*5fvyC}>!FUd-UNF3xA&6!?SE_n1bQ$=o7fX(7AXr0D z7DxeSyR(q*lD9xP_mKFQ7I_aDX5eT0#dUUdx@6sa^_LM{}>@jGM;XM=RvuA*^uP`sc6(?g02)?kc; zERZmdIZh2z9>W=QT_Q=5h~Z9|GnncPufQ=5G6dc1n42%Kd)y$6I)XN--RzNaEk%-o zw0(UlNLSpM<$c(klIW~`=oFXpu%`=CGS!b*_&LAn2z_R-W<$i zk{yWeN@Y^c31peVujQ`|p~vuZKM0?lAWLDUE`r#YX)9%B>^3`Vr4J{pjM;8?*qsP> z*=<-#WkJQeHhI^LcZfwei*UQyW3Ph01O864SIX#;_&$m6mOQJYJUxk5dSvuZbMu-o#wFAu?zlTbGP_+!{2W1k#hG)o)P3x{XQb$ zTolepIFImN>4&_8_aVGP!h0pWKT5w(!Uv-8ehD8$c$K6-AmRHER{eSq;rq<{1+M$y zZ#N$hxE_GN9}E`9;z9Ty0&69`Z9S}d8}Ww_pD`asSqipCBzy?rM=&nyBiuTS_;&NK zlzRmJ_2v;N_bB|E&7uX&#d_k4gL}{Qc&r_>W1NW0K}L{5#F#5`O~zjClg} zj5q{ddAeeMP-i_m( z8ntnRC(H?f@uIxD80EVp;Y(5Y83{jwuxj5sB>avj{4B!l=CcT^vQ2q!M(>}K@N-f4 zc?mxsgUa{w_Il+)J!WF9u_P=abteRc5z|%pr zxN12sW*|1y?7HRJSFbn_Mm(z_aW2w$2+v#BUd)&cgys>tZY>~m4Wa7@`PRD-TCg%; zITnL1#j=wp@Hd64jw?&l{OQwY48^$R{Wa6YDSNV3o7-Efd%67w?yor8LC((xYsj+O zRzVolLbX`2gHA8xv`+>D?p&!}h$0H)uAO_4Gw5wbGHal9q zs%1zKSm2zXTYOxJA&9Kf47U!pSsq20UMzl76kZA{y#8vVSoKOizbHD_7u7S=~lBq7@bCZV8ebzfV}0=p)`rp&2A*4kC!k?W2< zUAiu!as^%PwPL;C)-jSnFT{jG zZF*YzbY^nqcRtO(zj-uRH-i@y41%*Dth(;JU>(7(HY#YkM`25isry<2T@dtyFM*Km zYm_Ba6vU{yQ=4>yH6gf&0lQxWd(z=RbaXK;97~I$ln(YdkX{Nd_%t@OOG4nmykgMb zM44vl_z(xHFhy$4lwGjd>|hh&A`@+2hfHTz0W^BB4xz#>B@93*tpd=?UIrUOc&C7q zwMqel1puuEdTyWzL>+>p8l^N)#TjC|m%kRg+`KFgz;TLNW()?(r3ysO3Q!Y*B>D^D zZP2ZUyviL6sE7xxt3pBh?@V1JjVi77Ffc z(XlYHs;0G4i-L3>T=S%A>Be#m=qm5@5EVcgAV0f7m&B@>C$ow;rV7hiq9+Ijh(T^B ztOFD5x*Ke)G|FB{0u%*pOj0nl^#+h+9r`t#8!{!UZfR!L3)Zt7Vyd9ITd2+3j#C23 zkP6D6Q^i>ubi*yzP|8L{oQUh_;g-(tE6r4E4ri_hF}~^seVAxA`jP~L*=8`P@)xXY z!VOs*aH)0&>-11f%{HoY=suaL<#IsIoK(RL*3UEm8>pC8o`mEB#IvcUP&6pmc3V`T z1;g)+zqg#?v{zV>%&e21jukH`=(d(0VtM6i5u<{BaDt8T@`*9J??4*wj=u`CLeC)- zL1D-G>Wt7>L1K)3ZLU!-pcD+)N-s*)V2xv6u|=3HxWx*zQBH6d>uGjYacau0d&Med zb;X8C!3y%2R3*@dLhO~!T&UNmk12udg3Q|x{VAEB>Vm*^&K`RIi z_hy3~VWUI3Rq5LPDD6%}GDs|tz;?O5%w2=v=mweTscCmO<$NE|HR!0b51>W^j;ZNJ zIO{bCr3<$SciIc4naMmUbq+q?ft^5!6cG z%%F8_d&bKkuFYzT#bvE_tHbKNnbv8#yo}Xt^~6%Fl9=9DOrON8j>W8z+Op6j_rIF7 z2CTuGZMwX*D1Tk7{PhAuJM_X$xHd}e4y7CRI;qEkez;l2RzTflZKLMc>z227Sw_^V zyCruI>K;Pf+hh6e@sfz$@hW>_m36PRGlFWDv|u;}!)~t^srsxvvA7X0Y31;rpGjGJ zW99C%_FD&LQb@nrI%wS&L2^IZ`arC$4_XgNscWob-@~s#Y{-5P1eO&+$FCUxb#dc<{eW0X6xBloGEcztmk5J z&r94^>xEd{I{|+o)~3s_6=Y@--Zu0__eVI!cUhAdFWi;y!C-t>JLf@d#w+};=TdUd}9on4tit}z6JR6t=6~2(tW#kAA0G2>pQIPw7$#wcr@1EE!g*f^@&*8 z?~$|*O4{#@rF~i2_K@{TEbXhyFzI3I`(o7M`vu}d)(^yT{2*HT$yiH&$oiC|dc^u+ zqzdQyk4W5M>qlcbe@x;eYJ!7zfu6rS$RBrwp(WqIF(D z{%_Cagc0;}XCdMn&|p)JHlXpQdVIDqgC&24Is!wIQ8NxfG!N5NMkJti%bqTR%?~oO zcDe3%=W>xq5op&+}rYjZhjBAT`4 zsEL3`49Ud?C)Tk2ed9;~g@4Ht6Q1Y{q=BI}!zflQ1Ke8}{!}W+LiB;8;QS~!3%@Iu z3&qm2{?|>QUo?bM8a@aq6nLAh@QkTql|UAaNyu|BKd=cS2ztdC?MGJ^VF;=~Ml?!R z{>_J4{U--M7U zVO`s84AF=?1Tz}YH5|t2qU*)dmZpv3JY)tceDch~l6971`C4(ojdJo$GiTmlY7N24 z87}L$5G^W|TJ>@)GdTDkMnjw*ffKY(&DLP)cNMA+pwU#QQlvb%dwQ4QWLfa1QdGG_|X#ig?>0*GeimR5F%*CyQmg5wo6Xj;5w~HB zr}}JpH-$D!w>WJp)qv1*XicS8r=iO-sJ{$*5q5g0%w~;A*g7>BK)4lhmt3P-Lm@4w zSobtu-3g4_i&rSzgzjq;a62SYK(#Kz{uY_TnsuW{y@lL^oLUl1~goxHJa(vTsn^B>KVH@Z|9A1TLCg5T4oewh{pa5=3Uuvr(iC}f}zaU zW7kEs0wc6iwFp2^@zTDndI!srGvtQl`c>y_P zVaHD1mRO=x8o>#|KBSA;|3Kcj05xU>o0g!`l-SW{o?eCkSafs)hd2vJtD&C*o8uZo zQkufGC=GBBlX9|Hrj`%{VpVj7|o~znHWTiljk4G8|P5E zbFJip>y1)q!$Md`w9W&_{rP<%dBqMqz(ke_YTyR6tqUmBZPA~0# zE0Wl#_$d-t+p&q(pvcrr7lkQ@PFx98IhdtH@jKXyyHFpyW*pa4xg}M}6p`4}(;t?tj-YWfSI9B2p3uVmYz&I@pmplQ*8> zY>>rQp%~{h(FmD1SG3sLI>xR|dB7NgIShmk+suG+G#~?e9WeOyA!C;q(X>$j^$Z)e zaArXYS4&fqB!KiOi*pXO=1{7C@Ptj$2-F74|F5t8{!q|j0ht_PYh3MVle63^|7e>z*=gyr` zL@UmOHDTa_QwB?~ry5{EN1#K(s-;tvrPALARYgou!JbubSfkQERw{*damXpqgRIkVKPEiLFy8 zp0X!lW^(&BcATO`8pm2G)A}5!0_Dyeumc+qWVu8y`*)8?G_r=oh)XL{$T~qpXQya` zSg=4>SJG5s4H0(!fbJr_zrinDM86*6 zu8DDOZ#Y9Gj<(BrUdQ?^AoF%Y)eOPa6JV-Y*cL;?XVeFMvaJ}cd19D zRIjKZ9U#7=Mc9mrH7k}}m!}#0=?3>SJVJgIEyO^$0DzsS1`oA^@J@L!MuQsz^#ip5 zzLdFKRP%lo$}tE){^kq711us82z=Oxf!)TC4S?c6OYR@Z$^H?4=s1WL!vKm}Awk4l zkC%=O3y}p$%agyt4uG8AWs;kj6nkN@O^duLBN~8Q1XZV_7ln^R@t30p{R2?07KtrTj zkvGmlNrQ@!K)NN$A&MsSc23qH?2~bTSHMvZ!U6CI`-p{(L`+2qSdL8;n%N)@=zcwW7mV6k)8kP_nD2CiZaTR1rBI4A8rc{NF=IfT2;^Re}&J996|R z5muxM57*|9L1br()Sy_SeKI7=1`~+DE!C`IU;cg&Veb1bVIRVRasR#@JL2`m;JX8B zM&dzmLcLV1=!b<7BgJ5gg?PKU(7Nx>wk{HiPD|a9)&L8ii$zN)wWh4j6 z<_(ZSHG4VHNqRsI^o|yx$t3UuL~rC$Dsk7}d!~53RB2Rnq3jWz0{zByL?)!1B_c%- zOs`Gn7Tc$sM+bJ;D8UIV~>v z&s?aT9}9&iy@o;5A_5S8B`(U2%w_ZHT|-o03>_M>7GTmSO%06Am6B&?#UC$-+%RVw16Dw>AslbapV+ zlpX?z2qFZ&0gDz;oJrYOOc@~}BAjVCC?}f;{dbQxRqPTYf}3%EVS;K?87*NSH1E!y zmt<0)42B5&*#A_AP7wO0g-L8xll2wzr&Pfr6bFiAS!YApRYxu zmIW??2PClyP>e%IL8`M)(Mk}2Q&Bj(nQmwDuuPiOrxBliP zDN`F!bASmp7j8uL2dDrh(W6{+T-ZxWdcvYMB|Y$|kI8#WWLMM}%FDOq=EYxsu-`a`^X?Be&P_>gR_{9loH~-1CYcYo1A3~4) zVtOwwdwO5q-+EikvhNd?D)Qm}hsL-CS5q`;Zd86HAnXCm;kG7k6tv+$Joe1DfgM0@kwCzeM&O}GfH<1`_|_;aSqq@v5^L;2 zsmiCwQ%I3`wj1m=i0yodJd;8Tuzf(QsRdXSYkOB(T za@jwR_Csq)Oyp`Nk)a&}+`Ce3=}badMa9d+K&=%HZHbH=Qzq)IeUDBchuTA@W~Ll3 z(dz+ew%SZb?2BbaS121$-iOT~_CnjY;!<$2ll=YH7OXjLv62)BvYJHl@c$3eHbz{g zQ&X+0=B?|$Lxv)c~se?iU}fw>w6mrUWN|fc-#E$G@zwjWsCdhlq6M z=}{mq->WGHT+ms`__hp|@PIA5jD%8wL#ab&3p(QOYQ+Ohq&PIK* zoK}a1A;ruCRAWV1p@a@6ptQVdU&DZ4IpyN0ZJppf z)r1DNX?vVq1$@wZHdEX#pDNX;Qe5)%R=7y1V_obk|2T-g(*0w`DjSZ1g6SgyD96B- zqe%$$oy#v(Q9hANi>m+B^6yqXtHgJ7ORjlr47z8YD0998E10umRzm-7p%RyK5~T zn5=|oC<0*^gr$Jh5Zx$k7KP0MEX!-stMU}i=do7ust%Qwyz5DAMWQKEc41(P#6F1c z3XJ$@DT)UXIXvW-iw^nupBtxL4@09$o~$*n{ztDPst{!*c24TMri4$x2-2{s7CHqz zxM{V&rCAPIk71UDo|WCPuTV^-I4AtRXs0<#Wj~f1qR7f<5NEFU<@Y`8kG##EkmHN~ zFOf{kRu?T6MGc2R5Rxvb3BhkRh=ZnD9vd;ty0>Z_dPNNmxFRjoDPiKOG8ny0kefi1 z@{ou2i9qaY zMU*1r0%T9Dr+WRQozTI48A!>Y(hbJ(WlA@tYY>|vv@;W2C7W6`R}fY8X*?mw53Jd4 zy;hC)I`*f-=#5t5*f($*`=fM$&C8trDLOoVO)jVGaojP7F2EV_Ec8u~F8sv_jJA&5 z<%-fko9<-C8$n^m6DI`zyq?^oF*|WO7siTWdGGGsdw1_-l-hp+K8736ppetqNi567 zN^vK*cEGqK$BErMHv5UHH`=1Fq!@q;2|?TYWGwu50qJ(a{4XN5y;F%1dSjQiJWyB( zGZAUR)Byhx;_ObE`rlSgu|mqu03w}|G~|X5jTe+JUQg~R{v75nyH5=)W7&g3S~c+* zJ%37A{z&`6L<-}8vs5u+r;?g)xPe6Tr5Z1!JE0gw-!Vm)UZfk@5EHMMoJllBFyq-U z0V87(y+9{i1=+VUsr#Y^Q$vKA^IAP;z8~_adTe3vs#N98#SL!qO zVExpYdtJ8Qe+g@(h500BT7pX^aqLl}>=+_#68&6h8glMRfQn_> zR+u2-I!L>8qJ{!=UO^FpD@!jBfa{RD8V)0gTPy`bQC+DzUn|paNyZWw2|DfK(nxH3 z1^+{Z1T`Fv+6qQO)7-8jub?UcLsj9~8MRxb*kWjeSrq##gk*=phM0V`{)OEBlz%c5 zIOLU9v2Z?7qs|Ggh&K1)>J18*6p90j88v0!0R{4SVR4?pPA2YxR~I=LyHvNTO(hlC|6~^(?67tsyM)>|PVI86 zdelB8cLV_`FjK$~4d;iS2zq!nDI_>S8d@-CjMI8M9bWsVu5UO+-&1r>(;1+1hE6Y? zvvkhUIYQ?Goe4UZ=sZK`9dt}OFVOh_ou}#i0-XY#BArP(7M*E2+vrr_1eq%?E(yXt zkVVUNYK*Saxk|^O3kcVZ>RH}biRwu z$LV~6&IjS3J-81f*fx6l^toroP83dzO^lv84bKE_#T_f0Id|gX>9KL=WhBLotDj=F zk;vhJ7&UU?sj^=|jQjU#+^X4@;@yn+W#4G=XBm&!uVymJCn?(B%AZMQnd2+jPRaRZ zHmytkYDP-S;NC~R&kQnN{BLBWM9G=XemT=8ITbu#$#gVp%XUX)3B)KzmDNL^hF?RM z{c2WVRbpNWb0WD{BMMamo*!gz(Hsd#}o*1OBgWs4cd+|5=t`&{0PXx3tN*4 zVF~*$6Ng56IF$;=Q$wk-O_wZiYJa>{EBG}Y#>YE^gE$lU6^vP=lXAb>#0s>y1=m~O zFo)#*s+ynT*K?Krj%4Z?XLbuDWp>yfbl)Pz#}UHr?aqNcFV zHu}rJm&Ff+BH`W#hnM=ih|7~-#uY#MQsmTP!fGoeK9HEk<;hm&#g4@kuaWuw`J9U8^LhRZY7Dy5n%<$X9@a)F38_ym*d+=O^=PEpV@$AJD7v{rV zgHRuy{dh7wfae-K2l4F3b1j|&c&@{95YP2^uElc$p6hxN@NI-|J)WEJ+<@n1JU4y} zI12pSdRDP7a`BXKE>EJtic1lmW0t`Zohx+a=#=RQ=R$VFIZvkwXClUI2ut!j-#kI* zBpvb@=34NJh1)Lfa$aGAGjv|1!v%bJle3oo?R5UH0c`R?cLGf~h-H8P-#;0`6GQp| z?tRCln@WyU%H5MhHg-}6DbB4d=~6nyHIg*_D_DbG&$g$&jP>P9SV8}Hwk!3ISlj*~ z+mrfYc2(*Npml!_9^muY)v3SBu1Wo`Y=7!+vjeIBnH^01O?GYSue0k?f7P)*^;&jA z>Mye!Q-6`&l=*XiTZ{~>0Mcz}&AA|Gy#XHH1JC=)t`MP#%o!_rGXtbZorT1Dn#~qA zmOPmAAvmxH_tU?^E9Z~+f}#X zd-W#ejzp;+hL0trorjT{Q#=(*jqA1N+T4>*@>Y9H_VA6zDc@nW-%QGl`kk*NmQstj zW6XIc;=7&ayq=d=@!ovg2M1zM==M0(`)(lYt(Kf!sILs)nkvVHQ=ZeEQ++MU;S^tw zTlqJl($>{%YqM-=^$v#JKSMo#CTZ3zw!M;)RJ*PIxs3Bgyx-$~QA*h( zDF#@IH*zIsJp?j5xJcr=c9A5PJ(9e#F+>2WWPOx=RJ5nj3?W02+w=*e1xQQi9}uCe<|q{ z7rWR)@Qnj^c4`dUA#X5FY$g48w~Oz*!^nF?<=>FFky`4O{0XGFy4Vd&=(=)bsoOh> z6p!Nf7)K7zJ&VMEQT%ow<`~Be{^R)V#_xnTcID(t$=XL3Gglr*_z7$HX73W0nq4UO z7rj&PJ&E5__?^b@41Q&902)}#*bAnhuj1+(Fjl(yA-$g5TGbJVWd6y7=#>!(3t)59N zart|P?#(yofB9K!?<+~fufew}`j`3_`#p0$`BKX1Lx1jD?8lcT_IoB~B(8jZKgQtd zo8EKC_q=sram{?3b&x*mJ{`VChws0cUK(g(I!ZjW*mLCt{oM0X$~}Y{9*_}x z=VEFxjW1U8;CmFS7P}UE7gsM1EcRIs-s}ebeE6)H;kQf7p7@my{tL&Td>O4;$9Ppl zc=|xuIH{Lv(d{Y4nKoV;JLBc-6g@YsRM6Ou7`SKIn1QKgS@EQwI*q8d-hS!v?I+FR zzH5yPATzT_;$kQfBCq&>6SJF?oVQ?)y^d`i)w3@YA`P&p-l%$U+zUx z90oQ*7)OnR2cC+Z2WCz9W%3H9Gh$%4d{|WVm(g+!~B9Iq)SJAg) zCzsyfv1?qr%ZT?4tiB|C{?+*ik4R*poJ}J)fNS= zh5Z}UPU8egId*vh&tsxvVYq;@RaB~S3QTPi6xg}Bg*l5VFA|ATF|RiAy)Y)5opCit z{5-9kgFOH|Rn38F8>3GBB~Bj3kX!A_!v>!f_6cIl_Ya!`%DULXY?|wIBsZh;ID(Z79+K2g1i67ItcilT9bzDONsZk25g>cZ>zy}`7OU;{{`s54h< zG{Nn~>cE-*^mucF7L}>$#sf6Jb%HA-=U)A<*$FuBCh5kpFVWDmDR;g(n_3Gd$&wls=Ns5} zH|S9LY4*Vq!rQCMUa>%_7Pst*LVdVhsaQqXy2Rw7Y!yB0dl`HmoFL(RHy*f{IMTX; zjD8be?}Ov_ovH%+V2lAcuzeW*Uh2^Aa9MKRxD-pPjDbp`+Q9X zNNS7^dK8IQ2&^>zCWSEbt%YVmhqMk*p6f?3UrN0eL6BlWV&ra5ML~mkHYEF+4o^jiJjyn zX1OEfYNb~znGf*Ers8#FK-tKmCh1so#5(a`=#!btNgHJF1vuB-a0y@WC|n#LD_j_T zwlIGF%!$G?=Po=ocEN0?<^q>iI6uIA6hz;~WEKmC6EHDl?&g$s`WTd1{k`-ZqVrpH z*3pp}xn8gE;SKq?8=V`-EN2ap3=b;R#Gf(wf2H%^==^s&9Dj2);pKA6KIKVBGYis% z)6`}^-?Sot*(Y%v0l{$F`EwTZ0Xlz5=YP=o3p#&A=P%)y>q7gix|yA{kjB^`S#e%t zqGkGkK|Rh|f2WYCoQ6i=GfCW4jt^vYWxA4GqOOH*;1)iQwJh8tigLc`$w1ALQs$4$ zt!%mrdf?TOPJx;QzJvRZsUAN0-_$vzpisuUFJ^l?l3l5;_AXHp^K)GC!LC~wQQk<7 zTV2^K(uhJioq1z5LOLFw@Ji{FTFpZhPfWd&>PCu4xszS4+<{^zwH7JU#l&psS##Mxp^`XkwP5tFn2A7fGdB%0n5n3z!g&1>v#=VQqBAq2VO`fBR!S})m+F-!N zk<|Awf)ZPknNbqW*E3G2!3XL45S?$JBb4;R^a(BdX8Km5+rn%L+xcxw^6hk%S$w~n zai5?=>gNbqe3`yi=(N$v(0P@P5aD+Eeh5y`Bi9p5Y3B59#{V|syTzcOqaR^RCpXrJ z?CSg>ldYvAi)=rALaIq$oxh{gBw^yqF>VYF(Zc;bI9-Vhh{hKAvLB}pzwoF2>Ekce z)^RJF?77v0bj`QTG%D9>yleiew;M96lN+M)R9p|fD!3_Dj;clK2LbF!t%lqB&sz0A zQ=r5R2;+-{9fQfld&2)zvOn3|-n*^8L)tfmrnUSLxgNvMJq+j0-y^i!J5yGdm9aal zHoMKr+8xvKDTo*GaZt0<>V$fvJLntb0q7=q@OQ!@7rRJX%md;1XmzAY>~{H+!C*|- zzz2q~Cm(n7h|xdjQ&t69@(SE{q&{hp0=VHH{Y0bV__;%H7L#s!Xwh4uqB2D*A8mOz z(^N?U+P1U2Ln&la!F=H~xvurcP=delgcg#?mr)Q6U+0}4$9pi>L>ECu=T{k#Qd<<> zWNZ_=1hI&*&jQ8WUU)M2ZihBqzL`n;TbA`%i~SPcI6gVt<$RuH{ym*9(D@>ruhRKP zI{!fD%XFgY5d)x!-ftku1fd{{qtF|RYpO$5j#P5>Er%&?Ly$BgAH&aG2PZ@jd`TAF z`654BNpn-1^P6~ic8Y@#{?X5AErW6zp2ajjY%1TU?emh*w23DE1_ngFv8&#u@y*?B z(c4V;)=R?-2fe!5QD2G zz1)%LP01jvIDRq)JxIn$;iYcC(2Wnv;w!$bcr5tSY(`X7FaQn}lK=)jPC|%c5RI;~ zYRLwKX%LVs zN+55Oz2)#|-SAO|i-Pm}bbg%AVk7_0!{~TgdFs~WJkIubt2i*rWaqz i2l#b>sqRSic5ln3wy%yYqG~Zsb@V6u`#M(jWd0A0D;5s` literal 0 HcmV?d00001 diff --git a/vllm/engine/__pycache__/async_llm_engine.cpython-310.pyc b/vllm/engine/__pycache__/async_llm_engine.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecb32f47c91cbab6b6b42511f196b759dd94a9cc GIT binary patch literal 38104 zcmeHw36LDudEQJ<&$+X+2Nu}HJvc~WNn*g86hu-40T6;nE+Args3mGJ*zN^pF~{O{ zFF=fEB~qYc*)eU|Nqo}fMPfUlU3Tp95j(M!_>9lQSK>;S9V=Fnic-00i6TV;`M&>k z_ssMxC@M!emC|nQ+dkg$zyDqT-R;rQLIOW`{`hClRsSTB_&jg=e{#5a1b^9+BF{E94mBUO9+m4&jl<0&))Bd$ZX9hsWL2 zer1`N<}WWhe){PaUirE9MMpK-H5(v|KIJSr%4sb)%rR^DlaD&J<|(J97S5e=F0Am4 zzhgl;HP5NKZRJ(Z;JsTt?_91fuCx}|)@;Tv9dQ=gHrhYjZY|c&`o*KIvj8@@D?HJD z>QHb!@?_g~8o^EJWKE%q(f!z|g>#O*(r}c{0076MM<-UiP-ijLQnEfJn^K?<4?sZIpv`{sHnt?C)#JvI_lAy%O?{@UvL&yfP86!4_^Ze z$4)*y@EjR8)mwFrUA^N}Rqb+hp|Rp(NYHNKq-r;pJ;8W!zJ1qs=pp%@GNz51wPZrL~|a>N?qOH;|1Z(Ih6PG_Z&4+A{hR0%!Q+y@{do0(%6b zzw8E$B}c|MESmI-nB0O0AdB!9pLT0!IXOJ%1#i~$w=J|W2^PHSA~08#k*uDLR>8RK zEGuV$6Y%_{8iv~+QMHzhoGU<;pTFeP&R5UWTlNCiRsT;CH;>@YEh2Fe+|jX{a}65) zRNovr;?(S;O10G-aG9%CTeYTBt@?#(wb|xu#C5S+y|7Yi=r^w=)Ptx|lD7^Y@S4j9 zmg~#=Jpku;`|oeAoNM0MymX*ZKXYLDvUjfCI&iVkXdZB6O&rku!-J}RP+7jL9zb=s z5>gXME@39+iEnx}_q<xoUJq zx{0;K)x>JbOWH;^_5Fz-Fs>v&l{6Ecv6NaeFYibu`0J(DGIr`}GMV@Y%AHE!efIME zlZjOvc%8|vxoUXnrJQYcO*{RHkxX=xUDShec{_6@*)=}}q^_EuFfS!9C1$gopP4^b zcgx}KP{wt;)wo=)W4K$G+U3QzDtqUg^68-eYNZvR0+YSHyl?h&nf%36+%JoI53#}@L0O6(c z!h$1PWkp`ib&k#hCJj=Vcs`3Aw`J?L+;i=fhFxws*g4CKSj6tR@})Y4cEua4tK8Xh zREC+YoK>|2XK|&0&b#MUJZ#@*%Xa%xtJ2vXWQw=5SVLd9FXsSWwcc86cV3zYE`m0J zyp@*o!ZOy1V>d2W%7W{CmRknuW}nVgs|?qE4XWwj{)kHGnMiw$0@rW zx>@!JWw~|+r$RI|wFM8ey}VFcah=L+fm6<1X?Q%LTnA@XyCv(z&&p)3xBWtdPS-CT zuHn?ciq?zD&*=TmH`yOQe>6nApO%R&Q++nA9>%OyA3);gqssiEp6k_%wZ@9$C!OBH zQx7rg6!*s}r%QDq+5;}3<=%p~qN=vt*c zX%+9s?FrO6zL^yyiA{7=LaV3AUa@HPJPYCOu;lyACDV^IpJ4f8u2I z`25kQ4$U7wF*h6ZfuO|ukkMoEF}#v9J7W^jD z*p*WW^$}doCI!aVk{W+8)aAmd;2^8Zp~@J8BO7KIXpk8>84lyZCCo1wovnFkb}e}| zaU7%h!Ne-2rXFFnxtsJ-=M6Retc-IPCvi9V3}C*H!ij5M1}ny22;yi&lYR$oJao=WUNM$Sa%EgG z+9fZgpM@i3`h~h%Z{chr>oH0aymw-l9{Ct#a97SOG-@sgR>eV!^+|9;Do7SFp0fG6h>m*og{A{y!dAqV9@yR>$rTSfbppwg_>;4Z zRAM=m%NWVbbt750Ze-1ClZJ84NEO~O^5aP(HE9&C8DmpPDYt-f13!7(J%T^Cgkc&u z+jiTa^Khn^a+)QqG)}t=PL`aV$J1;+1;)7G@8H z(@jpQ31=+GGih&lIb)3nc}ARqJ>^W;;rkrU#Z7oW+4o-FIU7*_v^|4&Q+5h(H@|A2 zb(YSfoe5{7GZCGlz1p_eTTx;Ypr3YToXwjP&KAN;n9v&v1x#zJy~E!5a>CjM5x_3| z-)^UHi0t)C!QnU$;^@4iI+askFN0GJv=Zh6NFP%8C6lioxZf)0$<0vN*@c7SU2n~~vl)`XLG>rl(Rd^bikTjyT0 zl)M7QMx)a3Qw^u3K{$=~aS*WV@xEI9R=&H4q_b(LZOP!LK&Z-2J3zE>aYtwBBe>~; zKUztuQ6d4P3wSYdQRK8dB1XuVE6F2?=ku%B43P)ttsg?9+jCBP82`wQD7v<@l8m1 z+*wnfnU|4W`f8sTyDVR&b>ADnDC9DlY>5$Z*nWMd3{pLslY|yrI|;*k-{@X9b@uABp_{t z0DOrLUSuNYJWKl-y)DE?W)o0M5W^7;F!?HqxHCvH8JzGS7mG$NRlr%FOXhHwyWY8J zc&OH?N0XE(RKy>S2uC9tk=?lIrc};LtU=Za98-LN5upenSO(m1&CV^rU%tkz^C%dU1 zhE$s5AD(0G1#|H(FTYl>IZuhhG_aQGJblQv`{o{Y1Mp)mHg#?cl(7Wz3>;LvgY3ZT zlWjy>g@{cw8;3Kk(#ZuwU+#>TYYiMW_GQRRAZNzO_DS?mW?g3jjmKv5h~7AIk2{w| zK%lsJ%h(7Jq7-k*BPb6E=qk{$(HMr4TZZpNytg)M@T)aDI@kgkpsw>}))X7wk-p5o zg!k?iB!%pFvLs}nnKZ5&M($c6Z5l;P{LYrxH18`8b}O1i(U7x89PVg}lfGVWifgV( z2GQj9VjI|iC_^)F7VPSpOBtC67-rkRhJrsVgZmEg9Pb-adqt75h>e^UboGtc1}Qf< z8gsHtX0tsEs|$ctGta6D6CFE|FE3o+3&W$zMvIU#+RuyE?fOc|^ z$rbW_4EV)j0%lPHWiBSv16^}1>*bd6vYd>y0yq9-qE+acC^gbeJ&5JHR_vyO<(4K5 zrgm_9TI7f%J5L;Qu$pA?Tncw?3J@X3jP2(x7u^{rT2;gAurfQUK8?=GI6(XbHNj=S zK&ijx@BI{JpI#&~60-RpVF6AuKWo?Bg_^SUBFl)R&kk0NU+kystceh~5c{q^DLA73 zNMF|Y^4TWPfO13+hlt95hteX3F;k!kv7YdItzeeGAm%U+iu&0bxslFov6=90wGK=S zo*-OezsjaaCci`m8bWNBRscP=%yxpzzxSBZtIzk#oG1?Cg#8Y~5n(c1h+sRT1b2j>D^oVnh%ddeHwmUakk6 z`WtwGX{&`3#bjq^42XE2v2#6wB6AukJ7SEjBAujJ*%e&*DTwA3*=5bjj_~R{lTR>- zal1`E`$i_h=>9cckug;y)-~VDvsjHXA(N@HOxSProlLmer9*vFPxf%(aoc#yBgW-X zK_0hY9sG%72|AIBESmVi#Xc+oR5?U2?!Tqa@_M!ZVM9tDOXI zJ%Rrl@P88jr@&E<@-6N+kn6?&DYj|n7AH0D3_0;8rDwRjuj? zFVmd-buL`co06TGSPvq`+Am$KD=K3G6`8E-h!{aF2`)4)S=lk4!QROo6HH`sGY?Y( zd=QSpC6XRc`U$NGDiiNA-3+Ntp)M0^Sssk;97^W+b}f(NG_{(N12xsnR6(gCHM(i{ zA+LavBUeD%o;TG)C{gTYK+6L&xE@_9sW}|MWAYaDtwC&z!`Q?T3w0v+u+&P@o&Qh* z!cVABaBb=9wF&I;-xIYziDN&D(i_N^t>(Nb*0Hn^f1B_(P0D#`41Y7-Y!|A@SMsa* zZd$#J@({CbPIU9tEx6vgwyle!A16i%qF9m)gzvji?{?HsASK-z8|cpuBTv*H5G7%m zpXsK+pl4P`x+7~lh_eenj#>n3pGms8pyprmc3zR~y_*DKk#c|2PPdT921HBcrCoOB zys?Vfmv(nSh!7`swu|lHl?*UbP_Ls0uLGI_aEqMo7e17*^W+=Q(w=UCwFnmWUQHo? z8tno@J!p~ZCe$gt!u8EvQ%D_*-V(S4Jh|m+0%i8~_xCeN5IcQ!^u05fSH?!l&lYNL zVZxqt6wb&db-GSDG*SpuKwNGljI_?q1<`s58lXT#EVL<=h^|(+mhnT4>!r4WfDIHC zG_%m_$Mr`LU@W&?w+>U6Kq0IPLX;6{n6U!i6((!}Jzl6%KJI~ZQ?Az|)B>v$5O4s< zrNjZU=YX68<3o!OfM4%pq9W2;I*ejbWgaa)%4Ztwh4Wzz?j@3Htl$8o0|yq`%c8n< zTDJpnVg0&s7#ui_s4gONW!Z+*Cjc!pj}hgwKr00Tp@Ul~L&gBH&t<7tM#t$1ga$j; zKTv%0RR>F9^g$5&sRi$ax!Jrw4MbI`Bwwy#C#?a>TD3u2qzT>ms@JZv=g^(`ld(rY zgJ0@BBSzE@qI!QrYo$pmfb+F;;5w^rqwTr=Bvkz^a6wviS9M`V;%^)xa3SDw{Y}_u zy{aqrF*YE;Sn&B{Um)=FuvE%w`HZWsvbhO2#AqNEas3>0S2Y?FSw&pw>O5!JpTeC? zn=0&lmV@k_d*tBZ6!iir0aT@0Lm!wu)XLCioP~u#AQ%G6JG;-CS*Wq)>QMUvm-;aR zH;s{OgAgY$L3?5bWKH%;Q3eX8-u1`SfS}A(e+yMuQ&m!i%4tHLN(2&g*UCrbtz2|@ ziTR5fNR4P~Q)qQWI~b=Cp=dlRL|HBDy{u6nr-`*)mlo6yv#wj9enlULyC_6TfgoJ4}9v$&Vnh#xxp3BO2E#4L&%5!3C^p9%BEkLKW5#o)D zRCaN}kk6NF)>MdOv=S~f*Du;_v3OvmE~rWvyg~VZq4lRZ(kmDZ8mIIr0>E;TERp49 zm4a#4zX)Wqsh?n;H70+T$=^d_O$?#{Gfe#`pXVch)Q>ZBOafDo1tJI=!LoCI1$m4Y z3nl7ug>6g~(#3QJ^n3<1_+$!tMX-`&DM8O)E1J9~ZEswsuqKxSMGtMHDZlzZ-lZ~M zG)I{$b3J2H|CLJ@lg5iB3U!jXR1T~uB${9Yz#*`fObKZQ^@P8fujI0F4SoTz80Iw- zrO?W?3?L|`3Mu2oj9DYKwyJxVMLgkW>Mcn5T|WbpFKW1c9u28qVDb-{{39m6h$JF_$y<`i zBt*pb>l7@#1Vg_-*&}oFwS1p+i!wp=Std8CJU*W%`b2^nzgK%`G)FtT;%FSokN!ya z7YJ(j(+{@#$9O8t`Y^y;K3@~S>X%XCC;LDwzIAJ04hC~Bp&D+{@Z3VvIv6{`fCt10 z=5x)iyarPvbO6S!v?9jQ?$THj`XVAO>HzUgmcD>Lndkoj7ntXHQK)3Qd9ogIJx1~u zWd4MZl3L+X2FVFQl2|3CAs$a@@i+jv!%IT(-Kt@CU>Ufa-A%{r* zPzTroJO4^5C`&Ee7;0o55b0ns0E1n41st2K+>y>rhj=QNp{5F%%!6bn51#HD`_q-V z&cv~{heh0580F69@)=kS((;%3xE9_!EgH($yM)4$`Tf=?%Y|#B{MJ4xb23;f<=|+* zwjZma0wz=>hq7k4TwvWaCgc#0f4!>wywF87#`QPZ&SDJ*cW8D@gG5+q&gS%b91n;Z zoF{r^#wVhzNLia)mb2s5h;&wy1%BZfn(b*#z)#{m)&Y!-7hnoOGi_^9>I9Ji1K+^9 z*+PrqT|kD7g}H((Y7sT5J6PRV%vjpL{me?eVTYXw*%O|a12)vUW`C4~M1%`#EXp5H zHPf-`s{wXE7;%_;xCv;l*<#;5^B$HhSCj?KY1rW&mVcLB96VNKRezSN`Z2tOIC9d& zxm1F#Zi@Okh~Oa`3r@G<+l36pvyg2;Fw0Z2__hg4k-^`>mj@&tT+#;qWI2~{p^q(< z$6^q1rSSL08ji7k8AwX~HuCyKka`Q}x!*Uub8=d(cL@2D{n_*|yv;so5i1Q1AY$De z-%y7OsqaRkuchbkBdg}MEH8tb*)2B~czzGcX|W_FJbhL?Kg(O5k!qZY$Upb;iZe~^ zXF@59pI(IPgr{!d1MYr`R&z2q>Rw*`F%lSGXqgYqU_-f&4CfCp%R@}Q7fG63=UGHS zXVD;;W5CLya4qr4!UoVTnn1y@Wxf;&H46||W-`WGnXK_VGj zh=~T-d_zh=>{);)G!v96p;QK?3@KIok}=9D0nuvIl1z~|jFHS2Ssnz_JEa?x&x{yf zG>XN)oIEIfQNP7raW1IeK{8+wk;Vg39Wu4f;X+R>6Pg{_&9PV07Ge!5VXbDzKAl>g z!mggGrl2OJJp@#y!R}18{sHd$Qg-$%L>tvmg(@PC(0Y*?wc0(v@!>WOH?^|hx!8-j z6<})(b`ux+P#3&;pdpQ|_}hgQ54Mjp^_qLT@cY7;VYNp5B?Jmi!*(emh;6o&@(~A~ z837v(BQKo5L1uE`*he&Lk1a=o2v~+;)Y{yBVEQ^B)m6$*!T|@DZUlio+zHhF<%ZaJ z_hd6t71@4pJuGF%OX2Lg_fc-9JY+Ohdg7iqfR*w|q1O(=q+!;uMguGcL>%V?w2;4# zf1--?)3$SF<*c6#wo|naZ>)&`%e{`NZ)d@DY+^}0em=?;-zNV6g$(RtqNf5wkc0Um zHb(P$F$E$&S-ft-%psF3g848F#oP<{8~CBd_!0cMKY>Xbiu<3&(Ql<3)1rvqgfUsx zHXa1&f?xAh1A_mYoxvYS5E!e`)H`G4Z3y^bR90|CAQms$S(K!?w@CvxG14yKO%8_S zV@u;GXHr0poRdxwf^XJFW41}$Z-DTBV<7s^+KI)Yn0bc+d6=5RppC|JG!&n4rqL3g z&}5EJHb+k&xR)ne;LAF@ z05N^!($;RfCH`@ywFg)9jWeyi_N;T0Kw{dr+PC4!%>i86H|`d@;_P###2D{@eFxsp zI=9+)I=2C;yX?CKs@t9YfvO~D-(%kkWyyWclzqQbu^(^_ICnUAin8Q=faoqj6hd|o z_jdzhQvt@_kNbP@{y~A~Ue*X7!2{0y&I3{&xB?U~)Q1}7A^YK%;nZTi&)FX-IPS6A z((VWTFSh##+U0n@&pBw{Zy&NBjVV~}v6lh&;a8IZB$G}BJ$e8=i|~N^uLTqLqb%#l zu7isy`v~Bn!sUGd7Vfbx*zEsN=MI73z2W*k=YGsL!*h<>|kaOp|wynqaKIeg;&654N{RGwt zIn?_|-;)#eNjy1(_8-N}`>_2K?hfPb2<}eV^SC>TyT@?%wEYb3j^XZuxO>)q4tK|K z_aWSU#I|tvIN`Nj`=h|^6X@H=@c%*QNm1G9o#A==auUs9Ao!IcTp%7dbvI(RxNaXJvRi*u2@B$ol!5zbI;DJneVaJjP1j7-hidHRLO zDIO+Pkz|>$6M}xd&Q-lgW7pB1@?#*z$$zX~Elv7XjrB z$@e@CvvoiMj?KGWeu15YUxpq$>U2c=GGIdCF|f)e*&`Pp+|FfnE* zhZiygQSYN7kPE`%CNP_#@aSOq_@X$+(RI%0M=_nISQ&)t@=6zqY9aua>M$XNMOh6V zu}zw(Q@233F`62a*kP3+Dx@bp-OjC4gB~oSM=MPjq-iIjVKnp=xK*o6-)k8V*+D;L z+RekM6C$V{#vt5}BSZaboa*Y&necq?iS5Yughc8IT)WR9fdGKMM%4bMp#3$e{lya+ zD2e-INOr1i9<55TTk=gcXOke!OIi?5r4hLt`JA&W=B4-cS&y83p znnmiD{`-6RI9Lne^aw@skV|VPQFTA-oxVXfE2MvKSl%Baai(1$MX=~^74En1353n= zIZk=U7L;;4=%@<1n68nbNm&o&-w%+~t4v~??X!4xH%Yb`Tx4J@oXePy)#ePyc2n91 zD3|I?i^wI~r6LXa029lBB&Ts0f9`H1H2$_B54-{&o7JR(R7V>TX%?1pLijNCv?r8W zLLCXINfyF^ng_oRS}KawR^UYA8_J9IKs-gDAgP{YFDQbfxgmU(HVwf{JxbLd4QH;X zPa>a+DGrqX4ZPdbK>JHxyxRFBHH*E|Kc21g zz#Q<_gRE~X`Z^$6E7^P$2Pb>6z5vf=jk zhh~BnT~Yv|>P*T={5&H?h>X-4)o_Jc)sf(qPH)zB#t^84bE~3fNYsM*PpHb;F_?#O zDKJZVnf@~~Z5hlokm0|qlL1qvSHpi}hMj{ogj1)N>%Xs44Tip#;eRm0I?xVedYzf} z4z?v~V@`$YK-MoJt6zL-rA4{2#^pM-pX~?XOYd6yK-M={dkk#@8F`q;4~3G>L^_A2h$%34n47t8(pDVNqoJa z^44yUl+z;=i@U%RH2k4tm~vJ*(JfqH@23U`W@-}Kg8s=k1EJ@kL7SmA!lBeyg4hE) zG{iGB+oLz#4<(_)NiJoIayBB>=2yn68IpA{HOo?3*tzd8yd7w66IvqCcrf&$i20&` zR|sfZXlOI)8ekKe&dQSv6%DPc;!0l5Vaw(f?GMh5F_lmIlF@&=XQ<-X0tR>*ttdkQDRBB?L>~ zKz$o0R~b2TBhaZsz6{|RHTBe*LyV?13WYb#d}n8D0>ovx{Sv^Q+7*?(% z+he!_y%t{49=~TNol>7iUiEL0_@)gLGMfG$9$-Ps%i zw3qwFAnrvK1aWyH0*_ZDX_9TZVl0Bt!_f4z0V&*rS5er1LvX3shl$uvhv1G2xVamE z8|1z*xa={(C3pc)(BS?%;_5V$L4+w#Y>8iB)A$3XhRD{SGdFIJi_`<1O(#px({H`@ zXv95eRz4sff>D7OV3DS8mufEAPnssdVy)#eu!-=Lfvps{Q`oaW>;N_s6MNyj7!{Cs z;uC=iR}zT=%SPwx@hyQ6p^c)jve+l1epG_9L;-(hK>|qpk@e8x7i`*~Ni)Ah>~LFx zRs?(C&>ZUJRF{!!fTAZ-`6qK-Mn)LRo{a(s+XO83pFtie9UC5hgvL<882mbF(oU&c zV$--U(@^uNuz8BMv0-8p3>;pdbkWxlI8Y&j;!X7V8wX60b_W2Z9G^e1zZU2s?(;W^ zEiy3$_)K(m!~l(D7WD^woJ5U_)t{R zk)Btx)B!W6jZ0$7;SW*W@3IS=*3h&f3I+Tjx5c_K(A?0{pkay(cmo$)iYiZUKrJKq zl=uOvaSzU5WJkcvAx||!Pt&Wgy~YR?yb%~DB(7qdu5!B2`(<>ksRUQiRZ%l4IKf)HH?+>0rYKd zc1rdSy^Lgw0fU%OU%@-^rcewpAinBWNoWE(ookuGNN^5P6a^l9FbNu#qjHVxLTZfH87dOaP&gG8)Q#3MXVsOvp}Dd-#SqBZcXiAWrG z3IU`fc+|SXszYU!@gqXpKp@$29BLQEDOALaa0k)ugmi;ytsqnu-N&1V+Y_||RO@(v zQt&hb65JPUXhmRjgb*-T{y3!NGQgS$ zfy;$w4<8| zBU!rM*&iGKfb6ARXXQZ{@Yfpd!YhRI*>Z3+SdQ0Q{p{~(`!28lH;c_BUu5v0C@@1A7x_mRNb61F6vd2hU%A#X+4 z|2rr89VmjME0ejNgWh`^%xW?q*g?A_CmHbUnwc9CdB`;&M6QX#av~2{HmxejHPu)0i$lB#$N1Mb~I=phd^3aRHJ`>}{#JdGRBM!@gki z8&QQ8hu3fwy79BuYHJe7OS~Ff=Yq;<4rvK5O`1RP_j(a#Go71b0~<`>5VfJS9DyVQ zqj|Up40Mq54|VJHJ&B8OeaC4fLr;QlK$eiA*iMLJz#^h=Vi$oW=8P9X6ve5DP?sUI z0O9n6P6fb=18akQB?Wl}f9?Z-l5oLk8f(MN@z~J9Vt9K2^=H8rPr<|{p8!jY_yg{9 z9vsnaF_B?D`hG<_2L#3l;h6MtYk3>a(9v_$F6{RGQoFfU=qnSpe3b8`ZtO7-tfx`S z2;Hh#(^W(?SWVl2z)nAy=v?(iz0#HBJweOgcN=J!9E>f~YvWgWHDPB%Lm%v22sjGcEyA3b>C!c;OPlX(dfa{jUV(oABuxxihHW{mx5(h`%4fpEu=g^d=d+Qg*!@ z_rK@w{pQ4{6Kk7bOqv&WPKG(k0iNlr3GR+|8oqJ`!Pks7C++6l&vr-D?}feOQ_x@O zZs9=)95C_m|H4(&u$gaw5129OXWqsxYu<7-F@u`X_6)qiI1*vaBkGN))?!d=F{l;) zFZ}tliPmPE8o8ydpGjVx#K{3ty)y#bZ?mN>V10WR)L;_Pd$1@+KQYR)VhCp&W=A^i zvksNIalp|iNv3tuI3uk7*;ycHh>G@o!((SC2Y9k+E0%~U0hP=DDo4)vvHYw8Sy zJmny~t7BK@9-HrMeKt_1Quz$8quz;7sfff|8_o8=*)^NAuvQuMv`Ss=z?$GX<+9qq zHL(o|4Z@x;VD@lRGCrMvG60*?cNiD2_4F1dsEKK;Nc&b>!9LVG@Alv2XvY{h$*2%fV$^wuI0piPJ` z>mbxF6n~+-S-hZ9wbo@F-r-*n{GVeIk=t-RX+U|d9j?c@D{Yg6-P>|>?%46UqrI?v z)%oL39zF5&eD#$2o6L}bT|dGSTVszMf9lkH_3()&Pd;&U{^$`u4JSPa=`OLUPcx^< zo~lQAnMN0zu664TTv5dFC+5z|olLE=aB*ITZazvSqWTb@lMkIODTJvF4%74ScBR{$yFmO;oQ)i%)35Jlo_ zF})P|czER^Diqum?#6qQP6$-Dx`-BqC_Z#pNCyrJ@TgwO)0SX~)&LMYFFyj_e&i}? z{ET=nXW+dIuiv#IC5W@xIR$NILJt~8MEz4Hzlj7?!ee zNsU75Aks8CXN^OwOzMe&Q5$tm&5d8w<0@Dk(q;SEXBS{;+)n2tK$XTXCqsurG5 zeGbT9CF1}aPT2|0z#xFXm_UVQztLG5>{$?k28VFSXNk>GDDjMpAA}>vuFEiha_$BO zM^Sx|s-jmUs8|F6p}kQVKd{j0vpM1u?5FV6Iw&Sp`Z^>HP+LVpM~8+ypAhw&_6@0= zlyX#Oo{j)vef;5J--K}pRIdo`*8ob+0A_)mzT2QPAk}r?FbWjF1+2&RvvAC&T0<5- z?D;Uk<5o+C18fdf(xdft`IS~3$Sv1#5Wh>Qe&3yg+G{5qD z8~wZ&=tDoGa7Jj4y>~?$n>)d%@31q3#}ECRw8cPJSDz14-?I9Mn9%hs;8@K(=+8hO zaQZ3Dubq%!fTzptGfU7Iz_bh#Po`2d+XryKJ_;49OZT;RL{G-S2KQ4t(MJdm>Mm~A z_HUA*Cqxv=SbGZ)X8nW6^qkKU9=m?g0XFKbi*QNrNxO8uz;#ToCC&L&YU1_X_mzMc z;n~NU#}^xB0p<;3^$C#SP7+MpzVHideLp6VR-E)r>t5$?amyjn=e21RtV1=_VS1u3 zEJ7S1007tvYYxo`gTPp^de|re?o-A{<{`jCEYqQe+rC|Ir;sKsw_yg0yr%^@r({1o z9a$}E;j%pl@BT8`4;{9ZjE$ZWg9e*})zM8uOgo9qX5ZvF`htWR7@Ry0Km2fcPZ)h) zgsEJW46}N9xsBc3Jy2VQm%O7uWa7&Y1Z1F#x;$|a#IuK=^*n%aYcSykGn6JdQ{`j4%edF(C&iehsxhDl!NnTx8B%xA1v>F zmOLgV36|~$b`P!~;zDxklMcBbgGkP~9SBA|9hj9$CtK~-ew~3+VFBJe>o*D31On_D zm`DdT-BG#Y28{(%0%0SVZ8+0vy!!Y(z!Rl_K-W7NhO%J-GaO3qQzt-IFzbU-&c2~S zz(+72G>ShOqVoA+NRju{z$aSv_2J;5ST%jGc0WJ6Ze_6&eTaH~Xd(;&z6Xb7AWYEx z9?AjPCLE4~t^Og2R6CSMwnbUJVe3JD7$Xnib9iNPHt~Gso*1tE3iSW5pn%bBOTMeI zZ+881EW4+CNa8bcHHBM;mUP4Rz0SCsy_#YhZ8(`(jcz{}YmxZv3%gUdhfv89^0TL` zMeO`k1Hc7yMjkX;_aQ&^-@8QM1FRSef*>l&Hy+T!=z;DQ6D7ez*9yHp!&rbDIr zVS$Wy9j;{#kA)N24x-*4g9xI|;5WgX5NAyU%oV}8ut2|sY<>RDgDjAr(Y=YWBYrMi zT)#t}WCm4$uYNq_IQ0LSH^=jtH-=1(!;%YRg~Ok{xS&i{xRLw7l7JgYAqFwr7GbuV zMhZEmh`*t$Su$4fIMu+lU^2Py$CwlGP`-LzgaDKdB%Z)^LnA1N1)0a z;=~rcBFufwDM}Qx)D5kUdZUO7C7Q1h0>~5lj@xpW71fp?Gep}d=X}f^WpCr1|(%!T;4Y}qtbE4%LvqTIIC?db95rET49XOq(jTy5`e zU)nJZZW|GGcXSc$YiR`XzTfU<+~4kw0n!};>5p}1de;}e-P_rnLEJ5P=K+U1yE|c@ zZ+;`RZmXpWhf%8BEwL1GjYYZ2-Ex#`S9cfS8C~6t(c0DB4QOED&+?E0@2>8?@Vd7L zd?-TU5gX|DF7_Le5C9MXatp-X6_`QmQgLHunWgo`iZ03GB~vu<^nv zdB*h%&Dsmv)fAt;>f5G8LIBlQMob&+a!bL)yri+vgPzm?oS)DB*RL4$|(eVY?Gr$P6?Z(0C!p9z*O=2tmWJSJELg zvuOl8fR&>)i4nqQtUNFSG~wg$S~?g5gH%C$3i9^_bvN;BDz>5?#T3MA2wwY1ewH*VH~_SNGuxAAdrmB>A9_*o-IF{V8!?&}39y$u&z{K|-`py_Zm2yNLTIWfBZp}uB#5v^mXJ=FYo!z;C_w&>U`}Sbh>#fk&{y#J zFG|p_7tL`5NGL$wj&K>taisW23S^D4K(1#H5gcbf^v8(9fG|=e7`v8WFP$@HL`sPm z4A7LDZz6I6N@f1Xw%zh3VlBMVb5)g5eQd42v%zyv(d4yv&O5 zG8+jm$0Fh7xx!ZyUTPr;3C8ftko5g`8hs4yv3Hk#eqE?pj{Q)u^uH|B9GoG6Y)#La z|G!0=T739dFHBtjYjq zP^^f_xc@5t>M~2C@HkNxQ@iJ0<95)@dkDR7tunf zsDXItbEw51r)Nul^p5p>6Vy{^{=WfiF{MICWMjx_GG5IMfma_!T>$v55a7@-5PCy~ z(fbUrc-SKL^D&&&}j60WQVMdUnK}{o*zJ7*|UW=SGWx1|mgryQVx=w(N0S^wy`2*rLB^6qw!rG%{d= z6NL~fR?h*J_+i3KCg2Nf?@MN9YYe6RG1Ui9Nl3Vz2{a$V#Y+&GbFYBt9AE9kCnb~g zGG8*)gom%>icuikcGj|@-oYnFphRFkQte?Drr^`o-HLxno6v-Q!)jVc7Hs#n008SquI#o?(|Gj{Rgj@QwN-JT4Hq^h|(lAKk-6 zx;BF^w#LxhAH{V5&745tpv%72Kgj+h^s_XPLjU0;A(4{TorfPqHB-HDhJV+t7y!c+ z`4n%$nz`CS6l@VE56wm_M17o5s2(0?j6opZaOJ6CFh2Xoh_z$L5}pVJ+PUi5!zmi& ziBHkzQFxl56AM$xA@AZ|RYRvO7tCS&xfG*^-~l*HOwlRzBx~$=ip;9>&_2m+m7zMPMQHeI55

NSEjf;$z~?Cp@@ZmL>Gu6ej1#ZWH{1iT9%6`~uPKPY;RBtH zJlSDD>H5$BH}DNf5AeN~HoG1x*o%JAA^=zluWufygCEfhXy#dUNUtC0);gQdA@1ui7dE)&G}bf?x)-3K z%6>X;4K;;0zEEEWnGsS1y_lLH)B>;&Tlv%UuX(h74do89u@oIBLO*w|>nA{Brh@{3 zey<;X=FEFKZz)vyn=mVcJOpCj-~ z1lEbPZl6SG_&@lDq|z)UuML@_hNh3Gn!ZO(R5BX6KE(3$c}iYXTehhlqRK+=iYObx zq{}R-6=`9yjLWoZII8TVS!P)fHaYnS1IB6i`xNMH3NI2Dtq+yyON)Ct0qpRFRI^?lOY~N;8g!1fl_1n!Vt?;BI*6ZREB27yoJUze@ zY>=N!S{VMVfLogQNYKJM`8hPAn8cpv)G7+*ZwwNBXKcUIZrbj$DMhD>gD0Et0n@ucX z{tZ_uAJX8=`ONfRg0~-`2U4gK+tE@+wV~85lbNi9maY%2nm+Qo0N})5r82V#AHl9o z?&L!G7YLDM(qE*~UnZ~#XYHrO{}nXW5BAt1-4yt8ti}5xzlY(#9pm)Mt$D`DCX%&O zML24+y&*y(v8%Z^y{1z4^nNzZk)I|rd~xd;65BXg-f`%WEoMmEbL3~5;T9}JrrOWO zLc|9)z4|w(D?K6ZjUZ_e4k~nDl273SfRAAW)D(#j7vjTs;*i+z#|nkI z!eInNINr=H27+a;3w=e6QtdlAtD(|u&O-Z=(s{ZQ;6vCM@2BaK>0@O{c?b5wJ!C~T znN1%o%Cld^c5p~edtfLCcdXS5JZB0lbZt#X z7_QZ~{=(eWt7=Aq4%NpCZFtG`y)spterxLb zjf5<}NhnO4(aqP7EX%|lqzLg4?vd$|Dyt`#?bccr;Rj8F$F^pDp{0?<|2_)fYK6HPyI0>$+$#|3{Z5YdmToIow0D7F;O`-fuZneC33uOzvZKHE z@O@XQoHCS1TYgVfR)A4I9O*b4bOMfkgft+8kCQV%o|4rtnrGm$NHN!&6h!>gSV=`@ z10lqQ{*e}?SHbxr4dse=zlXQ4=)b5}Qpx}O>U|~3tY(Ep>KDO93Myu1V^zvkET?O+ zE%SnHrN?MZggZa}b$Vp7^CQt-{tbY4h|mTK+JjLDc)#k3pU#PUp23NS>SqhKU#;E^ zZQ?!KwdcVwHZfzuhvmqgcTt5q=qg1Zs|XrDASOmS1VRjZ){F;vFfQ0x z8>NsZV%b@Jr(r$Np60e8iRX7{{Gp953NBj_W^djNcd2yRJUXG%_=W;t0Zhzy8wV+( z-O$OO19PMFSUtsF9o3SLcUM7V8-6oKw)I3H(``sYUqG~la7GY9gRnD%0jm#@?gd1} zDFTCgy))CvjX@I7=Bw3a>^ijPs}dU@v~Rg>->odvkf)d2l1(H*=^V=tRN<~~ygoH< zcPyKeAt(W%2G#lN_k7_*ilK^oFRtSV>7LH`ouJ5{^*ajR5zwllX`(Q6Y_f-Xg$!KVI5EHwjN5+z} zf}qL@ZW(ZERa{9o(gLZ(D^|l2IxdpNqf{e}OC{{zN~WRXHX_5jIEZ!w?Qccsfp%ms zu4Eh8RSv%5#>2d;%d;$O(tM9$toAh^c*d9qL}tajMvNSS%$ zlTiLa1X&atZY`?T&yTkq`H7@qoRnlUSHNH9{24MUsuGt4d_!};?E56d$_o{6v6zrY zFzg&V;Z9m6Nx${s)@?0qe(Q$#?*kD=1L;Zmj|tG8Lt_$~aH3XWB#F(^t?v5GYrJ>n z?F$#L7Qc4!;R2oTAY4+cJAWrT-;Y|CK2Nzgs*6jY`3 zh|~97S-~k_2MkzKl)HM3(rqO>-pD{%Pg|zS`_%Ox5I9eOeRbmR!nS1A|A=aT7r;po zi9`G*1kn~A(Aja<^A>RDKo}z!{t(yIWMjO4dIgt{ zoPBZzhaYzI3heBLiTsY!uryG{?={p0?pP+3$`CNqIG{5S0|}ComWs)mErXm5xRWrC zk-M>KJjPFJS|Q;SUs~~wNHPbKDSU(yuEuzQ?dilhh?REcY{Fc>4MZmw)^4>I*h_|C zX0XIkY!Tr{Ai~o;x*93s9!k)*w45emRBgFv`R~!nCOa54Bl??M+Wgzc$e`fP^~qkawF@B{!@j9e$a zB}huJ9pT_3=?qyA#Ihumg!X;x!EdU+PFA8ZTmQ~fXd_7#)=njaSr{NO*YYpt-Hr z6JTpzH+iPKcqS%lxq^L!VMG-Cvw1HZNHfe5*HCsP3-Zi~(yTm^|VV4YtK$e9-9x_?-u$HlXKK!-jkAfzg z?Q~5;kqOCJ=1sq({?7QO^=w8_^6|RywCT9K&F~LRp` zSVm$+tXZUF5Z^GrkpGQdOCj2VD?g_4-zM;H0T6JxRgxGgxi5+PN>$(YSh|PFjGJHfxZk}uNG;{*;7C=lvCM(vzq^G90k zLzd(Ly-YnO2rwHSrqXK!-XK6?S)L(qp1>~>xJ2Mh0v{4!MG{fz5rJPNaGbyo2>cF# z6#`_)%ikyPPYC=0fqzEe_XzwUfqzNhUlCx%#JZd`8f%u=xJ4G;AJa2)?YdOt(n&Pn zHXU&S`~>c-il^{1fGlo&EaZK`zQ%V_9Z2Q00jNrRjR6%F1Wc-&x>Fs{Ep0>}Nspwp zG@Tm!eUV!mF^y48HP=Q`)*!jxBX_bHa=!=i13L#sc8zGGd7LClRLbw+Glf!i45l3Z zOk5LEovCSGp!B(EWPg%9h0qMwYN$2!?B}BgwQaz}X-u$&fQ5U;j+#-b*eVv;l# ilA}J?QreF-Q~OLyXaBD@W_;dy54RaeVoc?K{Qm$>Q_e&H literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/__pycache__/launcher.cpython-310.pyc b/vllm/entrypoints/__pycache__/launcher.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b67935de18a49c3de4f6cfcc4025c602095f870d GIT binary patch literal 3630 zcmb_f&2QVt73WYCNl~(5E4H)Q2F+~H02S7;o3z<LNNo7Um-qzwW^5)UzeJE%imjyV zmvx)Owr>N^ik-yu-K64I63_SaSUIjHHNU3Yc08BV{km>D@u_6qpVw_SUPu~#L$@pO z>15GgBnEHrrRT(7W*&E0m6y8~tFgHkmVZWb>fG+X$4)WQwb?vd0RCAmI~ESLa(%*j+&+1`DrNC`h$(!Y4gWJE>-Tfxg2DvD2-GQXS=%`=bY(ADGLT7>u@Q9p^VbqV2}xwm%_my zuMeL@olK;`qrFh<$__zkXHN~j`2H4Nc?CwE5oJsXBL{>S6Qgh5Hh%fRzNtzRlbOtT zUfM6|c1gFb34Di!JU_AUE9m2jUTs0YJcZE%asaLaZn z56Hf)+=(%^l=@TAkR_+JSLEeq*pxzlN2}VeX6Cz6<8BFz?#?Or{!-D<$3!^^}-~pGg>RDsoQ;muDoa|R8 zRnUNlFG2p(tTyqWKL;9Ar;8}Xfd$f`UFzB{_H^mMu#I29u6ssvesjFE{v?dza3|(e zWJARz4F$iRI|*04jLF;@gsL}QZsXW>dQq;>d&LNL#^wsmZNb%0q@9^xjz1=}PNQ?s zOD~P7aR91c)~@XLitGGny+uIsbsJ=hz6Gwc%G|E*OFl` zxt#2+#nH~%Uve^qd^v>O0LD>FzxiXSRIV=x{P+yFb;ZQ%3>}qq9iC% z%^Y9Q#|1s52CFyXtP{ra=IZOyzo896?{ByVhW#^$?=3?tKvRqRCRu`kyv~M6G74mG zs93g_HgT~9#C&Rzw~O%l0x&%U!uT3eur-Y&9ZJsV&WO(91-Ubo8QqIik9M*o0e!D~ z7v%=pf5*LOMziylPsGJ6Z@`Nho5KFOLbxb$0c=m@n;c}%oFQhUa3aRe~J%r^Y0@; z|8x+#M^70V_?F<6cY#bGx)dZnMwHn#b<#0e=|v6FgQ>?LooOy4w-+vOOPXuVxHfY% z*J3Wzgv$7nb;cx}1&*ZMOwewq!kBW*?2@WpsHp5^!6@6M`<6U2G5ckK)Rt$IGj)qZ&diraP*I+$_G_&4CDtr!{MY*_ z-Q_74QHKJcRFv>Sv=s``-+4q;2GVdEb+uqzS0>O$QWkNx>e1P=p)itmAo-1ObcIGK z-Rngih@(*QSCoR_ies8)G|p0}TXct0oiSB*yjNi@I#XV*(rpo8jiVvm6=8>WhcTF} zNCic#hI9tw)$vtbLGRwZSJX=_Y8TQnXDd{3kwj^zqqup`%7cjI9zxW5^+HX*25!R^-i_8s5RF6PYl zB6$?7;(c6N))0ULwci8&MWh7gLV?K4mmn@%?alSO!B(sNu+aO zF2p$`1*kF07qPQ6b77#5K1v+7U|b^dA&km3432+$rtLuXzWVk=bCSv5y5DPXM3Il+ z_|^BIgLj8Mku_ybORoUWQe|cLonzqf&kzSEU_U~bPH}XGGsx3n42g}6C0ZGuxi{x+w)bGLUoMqGm5}phaz8=x6C~Q}T6C;tf&xLlzPY)jMdCJ~ zNO=vSN1E@84{+86Ai0H?s3HZ}YBYe-E-Xc*{t7uk)@i5fPuj(1a?L{CkeNS_SDvD)sDR6@z4HF&& zML6i2)*&vR$ri^T%qqXCKn1_TxaU!oxJ0pdYYA{Q_LQc_#;Z0~gM?kwho z?p^@9>a|IMj-*6TWGUr5B*j4DKm6gUR3%R0Jew>HSaO+k@!&K zL(O~5dnI0Oe7L#a+%NIr#(`$VtTYdr2jzLBabNS0d8m2VJS@+njoIcA^N7U98uvGk znnxu*-guz-p!s0)A@iZ;!{)=yN6begeWLM^=A-7L&5xQNZ5}g^N!n!NW6k5{afweg z9&0{sJ}&VcjVGE8?US<%CZryQ$!-m%=;^0O_^ zw^(O`vs|kgPFtN$W39TpSgoN@)ptQ>9HE?9nTx!RF>$Qi1) z>VCD+ULyAK_+0yi6E6_Co?p?z9Z#>;ZO1<4)Y^8vwG?H9V;HN-Q8VJ9==AC4f@AlT zicmTnRZ@@WV)$IU-9Tf44PtIefqBBVI=k z{K@tz)-W2iosR3&FqhSfS1hcGuvGV|4fIrPFV?u?dq0Qp@Fc$8?-x=jCq+JFYA=E} znHgI%btmiOcBP!WojIN=rksLPv~@fCYU)}VG*v3*iut03zXE=9@>eXDibKVs#LJSG zvzP5*XUHCL%FfWPls$_0n3J=|@ttsn?MXaOMR_}--|6UgC-P^U5qlTDyYbzFbiiZZ zf!|^L-iiFX(9YfX8c0Li&e)>1D}}thc5d;Yz0bbq)vP&g?{y{?HTy&My|3!#Bw(44 zeBJ)Ay&ox4&JKb2Krv_6?Ft||h}Nc^oz9H2b5|;A@jj%@uwS%(2t6K_GBeIDU}#s| zD@yD}jj}y!A9*zkEbSR+dq%LZefv=lx7Ek}{e9Rg_5nL*AN{6=F(_Vv znRf1Q?sV=@vzowuXU{x;(A<{nN6>ao@N$=ZpZyX0QS2)pwU1rTL{xw=s6V?QJ@)k} zt7ZtJ0X3*qBO{8j*b50)+?L~i9}Ij9b?!#~aj9!Kd!v>47@p%9*^3xg>4g1wj3vN1 zgT4qCR(MqB31=TDXU4e)DLMN|`vhp^Ll~JF-vQ7!#%VY0rv%D-K?j^A{Hhib?DkSo zj9pNTeG)K{C!7K`oyPZ!z}Q2@$lFBM=z*|L*-zWgfVMw8Kwm@lvoiAinC1P+NN$&Q z51>Zb{ppHk$*&E~!s*mUT`xu}tVrA=oq>tNZ|9P}e+Kt9A z(0)w2+w~Ezln)-qTu(n;+I{xNq+NxZvt8Uow0mv~+T9BF4}!O3{|wqD?N%Jp?h9Mf zuHCwAy4?zjpM-8nZ^XigeLlv5-L@~-FM{?CBKDH~NyP30eB=#%d<5KbTYj;q+n=&O zjhcsqW>;iIGK%0k! z3l{B;{aNP@TlKJCxE)(12Z&pG3_Ce@?6UoHb`>>e!B;M!#!frx1Mtt=7IKc*oMU2X z+S%aAL1 z)pk(BNCdY-cxYIyrBz63Mzd`@4dXzk?RoWu#+qSM_NywCY|{#^o$7)@yYZibki-UrnS$2*{` zdAd4gJiBQ4?kXB(F6OB2*R4i<9Td=R5rvYjMv);N8&j1N z>$W18xpoUft&nue5mf7UGUqs4TkfNRkRZ4ykRoO7deN;RqgU!!nyP8x9BdjZzdK~x$AMY(V_QD>1|9`%2}v1Oochu- zI1{;mRrBkYWwN|kXzfRopjh28hBbdZa+q|&($SYQy%W*uYiSp^BHgRgAae^S7B z9~hEBa1wjrL8l)KgXcJQG-|{+#)R=IL}Gw8#EzEYG|?7|5s}Hnl1FzTEp=9_YO}0@ z%f#GUbwF9BZ5&u|{3{MbnEPcCk5(e?74NZryKg|Q<2ml-ZTJqBc9-d>bsozTv&J)E zO^(~w81@9H?9^5Vfyi~NXupeV8Dy;jm%qFUp{a=-0-YzTbHCTup1%xj*U~aKicS1O z`r=ae+ihbRuz<5bkg(RUc(`)gtN!8k;^DsbA~EB@*|X1A499ibl;k+A{CHQaPF3Wq zXub&Qfv4+F2?kGgWiU{?PotexXm%Q?t+O!Hwum3e6mP-#8bm^9Y~%ow5ceL{*MZ$$L^Mlmf*#G zs0mxPFcWdJj~&I1u-0A`O7yHnhrl|JStt!+;P>|r^eljKuhczoH%^DwL|2p$xEeN1 z$@|Vi4GX-z$PLP_E-ie(Ohhw4tVVTWL6gJdCs|nuPg0X|x$gR_0CElFsMaZl1+))} z+I{$n@xkSVu9w>l5Hu?F?H?c$qR9b60*l9NV&xoWv*eh4%|XEqDHT&h__;T(F$SG2 z#451uE{2^J(rZrDp>g7v@yTWEWt`cz5#<k{i70fCKGuSBZ+5|!+-#|X%z!&)G!dlNLdQe8#fTtFCv=xcfzBEyU`?SsE5srzwMALnJ#5n)6XBx)%qh994`;n2A-W^p29uf8 zlp_FH__ou~*BYmUi=M>Qor>;yjFxq^hGRp!Sp~q@_wCghme6^}F=(fu41Q9itEAQjG;;eCo46eo0q+9OVV0Juo zD^l%;_k_3BswK9-`9w+4BPX8;$C5={eeU$rCr*C42E%mnVO+zvhOak+;1!sKf>age z>6cP4wrVO4^-5;$!kJ3iJ&H20Ol4{f@3oZsAfDd-*b%?kInt?j4*T4V{KFqiCoqC(lz*d#gUe0B5 zT1i`xsr&89?P6UIS0vODHn2MoMb+W}<9pPh(g z@%;q8-rqogCAH&n+AUx??Xc6wQ+~?cRfqpWkp6nw)&mV;Hpn2%1vXM=*(SI+*%8 zVENVbSJE5PetJa?%;^t=1sTMT#qO z1tEU2#lBe~zgullmO8L0ou{@+(lW$Z+gY@r5EPM1_Sr7CgHcm$<20LBDLi@?+>ZgH z?r{c>F?gK86AYeYaDu^83{EmQ#o#o9GYp<)@C<`z8GM`pr^$Ve!Sf8}5LD9U@G5Q3 z@qRrgq+Thw6D%@`z#OBNr`p30UD}yVIww?5K6m2$`LF~Uj|#Ylkr$3e!YEo5J9cGf z2s&si{@)lT}T%+J+0l+HSIk;qrI!^ z+BOSgT*`mpwfKBE0EeN_9O`k3}V^l|OK z>l51d^-1l&=~G%d_dR`QE?u})*4E49Y;m`ncyk6Ia2~x81VL&gbtCO-H`6Pb^QqU; za~0h^hxBlg9tB+{vqbj+_devvGI=fSzQFjvScm&6{W?m#!JS|vy*^Hm`f_W`i9duX zf$IT0bm7|20>HMS-^kkOn`w61` zyz3zCcbUG3prUo*sB!xF;cxGlZnxEcgL(?yp`W;||BANUB?Le5SpP7As`y$);XgsW zPynfbL+0idPvgLlE7`#*Oqv*&JwY#Z-QIoz6?r3mvx(t%$!?EB1_f2yu#iC-L=u z8v!}X3btvt=x1#3kI$zAxGb%vT?09~j}6<;`vt$~7h)#>a2X9{GS{_@VrvK`@_xx5 z@=Ge8xf?~l9OV7sYnpq?9|>SSxAlv<`xtu22KjHKzNN7rf7CA2Q$b!tg?yl)SL~Gj z*tHaRTxw-JC8{xin#&2y{DtJ-t!#CvQvzC4a|F4T40`n#UYItqhU0 ztBKwqcGyz~5vHk=QZyAkB1X@Nf}(B{8o-(y)WpUYHP~m-JS4&j-07%yN<m_(^|oO_-Pc-zqJiFLqGF(4 zgK9cC#X%5KTKS#<;SAPe@S_wJ4O(e}qKEht$j!+3G_+LIF-hC#GCuADw5WJ8Ot!}l z&v(iS+H=U_8L|HmC0O@n&&m3wcjNq)Ex|DBP&*Ez9V|5qkc(v=1zDm2aY%8|rq-EL9Ivk9I=?;q1|Bta%Ub;)=@TS9O|V?rf5&Dhc0i55?E zt4xK{C`~??Kc%oz815g`rj|JUPz+)r$wFw-tS8z=CIJOVhu{VM02DMx+BB$m)@hVb z&~>ajwLi2oLIcDfWo<)yTR{|WzkRwh!4URj4hEt$++WuOMl`Slt+fxxvlCZ%fz`h`d zzyYE@p;iJ&bbkH?B@8Lg-1&KfMG=cIHketLSt=u$T2O75swuUe zw&kAv+8K`cMe$&mpI39_0UEBk__-_5F~c)1I%`+ZQ4dK01< zwTrupbD+YBf;tHrePt^KbX>Rx0Bx-6tMn``rQe1>$*0b=-IFk`SdHhNKV-;Lw??#y zVcAz^=OBrQpgNC@+WiQE^(V=uY1WgDNh_pUB?15wPzZ9O;SSUFfB}?=k@=WW$w`x8 zP6p^Hp|^#m!;yYL!tEdt2OANtDEm~V)lsM{s-DQY8jd)zr?^t{lwP8_;xvsy;RFjo z!R%KrAH}H;o1N!U0CDpSc*`Tq;K1)v{&J}rb19n8(V9~vUj4ew8v_|IHuo+T*v|sH ztKg<}7y_Is))iQQ9ZNl4I8=34Tk(;2lBLIz!b;WRxqFdZ85)!#=aJeKz+K|RrJ76- z5sm;~FnUM6_FkBe8|yzXh{!n%vCyMhp!9&2*3$Z|g081$5G%YlmgRFPQ^u3Z({JT6 zIIi9*Kn;fjaFov9+}q_sIqkY=C=uZiPwm5(h@zbAoq@z0Xe+6W48rsVR6Q%&Miyaa zBZpA`5>!Ke*4F&oqW*={7t(Ow&e-~^<&Auhx{<$``cm32P&G7_f@&xW*($S9SW0aa zgY>D?XLmr(DniOCL6UnoD7Ex2rf&=drJLyuD2i^BeW-}8;S`!qT{;9gPrt@GuWghe z4??!Hi?61F;wd<}FQu+Q;@cRuOG|0wjolcxhq}2pA$9IdZHypiA{epD_V6#lH5sv- zU1m(`j{1|qFyzS6Ma^ALr&^co5qlKv53~J^F-V3}!5HMkF?$>%7)MK^LGhXfc;-K& zeMZ}u*qGdy!nk$-&WT`Ba!zeb2gAYiuGGfPU|fD@f|->aHYC}VX}b&=_}}$BBXwg} zu})4|^A-mj)F4F`8&?)C+Sz2mx? zy&nX3L5iNXEG*oF@kX2oqD3SM(qBrr3$D)~k+*MpHmLUzIlJ3bv?0LzsVmEMy%gxl z2kr$P3Wq-kI|(S*0^~!(2d{G&&z>3(yA(H5J2bENMDTUlILar@T{wH{>@nCKuNZb4 z!hLUfY(zAUKs7_tph!cJP}RYn-0;`vHl^^fmb8~-ahHS9q|@OrYB+r1cwike9KSaE zL{d6TUvYgRbUKAb3jUX=FejJ*!>g?} zsk;Pf7s9gB_LA)t%=_~w`I;7vs4-L{K$Ut&7?+%9N1$z1x1zi%HsjBceUZVG($LDK z_4n5IbK~o|H8p#jj*|`V@!9^Guk-jW8x^k76e6H-bB0Rt7+l)z=`V^Mx(@@a%CE=o zG=f)vfzReP($IH_rR0*9sB&N)$r2n7W7B5u4-4^r(Q{mvkrj_q#<|*YAO9)`MSYO= z@>FVl-?jrvDu}6L6NJKZ(aIaUV+acyY0Oo61ze2woL(q3H$W-(D-8ZDgMq!`Ye@SV z;Ch3|)?U_u@Y1ch*KgsosbemV^Z1GhX^E&wI<`~S)4!x$;z2l#IiE)qDQk-SM1S!6~P5uBgNYv@iy9#7G*ZiS(E@rjMZp*&)pox%Wzt;9hNZt&mAxXol$V0 z_%n$JRemS&^}dgQ?8%2(&K>nNKZ8HqM}UAuMO~>?kTj(5c?pU&2pbUbAUeFTQHF?^ zy_wn=#x|6rum~Zo5R`)vh+-KCK%*OD5D$k%Jbc_Af-qH16n5$BxsCBw35vjBh+?rC zPh%?J85iZ?c?e_UP$Z@|CR$m46mgyLV9d@!_?m}6H(}?FLx_~;WhewE>^z=_ZXibl zsXc&fJQ#waFayP4;W{=+C zH|6uZ7aeGD)QePeqD@lN7>P%{r)(h!UOtrrD|L!WIeCm4yJPdwJUjD|29NrWIIZur zp1|G%LyFLPTN=gpZ;AWopJ){0<|yoQA~~CK$|-w6BPxJ;;$$p1}_q^h#Nd5Sq9_- zDzR{3z-kd`z`;BRw%JMmACUPikpcHWV4&f(qniEW+QO*@UV`MFig{v|YT|KYi2s5? zpGbIBMjHkF88;10rY_R_&ME~j=vwLjV#l*sG2FvR{|_l~7^FaGj*%b2Z90~&JW-XJded9-)R}pe?f+c_c$Z95 z#%2yhqPPr{2pBwK>0B(oOHq}_m(oKNLvy#vS^bs{$Cd)bN-axIm6C|96dl!H;cdJV zB6*@Zuv99iOB(C&TkcU8=H6K^P{d?cP*tjTazy8c+cnzvxkmg04rHEz#bANK!0Gh{ z(w@SJU3?eTOYB~q#BR4k_aAQ4>RjAvqV~+7Bv%TsD01z9OTe}*nQUnZm44!umXekT zNBsG5VDj10GFpIdji5Qf@Q>vI(ci>Ylk^J-=rP#wZ-UMhq=@f>dy%p3OiPRhFSmi< zD{QHWNDUHp^5J2F&nNNqsO*4HE2 zJB-GtSY(fWSWbmeQMvdL`v7d({0AlYs8UZ`{qhIOi`Uvuf!RF4)dxvBTfblGk7>0| zOpm+;@V*jHw7z>pv7Z{F^<&+Qd0Ufpg2=9o`gvR}!PVP2_YxR7Yz|_F7Tx6n5|nk+ zXV!#rE)~h>GPABQxXPg4*6Xe@P2{~9#;Ejm_c7RD@be6)yK;Ylfq|e>?9rQUC6TmP z@?{3EFz9YuvSZyq+Ly6odGwkt9i%L^*%aGvcFnz0&X?1D%6IOOXhe3mH>g|Y?sg-6 z6S7Fx9-%4I_^{HoVzu$&w);tx9t7&QkoGUpp8IJ;B9B;6&_Ep;f&P`LNueH0;t+`WSo$+1?{T205s4G2=esYc{OL@CB3< zlw}z>*CRd4bSNWrAE{zl-V3|7YWpQ$hu?D+FFwg(x6idkqCOU2kA*~lbovP9TCjS< zpHaV&4|1X|d?uiRurSw$WKJ#zejeX9rVxmAMGQ|i^rXtN@>)8~^Z3jAHC$%!%n+kW zx-&Sze*wAw9&LJT7pA6z#QJ7Oh^`5(Yk}DoEcdt3mivzx$Qu7ANlYGPwX@&vvR_hI zE>*x8WBsuUID#wNA5?OQBcEu$6iZBug?wcS{K;_mrXG+w!`bzRPQw=*x=U45lzQ9N zLWZb1X5GI>;C>ka{Lc38+CQq4{~1D&tOf z;@_dD{jW1!rNHC-joHk%I}jXkZd>p5XE>(~%f17jS2VW;+8DJqW^wBkiIF4g7SFhK@b$ z=Y%x!$kY5h(kn;-Nd$QiMFkESiYTz~CHGO}f&?V@QMzjg$;5S^2(*?KXp3~d7>DSa zf#3_*2lqU*_qf#)WN!jIe`uu~6sp6Qo`TLDWQnpLh6ca5Q35*~g^LGU8$+#S5KwVf zss(~XT1L`bXxCL5wCq97U58VyJi|pJe?1GM=cmXDL0IeO`kjd4d82-&>!+n|+nw)w ztK)1`G4WxFltu2J29jYqsxT`M! z61DMl2-A32z$Ifk)_I}MSU7PidS@h7^e0u8Qd-E)_c>Mlu*JQ-~BwV&|gj;+>Iv4oZk=5)htYKSOLLQzBtyw3o+V zP1Dj-+-%e3TikSEMSEMnwSJe7R^p5`=(!~@^r;Sbrc6leARhb__6c}I@U@xnZc6bU z43;96D$PG2Kp36Wu8i9+?(d)m^jgGysKW`scYg*^**eW3Vmv}x%yC3_Gk`BnbXbZC z!n8Svqu)dR!^F`9A|nN5`HPh{rMK5*%k7<^zAYCOyOz3V@F3e5WTXIB5$N_H9A(r8 zKM{%tm9KzKF=%atE?#u-!V!3-fN#;(uHynw74<-gnYr~7kvf%E%;TZuYh}M(f+ZCo|B!O z(j>Qz#6Ua7tt0Ku#8Tyk=3|;EbJ0)F&VEA&lU2X@Z)IuX+{jQWvh|HDzNCWGjm%AG zMgXN+>RKMuXkus!lBUTb@uZY*PS>*ME?Rvt?ANc_04J^|N& zSvs?daTD&O#|+`k>zUaH7T3q2u(;?Q!4-PNDvufK`6vl5afYEB?-DX~sP${+u)T&$ zZ}nOgH|XINl83{DzgWK-=ADMsfgV4ZF3gk&|D;q|7+*&u7Q)HPO?BfLtO2i07Lm|| zOY0_~@M|c#hDO{!hluh%>hV+Ukp*eQp}UeeQT2hrl>@;>IRTXrQ66H~K<+U>e;=vt zzh>~K7z`|8D)7F*Ma))8sr9|WkGtNoJy1l~n<{DdFQRO8#54C$El*)=74KET+voUp z<=DvK4x^yC6YqxDDi?Q9eWw!^;%Z7yPahuf#Z`)CLFW)`^SFLfrP0n#y8p5?f3=rO-`A%gf9lFbq-RlVIr%5i?;D)_m|b)y{~S`y8Qdd_UJHTy zWb%3lWk}iTEE?{rZdd$E7^K`DETz})ly$OQ$!)=O{{sMuSEc|r!8ZE5#T#3KH7Q{A z7yL2+a{nV*T;G|1(_f-8Ip;ot0%y($?MQrXZcZ44`zT@gCmg$(3AnmCGfKx(_oD*^#1pI)JvwXaV!QBk*V(=jbbX;?Poxv1? z-(;|lfx+MggWqECZ3aAcDK*aTF~def551Pz3O9U@)bWo*p{` zdV)|1_xMswTkrR)e-EYKg9H7$IJ~{1;m){*JL4Mej6*_wD~CJdIouh~;m$aYcR$c@ zXI#UbaSeCIbGS2}!<}&rcg8i`8P}$?KN#7eeNWq2dVS=viHtUc)`YtCAE`cYj=ZJ4 z3)e5{V*-8T3gx$Y%fX2Zu&8n)o6BX;&O3lDoqMadWG;euLLe@`2}pZNmU`MOZ7!LE znEf{4L!Ec0BOFYiTrOY!5pbv++TI#rE3h(R1b;;A^z==ez3p+P;BW^lYHz{P z`6i(K;RI%CMA|F8p?N>OGoZiu!PqbVuyg`D2wr-iy|+74dRN0`6J2Jyj5P>MzdM07 z$kjzGmjuhDH?@a3FKW%8_PaT(nOs6Qr5|ZKuok#l*y0Y9g_~pPy~5#gMlMBH&bYry zYX0X4!jfpjuKIWhjW8_uq_5HSU9l$eI>o7k0|^V`an7aetN4>Bh=pB=_%J5#2wH%f za`fLh#FtlbqA^w)M_}%dQtA~GvzEIgZ}9WvwHfAgZ(h&4F(z4YPn4OwAxF&RQFUQ* zJJ`uIgPp7}Bo(5IOEXN~#eljibG#>SR_+FgvMZ_s2Uxu0%H?q(DoGW?Y#Lsv^Xv6jtcP5`U%Tiehx}IwwNw~W?De3(dFJar!=p+4Ibcs9x zh#P=8C3Ocp-_&M6ChbxuAzO;MS4yfEuc_AqD=f|?{*G!-s<6gEw!QxYYgD`M)QBLF z*Q3m;m!xosQH^hwq#?ev(UUixs0~bxnk@?> literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/__pycache__/logger.cpython-310.pyc b/vllm/entrypoints/__pycache__/logger.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e603f651e5499ea798f828ed61d75700681fc115 GIT binary patch literal 1569 zcmZ8hOK;pZ5GJX|YPAoq9oLOxwD2Vb7HD>nQ-Poen)c9(;Tkr2(S=Z#w5?Q>6jGbk zhH|N0nRh^YZn%m&(0)#2G zgRIDEo|k85QsefZJbrR4pDm@SVLLi1%RDR2j#8Z}1B&etPazP==xh z))1@_tUdGtxeszA`!be;Q_pE)&_E_pjp z>GxO&p3B09TGmTl94=@VCa$cT$f(%{sL2(1MPCz28rl%f1c49JL#TIPmX*{t;@s7j z>mj&Lne~m#Pu=DXwSCSPd1?&jZ^)mI=e4TlRaQNyQ++1u2cM|rxq7I+o9Ed#^Xj5L zFN^u}JXdpB)cT?-v!XWhmbh7Up*P?Htf}zXdg!>`v#Gwhge-FM4q4YD6IPVSWMA904@sDS6RrdV(ye-fN&t zux{eo^nPT!75$+)%_xjE|z|OHTc3M_c%i5sD;E^znXherJVngaPMj4AJqtV;TySJIVIXazS zixsJooZFalRf=Vf@{sdq%QSB-8XMMn&@iq`tY>Piv6(gYipJ+lBY_%O(;)(Ks6PO( zqpnF8GS4|*OJfsVFc&=lz#Pp-01>&zSil(jC-m6gvCsaB0!G7s+1s$k{s<%XJL_-# z=FS{Ww`^EnRPeyG;b5FDreuJfK5I*@;Z-HcW~b5VtYh3on%r^vs(q(;OS;P@*=sE= z^&O?}&SHYew(~IEL$x~Ws^E(9A;z Response: + """Health check.""" + return Response(status_code=200) + + +@app.post("/generate") +async def generate(request: Request) -> Response: + """Generate completion for the request. + + The request should be a JSON object with the following fields: + - prompt: the prompt to use for the generation. + - stream: whether to stream the results or not. + - other fields: the sampling parameters (See `SamplingParams` for details). + """ + request_dict = await request.json() + prompt = request_dict.pop("prompt") + stream = request_dict.pop("stream", False) + sampling_params = SamplingParams(**request_dict) + request_id = random_uuid() + + assert engine is not None + results_generator = engine.generate(prompt, sampling_params, request_id) + results_generator = iterate_with_cancellation( + results_generator, is_cancelled=request.is_disconnected) + + # Streaming case + async def stream_results() -> AsyncGenerator[bytes, None]: + async for request_output in results_generator: + prompt = request_output.prompt + assert prompt is not None + text_outputs = [ + prompt + output.text for output in request_output.outputs + ] + ret = {"text": text_outputs} + yield (json.dumps(ret) + "\0").encode("utf-8") + + if stream: + return StreamingResponse(stream_results()) + + # Non-streaming case + final_output = None + try: + async for request_output in results_generator: + final_output = request_output + except asyncio.CancelledError: + return Response(status_code=499) + + assert final_output is not None + prompt = final_output.prompt + assert prompt is not None + text_outputs = [prompt + output.text for output in final_output.outputs] + ret = {"text": text_outputs} + return JSONResponse(ret) + + +def build_app(args: Namespace) -> FastAPI: + global app + + app.root_path = args.root_path + return app + + +async def init_app( + args: Namespace, + llm_engine: Optional[AsyncLLMEngine] = None, +) -> FastAPI: + app = build_app(args) + + global engine + + engine_args = AsyncEngineArgs.from_cli_args(args) + engine = (llm_engine + if llm_engine is not None else AsyncLLMEngine.from_engine_args( + engine_args, usage_context=UsageContext.API_SERVER)) + + return app + + +async def run_server(args: Namespace, + llm_engine: Optional[AsyncLLMEngine] = None, + **uvicorn_kwargs: Any) -> None: + logger.info("vLLM API server version %s", VLLM_VERSION) + logger.info("args: %s", args) + + app = await init_app(args, llm_engine) + assert engine is not None + + shutdown_task = await serve_http( + app, + host=args.host, + port=args.port, + log_level=args.log_level, + timeout_keep_alive=TIMEOUT_KEEP_ALIVE, + ssl_keyfile=args.ssl_keyfile, + ssl_certfile=args.ssl_certfile, + ssl_ca_certs=args.ssl_ca_certs, + ssl_cert_reqs=args.ssl_cert_reqs, + **uvicorn_kwargs, + ) + + await shutdown_task + + +if __name__ == "__main__": + parser = FlexibleArgumentParser() + parser.add_argument("--host", type=str, default=None) + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--ssl-keyfile", type=str, default=None) + parser.add_argument("--ssl-certfile", type=str, default=None) + parser.add_argument("--ssl-ca-certs", + type=str, + default=None, + help="The CA certificates file") + parser.add_argument( + "--ssl-cert-reqs", + type=int, + default=int(ssl.CERT_NONE), + help="Whether client certificate is required (see stdlib ssl module's)" + ) + parser.add_argument( + "--root-path", + type=str, + default=None, + help="FastAPI root_path when app is behind a path based routing proxy") + parser.add_argument("--log-level", type=str, default="debug") + parser = AsyncEngineArgs.add_cli_args(parser) + args = parser.parse_args() + + asyncio.run(run_server(args)) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py new file mode 100644 index 00000000..5195491b --- /dev/null +++ b/vllm/entrypoints/chat_utils.py @@ -0,0 +1,581 @@ +import asyncio +import codecs +import json +from abc import ABC, abstractmethod +from collections import defaultdict +from functools import lru_cache, partial +from pathlib import Path +from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal, + Mapping, Optional, Tuple, TypeVar, Union, cast) + +# yapf conflicts with isort for this block +# yapf: disable +from openai.types.chat import (ChatCompletionAssistantMessageParam, + ChatCompletionContentPartImageParam) +from openai.types.chat import ( + ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam) +from openai.types.chat import (ChatCompletionContentPartRefusalParam, + ChatCompletionContentPartTextParam) +from openai.types.chat import ( + ChatCompletionMessageParam as OpenAIChatCompletionMessageParam) +from openai.types.chat import (ChatCompletionMessageToolCallParam, + ChatCompletionToolMessageParam) +# yapf: enable +# pydantic needs the TypedDict from typing_extensions +from pydantic import ConfigDict +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +from typing_extensions import Required, TypeAlias, TypedDict + +from vllm.config import ModelConfig +from vllm.logger import init_logger +from vllm.multimodal import MultiModalDataDict +from vllm.multimodal.utils import (async_get_and_parse_audio, + async_get_and_parse_image, + get_and_parse_audio, get_and_parse_image) +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer + +logger = init_logger(__name__) + + +class AudioURL(TypedDict, total=False): + url: Required[str] + """ + Either a URL of the audio or a data URL with base64 encoded audio data. + """ + + +class ChatCompletionContentPartAudioParam(TypedDict, total=False): + audio_url: Required[AudioURL] + + type: Required[Literal["audio_url"]] + """The type of the content part.""" + + +class CustomChatCompletionContentPartParam(TypedDict, total=False): + __pydantic_config__ = ConfigDict(extra="allow") # type: ignore + + type: Required[str] + """The type of the content part.""" + + +ChatCompletionContentPartParam: TypeAlias = Union[ + OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam, + ChatCompletionContentPartRefusalParam, + CustomChatCompletionContentPartParam] + + +class CustomChatCompletionMessageParam(TypedDict, total=False): + """Enables custom roles in the Chat Completion API.""" + role: Required[str] + """The role of the message's author.""" + + content: Union[str, List[ChatCompletionContentPartParam]] + """The contents of the message.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the + same role. + """ + + tool_call_id: Optional[str] + """Tool call that this message is responding to.""" + + tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]] + """The tool calls generated by the model, such as function calls.""" + + +ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam, + CustomChatCompletionMessageParam] + + +# TODO: Make fields ReadOnly once mypy supports it +class ConversationMessage(TypedDict, total=False): + role: Required[str] + """The role of the message's author.""" + + content: Optional[str] + """The contents of the message""" + + tool_call_id: Optional[str] + """Tool call that this message is responding to.""" + + name: Optional[str] + """The name of the function to call""" + + tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]] + """The tool calls generated by the model, such as function calls.""" + + +ModalityStr = Literal["image", "audio", "video"] +_T = TypeVar("_T") + + +class BaseMultiModalItemTracker(ABC, Generic[_T]): + """ + Tracks multi-modal items in a given request and ensures that the number + of multi-modal items in a given request does not exceed the configured + maximum per prompt. + """ + + def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer): + super().__init__() + + self._model_config = model_config + self._tokenizer = tokenizer + self._allowed_items = (model_config.multimodal_config.limit_per_prompt + if model_config.multimodal_config else {}) + self._consumed_items = {k: 0 for k in self._allowed_items} + + self._items: List[_T] = [] + + @staticmethod + @lru_cache(maxsize=None) + def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str: + return tokenizer.decode(token_index) + + def _placeholder_str(self, modality: ModalityStr, + current_count: int) -> Optional[str]: + # TODO: Let user specify how to insert image tokens into prompt + # (similar to chat template) + hf_config = self._model_config.hf_config + model_type = hf_config.model_type + + if modality == "image": + if model_type == "phi3_v": + # Workaround since this token is not defined in the tokenizer + return f"<|image_{current_count}|>" + if model_type == "minicpmv": + return "(./)" + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma", + "pixtral"): + # These models do not use image tokens in the prompt + return None + if model_type == "qwen": + return f"Picture {current_count}: " + if model_type.startswith("llava"): + return self._cached_token_str(self._tokenizer, + hf_config.image_token_index) + if model_type in ("chameleon", "internvl_chat", "NVLM_D"): + return "" + if model_type == "mllama": + return "<|image|>" + if model_type in ("qwen2_vl","qwen2_5_vl"): + return "<|vision_start|><|image_pad|><|vision_end|>" + if model_type == "molmo": + return "" + + raise TypeError(f"Unknown model type: {model_type}") + elif modality == "audio": + if model_type == "ultravox": + return "<|reserved_special_token_0|>" + raise TypeError(f"Unknown model type: {model_type}") + elif modality == "video": + if model_type in ("qwen2_vl","qwen2_5_vl"): + return "<|vision_start|><|video_pad|><|vision_end|>" + raise TypeError(f"Unknown model type: {model_type}") + else: + raise TypeError(f"Unknown modality: {modality}") + + @staticmethod + def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict: + mm_lists: Mapping[str, List[object]] = defaultdict(list) + + # Merge all the multi-modal items + for single_mm_data in items: + for mm_key, mm_item in single_mm_data.items(): + if isinstance(mm_item, list): + mm_lists[mm_key].extend(mm_item) + else: + mm_lists[mm_key].append(mm_item) + + # Unpack any single item lists for models that don't expect multiple. + return { + mm_key: mm_list[0] if len(mm_list) == 1 else mm_list + for mm_key, mm_list in mm_lists.items() + } + + def add(self, modality: ModalityStr, item: _T) -> Optional[str]: + """ + Add a multi-modal item to the current prompt and returns the + placeholder string to use, if any. + """ + allowed_count = self._allowed_items.get(modality, 1) + current_count = self._consumed_items.get(modality, 0) + 1 + if current_count > allowed_count: + raise ValueError( + f"At most {allowed_count} {modality}(s) may be provided in " + "one request.") + + self._consumed_items[modality] = current_count + self._items.append(item) + + return self._placeholder_str(modality, current_count) + + @abstractmethod + def create_parser(self) -> "BaseMultiModalContentParser": + raise NotImplementedError + + +class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]): + + def all_mm_data(self) -> Optional[MultiModalDataDict]: + return self._combine(self._items) if self._items else None + + def create_parser(self) -> "BaseMultiModalContentParser": + return MultiModalContentParser(self) + + +class AsyncMultiModalItemTracker( + BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]): + + async def all_mm_data(self) -> Optional[MultiModalDataDict]: + if self._items: + items = await asyncio.gather(*self._items) + return self._combine(items) + + return None + + def create_parser(self) -> "BaseMultiModalContentParser": + return AsyncMultiModalContentParser(self) + + +class BaseMultiModalContentParser(ABC): + + def __init__(self) -> None: + super().__init__() + + # multimodal placeholder_string : count + self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0) + + def _add_placeholder(self, placeholder: Optional[str]): + if placeholder: + self._placeholder_counts[placeholder] += 1 + + def mm_placeholder_counts(self) -> Dict[str, int]: + return dict(self._placeholder_counts) + + @abstractmethod + def parse_image(self, image_url: str) -> None: + raise NotImplementedError + + @abstractmethod + def parse_audio(self, audio_url: str) -> None: + raise NotImplementedError + + +class MultiModalContentParser(BaseMultiModalContentParser): + + def __init__(self, tracker: MultiModalItemTracker) -> None: + super().__init__() + + self._tracker = tracker + + def parse_image(self, image_url: str) -> None: + image = get_and_parse_image(image_url) + + placeholder = self._tracker.add("image", image) + self._add_placeholder(placeholder) + + def parse_audio(self, audio_url: str) -> None: + audio = get_and_parse_audio(audio_url) + + placeholder = self._tracker.add("audio", audio) + self._add_placeholder(placeholder) + + +class AsyncMultiModalContentParser(BaseMultiModalContentParser): + + def __init__(self, tracker: AsyncMultiModalItemTracker) -> None: + super().__init__() + + self._tracker = tracker + + def parse_image(self, image_url: str) -> None: + image_coro = async_get_and_parse_image(image_url) + + placeholder = self._tracker.add("image", image_coro) + self._add_placeholder(placeholder) + + def parse_audio(self, audio_url: str) -> None: + audio_coro = async_get_and_parse_audio(audio_url) + + placeholder = self._tracker.add("audio", audio_coro) + self._add_placeholder(placeholder) + + +def validate_chat_template(chat_template: Optional[Union[Path, str]]): + """Raises if the provided chat template appears invalid.""" + if chat_template is None: + return + + elif isinstance(chat_template, Path) and not chat_template.exists(): + raise FileNotFoundError( + "the supplied chat template path doesn't exist") + + elif isinstance(chat_template, str): + JINJA_CHARS = "{}\n" + if not any(c in chat_template + for c in JINJA_CHARS) and not Path(chat_template).exists(): + raise ValueError( + f"The supplied chat template string ({chat_template}) " + f"appears path-like, but doesn't exist!") + + else: + raise TypeError( + f"{type(chat_template)} is not a valid chat template type") + + +def load_chat_template( + chat_template: Optional[Union[Path, str]]) -> Optional[str]: + if chat_template is None: + return None + try: + with open(chat_template, "r") as f: + resolved_chat_template = f.read() + except OSError as e: + if isinstance(chat_template, Path): + raise + + JINJA_CHARS = "{}\n" + if not any(c in chat_template for c in JINJA_CHARS): + msg = (f"The supplied chat template ({chat_template}) " + f"looks like a file path, but it failed to be " + f"opened. Reason: {e}") + raise ValueError(msg) from e + + # If opening a file fails, set chat template to be args to + # ensure we decode so our escape are interpreted correctly + resolved_chat_template = codecs.decode(chat_template, "unicode_escape") + + logger.info("Using supplied chat template:\n%s", resolved_chat_template) + return resolved_chat_template + + +# TODO: Let user specify how to insert multimodal tokens into prompt +# (similar to chat template) +def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int], + text_prompt: str) -> str: + """Combine multimodal prompts for a multimodal language model.""" + + # Look through the text prompt to check for missing placeholders + missing_placeholders: List[str] = [] + for placeholder in placeholder_counts: + + # For any existing placeholder in the text prompt, we leave it as is + placeholder_counts[placeholder] -= text_prompt.count(placeholder) + + if placeholder_counts[placeholder] < 0: + raise ValueError( + f"Found more '{placeholder}' placeholders in input prompt than " + "actual multimodal data items.") + + missing_placeholders.extend([placeholder] * + placeholder_counts[placeholder]) + + # NOTE: For now we always add missing placeholders at the front of + # the prompt. This may change to be customizable in the future. + return "\n".join(missing_placeholders + [text_prompt]) + + +# No need to validate using Pydantic again +_TextParser = partial(cast, ChatCompletionContentPartTextParam) +_ImageParser = partial(cast, ChatCompletionContentPartImageParam) +_AudioParser = partial(cast, ChatCompletionContentPartAudioParam) +_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam) +MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'} + + +def _parse_chat_message_content_parts( + role: str, + parts: Iterable[ChatCompletionContentPartParam], + mm_tracker: BaseMultiModalItemTracker, +) -> List[ConversationMessage]: + texts: List[str] = [] + + mm_parser = mm_tracker.create_parser() + keep_multimodal_content = \ + mm_tracker._model_config.hf_config.model_type in \ + MODEL_KEEP_MULTI_MODAL_CONTENT + + has_image = False + for part in parts: + part_type = part["type"] + if part_type == "text": + text = _TextParser(part)["text"] + texts.append(text) + elif part_type == "image_url": + image_url = _ImageParser(part)["image_url"] + + if image_url.get("detail", "auto") != "auto": + logger.warning( + "'image_url.detail' is currently not supported and " + "will be ignored.") + + mm_parser.parse_image(image_url["url"]) + has_image = True + elif part_type == "audio_url": + audio_url = _AudioParser(part)["audio_url"] + + mm_parser.parse_audio(audio_url["url"]) + elif part_type == "refusal": + text = _RefusalParser(part)["refusal"] + texts.append(text) + else: + raise NotImplementedError(f"Unknown part type: {part_type}") + + text_prompt = "\n".join(texts) + if keep_multimodal_content: + text_prompt = "\n".join(texts) + role_content = [{'type': 'text', 'text': text_prompt}] + + if has_image: + role_content = [{'type': 'image'}] + role_content + return [ConversationMessage(role=role, + content=role_content)] # type: ignore + else: + mm_placeholder_counts = mm_parser.mm_placeholder_counts() + if mm_placeholder_counts: + text_prompt = _get_full_multimodal_text_prompt( + mm_placeholder_counts, text_prompt) + return [ConversationMessage(role=role, content=text_prompt)] + + +# No need to validate using Pydantic again +_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam) +_ToolParser = partial(cast, ChatCompletionToolMessageParam) + + +def _parse_chat_message_content( + message: ChatCompletionMessageParam, + mm_tracker: BaseMultiModalItemTracker, +) -> List[ConversationMessage]: + role = message["role"] + content = message.get("content") + + if content is None: + content = [] + elif isinstance(content, str): + content = [ + ChatCompletionContentPartTextParam(type="text", text=content) + ] + + result = _parse_chat_message_content_parts( + role, + content, # type: ignore + mm_tracker, + ) + + for result_msg in result: + if role == 'assistant': + parsed_msg = _AssistantParser(message) + + if "tool_calls" in parsed_msg: + result_msg["tool_calls"] = list(parsed_msg["tool_calls"]) + elif role == "tool": + parsed_msg = _ToolParser(message) + if "tool_call_id" in parsed_msg: + result_msg["tool_call_id"] = parsed_msg["tool_call_id"] + + if "name" in message and isinstance(message["name"], str): + result_msg["name"] = message["name"] + + return result + + +def _postprocess_messages(messages: List[ConversationMessage]) -> None: + # per the Transformers docs & maintainers, tool call arguments in + # assistant-role messages with tool_calls need to be dicts not JSON str - + # this is how tool-use chat templates will expect them moving forwards + # so, for messages that have tool_calls, parse the string (which we get + # from openAI format) to dict + for message in messages: + if (message["role"] == "assistant" and "tool_calls" in message + and isinstance(message["tool_calls"], list)): + + for item in message["tool_calls"]: + item["function"]["arguments"] = json.loads( + item["function"]["arguments"]) + + +def parse_chat_messages( + messages: List[ChatCompletionMessageParam], + model_config: ModelConfig, + tokenizer: AnyTokenizer, +) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]: + conversation: List[ConversationMessage] = [] + mm_tracker = MultiModalItemTracker(model_config, tokenizer) + + for msg in messages: + sub_messages = _parse_chat_message_content(msg, mm_tracker) + + conversation.extend(sub_messages) + + _postprocess_messages(conversation) + + return conversation, mm_tracker.all_mm_data() + + +def parse_chat_messages_futures( + messages: List[ChatCompletionMessageParam], + model_config: ModelConfig, + tokenizer: AnyTokenizer, +) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]: + conversation: List[ConversationMessage] = [] + mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer) + + for msg in messages: + sub_messages = _parse_chat_message_content(msg, mm_tracker) + + conversation.extend(sub_messages) + + _postprocess_messages(conversation) + + return conversation, mm_tracker.all_mm_data() + + +def apply_hf_chat_template( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + conversation: List[ConversationMessage], + chat_template: Optional[str], + *, + tokenize: bool = False, # Different from HF's default + **kwargs: Any, +) -> str: + if chat_template is None and tokenizer.chat_template is None: + raise ValueError( + "As of transformers v4.44, default chat template is no longer " + "allowed, so you must provide a chat template if the tokenizer " + "does not define one.") + + return tokenizer.apply_chat_template( + conversation=conversation, # type: ignore[arg-type] + chat_template=chat_template, + tokenize=tokenize, + **kwargs, + ) + + +def apply_mistral_chat_template( + tokenizer: MistralTokenizer, + messages: List[ChatCompletionMessageParam], + chat_template: Optional[str] = None, + **kwargs: Any, +) -> List[int]: + if chat_template is not None: + logger.warning( + "'chat_template' cannot be overridden for mistral tokenizer.") + if "add_generation_prompt" in kwargs: + logger.warning( + "'add_generation_prompt' is not supported for mistral tokenizer, " + "so it will be ignored.") + if "continue_final_message" in kwargs: + logger.warning( + "'continue_final_message' is not supported for mistral tokenizer, " + "so it will be ignored.") + + return tokenizer.apply_chat_template( + messages=messages, + **kwargs, + ) diff --git a/vllm/entrypoints/launcher.py b/vllm/entrypoints/launcher.py new file mode 100644 index 00000000..5dcf50bd --- /dev/null +++ b/vllm/entrypoints/launcher.py @@ -0,0 +1,103 @@ +import asyncio +import signal +from http import HTTPStatus +from typing import Any + +import uvicorn +from fastapi import FastAPI, Request, Response + +from vllm import envs +from vllm.engine.async_llm_engine import AsyncEngineDeadError +from vllm.engine.multiprocessing import MQEngineDeadError +from vllm.logger import init_logger +from vllm.utils import find_process_using_port + +logger = init_logger(__name__) + + +async def serve_http(app: FastAPI, **uvicorn_kwargs: Any): + logger.info("Available routes are:") + for route in app.routes: + methods = getattr(route, "methods", None) + path = getattr(route, "path", None) + + if methods is None or path is None: + continue + + logger.info("Route: %s, Methods: %s", path, ', '.join(methods)) + + config = uvicorn.Config(app, **uvicorn_kwargs) + server = uvicorn.Server(config) + _add_shutdown_handlers(app, server) + + loop = asyncio.get_running_loop() + + server_task = loop.create_task(server.serve()) + + def signal_handler() -> None: + # prevents the uvicorn signal handler to exit early + server_task.cancel() + + async def dummy_shutdown() -> None: + pass + + loop.add_signal_handler(signal.SIGINT, signal_handler) + loop.add_signal_handler(signal.SIGTERM, signal_handler) + + try: + await server_task + return dummy_shutdown() + except asyncio.CancelledError: + port = uvicorn_kwargs["port"] + process = find_process_using_port(port) + if process is not None: + logger.debug( + "port %s is used by process %s launched with command:\n%s", + port, process, " ".join(process.cmdline())) + logger.info("Shutting down FastAPI HTTP server.") + return server.shutdown() + + +def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None: + """Adds handlers for fatal errors that should crash the server""" + + @app.exception_handler(RuntimeError) + async def runtime_error_handler(request: Request, __): + """On generic runtime error, check to see if the engine has died. + It probably has, in which case the server will no longer be able to + handle requests. Trigger a graceful shutdown with a SIGTERM.""" + engine = request.app.state.engine_client + if (not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine.errored + and not engine.is_running): + logger.fatal("AsyncLLMEngine has failed, terminating server " + "process") + # See discussions here on shutting down a uvicorn server + # https://github.com/encode/uvicorn/discussions/1103 + # In this case we cannot await the server shutdown here because + # this handler must first return to close the connection for + # this request. + server.should_exit = True + + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + @app.exception_handler(AsyncEngineDeadError) + async def async_engine_dead_handler(_, __): + """Kill the server if the async engine is already dead. It will + not handle any further requests.""" + if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: + logger.fatal("AsyncLLMEngine is already dead, terminating server " + "process") + server.should_exit = True + + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + @app.exception_handler(MQEngineDeadError) + async def mq_engine_dead_handler(_, __): + """Kill the server if the mq engine is already dead. It will + not handle any further requests.""" + if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: + logger.fatal("MQLLMEngine is already dead, terminating server " + "process") + server.should_exit = True + + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py new file mode 100644 index 00000000..20103810 --- /dev/null +++ b/vllm/entrypoints/llm.py @@ -0,0 +1,909 @@ +import itertools +import warnings +from contextlib import contextmanager +from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, + Union, cast, overload) + +from tqdm import tqdm + +from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput, + BeamSearchSequence, get_beam_search_score) +from vllm.engine.arg_utils import EngineArgs +from vllm.engine.llm_engine import LLMEngine +from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam, + apply_hf_chat_template, + apply_mistral_chat_template, + parse_chat_messages) +from vllm.inputs import PromptType, TextPrompt, TokensPrompt +from vllm.inputs.parse import parse_and_batch_prompt +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.guided_decoding.guided_fields import ( + GuidedDecodingRequest, LLMGuidedOptions) +from vllm.outputs import EmbeddingRequestOutput, RequestOutput +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams, + RequestOutputKind, SamplingParams) +from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer, + get_cached_tokenizer) +from vllm.transformers_utils.tokenizer_group import TokenizerGroup +from vllm.usage.usage_lib import UsageContext +from vllm.utils import Counter, deprecate_kwargs, is_list_of + +logger = init_logger(__name__) + + +class LLM: + """An LLM for generating texts from given prompts and sampling parameters. + + This class includes a tokenizer, a language model (possibly distributed + across multiple GPUs), and GPU memory space allocated for intermediate + states (aka KV cache). Given a batch of prompts and sampling parameters, + this class generates texts from the model, using an intelligent batching + mechanism and efficient memory management. + + Args: + model: The name or path of a HuggingFace Transformers model. + tokenizer: The name or path of a HuggingFace Transformers tokenizer. + tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer + if available, and "slow" will always use the slow tokenizer. + skip_tokenizer_init: If true, skip initialization of tokenizer and + detokenizer. Expect valid prompt_token_ids and None for prompt + from the input. + trust_remote_code: Trust remote code (e.g., from HuggingFace) when + downloading the model and tokenizer. + tensor_parallel_size: The number of GPUs to use for distributed + execution with tensor parallelism. + dtype: The data type for the model weights and activations. Currently, + we support `float32`, `float16`, and `bfloat16`. If `auto`, we use + the `torch_dtype` attribute specified in the model config file. + However, if the `torch_dtype` in the config is `float32`, we will + use `float16` instead. + quantization: The method used to quantize the model weights. Currently, + we support "awq", "gptq", and "fp8" (experimental). + If None, we first check the `quantization_config` attribute in the + model config file. If that is None, we assume the model weights are + not quantized and use `dtype` to determine the data type of + the weights. + revision: The specific model version to use. It can be a branch name, + a tag name, or a commit id. + tokenizer_revision: The specific tokenizer version to use. It can be a + branch name, a tag name, or a commit id. + seed: The seed to initialize the random number generator for sampling. + gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to + reserve for the model weights, activations, and KV cache. Higher + values will increase the KV cache size and thus improve the model's + throughput. However, if the value is too high, it may cause out-of- + memory (OOM) errors. + swap_space: The size (GiB) of CPU memory per GPU to use as swap space. + This can be used for temporarily storing the states of the requests + when their `best_of` sampling parameters are larger than 1. If all + requests will have `best_of=1`, you can safely set this to 0. + Otherwise, too small values may cause out-of-memory (OOM) errors. + cpu_offload_gb: The size (GiB) of CPU memory to use for offloading + the model weights. This virtually increases the GPU memory space + you can use to hold the model weights, at the cost of CPU-GPU data + transfer for every forward pass. + enforce_eager: Whether to enforce eager execution. If True, we will + disable CUDA graph and always execute the model in eager mode. + If False, we will use CUDA graph and eager execution in hybrid. + max_context_len_to_capture: Maximum context len covered by CUDA graphs. + When a sequence has context length larger than this, we fall back + to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead). + max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs. + When a sequence has context length larger than this, we fall back + to eager mode. Additionally for encoder-decoder models, if the + sequence length of the encoder input is larger than this, we fall + back to the eager mode. + disable_custom_all_reduce: See ParallelConfig + **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See + :ref:`engine_args`) + + Note: + This class is intended to be used for offline inference. For online + serving, use the :class:`~vllm.AsyncLLMEngine` class instead. + """ + + DEPRECATE_LEGACY: ClassVar[bool] = False + """A flag to toggle whether to deprecate the legacy generate/encode API.""" + + @classmethod + @contextmanager + def deprecate_legacy_api(cls): + cls.DEPRECATE_LEGACY = True + + yield + + cls.DEPRECATE_LEGACY = False + + def __init__( + self, + model: str, + tokenizer: Optional[str] = None, + tokenizer_mode: str = "auto", + skip_tokenizer_init: bool = False, + trust_remote_code: bool = False, + tensor_parallel_size: int = 1, + dtype: str = "auto", + quantization: Optional[str] = None, + revision: Optional[str] = None, + tokenizer_revision: Optional[str] = None, + seed: int = 0, + gpu_memory_utilization: float = 0.9, + swap_space: float = 4, + cpu_offload_gb: float = 0, + enforce_eager: Optional[bool] = None, + max_context_len_to_capture: Optional[int] = None, + max_seq_len_to_capture: int = 8192, + disable_custom_all_reduce: bool = False, + disable_async_output_proc: bool = False, + mm_processor_kwargs: Optional[Dict[str, Any]] = None, + **kwargs, + ) -> None: + ''' + LLM constructor. + + Note: if enforce_eager is unset (enforce_eager is None) + it defaults to False. + ''' + + if "disable_log_stats" not in kwargs: + kwargs["disable_log_stats"] = True + + engine_args = EngineArgs( + model=model, + tokenizer=tokenizer, + tokenizer_mode=tokenizer_mode, + skip_tokenizer_init=skip_tokenizer_init, + trust_remote_code=trust_remote_code, + tensor_parallel_size=tensor_parallel_size, + dtype=dtype, + quantization=quantization, + revision=revision, + tokenizer_revision=tokenizer_revision, + seed=seed, + gpu_memory_utilization=gpu_memory_utilization, + swap_space=swap_space, + cpu_offload_gb=cpu_offload_gb, + enforce_eager=enforce_eager, + max_context_len_to_capture=max_context_len_to_capture, + max_seq_len_to_capture=max_seq_len_to_capture, + disable_custom_all_reduce=disable_custom_all_reduce, + disable_async_output_proc=disable_async_output_proc, + mm_processor_kwargs=mm_processor_kwargs, + **kwargs, + ) + self.llm_engine = LLMEngine.from_engine_args( + engine_args, usage_context=UsageContext.LLM_CLASS) + self.request_counter = Counter() + + def get_tokenizer(self) -> AnyTokenizer: + return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer + + def set_tokenizer(self, tokenizer: AnyTokenizer) -> None: + tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup) + + # While CachedTokenizer is dynamic, have no choice but + # compare class name. Misjudgment will arise from + # user-defined tokenizer started with 'Cached' + if tokenizer.__class__.__name__.startswith("Cached"): + tokenizer_group.tokenizer = tokenizer + else: + tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer) + + @overload # LEGACY: single (prompt + optional token ids) + def generate( + self, + prompts: str, + sampling_params: Optional[Union[SamplingParams, + List[SamplingParams]]] = None, + prompt_token_ids: Optional[List[int]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @overload # LEGACY: multi (prompt + optional token ids) + def generate( + self, + prompts: List[str], + sampling_params: Optional[Union[SamplingParams, + List[SamplingParams]]] = None, + prompt_token_ids: Optional[List[List[int]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @overload # LEGACY: single (token ids + optional prompt) + def generate( + self, + prompts: Optional[str] = None, + sampling_params: Optional[Union[SamplingParams, + List[SamplingParams]]] = None, + *, + prompt_token_ids: List[int], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @overload # LEGACY: multi (token ids + optional prompt) + def generate( + self, + prompts: Optional[List[str]] = None, + sampling_params: Optional[Union[SamplingParams, + List[SamplingParams]]] = None, + *, + prompt_token_ids: List[List[int]], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @overload # LEGACY: single or multi token ids [pos-only] + def generate( + self, + prompts: None, + sampling_params: None, + prompt_token_ids: Union[List[int], List[List[int]]], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @overload + def generate( + self, + prompts: Union[PromptType, Sequence[PromptType]], + /, + *, + sampling_params: Optional[Union[SamplingParams, + Sequence[SamplingParams]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[RequestOutput]: + ... + + @deprecate_kwargs( + "prompt_token_ids", + is_deprecated=lambda: LLM.DEPRECATE_LEGACY, + additional_message="Please use the 'prompts' parameter instead.", + ) + def generate( + self, + prompts: Union[Union[PromptType, Sequence[PromptType]], + Optional[Union[str, List[str]]]] = None, + sampling_params: Optional[Union[SamplingParams, + Sequence[SamplingParams]]] = None, + prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + guided_options_request: Optional[Union[LLMGuidedOptions, + GuidedDecodingRequest]] = None, + priority: Optional[List[int]] = None, + ) -> List[RequestOutput]: + """Generates the completions for the input prompts. + + This class automatically batches the given prompts, considering + the memory constraint. For the best performance, put all of your prompts + into a single list and pass it to this method. + + Args: + prompts: The prompts to the LLM. You may pass a sequence of prompts + for batch inference. See :class:`~vllm.inputs.PromptType` + for more details about the format of each prompts. + sampling_params: The sampling parameters for text generation. If + None, we use the default sampling parameters. + When it is a single value, it is applied to every prompt. + When it is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. + use_tqdm: Whether to use tqdm to display the progress bar. + lora_request: LoRA request to use for generation, if any. + prompt_adapter_request: Prompt Adapter request to use for + generation, if any. + priority: The priority of the requests, if any. + Only applicable when priority scheduling policy is enabled. + + Returns: + A list of ``RequestOutput`` objects containing the + generated completions in the same order as the input prompts. + + Note: + Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is + considered legacy and may be deprecated in the future. You should + instead pass them via the ``inputs`` parameter. + """ + if self.llm_engine.model_config.embedding_mode: + raise ValueError( + "LLM.generate() is only supported for (conditional) generation " + "models (XForCausalLM, XForConditionalGeneration).") + + if prompt_token_ids is not None: + parsed_prompts = self._convert_v1_inputs( + prompts=cast(Optional[Union[str, List[str]]], prompts), + prompt_token_ids=prompt_token_ids, + ) + else: + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]], + prompts) + + if isinstance(guided_options_request, dict): + if len(guided_options_request) > 1: + raise ValueError( + "You can only use one guided decoding but multiple is " + f"specified: {guided_options_request}") + guided_options_request = GuidedDecodingRequest( + **guided_options_request) + + if sampling_params is None: + # Use default sampling params. + sampling_params = SamplingParams() + + self._validate_and_add_requests( + prompts=parsed_prompts, + params=sampling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + guided_options=guided_options_request, + priority=priority) + + outputs = self._run_engine(use_tqdm=use_tqdm) + return LLMEngine.validate_outputs(outputs, RequestOutput) + + def beam_search( + self, + prompts: List[Union[str, List[int]]], + params: BeamSearchParams, + ) -> List[BeamSearchOutput]: + """ + Generate sequences using beam search. + + Args: + prompts: A list of prompts. Each prompt can be a string or a list + of token IDs. + params: The beam search parameters. + + TODO: how does beam search work together with length penalty, frequency + penalty, and stopping criteria, etc.? + """ + + beam_width = params.beam_width + max_tokens = params.max_tokens + temperature = params.temperature + ignore_eos = params.ignore_eos + length_penalty = params.length_penalty + + def sort_beams_key(x: BeamSearchSequence) -> float: + return get_beam_search_score(x.tokens, x.cum_logprob, + tokenizer.eos_token_id, + length_penalty) + + tokenizer = self.get_tokenizer() + # generate 2 * beam_width candidates at each step + # following the huggingface transformers implementation + # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa + beam_search_params = SamplingParams(logprobs=2 * beam_width, + max_tokens=1, + temperature=temperature) + instances: List[BeamSearchInstance] = [] + + for prompt in prompts: + prompt_tokens = prompt if isinstance( + prompt, list) else tokenizer.encode(prompt) + instances.append(BeamSearchInstance(prompt_tokens)) + + for _ in range(max_tokens): + all_beams: List[BeamSearchSequence] = list( + sum((instance.beams for instance in instances), [])) + pos = [0] + list( + itertools.accumulate( + len(instance.beams) for instance in instances)) + instance_start_and_end: List[Tuple[int, int]] = list( + zip(pos[:-1], pos[1:])) + + if len(all_beams) == 0: + break + + prompts_batch = [ + TokensPrompt(prompt_token_ids=beam.tokens) + for beam in all_beams + ] + + # only runs for one step + # we don't need to use tqdm here + output = self.generate(prompts_batch, + sampling_params=beam_search_params, + use_tqdm=False) + + for (start, end), instance in zip(instance_start_and_end, + instances): + instance_new_beams = [] + for i in range(start, end): + current_beam = all_beams[i] + result = output[i] + + if result.outputs[0].logprobs is not None: + # if `result.outputs[0].logprobs` is None, it means + # the sequence is completed because of the max-model-len + # or abortion. we don't need to add it to the new beams. + logprobs = result.outputs[0].logprobs[0] + for token_id, logprob_obj in logprobs.items(): + new_beam = BeamSearchSequence( + tokens=current_beam.tokens + [token_id], + cum_logprob=current_beam.cum_logprob + + logprob_obj.logprob) + + if token_id == tokenizer.eos_token_id and \ + not ignore_eos: + instance.completed.append(new_beam) + else: + instance_new_beams.append(new_beam) + sorted_beams = sorted(instance_new_beams, + key=sort_beams_key, + reverse=True) + instance.beams = sorted_beams[:beam_width] + + outputs = [] + for instance in instances: + instance.completed.extend(instance.beams) + sorted_completed = sorted(instance.completed, + key=sort_beams_key, + reverse=True) + best_beams = sorted_completed[:beam_width] + + for beam in best_beams: + beam.text = tokenizer.decode(beam.tokens) + outputs.append(BeamSearchOutput(sequences=best_beams)) + + return outputs + + def chat( + self, + messages: Union[List[ChatCompletionMessageParam], + List[List[ChatCompletionMessageParam]]], + sampling_params: Optional[Union[SamplingParams, + List[SamplingParams]]] = None, + use_tqdm: bool = True, + lora_request: Optional[LoRARequest] = None, + chat_template: Optional[str] = None, + add_generation_prompt: bool = True, + continue_final_message: bool = False, + tools: Optional[List[Dict[str, Any]]] = None, + mm_processor_kwargs: Optional[Dict[str, Any]] = None, + ) -> List[RequestOutput]: + """ + Generate responses for a chat conversation. + + The chat conversation is converted into a text prompt using the + tokenizer and calls the :meth:`generate` method to generate the + responses. + + Multi-modal inputs can be passed in the same way you would pass them + to the OpenAI API. + + Args: + messages: A list of conversations or a single conversation. + - Each conversation is represented as a list of messages. + - Each message is a dictionary with 'role' and 'content' keys. + sampling_params: The sampling parameters for text generation. + If None, we use the default sampling parameters. When it + is a single value, it is applied to every prompt. When it + is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. + use_tqdm: Whether to use tqdm to display the progress bar. + lora_request: LoRA request to use for generation, if any. + chat_template: The template to use for structuring the chat. + If not provided, the model's default chat template will be used. + add_generation_prompt: If True, adds a generation template + to each message. + continue_final_message: If True, continues the final message in + the conversation instead of starting a new one. Cannot be `True` + if `add_generation_prompt` is also `True`. + mm_processor_kwargs: Multimodal processor kwarg overrides for this + chat request. Only used for offline requests. + + Returns: + A list of ``RequestOutput`` objects containing the generated + responses in the same order as the input messages. + """ + list_of_messages: List[List[ChatCompletionMessageParam]] + + # Handle multi and single conversations + if is_list_of(messages, list): + # messages is List[List[...]] + list_of_messages = cast(List[List[ChatCompletionMessageParam]], + messages) + else: + # messages is List[...] + list_of_messages = [ + cast(List[ChatCompletionMessageParam], messages) + ] + + prompts: List[Union[TokensPrompt, TextPrompt]] = [] + + for msgs in list_of_messages: + tokenizer = self.get_tokenizer() + model_config = self.llm_engine.get_model_config() + + # NOTE: _parse_chat_message_content_parts() currently doesn't + # handle mm_processor_kwargs, since there is no implementation in + # the chat message parsing for it. + conversation, mm_data = parse_chat_messages( + msgs, model_config, tokenizer) + + prompt_data: Union[str, List[int]] + if isinstance(tokenizer, MistralTokenizer): + prompt_data = apply_mistral_chat_template( + tokenizer, + messages=msgs, + chat_template=chat_template, + add_generation_prompt=add_generation_prompt, + continue_final_message=continue_final_message, + tools=tools, + ) + else: + prompt_data = apply_hf_chat_template( + tokenizer, + conversation=conversation, + chat_template=chat_template, + add_generation_prompt=add_generation_prompt, + continue_final_message=continue_final_message, + tools=tools, + ) + + prompt: Union[TokensPrompt, TextPrompt] + if is_list_of(prompt_data, int): + prompt = TokensPrompt(prompt_token_ids=prompt_data) + else: + prompt = TextPrompt(prompt=prompt_data) + + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + + if mm_processor_kwargs is not None: + prompt["mm_processor_kwargs"] = mm_processor_kwargs + + prompts.append(prompt) + + return self.generate( + prompts, + sampling_params=sampling_params, + use_tqdm=use_tqdm, + lora_request=lora_request, + ) + + @overload # LEGACY: single (prompt + optional token ids) + def encode( + self, + prompts: str, + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + prompt_token_ids: Optional[List[int]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @overload # LEGACY: multi (prompt + optional token ids) + def encode( + self, + prompts: List[str], + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + prompt_token_ids: Optional[List[List[int]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @overload # LEGACY: single (token ids + optional prompt) + def encode( + self, + prompts: Optional[str] = None, + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + *, + prompt_token_ids: List[int], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @overload # LEGACY: multi (token ids + optional prompt) + def encode( + self, + prompts: Optional[List[str]] = None, + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + *, + prompt_token_ids: List[List[int]], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @overload # LEGACY: single or multi token ids [pos-only] + def encode( + self, + prompts: None, + pooling_params: None, + prompt_token_ids: Union[List[int], List[List[int]]], + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @overload + def encode( + self, + prompts: Union[PromptType, Sequence[PromptType]], + /, + *, + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + ) -> List[EmbeddingRequestOutput]: + ... + + @deprecate_kwargs( + "prompt_token_ids", + is_deprecated=lambda: LLM.DEPRECATE_LEGACY, + additional_message="Please use the 'prompts' parameter instead.", + ) + def encode( + self, + prompts: Union[Union[PromptType, Sequence[PromptType]], + Optional[Union[str, List[str]]]] = None, + pooling_params: Optional[Union[PoolingParams, + Sequence[PoolingParams]]] = None, + prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None, + use_tqdm: bool = True, + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + ) -> List[EmbeddingRequestOutput]: + """Generates the completions for the input prompts. + + This class automatically batches the given prompts, considering + the memory constraint. For the best performance, put all of your prompts + into a single list and pass it to this method. + + Args: + prompts: The prompts to the LLM. You may pass a sequence of prompts + for batch inference. See :class:`~vllm.inputs.PromptType` + for more details about the format of each prompts. + pooling_params: The pooling parameters for pooling. If None, we + use the default pooling parameters. + use_tqdm: Whether to use tqdm to display the progress bar. + lora_request: LoRA request to use for generation, if any. + prompt_adapter_request: Prompt Adapter request to use for + generation, if any. + + Returns: + A list of `EmbeddingRequestOutput` objects containing the + generated embeddings in the same order as the input prompts. + + Note: + Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is + considered legacy and may be deprecated in the future. You should + instead pass them via the ``inputs`` parameter. + """ + if not self.llm_engine.model_config.embedding_mode: + raise ValueError( + "LLM.encode() is only supported for embedding models (XModel)." + ) + + if prompt_token_ids is not None: + parsed_prompts = self._convert_v1_inputs( + prompts=cast(Optional[Union[str, List[str]]], prompts), + prompt_token_ids=prompt_token_ids, + ) + else: + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]], + prompts) + + if pooling_params is None: + # Use default pooling params. + pooling_params = PoolingParams() + + self._validate_and_add_requests( + prompts=parsed_prompts, + params=pooling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + + outputs = self._run_engine(use_tqdm=use_tqdm) + return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput) + + def start_profile(self) -> None: + self.llm_engine.start_profile() + + def stop_profile(self) -> None: + self.llm_engine.stop_profile() + + # LEGACY + def _convert_v1_inputs( + self, + prompts: Optional[Union[str, List[str]]], + prompt_token_ids: Optional[Union[List[int], List[List[int]]]], + ): + # skip_tokenizer_init is now checked in engine + + if prompts is not None: + prompts = [p["content"] for p in parse_and_batch_prompt(prompts)] + if prompt_token_ids is not None: + prompt_token_ids = [ + p["content"] for p in parse_and_batch_prompt(prompt_token_ids) + ] + + num_requests = None + if prompts is not None: + num_requests = len(prompts) + if prompt_token_ids is not None: + if (num_requests is not None + and num_requests != len(prompt_token_ids)): + raise ValueError("The lengths of prompts and prompt_token_ids " + "must be the same.") + + num_requests = len(prompt_token_ids) + if num_requests is None: + raise ValueError("Either prompts or prompt_token_ids must be " + "provided.") + + parsed_prompts: List[PromptType] = [] + for i in range(num_requests): + item: PromptType + + if prompts is not None: + item = TextPrompt(prompt=prompts[i]) + elif prompt_token_ids is not None: + item = TokensPrompt(prompt_token_ids=prompt_token_ids[i]) + else: + raise AssertionError + + parsed_prompts.append(item) + + return parsed_prompts + + def _validate_and_add_requests( + self, + prompts: Union[PromptType, Sequence[PromptType]], + params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams, + Sequence[PoolingParams]], + lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]], + prompt_adapter_request: Optional[PromptAdapterRequest], + guided_options: Optional[GuidedDecodingRequest] = None, + priority: Optional[List[int]] = None, + ) -> None: + if guided_options is not None: + warnings.warn( + "guided_options_request is deprecated, use " + "SamplingParams.guided_decoding instead", + DeprecationWarning, + stacklevel=2, + ) + + if isinstance(prompts, (str, dict)): + # Convert a single prompt to a list. + prompts = [prompts] + + num_requests = len(prompts) + if isinstance(params, list) and len(params) != num_requests: + raise ValueError("The lengths of prompts and params " + "must be the same.") + if isinstance(lora_request, + list) and len(lora_request) != num_requests: + raise ValueError("The lengths of prompts and lora_request " + "must be the same.") + + for sp in params if isinstance(params, list) else (params, ): + if isinstance(sp, SamplingParams): + self._add_guided_params(sp, guided_options) + + # We only care about the final output + sp.output_kind = RequestOutputKind.FINAL_ONLY + + # Add requests to the engine. + for i, prompt in enumerate(prompts): + self._add_request( + prompt, + params[i] if isinstance(params, Sequence) else params, + lora_request=lora_request[i] if isinstance( + lora_request, Sequence) else lora_request, + prompt_adapter_request=prompt_adapter_request, + priority=priority[i] if priority else 0, + ) + + def _add_request( + self, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + request_id = str(next(self.request_counter)) + self.llm_engine.add_request( + request_id, + prompt, + params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + priority=priority, + ) + + def _add_guided_params( + self, + params: SamplingParams, + guided_options: Optional[GuidedDecodingRequest] = None): + if guided_options is None: + return params + + if params.guided_decoding is not None: + raise ValueError("Cannot set both guided_options_request and" + "params.guided_decoding.") + + params.guided_decoding = GuidedDecodingParams( + json=guided_options.guided_json, + regex=guided_options.guided_regex, + choice=guided_options.guided_choice, + grammar=guided_options.guided_grammar, + json_object=guided_options.guided_json_object, + backend=guided_options.guided_decoding_backend, + whitespace_pattern=guided_options.guided_whitespace_pattern) + return params + + def _run_engine( + self, *, use_tqdm: bool + ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]: + # Initialize tqdm. + if use_tqdm: + num_requests = self.llm_engine.get_num_unfinished_requests() + pbar = tqdm( + total=num_requests, + desc="Processed prompts", + dynamic_ncols=True, + postfix=(f"est. speed input: {0:.2f} toks/s, " + f"output: {0:.2f} toks/s"), + ) + + # Run the engine. + outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = [] + total_in_toks = 0 + total_out_toks = 0 + while self.llm_engine.has_unfinished_requests(): + step_outputs = self.llm_engine.step() + for output in step_outputs: + if output.finished: + outputs.append(output) + if use_tqdm: + if isinstance(output, RequestOutput): + # Calculate tokens only for RequestOutput + assert output.prompt_token_ids is not None + total_in_toks += len(output.prompt_token_ids) + in_spd = total_in_toks / pbar.format_dict["elapsed"] + total_out_toks += sum( + len(stp.token_ids) for stp in output.outputs) + out_spd = (total_out_toks / + pbar.format_dict["elapsed"]) + pbar.postfix = ( + f"est. speed input: {in_spd:.2f} toks/s, " + f"output: {out_spd:.2f} toks/s") + pbar.update(1) + + if use_tqdm: + pbar.close() + # Sort the outputs by request ID. + # This is necessary because some requests may be finished earlier than + # its previous requests. + return sorted(outputs, key=lambda x: int(x.request_id)) + + def _is_encoder_decoder_model(self): + return self.llm_engine.is_encoder_decoder_model() + + def _is_embedding_model(self): + return self.llm_engine.is_embedding_model() diff --git a/vllm/entrypoints/logger.py b/vllm/entrypoints/logger.py new file mode 100644 index 00000000..584ee0d9 --- /dev/null +++ b/vllm/entrypoints/logger.py @@ -0,0 +1,42 @@ +from typing import List, Optional, Union + +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import BeamSearchParams, SamplingParams + +logger = init_logger(__name__) + + +class RequestLogger: + + def __init__(self, *, max_log_len: Optional[int]) -> None: + super().__init__() + + self.max_log_len = max_log_len + + def log_inputs( + self, + request_id: str, + prompt: Optional[str], + prompt_token_ids: Optional[List[int]], + params: Optional[Union[SamplingParams, PoolingParams, + BeamSearchParams]], + lora_request: Optional[LoRARequest], + prompt_adapter_request: Optional[PromptAdapterRequest], + ) -> None: + max_log_len = self.max_log_len + if max_log_len is not None: + if prompt is not None: + prompt = prompt[:max_log_len] + + if prompt_token_ids is not None: + prompt_token_ids = prompt_token_ids[:max_log_len] + + logger.info( + "Received request %s: prompt: %r, " + "params: %s, prompt_token_ids: %s, " + "lora_request: %s, prompt_adapter_request: %s.", request_id, + prompt, params, prompt_token_ids, lora_request, + prompt_adapter_request) diff --git a/vllm/entrypoints/openai/__init__.py b/vllm/entrypoints/openai/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/entrypoints/openai/__pycache__/__init__.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5f09d236f58d8c1a162ded2f8b67024e863202b GIT binary patch literal 164 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H!eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_erjGxQDs4XW?o6Lettn}USg(x he0*kJW=VX!UP0w84x8Nkl+v73JCFs%OhAH#0RV+vC>;O* literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/api_server.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/api_server.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f95699019b7db329e8588e1ba9d3eef00949986 GIT binary patch literal 14834 zcmcI~TW}j!njX5*02&|w-bG24?Y3-r1luC58GC%w$f7CAqRpF0O169K>49*XB%p2p zKHZQ+7@U<%%~+mYZOvsXwV8~w;9inaQmIWPc}N~|zgJ$8O;SlZ$v*7n=6y(|;%qjV zow4%$e>VV<62~*C1gJiJ`kepV|MTB#xm+fO-^su5H`Z@njKzMRK6d`7_6#2H542e9 zLQKSj;>67Gt(XdbtES5DxEbfSW@`LSm`Q%8%oM+MQ|EWuOyjLO8Mn{uQ>d@FlXY`u zj`Nz6cl*tL&L^A!chDT*uJ!Br@I^8+! zK43n;`LuJyeb9W6^BLzM_hIv4*DwwD5%Uo)>vN8}kD8BiKI=T@9y5<|KIc5{e!=_# z=N0F;TQCcp&pS`JPnu71zTbJuJz<{U{D3p=PM8y%A9S8}Pnst=KjeJT{gU}5&JR1! zxX+r;a(=}5viqF*9Op-!=iL{~7dU^wdC`5ze91j!p5pH@=PT}M^R)Z2`LcV)JmXH9 zlkO|#EAEsz<(@Uqy3^*gd(J%PzG}Yeo;T0CGvHJ?Omd&YH8FKjh50^X5F~ z4?7F)qPfWV2b?$DC3A`MN1SDM#a!Y1gU$u_qIr??4>@nTZ<%j#{$b~B*ECJe8_rkV zcg%O(cg=VC`w{1Bu4R55`J-mhPFF73CGqH~nEjS`Y)dhPZEwZQReQvaRn`v0P&mgsAzt^xmA6*=XvKNa^Srq+ptvW4bjmF7OEi5h1m4$HZt3_#} ztL$sb3-e2uU#;rdf!@sxF8k6hy5;IxrwGjxG%s34Vy@Px(oor==h@P?s`eFLNZ(vd z*v?d~x>{aCbw5vky1G`b+LLn4Ls6cKW@qQZQj`qNz0v)ga>_P$oRf{J<<>2m+tNrO zd>0^taj)3cy6@Lfn~Rn_TU($CEN`=ED%=M9uQt37mQA= z7yYT4TX$^Quc)`+z~16$p9b#*9okvc0Xu8^waa$3+_XDA4(%+8zy_w>OSTZa*L;S- z;Es|AD92~mIgH@IY^^9}YqB^gigh4Dr~m$yJp&H(3ZkA4UZ^_vZA@peP`9g-Gt1ap zOovvF!m-_j-CaV}fn8PIS<{KUQuJ)bhQ*@4j)FtGK%-ePcA|1{rB-tm`K0E$x1z6H zE&G-e9w{*3f`{t@+@iaKBR+7>v9FacIfPja7ibf<4@TE)->P99R?~gYD%VRmah^e+ zWdTb)oUdhllEC#04%TJOwf%Lw;aP6O@yqxwS)wePN3S@JJC07}M!i<9`rbJ8SS*hh z>*Wx2WRSOjq*d&bm`Nmee{dtt>cNS5P)tWb3<*u+=^9V6-6H9xB5}+ z!`P}K6rtYKHx)nL)>_I|Y+Kn>{X{$2Qi-Uv)ON~GRCLN$Au4UE&76o`j481szt5x*h9$y{Dp+m9?<>V`o50QcT;*wIJa+wp}-~qDF^jc_13zo!A}VM0lI6 z65n$95*q(#X2N&t6ZLZagpa#u`zM}r8|&^#_v(aGzBEza@YieA2|A349fvUy9>zrP zFvjZ}LB?9G$&yWjF3>_q_EfK{%7Btr)X$nnoboCW;_CP*r&cOD-s$mPgF*(=Ch8WX zyK|ObK?m{$BxYX;)X=y2pyhKY!C?zw4kEEct!i87&$(1AT@I4cc57GcAXzF_OSV&p z%jc<=m#K|Lc*4t;r>O8W73!U7&EU&>90}&E>WZ#p)odbzxofJ5FCAs-b^UHeQSYkP z^O`(Itu-QGG^pQ@)q9XH4Uj29zPFIMq0))A2)PtrZ6Q~QZ>i5jC;Ef_O|_+lNF`z# z{#)_RSrg)%jNrUuSAmA~e;W_g+)?HWnjFChW^NVtiH?^juNORVsZn->#dr}>^62a> z28_^wVGU9Cinz>FgBxBIy$fl39xfSn@>N=K-;P#hGW-bll<+Cs(?k0XOla>7AW^Sp zlB$|ivUh27$3K&EC_E=;sgKu@l(22xUmtSaz5F}4;=R4}^p+x@ZSmc}rH%VqJ3-f$ zuJl%XGwvsSt&(ctqWXFzE#kMeO|2D|$6DG}Y%@{q^E2(f?N~c2w3fE5V#l@fC~qa` zB!mWOBH`yM`74T?_WRod+p*%HKUf)}GLbkHt0w&6_J~LlY4AtUV-nv7M5>kCR#7t6 zJ}5voHdC!s<&b~46{9({lB;o%zO8QRExr8!5e;sWY3Y?C)T+CdSPKMbOWjO2&!uC{ zbSvF{kncZcNTjDati}9?Hhv|<2K4ptcFg;pZ;0%cBD4?Gw_{c1YV5s@Ryrb8dixO? z4gDPDx88nK{`TlB%VB1&L5$Oiq|W)^;NsH4-1N%%=?lx&+=bbd8GKJ!XJ?k0fBhWfI$`Wr zZ(J>UhG+Z6r44L?EysA4OM4@rj6p(Tc2qX`?wJOeDkm|2`MNGcgO6W|9|s}LrVA@^Yt(>w|} zSNd2%NL`H1!CwVd7>hG!jbmb^`A~>4-OBJxXe96v>3;%yM@*750P!Zf6AcOlc>u-c zXxX!{DUEu)MuXA$Kx#HeB32WQ9hC+tu*RbAOF2qS2Q89u>}z(Zfqe%}^&|~v4(uhX z@+5;VEKbi)&REbBtmWyYi_=RXR~w*SIx>t?tJTeZ61&UQZr6cY^MaIJy;25M4s_pk z>#IP$U}yz7NS|^e#CaUP{4$0Lv?{cnAQ=j6K?+(12}trwTvx8H)*zYhJ|we`W)UH0 zkXj5cnH-}Q#FC{S#Oq~&Jnlz5$gY%K8@dmR!-0-Hgj^4Pn_YrNf~v4Q4Urhi3-R+r z$*DoL(JBz*0v!^gUxJPjXqSP6!61&dBLGHH$ujSlTa=fr=-e$~7e%43#~UPx7jm8e zvmvJm4~qo`Q}2kJK`L66nU4-L>d+kMYBn8Nh(_JR#pG)!tK;#01qt|7zY5l`s~JU8 zw7bbTzK~1)MAIM|sG5>N8B$G8XsQOTrrp&51ISd8An#_9V0dVOcAq8h>RMjWAzr8` zQ%T5BvfzW1hj@VpJ=|4)tf%$39?!%x5I?fX0Y(3By|28&Gu$O~q%c#Gtfy$&-#|ia z(22PaIb3y9b-@t9Zonc1OnuubSRqV(kYAtllkHT<0zvR00fPm`+j=X(j4ds|QdBYX$zCgakn4k7 zs3Ur75LfDT$w#n=IX9nRTGOK$pnCg}#HKzG5iVSFT}E9u4}7gteEj9bQ>1^K9)F_n zSfO85Fo1L@aVg{}*E)+%vY_F;Cog;B>cI z>yG*hKmVwqvUQ(5o*(h6Z7Af^FKq~Xmh!&u=Q zWS#-#b-ebXLox)(V)kAV_3HH@kW%?gWSi-U=)~+@ss>G(csWh{Q+QOui7Pa5Qr$w8 z42lMnH6QAmJmhQ8^b^}qFe*u*+{Ufpx)kb;b?Ssli}ZDf^ATmd6Q94MMhpr@jx7v? zYxEIbg|sB3G5b3zZ)8vkGG)lU*bS()@)#gw4oQ%4$>L{;hFcFb+9Y|L>V}ZqiHG;{ zI>Dva>}qJm+`CD7XCH>KO`lRj_4)`%hfdt>Zbr>R5tiRUakFnibPa0zZkqfSK;?%> zIx88BR?=%Mf01Cii;%xWpSLMFN(s$D{xT(duXnGDhY2IUOEBL<(#%X;IXS^w;qAgh zB0Y4ZbetY_z6TFsSqZiAc6=Mml5tU;m)`~?lxm_J!Kk-hyJ{7$K;~x!IUI#|pC?{W z8TW!@RIz)`^84uH$sGuY{p2!muE-iz0z~%*$lioWslNeL){m8y&6p1i3^56&pCA>r z4(F-{*^3bKEy$M3GYj)PJ8;A}fc9=;me&)thL5XrHAu3Nzlsr?Bea=By(fC))Z6#K zKYfd5iR&HRl9s}XG;M9~o*eGnh5Ou-$2)bsdjk1Hqq}q`F)zt-$E$1(lId{*w(hEb z0wM$qJe%qizH3h|>fIBDAB0fP8fL!v45NWJ>l^av$82KSTw*eKGF< zGB5;bB=KdyZ26YN`IZn&v!8dbJKp{qC;udk^#dfBGNW4Oru;0kHK_5vv;7#g=P}z) zquH{2nt=&x$DJmaW}ar-Wg+`d*8Bf)${lLhAs58Af`N_#)@6J9QVTMb?n^D?uc7t= z_CWq=0;8|yJ1ekILWh(!Qlq2+?HN{OsJNsGhbD!^8K~HBZFID2+?IwTjD`8xw@I~z z8Y@_dWgDg;fQ(M9PUi1Nu$Qbak={bx?;=E)NB3TT`bB&YA%cNdVF_Dh)(8^RG5K|j zza*{mT_7fzm~W0OlSW4Nk7y7%K0e+Y>@J8W0MmrLL2Ld1Ng>e_X_Yuw**#>Yl@mjt9(ZZy~xL;3%=m@rDWiiUwPTo$cj=p@>ogp$C| zoayHiG*TUsdmq8%WQVBkAh^1K*aC@*2&)zm)*BcaRCJ5>Dw02#d=B0-{rxE?5wB1V z+bMED9SdDvdlBo;0Q`MItb@ov*dXvN*j2`+{j6t%7-c<0CbVQ2@CP23b>)Ov3E~y#-Zwu zh>YmlR^Ne1U^C&5`Ul!$u!k$H1grQ5+lNT0msRvgwr$Mry*K=W?ZaI9erKFklID$? z2Sm0N|5GuM`#8R(e4yNdm!gn|m%{90)dnuShI9~3R4c&*hg@3P$WxZRDg`BcX=V;? z9SC1FS%wSS3kJhaFbDyH3x~befq)*oUK8C<$o-IncXKn6>ggeY%zFyS4NQ!Df#i6p zK$@q^L?R%Z25=FOOz$TvDI6273nVUxQz=9ekS#YPthzjdUgQ-d=4gj$k*6OTQRM55 zLrUpMiZTJvpzoE*v)0n|8yBXRR|;C7SV7#rRtja7IK(xq{(0(`QSUUhI1>5J$#V)& zyG}lGE404K-v*j_S%5(%QDGtCi`1NmJTUco&)fFXdI>c0nTUn<3$M(iZqG#HIy zQWr{9r7gq;VjG2eS(Rht*V>BIByx9F)CxIORbWH>WB6sfOgce^Ii*K-=O7i~R1k-uD3q^5 zVrQnw`G^pVYrgzz)T zj&OGmfinIHI*B!hXR3U-hTDxTF`CA>F&Fs*ipFZAS|Y3=7eEmfd&G^57mk=CU9TU) zSjd}8S5l_YC*MFbU#^y%hOi?W6dau#_QLCAlaiSy7i@1>NtkH%^F4)4J-WE^&!aTx zmUb>{w;iG#KVQGt_X%i2nNY+h&TUd)bEDdB^BvzQQEEx>NS$3Vo!@ zWGMIH%nov4)`FhafDJtbic*I~5*d#-Bp61fMk_?POlQM_%*CS92+i6s=DmtOvBvx_Us#^bNqno`Lx|gZF@{#pOSxS!G!rjr{lzSv)yBc|85l^!flj;1718dyqMixSc}nkUt#O zvK<-(#TxNPd+=eeB6S-sQ|kb^#bJ8}#qM*9*EkBov3I-YRV69+x5M#dJR4JbN+q9DQB3=ft%{xk!@jKc@qBj1@*4h-I; z)1Cf3I$<5{k}*}y-c8hYh!|qY*UZX-P$g&t2;+YrpL-u#;en+hIf_!9x zqm_a?1v%Dv$y?-*3+8Ybo?tOSg2ToOb59@EKrsjifdr0ee?o0Q`{DeApZ)} zmH(QOe?!T?MFJK@c_1`ba(O@nW9A?$Pap|wmZDoA|2b-Ul!n4PqUm~)HeUV?C2ZIm z?2w~g49Fx^K0!U?I$RaR2)bKuS~N0GD;V3m7}%$4<&qr?@RFELhwEalEuW;8O-e`} zkl&zWoRSt2+;*Cj{AZM$p@h!W9J*(L9&d&@a0wAt(YgskypZ(&OKQm%9mEvzRNbTg z9m0jLp*EAG`<#{$Rzl$MuNvTBM>FWOzJRSmsS@R%O)K|ADYJua9VC>{rgi2gK2yuvFRTL4wtluG^paTNLqB6J&I5>}TB#+)itcE|-9`c7O!xWu?@kU6x=*G;PB3YG6Kzb7G10D5w zGv$x95-muom4m2-UEGKKx=Il@h{33U%v2umk05;z&qH_~#$#X(h-(npAEsN%Zz{js zG4<+if-GwI_u_G?ecfoBU)vdt)6zTYI}@P%eDg4SvcR$;n}QLU-QWpt>J)eLB66P) z4Q?JTM?9QR)gV+gLR5XxX!aY&fyi*6lYiNG-}sU8R5NaD7UW++OL-ni2~YQjBaIP# zdlL^+8y}`k+V+AvZ)Uo#SpoVu_`1DusaBL?20=ekHtGmCl)sI3A@cT!Sl%nIRf|qX z>O6}6JsLw-n@7oM9obB}pmW-&5D2f-TtKyEz+c94_y2I-6CooS8)T3%v-DyCf2;$%h za)N6k*mk@0s|3e7K8yTdQXI@>IUYg^;CuZ3u#{=wQ+jMtB+H;6$=%ML*#R zA*p&j3o&Ixg{T5zM?X3R_u%hPB#ojNbhlCy_wdUP+c&i7F`P?| zVmHJ15c!v>B`bkRSgK)mk|cu26cRJpxWXYIsnt*vmn0^HqBu#l=1^2V-j$I(7LLua zPDM`+9tVDSp(sAssqJu2FVsi#)H|7v&4Z-mz-Q{)M zJQP*+;*gypR4D|fNxG5d*uC{Eg0W;{8u!$Ou3efYhiJ+N(1|=kJ?#tIozQ~>`w%4$ zQ({o^2qi};d6bgJC^<&Se^jlW3TCKsEO;^RTm^}q?1XYFc z%-RaYCZ8ktIfBQU|B&jq>=$5WuR4rs81{?h=Wb+5H))I zLq8R2JEuvQ@sdb0F$&7w`6q5de^|ZL5zd7v%D`iG{|A&)S~k}$Xb`#e_=u_^ zNJc%xqVrvjO3tcr?K6De1wZ;M8PEPCS;@zT4i9PJU)M$yJ&C7}9?XE$^aKUoWbf(& zD9h5rLO5c*P*2fYsm^x&&U__v65lWfQSiC045`{5 F{U4h^!PEc% literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/cli_args.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/cli_args.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8274a598579ca986e60e5e24da4d18b9540502f GIT binary patch literal 6923 zcmbVR%X8eunFlZ!%!9)>^`IojF#Vv$CWdS0;jGpVF_x@SB2kVhmn&1*oq*F#4kQdP z>c)^FJlx${`l=lEn%XKUl^lG?ACSw|{s}qblEa?1a?Ht+W9RpEV}_t5RZ=??8r>hh z{<`~n_osMzx^Ce2=I{Qy_s3Ta<9}&T{Mi^>#v}iLh8b;xnam2!wyEA$+fwgRyM%Ws zwEN|Dx$m@{ex+UMSKC#S@a?eHuea;{srHl_m%~PXx;?G>PB_z_ZO``S+H-1L3FrF@ z?FH4ZhKv2B_L6B_Ggyt)KQ~yNn|DqfmD;BPnPLq<8ad<)o90VwhA-{dY?jS^Znw{} zd3;;mvDgAz#Q2>0YOy7D3ZwJP+A^A_)BmjB?gi4_2}166<0$ciNV-Xn2K~MtF*gh% z?)qYP*ym9K#8{}$dpB;}T$S}(10HQ$_kcX`lVBS}Bo}*JwA|~73%Ymt0Po240U!9n zPq>Ghhvg)k?A^2fd-QxF#JnHhS{7D2* z)5>Nyxjc6E#J~q;kiD;VH={-LT7}o7{dw$YMcz+OLgsC$(;*U3O z#%vh!kNl*U&3`0tie!WNgMeq&G~9sB7;UR&vMRH9>5k1xXv@sz4zKVkukq@#v12V8%LXe0Mx!coSmpC_ zyUt8VtCr2tp7m-QUE<1S=76o(T~>?yK9>U@+;MQ`A(z>7tT;!U2%_C?8M_$&8tBmW zKENZtMl&igV`vCx*ElRC7$xSBC1w+-_mTM@CZOd}>5B2s+lS7`Vb-1@&fQyNrICZ) zTg=AW6mN~3hb2~eWd7VbtQ=NH<%f1s9#uxwBkPfA8uu2{#l(47899&4!&*{i)xa3l zeq;X9JggJ$s7_d)0?#%Q=T7zDQ`4Zok@>L3$|L7j#(ya~Khqf2!O%1gVNmI!!oa?M zWw1(Oj;4-G;M$tn&tSBNQ~x@J-KzJ#xo;%(ku@^zOpVG%B^a7PtlfNk;p+VX?b$h7f1g;a7q_Y0%h$m)TF@g+VfcnLDgYPSyt{HhSK$jWjM21(}l0~j0=HApP8 z@5ngH%3uJCTm#xuB@Sdl#O6A-f*$1xh2EqS;jWcHg1X zW_AEW$!t=o%!Xz6@{cU&CT*MWGLwrlG?kji*VdE%V0{n_yacX-C*IrrVXyyYe}6p; zw$}#-NiUAp_rkEh4kscG25|sivL53sez4vRgAQiN*5Dwkb~;@@3_G3EAS=IzrdqMh zx>+@CtBTe#=gb*%!L+P8-t=b!TK#6mvH_j@ruy&JHw$*U{B1jFjp=@2ZhkRE%h1_6 zovhyJXnCRE=ydK4{V@Mh#h#RHWiu!3JgaSvO-+=D1!P!anHq`~qDjq!H@bI`U^8fz z@cfw$S2e7P{lvDcukDicmj&DU$|=u0Q8>?aBw`v9;Rcq!<*LX5MKs2quzOHE>J^3SiD5d z3Yun3yiEOPX=%gP4q7Ryc!kgqw+-BGwbBT?BGQVpgqhO+RfPlWLZJZB@R2!Eur|%Z{^gWEfUl{)Y zP5hMYeZdOhNvWN5(X1i*(3TipcrY*h?bnyd4q1%gtE|=nMjA&uF z?0Dbn3eMo)13#2mT_&;MokR?IIPGH^rAPilB zgfkEvaFxp*CDOK{@8pY{T~WNu8uL3VebUHL*&88-6YNC z0p<&Lm1O=dMcUOoqdN0pqh3qP=p-)$cWT zC5$2N4lPDd^#*j?mz-iD4^?cWPXSvwTy2&Mk}pRTsm1%G3QG83H`s@~*~|U1OxxP4 zx;fw(hbETgdDGjo}T2kMxlVl<=6MWUvxJa-DAoSfsUX)--ab3WK`2?LDn;zFdafOOh1bsYPiOfP@h zgQY=3k)=W9i&N-+mm3IkFG>zUB@FqV3ktcyugYdR1qQ}(;*osP_doPyvhmS%a4k-h z3cC9}j{D(YK;*W0FTkDC*TN)?WWXFI5EfbK{e!gOdHsN~ki&^_l{@%vAjobo1}{dp zM0aEB+I9CBhol1Pp|V|^ga$45Q@p!fx}5gMT2~}|eBo`m#4B|paAhwZ!uaS?5V=8L zIbS*zF1tI}^Ug3*E{zU?&E_tJF3Q@OKoQ&kYN=H;cQZp$ME5dn>Uhr=S9?h^SZ(Hn zyCGEW|3Q_AlJ*q{gXGwgD;p+vt_}GD={e;rUJ~Esk@S42dtLxH`&+0%`fS`gTy-UJhm2RxReea> zjp9W4Ll5`=n9&7)>>c2_5p%SASn)g-Nc!sS2vjq0SMs!gMT~ewyiT_vtiranpC4*J zuFDzjcy3f1E2zC9EL-j;l7sJ>0^%Xeh_Zc5)n_<>{RG_}42Hm@TtkWE zL3`+9=`K7?A~Qb|bP-9{4#`21BZdrj;o3OjIGs74qe**%aEQR1{(g&xxF)N^>)m5l znU*})(q9`QBpjJhpP~zQh2f|SPA}Nw~ThJ7CGjqmwQb0kyu2F;L;eoxroSz z=n&F!Kh{xNg+*-oIh=3hil0@GD)@W8P`Qj&{-tLfRnT26{q+__ANUoLwmNXmYRwOK zAL$c)_**A<_U3iUaf0_++sKC~hL6=;G3r<-#E@JUR20b#6KP+R{>x4Oex4PQ#X(~c zpD89Hzk_gLhGGo7ekDgF-%D~Epj4D6Hn8(ny?#ZfpO@TEqA=*=tSmiUurj!FN~KWC zL#()>ZO_e}{HoN_f~aJ-$NT=8yS4A{qKM{xa(xX>ltFY?;x5dFsggi*F2nVXQ8hr- z)kFmn$V%9ezBIl--s(Hl&{bMpw@@jOVOIO#>c_V`o3}Qv3aVGm5*+F*=*mx%V~Ex>6-w`vj{&93KR6_;3992 z`!s&yI12u?Q?gO2APff>Pxr?2?XvFWF*RjHF`q-UCtoK$TlJ`P;(TSR z-4G7?g_WEq<8_?BjB$>8mgPLDSM|B(P9kqW|Qu`CoCDnSDP#>$88YQZgiCX}4 z@l1Xv#6h)^g#O#YdVpIj>Z!&N6`xC+>8oDfr6OJn0e4))Yw03h3r;|6XqxR=&D{>1c89#hw@+~z*Tcyf#02e2kdVYXrL^BvE%8$fnisN4hsbm~Vx1OI zp5YpLnS)z1Br2%dqRVrtC`UEA_NjdL^uGlyGGkvnVspenSgs?~nwc{5IU8s)a?3RSO0yvrP2)jM`~iksq1IvYt*F{8lANnt9h sz61Toe{H-+vXw3x+o(EJLOpBFSq+p?8)z3w3$HAkGX8Du#f6!_0>NR$w*UYD literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/logits_processors.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/logits_processors.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45a3dbe6f67697b744a54132902eca92b0c26d55 GIT binary patch literal 3164 zcma)8OK%*<5$^7J?Cfw!krXLG2`o&I*zzJEDdhnN0-?yTt-yf62%#+_gBXL+b}u>P zJXYPklDJp~jz}Q6_>^N1K)OYT{DmODW3D;r>SJJw`D&Kr(xj8P7gO6^)vtPdRb9=_ zh79f2-~X|FyTjPOXmGZ*F}RD*du3x+F zXFXfD@9Ao!%1veK;6tl)`Y>0u`%z{v{`J6SRhj0s_eqJ)Mwi!@KB~4dYwqi+r;MpI zSe&5G3BTo~-5B!?nSl-LA^D$9Z0?snY7bL6w!(tW^V5rrBEFcqchk-mDJx>L5v=xJ{Bd zx}pmb@{o^i{BKLHo~)|djhRUC+MgUy`&!wdE_)6(J89JS@HK|GkItCw0A@p`->`f~ zjKz*4IsDBraxwCx_`-pKm;~LS+5S4_gX53I^A_=JAOW2hfa_Sm``*T@BebrFJ!$s_ zs`jONob}Ybk-L(W{cc;&mF{iFZdEE%duBVuO!r|`<@y@Ljj`hn*yM*uiW(L z-Wth~QK~d#o`AW0(Ft9bizAoM9W~Nd5hTEJt4N<3;J98m3HPb*!^c%GeVC9~U&Ztb zD4@uqATU1x88hQs!; zoF8XW$sWhvCyii7hyE0wc^l*pIQFwk`vg4h;x4HBOK2#rQUFq7v`o?Qe;|5)KdgmY0u!+?h z-|>PJG=_r1^#Xo{2RsmZ4Z}KYR7oCYshJ|;8-ORUp920Z494s^djbD`*7=>-clH7D zvE!j$joo+PjCJ-1oK2=F4I$RN%&Nk4J$)17=@E5s5^?}79X^fdMzQZt?{&f2q`o1X z%zK1I{WElr(^pfR&ZesXGSa(ASARoii|r#3>Gwu660v%Cs?7o4cRyw2g3B!5i#Xf& zI2(DEO9xrrdxreJ@BQ9;!k<9TWBr$*zrWua`_kP79L7j!J7U+Vx2>j@VD+(->UwMA8z>;9NSRH1hTM=fD8Lo? zzL*pv9Lp>se~fJo#Gl0@%BErqbJ2c#_oVYrWWXPSQ#(agf}~Hu=z^gM(VAd(-W%b% z#4Dm#6@xp!gxJtUeCCU#BmZdBC5)U$ggFC=)3u{d?p4F`xFDm_L|GY4{T$it)JB|AL((X5pD7+fF!6cj;T>c!h;)M$ah!=Z#S`1KqrYhEsdfNN%W?ueQOVa-4 z7a*>OY|eeh7vbwUSD-JBgb=TT&&AQazsTn)*Si6Cj{=8ghBO98uVZodYTfP+%brDr zG@5d1qh9T!#KN4Ka-JrfHx+17cgW3}-q%e9ikIj;vTC4Ijui>sm}y}&p)3ADBf{IO zG#p!#u7dzINU4D-m7?Cj>`;7UI1%tr=7%T)ya2tXP2$39+S}2L6C{o-^i3DYlw5(^z!vWp{Fyt)I zeFIX2!CJ8#r*$H?iDRerl8SMXmPws%andDD^L_oAy6N)8`4TsNZC$r+ZOe8OyG-Jk zj?Di5xo?>bG;QnqN`7b_mjUp|MX8rB3~lV_%8+E zFn-RZcqDQmVnwWIIb!l@l%o~HG@^{h%CSn^j8_t7qLMU|l`gZZk}^}3ZnInRqvf=j zM!9∨;%Vh^&5e?tE^n&j%$&q~%9|@==2$dh8&<~ZwfbH) z%q@1e)o&ZK#@dLz)yf`>ERCRd1LbX%?dEor9JkZ<_E!wGY;Q4lRFii4wFx_Y>Ao9L zbH}rhYJWUp@4WUi(Fp%H<~0X`@28NlbQ3xF@!XE480fZt|4 z3;0>dKMVNn)^mWL6ZmPsru7cM?-2N{fSRRCNN_#EJiRuOPf;PZf| ztr@^G0$%`ZSvFu>;I{#uwdMfN3H&VJd8-7tB=Fk-U$QO(zAW%_cDMa@`#G*3(?nj` zsvxf-dG7#RwQ7KC0zVIU!Fmz!ivqs@SXmBWM{xB{z;$a8@S?y4D|#+6apiS#p{a?e z+dWmR*v>+6#-2#Ju_M)MZv0qjrtZd1mz=uWb-GlyRk7@*&MwqTwJKtX3spq$=EkG* z#rn}&Wua^{>u9Z7x2yH1i;BNuWg_l&KT&k-Gd0UDyJ=+4mgc1D#EFt!w%q;-a|&0A z<&ssb*VIJJ?LJbi*6PK&ZMmsu>=zeH3bF3<*A{GxZ=%uB=0?xhj#Heob%Rm2_vu=# zT&m6q?4SqPC+uS7oLy8i^E%BPKDk)3P@O$fvx1WD09&)2`q{<$!eafYQq^+%&J{7V zpa|OTI$fJvP_^lsQMX4GF$tBzA}T9?_s*R=L?{2HBIN`gciyf4cSR-)H&!WC5fooq z>cKE)=JTaeb++cF3#ZRcoh+Q1I&l`2_Y}_5W-bF+i)DL8s%i+ZIp}TTc|`1p8MPv& zf$bt{#_YJA0Gmx(#=%H;#O|^acFOKv8xiK4v|^H)Kz>|eX|HtRAXqC(_NhpGv@8v?O$Q^n5RNjrVh}$)z zNyLp;L33uNj>#&_6wBp8$#T;HD~M+bg%=l#WiO|zP_Sw< zg@Wn_R@|6VS8jiyP}C#@igb|04OVeRR5ybzmW*4VrJE6TA5$3IJ~>&hEKDwx7IxQ* zYR<0jez>wYU)fu^I$17FPcB@m&)2GxSIXteBq&^6Td0+)b!W1+U{{N!NsvUnHd8C- z7p|#|sBJrb&M<;>Bp%H~jp(ppL{s=>lEa3SIDz?2)IsRwwwaFMy~C_v1c5)(cFc-e z#;Z{?ZpAQ1aku|0nm%$WSYBpwwx*^_7VEWNs;lDEL_%#r4QeZcZ47!4m>Iq1FcDac zAp&%4CyVVyupd8%G?$9>8IrPcKc7G`zn>vRk{U^wq(6Cr`mBVR1bxOqZwU+Z2I@;# zT}&6c>piYiO+5o%gqgE34uWp7HhswkJL&?H;!1bpb#Q|zH?cq-?QgR$Dn6_aW7~N6)ZV@Y1v>tgS z^3ll04ZLhjEv1E)7V!3NwFdc}@am=xmW!2Xt9a-wtcLYvB1>yp+Rgigz*+k(d$zb( zt`}yDGvJojT%%+`e3h+&ZRwsW&Tw0IV@M3TTaVPEuSIV}Ucr8MGdgwCQ2US#`YBcH zPF+=xBKt}F6v-dVO4;0qPPjwhP_?TC4HxX{T&ZfGcl&gz<82Z?T-IBJ;|_cIviB6Q zp_FE9$4%?hD<%7?JD}sUsIE|7tYW8eqxZR)nwl$COG`!Wsi5;YRa;zedrD4Wx{M71 zr6yLjk~?QdJ;orUd07HnoHA@hw7BtUa9nkaiN_h7U~rNFXGh84pJI$_MV)34@JJmK z7V$7L_?9z@ARQUOD&ep3e->*;v0TSFI?R{fkDzho*l{Zg$~4S`jZIA$2Ud@jz*sjnX*xfTm+$DRHYuV6GhvbN!@EuAh|TV-49!yH9HCx3l(uq;fT- zLD2nfw@~HLqOt^+ZbFFe?v$nuVHAptLmg(&30ul67P5I@j*{LSQ8ZN-P=id4pbwjRv5sx8rxgt?TY;cKUVWDD%MjVqjzi?| ziugH61l$xbET*kIEV7u+!@>}g{*H*akaO$cE^voC1=&zVQIFdT$)d7QFVt(7?W*HK zUV_BnLjyB`xwH5=6r4EZOmsM^n7ZXFLLVKw9+Ctguy8{3$BQuSdjwcAQDd&1L*<>6uJ%P)utZR3K?{o>v@{W-yd(u$hxdLFthKb0K3&Q}UKU zQ>HddZ0mMLV9Gi~Ldsq9q*JS&o0+#OMIR3*YN}GKtIcS3%I&diXGWFe$YmxSDTsqa zm$x^%37naVj@xqy)f9BLo5Y&n8L4+=M3I&?qGTCZ1UX<8V_-8l!=MvkTtvo;_&I3= zT<+OuE(R!heYcQRSSbk zI{pBvo|0%gZE%(~$xBh8_KQsGG%}L=5`O+FH!v!utvsqca>G&i%nKuvQ<$u7|5Qf8 zkITq<&($GORP;I8Iq&wCsx#$9%hp8hj_`0)sxHYFEta>7aL=)alln|LjAG+2wABoRHmU2X?;HGB1(JCRziLTZmFr!^Fl+$Ov zKEV9~^W2v(vjI$5HiNUApU?AO4Cgt&9ZIbUa|rOTypMArwA~4NzqBW9Hi6 z?E&W6qt+ng=ygb4kG{q1q*fBi`$MR&%iMrG-9Mfe6Xr(1nk)3B1s9Q8H=sn9IgS##)s0fi4#4Oe zu(?U#3BVdlIVpdi-HW`w{&?S90!ARaS!Vf#&3TM=Jl?w0sAiRUH0y#WW4s4>Pm zrIm4%>N598tve*XSKI* zuqGw<;h_I}WR|+Dz1IB@7#@)t9+0%1(AP8VQAvBqE4NS5_DkBsUfLs)c0ke|m9zwO z_k8!DjEdq$Qujbccfc!iP|7@pG8(s!dFh8F{Seam=Ho%%ANTSPOa5WxlWLCGPXx1f z#4GWHlsFpHeAG)nCh5nJF0>}SIqv11ki6r;J14#Lw@CVlApMk={-mUz4AP&H^ln%Q zIP$m19GwBSr|f>%2u{t$#Xb->pY&3mLdr9+5X8;X$Zr|TtRdEdxOv8&qMhK?sC`!3 z2VRN35;dQ0NEIuAq`tKwRcr;4`bg(+~t`TSbEcNjXB#jpu~+kwrzPwF6fCDMDB!K;;%9 zD2N23?86y)u09{a9FIm04k3;^>Pu*nj^W`XQmBcAz)hFtsx@U7>>89hY(nGoygTGv zE-e(C1$(9x9@aL9fdwiYyMEQSA+k4R-;AnPFt_eVT|th3(y7pV@ay+Le-f99r)F~~ zYwr?`d)2<0^Jpb^wNx(WrtKVN6vP2L2U6Or{@Dwwygp=h~Qoy*m0QUuB@ zTeSIkXv?P2B`vMRZA`|}2bvn4v#Xp^pc8HJlo~W$gyqZfYn-7qfb(C*u=2U{^CcJx zoLn&%2!J_Eo|T)vmTM|1)&w4VOG~!OX?1n3CY{cS{v+?MLPLeH8y?Pz`$5(0sn?QY z*EN=;dLKQSs-fM>IUG6r2-Ag}IYSg_dxpG+E>qdRTTr@7a0)ue6|2{B$IqU_V0l32 z@NC^XjSNqsw{>UyDFW8!F($NH%|XOh5c=nU5Y8gjfQKVd4LZ4NwMAlb4AVJ_nH?kA z>P>hUcXIw(&0$ozQaztLDkcz;f#B8~olGv3pShxwyVwzV7xTd}#4hL)xYz5YuLra5 z(XKb>+zvZGm(M+M_S{5g3h<}{bb`gju3sg(NdrPBn0TVTY4Ywj*Q8VFi{1 z4`7WXM|S_2BPVk^l#O$|4Xfzx;^G`D1eJzlK9SD}b)zE6s2uh=M^gtR=R+IsA9oCCD z&Y`Pg;H71I3`@X594GX#+-3WklXtuQ*PW%`I+A;`RJ~LTy9x||B!y6=f2i;Z&Vb+| z`P>n#F|xH{)geBRoV)U1{=o-#`4iJjB%C#jh3pHfN!E*P?c#doqN%_#Nyc;qwq7i* zCh{WcIS;aGg}($C1PMSJFYD0lPY6`C zmaS|B414Ujfe1PhygS^~SV1>%`youVkl}2JV`C$F`_c0`Tccq@lAkP&ok69j@@}73otLn5qq#m=dNmN1;_MS^-qOzL_W)|+ ze{Q-s1I}&v{e@J4w?gvta6f}(BHjVA_GCv9`r@FmyW)*;Obc8EY-c<-nHw)xc9VjO zb?g_2%-Cw&UF~(*Z@S<$ef#{q_WkNSTrotB@aIT6>J5o=9gHjQuJxJ=Un+PLd;3?< z10f5lR8yroW=pRHZ-BWSASjHdVq+D88>nOykyjt~cYJnql!0Uv^|NJBk>2ies2 zXzEnO5|%nPIWgq=i%w8%c|0&!cVr4dmq^C2_oKnQoAOiLL2nKVp#bY9!7CS4b;i)5 zv4qs(dOe01OGMQ=#*k)Q zkJn?%v1OKyFUJwWnEfhD)i>i)>aU?*^%8=Kgqaq}^J>Ye&#UW9|0sjM!QgKrxS7jleWT_x58fq_Scym|PYmIU)rW7l}f$v@9)*9p|jZ`!aZ-e2e z0dE5XasHCuUD|mRtSt1&@J<-{jEiF*H~E^B=W znpC7^KNz(Vcn;v1#B&f&)O2YG&lH}+c(S%tc(TS3JXz~%JTrK%!IO2b#j_93Q9S$c zT!&{C&-LmG!Wc*(f|P;f7{WoMY`}A9*+4i9Xd|Ah@Z5wa$D6}*HJ+RCT!ZHro@?>k zqCQ^VdTCoVgK#_2Mwb%^No3<6jD7&7=}u`A8>ZAeP=5YqmmBBGb`!#d-K1uqW|znL zq!bQ9nvI!*?aN&WG5riGQ!g{P#^C1|2!&GFs6NEt!wjfDRqtc)cNn~j!3_pNu0ot2 zBlvL!KhNMJ3|>Rv_UNst024KQyIRm@DCaMV+Q|(!LKu*e)LDjfee48k*l9z8>hyCm@QI}QG|PlboTGOK8wcc>Hh!1oT9*AF7OL-O;H~)w zqF9h*A4#~WV2vAn(PT7kq+za$<0+xH{*n8gbaygp+({b7_mVN=jbz;Tt7O9XZZc`y zPIejJNv4dyOm-XJPNt2&NcI?ip3E43*4JyiY}|_X8UHugZ+t7BHNKe~F#a?-X#7cX z$oM~SWco&OmGSk=2q_(!!ef5LDsA%cYh z#}oMWaw(H*M^wg&oQu@Sm*Lw{k0Ip&#>rl;gKt~0_wudiG30{RN0*H&QMIcMUxMZ6 zN1?6zc7A5Ays#1MZua92>Vj7)(RwAfhUc5^&SI$UZq2^Dr&_O|-0jih-;>>cYbzxaqeQ z%Zs-7Whq)+siy6Uc#Ppk>baFeupE62)N>=M`XpxDh&p}ssHG`@enHsF z|2oRa5?R*lHfaS%*CAP~4e06qkgidjjk>xN;az;6$=5<|-_O#%A$#piHR4;C5E z06aTVe&Df0Z;rv4Cbwg}VY?jPmGeQ7jK*`&82Kq$NE)BWYxYK_K4A!NS06xp=|EGv z-Vu(?w>F77g;nknDxI#?=kqXArc0m{Xzrjemo*R1wZ$fv zQxW9X>FM(Z6r2gA?1xDhH0r0pG!0`g%KynXQb|oDeQ`YRkX$I_$M1lN?!6m{bZn!E zUMS=T4C4<02z^9VZ@?ycEl`74!i93wywFS3Sr^btL~X*1YZjuZDt0-h_F$J5s2+R8 zaQ65#)0m37lH8?*65%Vv9oadKk||*&F$?xa93$e+*&t15^jPGDzUzr)I1qz6QD!-g zgGJFwU&j|y)#!`omJMefZ}bEVDKpiu*$ElspCvC3mVIDzxQJD+cgTQoV9Do~OiA?& z6COM8h(~w}z0#@p#_kKk8KW&@EU&BbUV3bdW{xqfx(f7lwv|62o9 zMXL~~{cCDBpF0<5GRvjQcJAW4#^@3~wm&!KyCID2YKZU8y=#nJV}Z)GLQv-V^^4*I zaDM45H3HJ_K-ms9!JvYmbLQ&yG%as*5H;MaUQHZSO8J|G>9E4b0>?%l#PXc;iFh`{{M|A528ZZzBdf+ZG3il&+f+O z#y1Rs2j4JKNRN(w&q(mO{(Jg;q!Ttqyr&Q2DfRj=>SsOZS^Cc4fN_5o&(t9L8BL*G zgu@c9l5nqq{>jK^FzBX#B#p!PIae@6&Avh0Bl-P>c)h|8gHQN9!2f|h>9hRwB~H&W zc&pI68M%OcpP_Dkj0HG}eEJG?i&sosz7L6+eSrH>Q*YpH*DdvAgL?G0BLe|U|My<_ z4B?X-i~d4g@E7vy>_ zKeW}nPg!sf>=?@~EbMTP8&-1>CTl5>k3|bFL ziHA`_W17B0Oy4Kzk04!UPHLqO5%V6Ayhnp~=tso#{gS>vNT=@*(+^7efgqi}L&Mfq zd=t!G9~8X7U&uNPcjzPbWAL@%*T8Y}keBi}QusY^+&pX_;g`Vhuz5m%0W6*{M;lV* zJ77sY){rV+154_0aTK0%2io-6;wOVoCBz+JO1Sk2&1hf3YSo%g*lu?nmknPy({^($s8Sl#8sL2HER`nj8mCw~zCSlE3R|t; z9KTN4ILxAPZ#;LM60v8!hBU0V^&{|rHs=tw!K3%{ex{pSq+1%rRd;I|lro3!lD-11LA-cest9yZsFADeWRL7UX5%4xsJYS!7%gR(|#1K*&g_BsqcIGOo|pWTeX@MGY7 zw%dxqm?MUtOElGlMjw`^ai-Ub!{C#w_rc(^2?igg_rpX&Lk|rerV{S;fIDE8h&W#q2Y&y9vXgV0rU+#Gyo0Zr1|nP?SHI|2A=hJ zj))P6<{cV=Xx^duhvprce@5}7`G-cG&FZ!K*rhGiEW)iwTfdw{NZf40b0eNG0(oYi z74s%?Q{uX(Z9wXGFeh)42_xipzh1>6|Bk`$F$hKi_xw*YK^C82gd(}Q znqF=Ny32u$H8_^3f6u~#jFxpr0sC33JBM7f(GZh`VCtJDJI!#+)Ba_ zlkSH}=37aaV&dP&6k~iPIU4_RXpYe^REYsgZ!iJ3m}AsuIov-$(8QA<>V`_iX8_c- znkZ6Bd&-+ojW`sjg`hkXHR5`tbW$UZ`zoPjUP=K2ObgY>*8Jbm8uWMn0}riI_@9im z%*B_G_EVgTVMGk5R6VVkoD`rXqe{JP^moz1H4UD}Q6fON<1`{~s!DybU8N3Wq*@i| z>c664_1_q9(JgK5pqg%1aDNU(+x643YFjY)1(f`VkHKc8^e7z*jHQNa9en!H81!}I zO-Msk0n*eT0#JX%fF0_jiI5}duiz#1Ur=IcqJ6ejFn&y&Xw@@}vxyxHQVh7A>W`r1 z0MD&Xi-n?F5Cz2ROcMqXY953$i1qws=4KeIXE4fO9RraPFEG|DCCV38>Q9)_Nkse* zGd|4V2iG15D<;1ZsjWc)A}eQt8EFP9a$b=;I%x`e`P$bQa4l)&fXi4&ClcGqw67!h z8pH&L`U-pv7~AR{|Fv2m1E^6YKt(|&Py{rq0fv#+tO&q?or-`T@qnLtNDlHD3H#(Y z&tA{q*Yqz5=rDc`WwRgUIB)6WeF6ueq1CR9i6PG zzaSWPJ6tK>M8>bPw`3cQD+LO$m{wf%y^-lwEZ0VxWL<~xbFLt0rA>Uy<*S-f^0^|f zO#pInJp!&&^KSM@$wdl&P6jztz@(^jfr=JqP(4&$hJ;^An^i*rT&sZ$fSjFe}`#; zds3~s%|O`tGGoG|J0bhq$o(vl&C@+^rWnP|6e%MOo!{Uc#oRU&2g{%2-il%%DFGxI zSa319yhh(7L5*DuXvVEBqE`W`>@@TwuzzBa_8>h2eo4)tms27Dz()$M*v@Skdm z;#bUUj~oTIPFoshBf+N;!XVrp`HV)tCP$BnfmWi}htV|{2SiK}D&W+qHy8-r++j>M zci9Ssc!VfcGouq_d>0v?CuLAR?B^gjaZ{uR~DWI)a7>8C4Jc7tgT*Z@^fv#rd zTX^JZo7eG{zJg?3JCU{AMe-Qxb^B#?hm`Q<+=TLVzZ-JliQcEWw8h?W6LR6hRQnwU zGVAQs$~W8y5pWXawu+8K5XF}CRkVMLY}~)Nh8LIMJ7(s!A(}LG7(eHo2pXrKob%0O z4w(9GT&NeUN`LCLN0Md%>aWm=1_q$l!}pl@3kl--bPZ@i+_UStIt|#KjY{daMiIJjy0Y3X0nW@I3kJYxGVr1ch zQg9)Ntazi9tR>1ddM#}Z=d8Jn4l7EL`BqdY_QtcOYCvDvsKx@Gapq z>q!8Z=x^n(_qvQYol=F~gWt{?JCQ{S9%3)r|aUMo(4#~2SB=my9tr8Sz z^zBdnSM~BWv$qS!F|i4ERfEK*zf(O%ft2e5yRt6|3#O;q@0SQy1gAnXl<=S^@J>`_U6kfun7ecOpHJwB8ib8}M6h zAiZaaNRr%-z`BXam033~HIDSLXpFplEe zF>!=(yZz7QoGIm?j*cYXho)Dfso*9d>>W&7u{+MemhP;`{UCD-2ZI5aW;>C<^_`rl z^EmymqiwA;(G5ibxEaDhhvEPoQ@)Q(umJ0%cvvk(Ym8o(+Cv=_#o%5h|D)3Qs34RUVaJ~7Ge+PwL3gz#@&HqcDY_`oa`x? zZPkyq+Ar7H%&v>bDaVZ+1X;4*zHB~?qw2Eytz3Al_s zGRe_;JB*Qu4aXHzR~~CCbJ1h<_EGH-fVF^=CyxB08W6<*O7VElyPA@2553#Y=<8M0 zreIk5RzePQBGW2IglmHc(8Cf&AlnSz(Fb}Nd8eU{U{D7dhKh-W$iUQhyA@q(C1Olqlg7up4R}Mdd!nzSIjV14*7LZG z>N;Ey7)}}s=t*P0^dY>{7}g>8BxjCiqjUtb#*~%EL1GBU;WKSsLW3E??Rg%fj=3Y9 zN8zopwp-A0>~Dy3c4FmDuSf1vVW%OtF{Q23!Ex_xO`Xr(;?)v6|`iGLnLmoMp*79h}f@nUz-}#QKM>|alS~x;|L==(@RVJNgbpr~Y zC4xv0fqKMPZz$%pAu3Rsy{`pP_q3}uUV)4&KC!1kK^CY+dgQi5ts3b?9yQ__xWDnj zV_w^uwg(!NXZpNH)k$9~1^I|=>%DXYsnUuL@d4ru~IdWYQ9c@8^ll}r7s zJpX2^QJ(j^(jbNN0H~#KtzK)p5>{4wCoPJ|YEKX)vNH=8Nb~b?()$vM2+~CwL`x$Tl4aFic zBD)!pUc3!$N_I58`yN0WvTwI*Nx1FlAX$DE@qocTz;}D4ey@6A~ z2>vLkc)97{56fz_+F7$2?rxn}4IN0yYPwo))bm$Fn^wt9Sd@}ImUBL+ReJ2wjMWe7`2J0 zx~u3-Ly~4J+nO0mb32{b%c>UklAoDhtX_U|v`lrsTP;jxO5yJgX;!m>Vdq|%4Cnh^ z6ir)D?nU$_P&E0h1?!7>Z8Lzfnz8Ue6Vi+?e~7&dheffD@kx6a9EI%^_ZGlhTePiq zxdTt|gBQ=PdWr_srT3(B_(yvSF7MgRi$8MOz0v<9W?H_H5$h{%l`7l$w7fBO_Wbev z`1>08#HXsSN`apwEhQHxuk4-VL{5hLgmV!V6-)lqM6-LORONQ-Kfb{~osrZ|{cdbg zm3hazb*+P>w;gzBP@a=RiF>;k9y@Zgt-$LNajsWsJeo?S;;Hy>Ofjbo;{ivz3&xWg zZ7V&i)+t&cfy5Q_eO^Y^$k>eUNfg( zmLaws8^Z0KJ=RV%a$N5;x;)14YaE>>0~!rS48_#8kp4`H;!BHVd>5(#&k2Bg+#O|fVqXqcC_Mz?a4f`@A_`?Ut)}tD_=Efhb8u*pD(ly>QkIRdYEXP{h)kN%Rk41 z?|YcvHkBx^S2XLn8xI z$(Td*hQru{ZvGyJ5z-+#&(xJWKZY7RbiRQOJ?#M$q;+pqHBpZ4ZrfGXEmI`D@6_k} z(YxPgpOfeaQI8%oZN(m|7f}$GRu4AzM{fbLuJ59ge*pFV5!zA|5EY@7Tj@nE)WRdD zYenmHO%;z=#Rd2*ctdx)%Se+?O-&uewOR9bY5GA__jxv*MMM!=v1w=53R7a?5(cPt zMEnpYLH`nckNTPmReTlMp`+>{G^OP*w@3Pnzib3ypq;6Q6DBL^VfM3&cv^Al_MylZ zFs5MYAY?IZ;-Wf&7u^)>B={HvS9uu(llaa9{vUdiARp9(%y|ztToal${OL1uK>yLK zJiiTbz!~txXEzF(sfBA+5f|XkX!pe)zAXKvTcSCjn<&8D$ahhr#Zmt7m&&rlI{x95 zJj5#huX9P=g~fUaw{Z-x#E_I|_6rlwGjmvSTK%Ah*Z{pndDxYk(uCYDi_UB`fiF8T9Z@<*!V@9oH+U6DUhqR;Nnv&cmT zvkXcMDhz526oW+uR~USq!8aLvn}Ik}M1@`$(4Z^!PAxZyX-5>iqP~|yF?H3V#TDhI zXa+^oCR!xX0LW=rj(H-tizq3Al?V*7w+dG_`$JwTckc^=;F-j)TSCU-*+k^sk#}d$;rCQ_gOuvd_DP9uDV@xAWiu$BMA>*YjsHoM>dvl4 QZG+ja%pls3nuZhq4-0M3^#A|> literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/run_batch.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/run_batch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..acb744c7cc3bb2f0b88689f9eb77bc79d07c43ea GIT binary patch literal 7898 zcmbtZ%X8dDdIvBV3MV!y|q9MHAhpa zp9wOd?&z|w2ieeYj4=9}CWg=bUrldFQ+wj|Y|THRpBo zC)nf{igSS-aH=0D>>xYzg~ASPXikky@mY45&o*>6&1SyPor@B3grH1q80;uJ2IwVr zoKLe85_bmBlgtM6J1;-Din$WMynY4G_gHlkxXrAbtI0WK zaFxHsFK?(cV(OB7DzjgaEX-Hb#QJb~d0{E`;!cE-xfBb(xjJ{Jq9^%l4?REjRsx=w zH@qN_J^i*H#ff>R9s8}O7huUm6ni3e*W$SCMqE7L0zf^!&%%n9j9rT!HS4#UtA3MT z6RS~DYh zayRHsCnnekEo!%#5x?ohUNUuK&5Lie!gj!k>_wi=B@?&870wtAyaOuT5~3wWLpt7_ zHg8^=Tf)hJM;ie+wFkJf3ua2!y@-F*Vmw&z;xzz@cO!3=-)J>s{%MTkO}!uRPkme} zuA>w3X1w4DTw78QUX!|<%mK9tVHyMFi*R2z;Mn7n)lEZTYPxM!4VsMmcn`&R_ zDeEfBFr8&%jTtPrnN?Gm$?_6r?Sd8J3@fq{D{n%-V||eADO;mG#$p4SJf6$kh^K8f#|Qkr8B&+ftU4%F+4m=?CDhonBa#nBc{9e0znsX^T#? z$wiG$%y?86z7(vkR`t8ZYSkw$R~vr7m%D$zyvA)TUb9=m?ufvSTefFMU>C{+aIe85JL|@ZSb0L_dPlZ0qv$2a$=O=aYr<5zhQqu*8%med5pQM9L$G2K+3^{upPg> zEIRxXT$?xn(N1!VH@r>|i<219eTCkqRTzY7z?H#<;aVYeE@MQmW#*F53X zW9Sa?cnF-kIYf-WD@Y9BJ`|EcZXg@0HOso0e&~vQw;A}MkGbx6waS}h^r|6;Rjfye z6~!&V-54}>XKu+kdjX&25T#CJ57D6H1}MOxQnItvDlWa7yI7Nd-PjxT5ae1$0Mu(iqD4@BEF5DQ1WpMPivZ zG+Rjq23!;X7ZhkXaB*tv$9L=TS0v-oq1p2Mt#ol_`!!79a6KhO9l_u=vctJIiw1 zXlM-f9#KV(S+Kx`1tjc{M0iOw!* znnb6k=*l`My9`o^Hc8c2k}6fa4g^u9hJBBPd1?phmO=p~f+{>u)cJ}gE&wAjI&B7??oUx2b_#(A6q_hA&=_i2)wh2O zRXga3ky_C`)DdMA%FAoIf=-MeV4!U&MWtPo{md3F?Vi?GV;wYSnMOj>Q;P~5c6QSM zi*tbIaGgfVOxaltjrI)&66x)f%<$a0+o(+5^Gg{G($Rc(8u=sJ|#{Ms=B^w*x!Y*LvDO-x}il5Q@a0RJyW}Nmr@Ley}Cw(r`ue?n5LI^!Fn? zKYVF5X~7Qe-QBn}6C=bV*@M(&eqQWndYOT-wdaefTpx)SS3)wM<|rn+GCTX!l))K# zi7K9EJ8`4>U05NLH|s5mhLc=0L<2NxC+bV~El=1k#ef}R3kjO`*+F$uHC1a{gUGd?jq8Sb zdRr{fembmyS0`N^{q}J6-vHg0s|zo1^&)X_9b5%(b(YyunZBv*@VBrZPnv}tj=Zt^ zUcZHVbGvbOxR(Y03~;ZUa<3VI#RgAlDLvXe}Zvz5KYRV zNwv7GX@B%=JXcgtZHpffc5WDyCV5WjpBqN2d$^SW(oWKK5d%uPVs%6LrSjJrtfv-d zVx4U32*xs-T93G+6USg%v-3~06wOqh7UWA>)#?*1DL`0}d%KWco>)UVT%SGB6U}Ey zzTQF583I6w#4vBoN-8UcAYUzxP*bO7f*O*tq;UP(O?UCu_wU|XT1GHUk&*Z@O%&ya z-87we@zQ2O?;dy4D;QCdV*WYSrgx}WDj887?p%MS>Bj%)d9jAsJ@-X4(4GB?mKD)_ zuEy}2J#_xp)}z5JjUO`7`3Eq>$2R(!ZX%u575OE$nAvwU>#(*@m{EiLUWeu60?+32dTVXT%m{h7xzJuftLoD7xtBkREeh zkoOQDqd^R&ue8`B@d-gZYADF+o?A>4D?5b5*v(c7I^jzu-lHC)W|f*3G_pWJ`%tR> z2J;cQP3^PsOc_CVDpW;8L7HZMZ6Wa9BW4-%sg~Cf*qa$$6>Tis<06S#a*-Q&qJKq5 z___KsiWQZPDlF>t)uxWOx~d`!(gvBIQLRHGmF;PPdFf?C0WgIs=FS7Mtl0*d;OK3^%uiXpkLZ;_M0Y*{y*$C`td28wA?G<1Xh16ezi9? zIK8Fx%B(~c%KQJ^D-X`FGJPXHi?g4@_&iRHb8cv(V1(RGWei11r_`Y$8f>JjqV3^5 znHq|rNo=DdXjWlpKF)*BP-pCn{AE-7OYJ@xVU=`;!Z*hvO$b~HykLS3V_A}wAYf>< zi{d^Yl-p1;n+kI3RjvkzU@0Ph=U?c1CK^N0wBok0-JPb4u69+{nxc2m8KENiEue7W zqSD}^a8c2nV$c%aFtCiAv9>H-xE}M`RJcgWBjcOaKAjSZ8nGL5@!iVR!dZ?dvNx_rCC}wJIo&t^hDhHaSDk~yBt%~L z@G2uwqAxzA<`C^w8hKNf3RQw4I2nzLGJ1*bHybU-l4V61M5J+CmeQAYl8xGdAB!>C z=Ro>tBtJf-eC~Qg(Ip~z6oJ|ZESWS{&Qu)+7}JuI&sic92xojaxpUXjn+eJ$5WtcH z^jnUx3Sq^?krzv{vVi4`r5|UVu$_Jw$vD54#x#Z0bEnkb!J^8zpkURRe2E(op08k&kf3ckaH?L0ynX zPH`mhbbZpBBnYw7a3^v~!=tQp8V%flbTQJq#Mn(p&D2Tx{t?6I&(I(uKqPKzmIkK* zw^7bms-_vwjI5y+HEmeqp*SY}GyWT>qy9HW4(WL21lALVrCD$~21PyF`nH9gbnFdm zIT}@(^k>Glm77$LZ5tUAd4Q#zP{p5u6v`!uHrE|i!c&ET%GW8!K!Sa3Va^`b`k}vF zljQvh^^WXQxl8k2Muv>aey~uCiSE&TFo#OODsPL=hz_07pfozv{8lPP1}f4ZK~NMW zC?pbON(BW*X(awV^qdK@Pic*l^pxsxWcXyt>l~tRww6jv4WzrB*bgG9+Oj(69NU8+ zqdBY?Mrf(JA0DFY)cI~4;hg-%Qy!mvUF#))TyD|DWr9?(kXd;RBx4my;GFXp?rv= z^ugi}@E;ZT2e%fN=I+ePvZQ#M@L!|mG&PiTN-K~?YCa7qYLXHu-lZOy0`VA)Qy$h* zhUIJVCC&UX8We&$&0S%fR5___=2e&m6h+>l@>@g^peYM2)6f|c*>6!D*YK#eRy1ft z{i2HC?P@tQIWuJ$CDI@0*0xqod+Er?%8{l$(=_XudBR*IeJ!enZb_Y*)bwBfFY8H2 Al>h($ literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/serving_chat.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/serving_chat.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac60efec61c37a766c3192cc3afe429a9457f9fa GIT binary patch literal 17201 zcmb_@d5|2}d0%%=PtUP4v$M1J0$2=gEVu`Ff&#$<1PGo0DT0U97Bw1d_wLSMdS>yu zm*8&oNFb_r33ZSGV%{tcJh4fBCl;e&c5~?OS}={Ac3jG5q|0 zrE8j_IeJ60)l;{1`EJ+-o<<|#nYQUA?WAYfmY1?qUfNE389U=;?X1o=3DoJab24or zowxHcoop1mUb|POtwz!7v-@N^)hK!WcE3!g8w1{;Jt)(e#*jB`56g76G2)Hdqu!W3 z=IyX|csuQ#-Y$EWEbD2Ed%Nx3GM#Iby*>6GZ?C=A+h_0d_S^ft1NH&$pncFgWFPVl z+lRdgd%`98#$(<|`=s}{ z{kVKDHBNb_?bF^9_7n2Gzj4NU(tc8=2O3X#Puov>&)Cm+&)Uz*yurqE-dX#scg{X1 z--jB{doS28c%QaEt!wV1?u)Csef~+!8FohA(VP+Yfz^cllB1v3CPu%_Jw7*)iV7$F z<>u^@ZqpUjpe3Tbe1AHKU(=_mjYjoC!;Q?-wb>v_o~bpfjmUhi<_A&w`Ng2t!uy0C zrO&&UmfYs7i*)kkCbE%8y~s>IK(gm-%W)g0TFtrIJYLnX&pmhciROH*$wqx=Kiz#j z)u_2mw#upE_&pi*ql4F7;a55CS=aZg^KLX$U0iG|R~F_fvkTRr61d)CqZ+u;4wdf# zRZ(qhD=alyRcBk?j>W3*U6tvn&i%^VQm`akAIRk6**w>ppLfMXDH=S5@lLfcoJ)L= z|AJ^;h=%T%?MJ)roR295m~FIoYw@|N=Q=O7T8&c+t=g>nLRD0~?IXqrZ6EPP*I#Tk zeK*>T&ykoznAxaI0QiXIl@dtWz`UVgH?N=sWE;g6f&2<}4S3bQ9%dR;NN< zzR4-EF#m)QEz#{Ox25H9Jhe{oSvROU)u0-syX`YCbN8Qa&b20zQU39&?@AJTp&Bei zh38r?p2X`?0}F@u{PT-$^W@X#UGaLYIUn_5pS;E3q*Gl4q01V@m)tjkms%IyrmqSo zj3~>ou<*VsqP~sv*=iF53zFxS1|afUvldhuiav~}xIr7wF9nNBL6?NoD~2dOjkas++jZ|oUekQfTeoJC@B!uA_!|D5e?0PSgN(AUja$ax|IdD>fks;gFqe`ApR}c zJAsPKLU$FX=JOysmCIGpL)F0QmpBuWO&FN0BQ+%gj=QD;g%Q}ag!bN%s(DlEX?&InyXPd_^-gdjglk(Wr0Bap569a}LxuCS35sW+jlv zq*4VHGF7QLe$?kSNnMrdQqV$ntKmmId{h?I`N_MI(N|_AYmagsuY(H68TUa{ynxeN zQCW*PuYQylF%PSV7A~J$qa~`WfCKGEy*Ldxz7=o;o?o09q3)UZ=tcS2%@=`q1jHyF zC3p;AHVMGL6n=U9C;%n!^Zyn=psi|kJ=8-ZOoYkM3R7Xm(XXa-Ei^-|VmJn1A~3`B zs*Y4L%mP}WR=32yAhniu64!KI#>mac+^1GDL3XVt?77DJnO9ME9xEN>mT|GSAq=5B z@0elFmoz8&l>~aty$yk?o%=%WvVK|Cp0IFJi<7`kJWgrx2#>lOkMcOZ` z9V@rwU0<&G^?*UXBXr|fV7&7aOc6TTy#BAh?4%& zA{bb@QjrX}B2Hs9>>W!WDmf7*6m@lk1JTb#dy0UB5m}09M`jK8i6j#-%|3ePT`<*6 zbewtOjagSNMU;7IdC`@#n#f0{?>6SP?-9}1zEN__trQvQ?sYxNRB$+|zF(q!MoeLJkzLMm|0v77Fx}j*BcFQ2BL#lUToExfj`s2Emy70 z;J)HDCk2`5#bxmf(6|RbpTnl}hQ9G{-LTS@p4F{{rJKk%*R7<*Cw{s0g288c-7t(> z={Jpa^P`oq+gQVNXVs%rd<=OtT&{FpEX6fudBi$@ydHS$zJZ^A0y@T9I2>zQs9)2c zhK})+)=q>8M_Q*9o| z#9BHuyZxntOsLnhJkYBN)J(Ve-h(kTanjM7DF>I*JHVjcb6Q)~+u0x&rh@#c(Z~$Qu~N?vH5g5=JHa1di$2|2?n!BMQt-Jm_0>%T8^-w*Clxv24eaPO*yadKgQaG#v-*O3OL2lodL z%o%dV*D>P}&Y1aY$1pNx1Wfy{CECNy0(0vRh6BL67nl#>iqq?#az?MF+QXoiG0@)e z_w?oe5>rh6lhB;R^G}0^!eL-~0w-n|eLf6){~WC!!JZ9ctTcA6c+EPmtpJC$N4Msx z6?s)%jK|+ImZ85(>iSAq5xO2E=AotEtsF<0>B^S8eZg>gu;*^6f zy0f*p+N>0hr^_!b;I3r82V`b(tD%}grrVm5)mF?mVR&w(UtTV_N`r;6yuQi@{8`A; z-9G~<8febr=MMnPgUDX9+F(>`+BNORj7!FOO|TSdL{{B^&XTtX!QPlYYMZ>+WU&G- zp<** zS!toMtm3R3k-|YN!8}~v+bP&fNr}=5M3cpq2wZ1RWK!f=dF2i=THMP23#20`TPWpd zQv{+i9jyDu8Wtk0NC%LKtMAG>Bh?pp@zC{RN^N1q+Beu z$`d59m19Z`mQt4*#04F?qFgP*p2AfD=~v6#-*RgXc%Zv7Gy#ld#XDJy<@BzWAQp&B z0u^Y@@MEz)MmHLYtH)w}M}U~nWiz`dcY$&oFy$_+JW1slbEo#qi_A+eidsuRx}A8;&VkO|*^5e)6wk6a>C%WH6@c4=!CLrBL*_sbEKcK=bjx)DJ zAce;b3MX;f1Kjtn)vIz5>ixVg`dIxeK$LO266;`mTqSB)-`Gr)z=p=!nj+*4Wl5Z1 zKL^-P64=11R_suh7n@}4Le;4+`9WvD>>))*m{o_?YE2kFgv+kx*p*4Di4u@m#U$S? z$DM^fXy?7^8?m6?aGOyA!0x5+egPX@Q5$U!Lc!ehPDKH!x zc!-sIF%AqaU{6hV7KcBwu*R4~BpDC%-VUp*sQeBegavk7?&qc&b4!iJM)|~uc%C@% zT#HxaSZX^JZ9>?V#4M>-5?Yjz=T8#Z=UKntdGh?krcFC5S(pR!u5S+j?~cKAquD&i zdVy_Iaa%FVtR-gU<72Cq*)BQmTowEhbpJ-oQ0(6A8`H4{#9On%&KdxuE}xC#<;BtB z?X~H88oKk1Lx%6a0>)*$Sju6r_Md)@_Ui@oZLFie1XZb= znS}yHEQ+WJ9M*Fzm;W3w{b3h!=T=Kq2DW-&8na3+?C8fc$3O^GFN ziBq{-#e}M}a`=;8{j-!wI)6;_cM3m$3;=?mCa9cImC);kqr(yf+ZU`(iF4mx$6sX9 z;4_gBlFLa2noRM!v5D%pk;EVwsXR0t*2FHxKnzNF!R!k9%Awn5aL0DyZ49T#`}0gu z2$F<8OM6l0?^cL(wbtqz^m|Hs)7(rq!agB;(+A-Q0C()*oXgV6y z%@N0J8hmR*{hZgso&i{UoMfEJLoFx-6w9q3wQ7ianX0GJYj2Zwq291Jr?pN%h=u|S zwWJsp#WRi-x9*$Q;#8?=Iw>f+eeKf9AnLpZ{IY~=*)VynGcJ@Gr_-Vz3cwL406oiB zA$|`)`5X^&Yk4Of4qT)1t*sS;-f&=*qYgTma1a)&>{k-6KqYZ{2w$a2ik4p(pp8qV zsFc2Etb8pr?%}g`FY=#4O(;1-^&-|o7rz?xF^#n<;8{|6VL|3Xc{`>dKMQqY5g7E# zIS)gz?pf7OYp*@t9)T(Y>uh^87z{^(p>Wu-oZMBc6#ny9&GuL@3`<@H3dk5%CIv+_ z+1`OQ9TESVus#|;$He~%c2H@i=%w*>tS-uFyp4fYTS6PH9woUE`Zdua%vCsG-9bsr1EpR9WBC7x&ASyVyr} zh9}!2@yvEP16S#C{Q>hr>x!CLH$^r3v;S|idr!YK=_EnDL-jqUf#Yje+T-DPeecG) zhk33)MtBExeT=*h>0iWiKX+25$DyptwEvS(PX}Z5^Z@!fh~GY`gzkshY_@kx8sClF zL+x_di`Kst3ybhIBZ|`ZI35%$66N?&v@Zta`i$2nTU&B4&?jN|L zV1DtAc2Z;L@43!6?StIg@F0J~+Fy=(i4*qsP&kSnFQdoN`qZlK=od}o9YS8v%`-=| zCQ90eu}6S7jnlw>{F;GUUz6>>7R;<2We?%uQ4Msyqo!Rmz%3?_cQ^CeN5YApBS>NFVn(Wt`-XJ>9_R+ub!& zDX!U{{Qp=pCqbG&0h+($U;6O}c$1C1mk^Wjcuav3Fx%VrWwOJKpsAHvoCI^fpL7PVPOi|D4hT2YIv7B+}9CXBU>57hn1|mBcdZuv8b^fFRvKc>`FnUfB5wC=bm^io5kJ!SI}9Mg0xIe_=FiHYfZ;} zBTB+YM4vCDcnFyn;1+^Opyn??)u5*VeqW;o(Yke^4j~)urESJRLV7;@C@D=I*ArbC zRf@E5R&7kDBckSlR?yXep+x7QH8qwTx3UCBdLaG=5ESna5LKwv%(yXpy5>kFza`zf zhffBlnxw z8on|*&^?0?4hZ1x>$TPrG#j{qVG~&rFj!F~#2a)`ETKx5HY#rT?Ygx)S$1#Sq?4gk zjWs#6P>(3rS)OJK(Hn+DOk6`1@pG(~R2e_YjK4|n9zfLNbFT5;S>nf7@Z$tOK|t@D z_}c(ca*3XO^!k&``g;H?Q&Pt%!-7L{lzt~=YN9cMR)DU;Kx&(KQ2EGymhCUIOEaip zRrRK5G0P_yB_PVtfr`J-T57-!BKuOl?ye(KdZM5&O0Q6qdvU4BzMhbJKWzY_31IJz z^%B*ER4-Yht7HqoscqN5^j*hsFhI0@%EVhnb%{vLvH0J{()sC;q0 zov$@xBISbFGCBa3Gm?T_LQJR%&G!&H`1|ajcf$r)spAAiri-&8&1F#n;&7BwMCq&j zlUw&?G>ir-oAs_PF@K3=CJ9aKBw~?aoXU} zFk8a5B6IB?gug6-gOk*31O#{pli@Lx?>(FL+V0!*f^7iO_7+;;WbHw@N8MX5-k5FH z2=PTAi18I=mSFOw{B%km=vJC z)099Z73%nhW~zj?{6}6go5-1%7;C)*(*z9-X`RN60iys6 zSifUJNq=OfEX-~-eA-mN4 zX_mmFI>6_u9`wO5^FHf1sbCN)9dhf#2n1q_G|UN^t1y{3Sr`UN%}}=e8A6AC@UL#` zyMdAssHHRr6rJ2f!~Zpzq8i9tU>H9Oe8OiB(@F@VOq(W{Jy^}r<^LHA^n=0J+72h* z73i&Cr!v9B5dczjH?ZoP4K87SeHSr@AH9It4u*sEajXoWxFvBNtnbEKIiAnJHvDsP z%wEO`Y%K})pjWES#jp>U?{tdIG;*l$@8c7ySs$MJof1^Dp|EG|Ksdx+E#8(L30yrc7I+H&_@G}5#eNjM#HI!P1QbH(SLrH+xEiN$guMs!^ zD_@W@X-7uh>|@yKK`2j^5y(m3q~juj%0v8hxke8j+^c0J3Dc(-3vh_tD!(Y>{&1U? ztB|+hM5_vCyC6O-C4a<0LB@~e|0vbD+`Bq}l)tuVQ@6Ra!HqPE=+b2T7G2tD#DY-G z#(vPycnRjc^U%W1K}Vo8E!Pt1&oKQj2+ntbbZTc#kPE)CC>}|nb3ctz<4se;O|!mo z>IZYut?ReRyWQzkypJ9yVwL76I5rg-rP5IF?uIOOB7Wt+kl^41Nflf+I|oU0Lmiik_%B&k z;`JC)FR)rc3D6XhrE>7kSn|&SB1=iVc21H-hs1xIjRq(+#uu*CSLkC`B5}IoZrw(k z;!QMMH9(c$K>{*lmNFz=Ms)NPC|}WSWKgz1j$TAbES*3?p|oWtEy^oWE`ii?%UCa( zY3Ybkl8=;RAa%jvgtGL_Vowp0MOGg`4kaT>WjTFZy0s|(STY8X@*G~ z(jv$^XH9_4AJhCv{!vEaigrc6Vq8fexEq1z@1)5?)^xgM$lImUX$$jV!z_ON`1zj) zknuE=@S^B#!k@hEd;b{qH2caM@|aL^fl6Z)M*nz)5jjUew+2aiq0e{5yrGPX+?{Hc2Xz z&Z*CUaIAI;q2&0)p`8xXGTIbU!x|h>koFiLZiwB8ZcWKZ$xN7BV`zB>?!OEO1`(-F zQW6ZvM#@nZMe}#; zNenTQ);J1@9UGg9_v(s^^DTroiq_Knf(&!y)!FTj2AGaZKz(k24-#M&!iOLDu+G^O z@%XB%Lbxthn~2S8l^ynqxUQ$qlo13ZreOSfy#^wPMb)a1fp3%Ls?3PT=#Gnq8>38? z?+|r|8gUE|`aonDBQ)a>0>q1MorL<9Vy@;k97xy*mH{7POec)dcj$RTiU#7CVrOk} zFfL#_&fz?K4o@YnNx{wM>t7|1{7VAKVSgEUc86n?vM>g%-;s-cA`|xSaR7=!TcYvN zySy_`;psmI01jra#)dAg6 z)Lk}jH@8~aAG@fbG^Qg-urlbjJM_kwO0k32#k>F_SJLi=*Rvx*-Jr(czC$W}`KOjm+_>~MT z#Fa{vtuWXQpHv~8t5hz*e5y)BFRBV&ZX!i+m_Rx(s5yxVg1ZO?3H}GcGX!G`F%{S5?mp`lr?b=L6P871P2JdNbn{>lHfAI{REU{1lfGl`y?ZVYqOp2jV0e# z8W4|)@|N7q6jwLO$6wX(yTwXn(*}H+RpbeK7^$Q9`TqtWjeQ^kw!Q;M!SxoRuWrKj z2^MYK$fk@BthDic$T{DGME+ewiv78jD}2Y8Hg9(7LQsYXoc&1qV(Af>EI0LFC5;bZ z>Z9rKgKRqceHsJtS>j(4uRTBsYU%vs!bDDY@%%uQPo869R6jotlGTA7Sxw zm+c;Yfdt<$dATGPd#B7%-}+5UWwwGU2({!_-ZGfd-n+SqjOhhmpT+0az-U^1Y^DM( z2R2tkouD;~6Gzgtcf>{APoJix@pB1Rm#=85k0_T2`UwWuYWTKR5DDOAlVog{Z!2xd z )nLrWs0n4zSTQKlrxOB&}rs~mRYOzkEp6YL?_OCXQIex@X~A7ttffPL`e#_oK| z6)&JXi1PRyVOx0yC)vUNkLy5cvC1LuizzmmW_`7Dn<8W5oOnTa9PFIzv*QqyM~Y|G zR{OFoq*F!9-pBI1Z09qYp3%!Zzj0rTuO{6+fGinyD+6uqJ+c7`n?5c8$oOzyawn%H z!HzM$_+!{ClLxKIEGq}rV}1`_`l!X=@3 literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/serving_completion.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/serving_completion.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4fb40e4af8a355e4cdcfb1bfc02e8e3d5fcbe5ba GIT binary patch literal 10645 zcmb7KO_1Eibp|jP3a?S`eY47xQ*}J^m_HNNO-`V5swfB1a?0w#Td%yRT{gj{=IyLWreL&=k zozHns+fR#psdLapbfnwhw#H+0S{;+s_MHrSpP!#6BYO zHCutYRO>-Ek6B6_phhySDAWVsZITo_Uc^Ch>ItJRljlC^<7>MyF4z4@|iFx znkVa>PW@uXjrCKlMi}d7TR|9`Z}-Ah*ROYKYHXf&FZW%);nq@d`a-{lrs*&GsEzeT zJqXbv^B%Pj+1y)Q=5|hY{bp+!HFEH?XWx3wUvBv>C}VHEJ1U;+v|OJU%EQ$Js9i^=0>r-|W`(xbSK{aK-YxQx8|- z;@R$dCs6EnTv0B(-E;jDXU@C)!&)drJn3E~Lih-I0RsyX*g32fkkl~eN}c;4&@P_vS1)ugxqfR6o2JE+p39eAr%v+c zv=V^};vBF0tm`@bev36oO1J(dl0tyPrnm}e5?f;`(|)9~l$&C@tFtstf-$Qw1E(bI znsho?M&z?1Zz6AjmUVN@;ti##GK=M|7wtUDy9HKoi>&CD+_F2yN_2wUiaXv+%_>cU zmD$*J-JTHbD(<8^zMm z;R?1R36*uFtww63MX5-S(vcBmn0hlqN)ivIovUT|(`d&JQnEKs^W?=VRsWyGx#ul+ zyemtc*2SgXYPiz%mp<%tyd~%_zS`@y{4iMR;sDoMOMyJWSi+&oTI{Xz!}nSsvq&|Fu3 zsQg$Pq{D1z-L~#1>*~M=b5WY5Z>ob#r19es(ste{EGq+Z?KU_SY5cEfgY3Yfk#3iw z>}Wo@ungK5gdm5}E4apSO)%qT8l3atWRzc5PbnX)p?_geMBAxwIx4cvEJmJD1|_tf z;cJn4dp0WFQ3qu|jr^Rv-cH$ zR#(E>>ZhtQ0M=P6(7uERqRDlv==+MH{1J9_F!{B~^U3{+dPNC87nv-N89&`pzpD(s+$mwSs+rz}F3EU|7QSgQA02f5pHIRUbzO%)R?HGo_~cf<|7 ztKo_($L_g}R{&T93F zTV>s<-wmtKy1gzB1(#;G)9GFT8mKlgnLc-qj^;xTc@T}8&DPcGQ#7+q-Cynl9#w^T z35aE5#DB>3;XxkaCVACHv~z1a6ZUvKn%QIAT*k36%38Vs&3q7c(HdH-7J~tq3VGkh zBDqdt)ug$MD|N=4fYz|yNla*5-r9uh=w0<75S7uBuuEFWp_ZO{*h&ty_u6TKYt=yD zwJIjv^|7+ale?^Xr4_CSus#CJRUoJ8V*q-wrt??vT>K=GxbQHRGkd(%<*jhl&bI=m zQHR)c0NXIvoZt77Ck^7Pcvb>#ae25X-Vnwba{g=M5};Rs{eTaAw-I7Rzkop-xE}h! z=qTeF%+DM$-vEI72nK^`KTdrK$CAmEth;{hu0croX!WjUS$B&v|K(F}pL^{C%i>4S zTO=^h1)d;!ryX>CS@n6;$CJ(aCD#cAa(9-8e$KB@6&OggYem&k*WP%Ng_Y-KtF=DX z={D+};P~QiWn6pF&O412w{c10A^|V#j7uJY%WR7QL_4As8&V$a$z%*c#qLWlL|~a% z_v?VkHF}T~P(-YUEtF@eoDM<=BYy*Vd))Ers~257-zM0y3sl&Gc1cBu~pcoBl?NI|tH!|v9y zBSqe>fE*;w#*mgH7?=zjS0H~Ro8&zJD3SBP_tFG1iQmEZ(46)W$!Y~~OG1)XzLTg$ zN=iuV+?Kh-X0qQde~r2s^&T)7i>)u!JAGF;X>kgY!Jnd@B}`ze=wg3_(Ab%aa9*8m zGgke0ep9DDCR02|vml}3RV4A=;YsNV2Mi0pMT|!K9A|_F)au&Blf&aDr&f}{n)SV& z6l$CprM8_vM4EVrz5~f=%VXPZPh4 zJkCL^w(LTpnxtsh73MbU{Z0sJyPBvvyY%R;mWGerV&v9&{13sEpQ2H-qfukkB~20# zvk{Ry)Tq>KA+kX0<6NwVjg>fsY@8iY?defgHju~7WwALrJ8#pFLOr}iIiU!~9y`h; z6v1}reC=AfXsWturcKpK=>XsQMn08S^QxtRs_H3CHFfO)f$~a91C4UJ{NK|z%2`7* zRAa*sHO2DgA+wmSy9CfsV+3rs>1} zx@!D7U(o-(_`;lKUQ22=K;1BnDZx$q%t*_5$vz3M-xT3YOyEhhjdH4}=F>%0zh+`y zx|rpVKi$nw%h{q@R^KRV3? zTqnsG93x*u1Ezv&*H^g7R9}<^S*9(kQFcO!4B$!Yp;C^ew$$ZC9knXlQrF&%v}b`C z*Jgk`l>~Vf;SH49@Bxr&!26|kYEU5x##H&~P^UbwY#}NhQ;<)?ulNKAf4ptLE5ID4 z72t2RouT%DEzqgjHuf=eT5by!o=!p?Zx;r$XjdG}h43uNnTKCD5drzb7noz2o9ShhnKyM{c(pwS ztp6O{+raoqZXEu=G&mG!mY_aGyELl|7HCA=w}lhYLbQ#pX~cJKrUu&u2lV5IqV3VV zGwG+8LicFCJ;hSYf{p_yXlfMi%e$*KFLma9kcK{egLmy4}7-?_#|`T ze6(jBzT03gc<$Zg`967aX!o@}ep=qilVc?Tng}kH&;61BtU$4^U${Gcs>KA-mG-9! zV+(bDF{q^zJSPG?`LNp~J1nZ=X+nJ?d$n01&Tl+Up%Vdf5dwroYzA%!Zt0fK+^fJQ zpvLBiR{?su6|6v$*8$1mTtFkq0^*%oi^ZA5FvV$MvQQSG*Wlw+JhVIs$R68bcO}Wj z#KRHd23jw;R~wx^bDciox*>m)R^ktke4OSHX8C2P|%#=48g;a7=CK*=gKn0FpCL!;H_ zYt-ydD4C{Y3W;4zplerz+ylGN@{`A;`P(8$+w%L}e&C2F0xYQm#SyaR-=c&e0nF)p zaakHp*{p@IGyi}_*pbXg4kYjwq16#W;a^3o+P*EgR@DKBAGxC3|YEAoxhj25UaH>41!?hj4^le^;A;{ec z>tbZPl;kPUDWxARH`kwbXmbNhD7>v@itj%{5^(ae5mb;(n)$EAc0 zXC|Vn_Ov8ECeOSLzOhanHys;+TzDNd_izROgG9SFkt?e7w^Uu50-B(|1>0^Jpz3AV z`aEg~omjBuzyTY1O=1~g?SUF};0@$AOilmH(v6hhpjm`zfKjLwAqhh@@9E}LGKMz9 zGQ&Hyx~FXb*%W|iHY|)`;xERaIX$p|rVK5wl~wI$+ON!u)M~_q_}|krgkXxAncD1^ ze^5>rwK)|i=YgfK9eyJA7{yN?2jU&9Q^Q0u{Lx9GnCD4c0hFymaYsbOvbd~>Vu$}Y z$VIufGRQ}HVFSm7ea!(W6hPMyRm`4?iwA7K4E#IJ)uqt_2>{8o+wJxI#;R}Vo4?RETPej7v1s@2W zr38Dhq+dbw%b*{{ElX_^c%b5EiI+e*8G&+4pd86n<$o1P3~>W?UQ6tJR>T<%(Iy2O zJT5H!cxWMRYXT2d5PvMS^SEd|RHolFL>#gxYLFd;R@KW%sFAcILA&Qa3K_Dh`a=leWDsd)k?+s@QD?w zfM28L*QsS?Q$a|LeS@k#qU76@jMTDZ_8p>4QbL-2QxWj5P*G^qKc^hMg{nD{7^xuHp>S8w#pc24S6 zN9YTomyS>uk&wH?FCzI4Bt5u>jBb%SFyU0>>1Ya7L5hN?29!je)CtvUMR|b9RrxHR z7H$n{%0eMX#bDmoGr~O~1t2-xwal_s#M!16bo9xAk%~Z2?ybG_q+>jE7Kab`TYY3H z5}UF>V!%H{(lC$=e`CmGa8YzAsK`TZ;Gk#du;UVkJp{B9fo6_Ca}LK{i*kUD1spLA z2kyv6+HEz;1AG-fqSqe#St`nlml;I_-BVGKg6>h~c8XqnAeSBH{AU10^WaoMKR6Ll z{v(Yz;GpP(Vwk4G0Z^^sNIt{|VgH2B4dsSLylUHhJPS)+I$zYIoF8oo!lzeWNq zSrLIR<~HzhGC5Y_wO^b99E&pvYC}*{TYN@>0z;tQ-p45}WJ!cn#EBgppx8Rb@Wlh! z>Eji0PsxTDijhS);8Qw7Li+C)aYoioKS{cO2iRL2#Su5h`Q-+FHQmk$m?ZJFT@-3! z1X5y5onF&&(uy^Ue+jqPkk2k2T57Am0h#|TCEuq8X^O-IQm<7&sEtYW^~jkLOF#$9 zE)g)=I*p1T2;ekC!L~J`1$J5HH(dt1h!>_w0Foe|TzFGHO^exMLFo3N12=obX;Eog zNev#;EHz z#jvvGC;38Ntq}GZDJL^jWN5>~=fi^)RQC#PgI8l;My`ML=_5kN6A~dz8@0;}m7# z?@>~vWIrXdl+-C9;}DllQ#8EQ7=FOR(^R#a5)u(kiyIe`7vJ<=z;U)9)90y)*n|Vf zJ%cOw0TNTua>gfE%+CNZ9~ioJ-!injRz~~OFty)k+2SYK_^@_Dx9%%n*N#xRh}PJn z`&wGNr=f?29-8Q(ePUVGuhFgyqawKe_IB1M+C}{?wLysP5nh!~InA4NQCU^ zq-yk?!6Hs4zKRmkV^<%m?TW7u7Dr6PP7BE*R3p(3X*%hF3ZW8WL+Hoe_at`1j~bGv z6VfaMc^k1JX--xsncNPd_uDgroi4QI-t@I=!Gf1L2>$Fs}n2=wL&kn#L=)V(N0u}w*lgn3%?CMv60M~4wId~ zu-bDkkQO|1Du~m~PPZP~B}wd^O};&|Cx{^J?2z`lFC=iw0ot@Q<}rjn@WIS+vY){V pNKD04t3YE2+`@q(=Z*ePr)I3Yo)>tHYRXV+CfXn<4cBTy|6ec%SfBs^ literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/serving_embedding.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/serving_embedding.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32f81569540b34d38d6768c5b5adfd1eaca55131 GIT binary patch literal 5336 zcma)ANpBp-74GV7dWN%cU$nKgdK{6oWkq%z!?7&MHXWH3EZcG#2|CTGBH2_gqv{$- z9P|W^DZrN~$lVA)9t|Y_Ajl=~DaYLE>Qj<`AX`%2tDYH>vJIq(uBp9Vz4!Wi-(y;< zRWx{B`s?2}NBHy-0&KzZbi*_+#6SQJDP|my-8IsMN{##H?8VU zG!xHyv+Zk9)^eyAqvQSK_*&6mOf;6x=dB8 zL6$Kt{Dj?Q9EQf;PbrJ8q{(`?0nO^QWFt)2l_+EhuB;7KUQahRV7?`r*WxusX_#yb zM&$T`o~)B58I#lB>$)0~Nl zymFKAyKofB8786dqe2W_Hby6W&=sAokTau>fm}sD>^vPazQOzeRp%>Bf$B0j#)E{W zvES{6RN85=OM@lt6fftmr#ub>EVtI9G!W8W1Nkmpls0Fg%agW_HW@qy{F?Ayfj4^_ zikwh#Tj!-*+tO}gONZ7mLAG@4p$0V{>bu6C*4Kr(YvuYL*|n*)2gVkqoa_*EJTSby z%8$Tq{3sM&Ny+i@g%W_ zd~Z1l*OoimVlz#a??zF)3{JzhJ877RY&iuB2H`Sz8=BA`?Y-35mUSPt+3&(zW(XdR zX%mM`8BJ2v`B~_Yjr9;5+UG3u*8)U*g35;0w>J&P86XF6Q3G`OC$)Xk(x@hh^r_}R3|VD_X7z6}bH z&4ZhPW-_HbuT;2o#v5>^w1WLWAzC@g;wN6?9i;tbT}>7fM+>EO=$o`8l{7=l7%=mDl_ zk|wc975*xuuERrgVn5L>{4M_MZ}}@QwGHo~2R$60EP;Y)7=s}G5N1%FnbZJOwgHt* zR$>m|wey9xPN+rg2UV|JF3~1+ScR5hbPQ%ySe4aSeSqv^wEDpE8i4;z7-_DXv`!m< z;NxEa5_l6}_i;IM2*a+8zR?;}80)tc$jWNrwrvH$vWYn37hN4d!c5kO>MFpmjYxwJ zdKJ3411};_hAI=w0M@ z;;JkS)QG@;58O-zL=G%QYoqpt5BfcloPWTZtpdvb9yGS4Wo6JHpgLl-(k0YZfXKY7 z^^F|dsZ025ZXnig8GREAh z)WA+FSc6l?ZWDeL>wPD&)J`f!0DP5upF+EugA@Q=6#yq~tF~LuDS8jlE9kob>l^)MZhxs!J8ytwMt>Z3o65)YW?suH`4}xdw7`SL zlx%+x)704^pQBc4TQjtrH)(lZ3kl2}huLInmR25G{fWH1JC{!&T;54nYtyja_h=qJ|2X1gnv$bD=F8hKjBHL;0t){&a6KpSqy+sxK>R+|ty2X7ZVQI&ZwL-TBugEuRL6clxvWEZ!jd?|a(k zd)WO^7)(aDT|X^r~X8?fr8?nvO#r^puB2y z9>#zc{{MGs)q4ND0=OSBWNPl^8!K)W#$gn2SEO#?#OE^XU#e8x!E=)_cT zL}6{=25uGyQN&O%nxhXs~u9P zH|fX0_8LhSE zg`k7Kgu9L*tsf3duBKuIzYp*pP>9MHq+Kvq{!6^z%Xq=cC+Hc)sF4ZIXK|+nDEOc{ z`9)lS+>ay1^BF9VDaeva2g1~=Ul|%iZPr@gZ{ehCSRj=LQmkxH1UMl71v63{X`eFP zz&$G)!^A98mgnzbZ+Tx|^~1{YN`=p5l7u5VSm?)V41<11F=vTod9=AC%xeNo>P7-g zG)M&%hm!_$^Ghi8?1-Edw7GB`Mx`?(SkJPu=FsJaJmpi^C;JN&`e$~f0vyaCCb9Jj zlsYk=+6FQ*Q?KY%9068lJ~eHfIHvv>gY$&p09C^^(0XD%ZCZ6$Jq+9ScMi-|`h_a*Nhedl{`9AgQ75o3oTVEMtg)Y%<>sS0NLQ@s zo%<0Y@a*mQn{cy3SsZ3`=p|R+&0HvQl9k0E>I3#;b_OhpLbg`4TE=Scr=w3_ArivF z+d-7ErNfBn-V38h0mr?R-(KonyxbPyU4Y$WL!m4tNdQ;{F@(ut^8ls{NYfHxSHS^g z>0ZDSSS-y@rO$=az6nAe0{zr86gKVxvR$G(E`7poo zeOd7bJQ3%A0KQ}Bk8M-`&bI51^hxtE_Sw)^F)QCK z#A;3JMScs{V|L9~p^!GD)u2?4o>0h+T)U86$%#lJEy#yD+x!I7At{G{ut4NhcL6+9 z>wR`#3__pEvAr?OjUiIw|5%h_yeZXFn0pCOK%{NBF)GfR9Sn^U>7~NvN_(FBz0m;= z7;U0ds}zb!U!he@_Pnz{wse%CtF(kGdrjhsLhixwQ`l2gdknM6X*>Wi)vKuft_Y({ zA+d@=yp7|(#o~RORQ(A4Yw;1LyDL|-;y)Xo;NV#-5S5i#4mhwfG}>4?uS3X#oa7Bm zda~ literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/serving_engine.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/serving_engine.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f33f7a0d99ca7e6f5a46af6d9aa158e91192f745 GIT binary patch literal 13793 zcmcIrTWlQHd7j(O&JLGXNu;Rj=wj2frKKgywyY?Msavc>rWJ`wnn8@!YR`~ddS`ci zW+fqR6rq+C zH~v*!74KSE^L1BOsZK8&e#%X8*(jTS+D-eGYxx;BLPx+JXB$xZk zPy5ff&v1F5e84~G9^~?1IpO9K)a44AfGl;k=WN zc<;aHy}9HC;rZ(PyeB97qb;Wv3gM}$k6CESq&XVipz5?2R^RX{rMlPbHFUIqS* z%r|TL&q!I77rmfXtpr{)dcIl^=c}?XEebX4w%IKcud*OT6c>5hgIs&TlYS`(u$XAm zY%(D&vN?9VRk?1}gUzO9->sJ`7F6f>iayFlTbE!~H zTJ#{4(O@1_W03=sf_(n2A`f8^4DP);75cTQTB&w0EXa8;Jouu&wBR4|-RP$%PeCH(a;;jbguzs`=2Z%%slb!hK%TtEQk$zS%cp>tp7+QM+KQ!RRjzv^ zJ>&#BV#mW>T)BWf%1N|DcDY(CB%7Acpr*a{xOAB6jsWDS@LcolkrM{g@5312raaTr z#}zQfNEnie21uyD5D>Y&HiKCa4PFJahy!2_(S0&45706W65v%$QR!I%M+u|}v{=xw zjTg{BTL~-xqMoI0RwdOwdOLg3y1{m^3Eo0mriNy{w?mfLAf|O>LX@FG(NO%G4@rIV zB@xVg*cOPhkq$A~a0@S?^%Ho4L;%zwRZ`u$4Gf}^{(_@=djmoh+Ba39d%7?{|CGnu z5T=(#$r@M26_Mt0hRYVpHtI57-`v1glsQ${qVL9ln-y8Fe@+wqV&I114v0Z-aB&Fl zL*B3$#xvp#qQ}UbB{qr8H&X7VNAws*k1b*pJvNImZwtnxy^(Qc&Jg5t3>UtLYsb8; zVjIZ4)f*Sv=QJw8+iauK93%<-$>SLAcJ9B++aY#i6%MfN^mci>#U8Qurb@lBmwjFz z=5gk-V!wC-WA?PhJP8`?LXVBd(HL7yh^H`SFOQi_`ag}isMQkBhy!Tdm)V{JGu zFH!R`F+VR}0OqH}i`f6s_S!%R@I<@>>W?ATc-ni0-cfT5HRJgG7=FP`kp(Za;<)%Y zXm-Fmm{G-yIN{~QC%j#Pc=$4NY8X_X7AJ3{5TQ(grq6ar$tiIfV-KPBa7XPKaTc}D zC4I-d5pS3!#Sotqub|~f(!aCiocI)4o)@1M=Wi(P3tv$T#eEU3;QU!ici5)hcrQ5e;=1GBRyJlr19|NSsR$4JLM? z5Jv|=G$7;fByqm-1oXLRuv#lvp`!HT$>4#CE5UyOgvzRNTfL*GN<)2D$!iS_P;Y39 z27Zl(4wwo}p{`PEdR_sv8ir7oRC$0ETod{YE6kt{=HaGxTH&4+_p}=+?pZ;5s@eYf zkuPW~)=DPqyPa)h7W;*HN4q_6NBP?DiX9H#9%|Tk)Rn$QpGc1@E7{K>EI06P6#;t1 zI-~^dTdIoHs>&D6j9r8r2moR6q%}p#p+3i@MzUu5#O;(j8T9rQb>lYrrnY zl|@ykH;u1rZ)%uHVYb6;X@JXZ|q}bFgt?ZiJ;hxHpaL0S9+5v~Vr>ALyc}B6arJ5%x9E*1$ z$^FTf0iu4t@ODDavR7eB$|tE|Fy@WGHaJ|2`ne)ery$CBmOdbXXXQ+H$o+m^cqs^jp7EeGE*nOsEs+MYMHvihHZCO7z z`zYYCR87^^3_O~OdUeMG(;y*Ap8MvSnYPvX=*B`6nX&on(mJw5qsi5sG|*%g#dK8kLeRxvo`fSSUq;Y%54^*j5a#sIU|2 zA}l1Vgu1AQ#;P)p^F+%UsE~fGB;;ibi40k-hLH^sO^#`kDLIOUSQw~_e3}3$EbPcR z{J3`85yTASV3oNzgZ4U}UaCip@c@ebVmD6$P$A!t=7(_S@Cd7KPC_P}2a{!2W3zAMU^LRm&%ak}nt)xG3cuoc{ zrVi2MZcG!2T#vF5G2TIaFaiK#nC1tDnuW-M91zoWyQ09KnMp?^-qlcZSHw)-MU$QC zO}*Q=>+M~{t55iN5x8Q_r2`Z5vAO0eRKXh9&A_jsJ{SPNnho;-ulJ$31~Y3={sR)i zH2541|2#ls;^;tDWDGBp2Ad6}ruf8DL{=bhaZE#?5SKT1+lja~b`>pa5N+8|Hr@vQ zU>ksT-PVkh`GK7_O!J;?8HRb^Hpv>)_dSM;xq~MQBFO=8iX%$cyBgk@2`qt1II{REYp=&dfx~!l%G8mOXIG{b7sVa4? zP+sy9GI1|iMZK&MXdnAXouVy8>A}k9v8bxd%Onx=TWI1lpphb@XFlxR^oFCl_CV_= z+rfLFGh&FiOQ;RD8MGIn-PCXEca)Nf_{V4%*p)sLrFbECAX@D2!bn~f?0tqcWkWMJtudIRCvBAfkdiX3gV zkdvSmoty+(_Hm1{h_IAKenSzdTe>jA)S@ZUH%VlynRVts$IO@TyhrAP%|U7f(chvH zD?U^0t}4%1$_`jppU+oLZ4@Z($HIdcH#uY7$8RaEqzt}B0Tzx z{32$Q4FW3w3DxAUPzBSB<%G;N^DMGIOJ<14+_XhZGqH@(hchlCtQ*Gm+u14EX+Fnx^}WiN^9E`#Up;74;PRo&Xg*(B|1~# zJe-H6FTrDa2>nqxndy+tF98j2SbmvGgf&Wg6>=0p{wlrMF{Ti{wU-sU^OdUffxO

1(Ks zsYG;mZNwz`2}T7rfR#4Y0URqph^ZdwTu;<)9LyJ9>gecTjHR&`Aat3c0a$AUeK9!-n5Iojkq=vqzhA9wC6UI%P*Lj>7rbBB^ z$LL&V8>|8x(2^q0Bsf#@^g5iWTa2@^3w_Xkj9n3CFLw1rOT!pbum+P?lBT|RvuBatw{7Sk!PjoNjeMgDNU|Y=_LXb{mO9y6cr%Kg7JyT-Hfu>inlbyjqPh5342uOCWm|%e!sxU~SU$_Sf_B}?BCKRC#a`I{z_8*HvRC(w|E)jq zI2=PT(IcggW*TYpJESj&48gd)6xawA7Zw`4#`cyLnF7}u%s%60iW!w^vo&r*zA9#6 zN|?~Ibiu*CT3nn>Zu6%^>q`Phc$7_qrXXk4w#;PN{X&?tTVtn-#j50fVb!@3Q^1+N zaLzft5Qeqj*wj?5TnK4ct_qBWQVynaN>q!%RH0TnD7`rkX_}%p)$$j?RD9GsaV6iH z<$&WAiVLwUIK@K6x#~Fu2d7G2pmYjS7M95}H2oD0ALL%3^*k`~SS@^nq6T}w2|1W= zzbe;DnwvqAJVtmnO0#=JKIsoga=rmTnSq`3@*!e&i`rXaCfP{z1U$~hrXk<5Vl!;f*c?!(0EYY#5&Vw{{0UVcq@r@mB;Zy9hf_*o%_lx$ z(uytlZR&T2KsSmu_FkLbN0(s<7P^%Q_G7f2!Rbk(G$|PiB))O8Zzf@TVm9v8$9oJ# z8&8x*8I-;nSyM+Pw*5#LkGR#V85ZcYCN;M|4&Fs9k@D-n1$+8!{K!8gz%e`>w|hx3!bi!j(O)NuP%Lev zkjz8s9|=9DXWw-G>>AW{2-h%}P9>{&f?oo7hZ5nCB|tS8!fqfZLJM`|Vsy&9gsCKj zpSpo_-=awwn^jdl%{_5tw@3kU8rhnTF)TyYO#Sf5YPjG`aB`C_)g~N#f0Vq_^~;3g zF9ji)9dPGb2}iqJG<2@QVbEoK9dnN29!_#>nEZJT%YTn@6!5{y>zO$DLo0eiV}p&v z!6ilh20F-J27pUDM`aRZ-2-1qgZIAkPWQ&?O?~PDmhdMh+XJ?{5K7sp-z$uiG#uLjRgU8=*feLV~XK1OP zBoGoHS&aHlPM^+SJoA~$XD-b~dZDr$XD+T&2j0nFQi-?YB*~mdF7g(Ml7s$8@aWAr z$$XGC4b?(4o7Iq!Qjw5PuQPuivHQmnw_xZO6Sv^8#O;_@!f8o9Lg3=7+zF@Rc_M&I z>k>Qp=R`S5r#({A$? z$Ujpa5V|r)`M;KkPz>4@5&3<3{c8f8)VfZk9?{sFh{lI3eZc&yhH52E^^jNi+ZlquL#x5-ti@4r6 z=FnGa9s0Qi_DaAgIYHcEZmC>e&cV0+0K+;I^s{)SPr8aR&GXbs-v@M_XQDyA8c0YF z=hGe^WtJ)>e5VSx)awF<*QxJ66KA#oxpsX>x4?w~WD{{;79VtQD9sx6-Q7G{hog%H zWLW4~#SSGf~p7_;k z>v*lpzo9wV7?VOF7-}C;|Bjkixp;|ecQA#rd6mCZ);oWkPakMZUa) zBV_gdllDj})2?q->PNbHvB7sS9lU7zNX*5CYdW5n>HLo<$20u{l~@2?LFqqeIwHna zwU>k&Ooj``*-7>6JDk-dpza82`r5gmEAdn?+oIoFemk6|uK%9z_ zzejaMVo3oUAEf0UQfVInhrnI}-vWrTbURxrQc9;Pc0!ps;{5xJq4^1s_N0G@jrp_+TkopZPNwWhA#W`q%_l&DuRXt-Wtr z+RwGjz|RH_YrF87!~JF-4Sh7jzE2674Ql(TpQ+mqv=K!2Ynu`0BDMBV8|)mvIi-D| zVID1QzpsrpTWI#U#lEKv#dx#_JYUNGKsy9KcCXoYpw&0QvaPdqq@xud%)GDd?QGrE z+q$2y4q^WmKDIH}wEHI3W`YKqW&d2;tr^w>+Dmh8P0v2i`ltpm{F=6fSE^x;yc3&B zyjv;_4N9W-*PsBtx{iVFF|`<18;L7t8J_)v*$8rP4S7@`rLe0TOC= zn5T#j-Z}n%51dj+c9AS*SGy;L<+4Y*t@_!$igIP04QQdyV5Rhr0BsQt0yNBySS*Qe*)bV7HUkXd*AlFPaNVXEL&HcEE zzl39Z7MaPG*nAxUKK|xi{u7nhkdXLv=rJoZ>2YKo$E7ho%0B@$R)UuSEXCq$%Kd7V kt||Gd^0=PO(v>Cs8%lmHP1A=|Efqf&P7UVTkTH__KavBL2><{9 literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/__pycache__/serving_tokenization.cpython-310.pyc b/vllm/entrypoints/openai/__pycache__/serving_tokenization.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73a6915ef8562a09fbb19e51c58224505d797df9 GIT binary patch literal 3608 zcmb_f&2QYs6(@&WE|<$)t-iHU9IqWD#Ws}`r)U!tY2dFkHDD`&)m#Pz2zqBEQQ?rx zaMp=bZqZf-a?+*OqJVw$q5ncJ?O(xbPyG*C+j8>WkgKo6=%p+0apt|5;mmu#_j{wX z)A0=WUHjYLckeA5#=o$0_Gd%qIz0J5K`?`vu^~)q3POpnsAZy##I|VAhHhK2Bbv0S z+ji`V7HtWSdZJC+!l%C0H{y=y(yneh@r>xvo|vVxVvf#_Ji`w3dm&7t% z7Atf`^l4w~-1s%IN>_Ee6~8Vn(M!7R#cznq^s;I2RX#X0>6K3m)@J@AgZaFFXwj>z zvtCfm^tbp4N$ohaeA;)o|| zXqQW2mc{$Q?shQR4OO7H$l_4(a<$S6SXGAc1>;yQJ6l{nj1qKIJ4zxI z#MO4qvUjiY#B^neLu8n+R0QL3#6}p0v%gN=$$t+GZcvjMlrWQ9jBuM-yuqEm!ED}y zmfJU=-GFur-VVIIzOn6pV{DtO$=pXiZL=2lna4Z4%V)L;SZ$34oA$smt1WXUmJaJa zvg!OchHcOVK*>zmKX<(P5Fuiv>~M{UV5A{Y`U3G1j5J`%-UIMjP^mKTTVDEcDnrzO z=kl^EtFYFNDm&*;G{S_#p9POwKiJN%gHXnyv1b-$K?z_5dqaD3=*T{-Bws_ZisE$;bS9Ij$W#!rFjHLS@)BsvCga=T zI9AkEX&MLPoCE$F`8eaUbb~<0JCH40nvw z&x7Ep@oZ~DiEJZ_vNaXT9j?~i7vo)VO?eP!6!rw1;ZbqGsG za>&+XL}10~uM3SPJpX_v2a+@X2=0Gu6y_824zT5~jR`3TGsdQzDaaNusxl9azZicd z6AQ)>9QRu?u_ukf*s~tng?%#5QH{b>&LNQxz}B2Jm0Os5tpdV+Xu%wJ(kfc@H@w1= zHHs)!ma*4`_Y6EeW*s#q?WCt>!RDT`*xF*jsJTfSEb{=o zHUQ5C=rxXj^Fg-&fOLvBj9FyPkv;JXpEdi&q@$LKj#@4{2*AA+<{nv)S*^`y4G2xL zcJSUkfL62yi4n$wD9OeuAB2(*qMam_oDJAm0PN`CZsay;h3wC(Ir4DpZ z;kaCaY*bM)=D~IZhr z+8lmA&-f^UujsAjv{#ST-5_E)kf~GI%El-iC#vj-@avi+VxA29r4Mz0kG`mM5=yaA z&Icmgf572%XYvx0x9p&*_6ww<7v)ist1uaHIfG}>$&H0w{uqZl6(^U?uR@uCOXU)p zkXU&I{9Ah;>sZ3!&PQ+ngs`uKZDk=E3cyVN2s0QSdHq# zLAsRQl>Z_o-^F3qa9Hc)G}Y$UKuNno1T0ixT@-1v?#mBxieGzKZ;-b13!N?B!d10+ z8^_IzB!~KZvj6u%neTxhf9Tn+X`4QA8{`<-V9xR^$0Wox-ETe11{*PLXrT>z_~$++ z?rGnB?zj#)VUicNw&7^!h3nYF!Pd)z;j3BY!tTy-2{P*mm;4zz|35B)jBG$In|n^3 z%|vH&Lo?F{s>!TF^H0cJz%iOJoF^6(m?pC!vt89HT*!YD^4ZyxL)e*I2XXN3XBs+# zx{M5>e2}DS02L!krGlK^PUX3T)}b7l)ytDV$CVKL5~Ebrkq@x-OB6o^QO<9s>JD7F zxB%|wOkZ~7M`*_T!?1OVJ$W6qH&C1aP2R-rEfCbJ5Tp3l>LO%j>!7fc3)k;pMvhqj zA<#Gkfr5V+bb!TLZNHAoy||#+5}nc!~xJ zEIo1#1>O?mJc>4m(t>NKu6eSDI=oltJe63W}@rm>3O^xXuR zkS>33LdA;OTXgU}YjvAA{pzW+v;#HIYR@A1(YIeUe|j6K4pBu)2WjE-$n{1RQFBEv z1ph%KIpm6fLKWyUt)CJkvN!egxazn%VLkz^`~?aO(wXR;GNJonbqW0lZY9+5>KOJ52@E)c^nh literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py new file mode 100644 index 00000000..ef279d64 --- /dev/null +++ b/vllm/entrypoints/openai/api_server.py @@ -0,0 +1,585 @@ +import asyncio +import importlib +import inspect +import multiprocessing +import os +import regex as re +import signal +import socket +import tempfile +from argparse import Namespace +from contextlib import asynccontextmanager +from functools import partial +from http import HTTPStatus +from typing import AsyncIterator, Set + +import uvloop +from fastapi import APIRouter, FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.datastructures import State +from starlette.routing import Mount +from typing_extensions import assert_never + +import vllm.envs as envs +from vllm.config import ModelConfig +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.multiprocessing.client import MQLLMEngineClient +from vllm.engine.multiprocessing.engine import run_mp_engine +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.logger import RequestLogger +from vllm.entrypoints.openai.cli_args import (make_arg_parser, + validate_parsed_serve_args) +# yapf conflicts with isort for this block +# yapf: disable +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + ChatCompletionResponse, + CompletionRequest, + CompletionResponse, + DetokenizeRequest, + DetokenizeResponse, + EmbeddingRequest, + EmbeddingResponse, ErrorResponse, + LoadLoraAdapterRequest, + TokenizeRequest, + TokenizeResponse, + UnloadLoraAdapterRequest) +# yapf: enable +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat +from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion +from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from vllm.entrypoints.openai.serving_engine import BaseModelPath +from vllm.entrypoints.openai.serving_tokenization import ( + OpenAIServingTokenization) +from vllm.entrypoints.openai.tool_parsers import ToolParserManager +from vllm.logger import init_logger +from vllm.usage.usage_lib import UsageContext +from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path +from vllm.version import __version__ as VLLM_VERSION + +TIMEOUT_KEEP_ALIVE = 5 # seconds + +prometheus_multiproc_dir: tempfile.TemporaryDirectory + +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765) +logger = init_logger('vllm.entrypoints.openai.api_server') + +_running_tasks: Set[asyncio.Task] = set() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + if app.state.log_stats: + engine_client: EngineClient = app.state.engine_client + + async def _force_log(): + while True: + await asyncio.sleep(10.) + await engine_client.do_log_stats() + + task = asyncio.create_task(_force_log()) + _running_tasks.add(task) + task.add_done_callback(_running_tasks.remove) + else: + task = None + try: + yield + finally: + if task is not None: + task.cancel() + finally: + # Ensure app state including engine ref is gc'd + del app.state + + +@asynccontextmanager +async def build_async_engine_client( + args: Namespace) -> AsyncIterator[EngineClient]: + + # Context manager to handle engine_client lifecycle + # Ensures everything is shutdown and cleaned up on error/exit + engine_args = AsyncEngineArgs.from_cli_args(args) + + async with build_async_engine_client_from_engine_args( + engine_args, args.disable_frontend_multiprocessing) as engine: + yield engine + + +@asynccontextmanager +async def build_async_engine_client_from_engine_args( + engine_args: AsyncEngineArgs, + disable_frontend_multiprocessing: bool = False, +) -> AsyncIterator[EngineClient]: + """ + Create EngineClient, either: + - in-process using the AsyncLLMEngine Directly + - multiprocess using AsyncLLMEngine RPC + + Returns the Client or None if the creation failed. + """ + + # Fall back + # TODO: fill out feature matrix. + if (MQLLMEngineClient.is_unsupported_config(engine_args) + or disable_frontend_multiprocessing): + engine_config = engine_args.create_engine_config() + uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config), + "uses_ray", False) + + build_engine = partial(AsyncLLMEngine.from_engine_args, + engine_args=engine_args, + engine_config=engine_config, + usage_context=UsageContext.OPENAI_API_SERVER) + if uses_ray: + # Must run in main thread with ray for its signal handlers to work + engine_client = build_engine() + else: + engine_client = await asyncio.get_running_loop().run_in_executor( + None, build_engine) + + yield engine_client + return + + # Otherwise, use the multiprocessing AsyncLLMEngine. + else: + if "PROMETHEUS_MULTIPROC_DIR" not in os.environ: + # Make TemporaryDirectory for prometheus multiprocessing + # Note: global TemporaryDirectory will be automatically + # cleaned up upon exit. + global prometheus_multiproc_dir + prometheus_multiproc_dir = tempfile.TemporaryDirectory() + os.environ[ + "PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name + else: + logger.warning( + "Found PROMETHEUS_MULTIPROC_DIR was set by user. " + "This directory must be wiped between vLLM runs or " + "you will find inaccurate metrics. Unset the variable " + "and vLLM will properly handle cleanup.") + + # Select random path for IPC. + ipc_path = get_open_zmq_ipc_path() + logger.info("Multiprocessing frontend to use %s for IPC Path.", + ipc_path) + + # Start RPCServer in separate process (holds the LLMEngine). + # the current process might have CUDA context, + # so we need to spawn a new process + context = multiprocessing.get_context("spawn") + + engine_process = context.Process(target=run_mp_engine, + args=(engine_args, + UsageContext.OPENAI_API_SERVER, + ipc_path)) + engine_process.start() + logger.info("Started engine process with PID %d", engine_process.pid) + + # Build RPCClient, which conforms to EngineClient Protocol. + # NOTE: Actually, this is not true yet. We still need to support + # embedding models via RPC (see TODO above) + engine_config = engine_args.create_engine_config() + mp_engine_client = MQLLMEngineClient(ipc_path, engine_config) + + try: + while True: + try: + await mp_engine_client.setup() + break + except TimeoutError: + if not engine_process.is_alive(): + raise RuntimeError( + "Engine process failed to start") from None + + yield mp_engine_client # type: ignore[misc] + finally: + # Ensure rpc server process was terminated + engine_process.terminate() + + # Close all open connections to the backend + mp_engine_client.close() + + # Wait for engine process to join + engine_process.join(4) + if engine_process.exitcode is None: + # Kill if taking longer than 5 seconds to stop + engine_process.kill() + + # Lazy import for prometheus multiprocessing. + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable + # before prometheus_client is imported. + # See https://prometheus.github.io/client_python/multiprocess/ + from prometheus_client import multiprocess + multiprocess.mark_process_dead(engine_process.pid) + + +router = APIRouter() + + +def mount_metrics(app: FastAPI): + # Lazy import for prometheus multiprocessing. + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable + # before prometheus_client is imported. + # See https://prometheus.github.io/client_python/multiprocess/ + from prometheus_client import (CollectorRegistry, make_asgi_app, + multiprocess) + + prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None) + if prometheus_multiproc_dir_path is not None: + logger.info("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR", + prometheus_multiproc_dir_path) + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + + # Add prometheus asgi middleware to route /metrics requests + metrics_route = Mount("/metrics", make_asgi_app(registry=registry)) + else: + # Add prometheus asgi middleware to route /metrics requests + metrics_route = Mount("/metrics", make_asgi_app()) + + # Workaround for 307 Redirect for /metrics + metrics_route.path_regex = re.compile("^/metrics(?P.*)$") + app.routes.append(metrics_route) + + +def chat(request: Request) -> OpenAIServingChat: + return request.app.state.openai_serving_chat + + +def completion(request: Request) -> OpenAIServingCompletion: + return request.app.state.openai_serving_completion + + +def tokenization(request: Request) -> OpenAIServingTokenization: + return request.app.state.openai_serving_tokenization + + +def embedding(request: Request) -> OpenAIServingEmbedding: + return request.app.state.openai_serving_embedding + + +def engine_client(request: Request) -> EngineClient: + return request.app.state.engine_client + + +@router.get("/health") +async def health(raw_request: Request) -> Response: + """Health check.""" + await engine_client(raw_request).check_health() + return Response(status_code=200) + + +@router.post("/tokenize") +async def tokenize(request: TokenizeRequest, raw_request: Request): + generator = await tokenization(raw_request).create_tokenize(request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, TokenizeResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +@router.post("/detokenize") +async def detokenize(request: DetokenizeRequest, raw_request: Request): + generator = await tokenization(raw_request).create_detokenize(request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, DetokenizeResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +@router.get("/v1/models") +async def show_available_models(raw_request: Request): + models = await completion(raw_request).show_available_models() + return JSONResponse(content=models.model_dump()) + + +@router.get("/version") +async def show_version(): + ver = {"version": VLLM_VERSION} + return JSONResponse(content=ver) + + +@router.post("/v1/chat/completions") +async def create_chat_completion(request: ChatCompletionRequest, + raw_request: Request): + + generator = await chat(raw_request).create_chat_completion( + request, raw_request) + + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + + elif isinstance(generator, ChatCompletionResponse): + return JSONResponse(content=generator.model_dump()) + + return StreamingResponse(content=generator, media_type="text/event-stream") + + +@router.post("/v1/completions") +async def create_completion(request: CompletionRequest, raw_request: Request): + generator = await completion(raw_request).create_completion( + request, raw_request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, CompletionResponse): + return JSONResponse(content=generator.model_dump()) + + return StreamingResponse(content=generator, media_type="text/event-stream") + + +@router.post("/v1/embeddings") +async def create_embedding(request: EmbeddingRequest, raw_request: Request): + generator = await embedding(raw_request).create_embedding( + request, raw_request) + if isinstance(generator, ErrorResponse): + return JSONResponse(content=generator.model_dump(), + status_code=generator.code) + elif isinstance(generator, EmbeddingResponse): + return JSONResponse(content=generator.model_dump()) + + assert_never(generator) + + +if envs.VLLM_TORCH_PROFILER_DIR: + logger.warning( + "Torch Profiler is enabled in the API server. This should ONLY be " + "used for local development!") + + @router.post("/start_profile") + async def start_profile(raw_request: Request): + logger.info("Starting profiler...") + await engine_client(raw_request).start_profile() + logger.info("Profiler started.") + return Response(status_code=200) + + @router.post("/stop_profile") + async def stop_profile(raw_request: Request): + logger.info("Stopping profiler...") + await engine_client(raw_request).stop_profile() + logger.info("Profiler stopped.") + return Response(status_code=200) + + +if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING: + logger.warning( + "Lora dynamic loading & unloading is enabled in the API server. " + "This should ONLY be used for local development!") + + @router.post("/v1/load_lora_adapter") + async def load_lora_adapter(request: LoadLoraAdapterRequest, + raw_request: Request): + response = await chat(raw_request).load_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + response = await completion(raw_request).load_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + return Response(status_code=200, content=response) + + @router.post("/v1/unload_lora_adapter") + async def unload_lora_adapter(request: UnloadLoraAdapterRequest, + raw_request: Request): + response = await chat(raw_request).unload_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + response = await completion(raw_request).unload_lora_adapter(request) + if isinstance(response, ErrorResponse): + return JSONResponse(content=response.model_dump(), + status_code=response.code) + + return Response(status_code=200, content=response) + + +def build_app(args: Namespace) -> FastAPI: + if args.disable_fastapi_docs: + app = FastAPI(openapi_url=None, + docs_url=None, + redoc_url=None, + lifespan=lifespan) + else: + app = FastAPI(lifespan=lifespan) + app.include_router(router) + app.root_path = args.root_path + + mount_metrics(app) + + app.add_middleware( + CORSMiddleware, + allow_origins=args.allowed_origins, + allow_credentials=args.allow_credentials, + allow_methods=args.allowed_methods, + allow_headers=args.allowed_headers, + ) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler(_, exc): + chat = app.state.openai_serving_chat + err = chat.create_error_response(message=str(exc)) + return JSONResponse(err.model_dump(), + status_code=HTTPStatus.BAD_REQUEST) + + if token := envs.VLLM_API_KEY or args.api_key: + + @app.middleware("http") + async def authentication(request: Request, call_next): + root_path = "" if args.root_path is None else args.root_path + if request.method == "OPTIONS": + return await call_next(request) + if not request.url.path.startswith(f"{root_path}/v1"): + return await call_next(request) + if request.headers.get("Authorization") != "Bearer " + token: + return JSONResponse(content={"error": "Unauthorized"}, + status_code=401) + return await call_next(request) + + for middleware in args.middleware: + module_path, object_name = middleware.rsplit(".", 1) + imported = getattr(importlib.import_module(module_path), object_name) + if inspect.isclass(imported): + app.add_middleware(imported) + elif inspect.iscoroutinefunction(imported): + app.middleware("http")(imported) + else: + raise ValueError(f"Invalid middleware {middleware}. " + f"Must be a function or a class.") + + return app + + +def init_app_state( + engine_client: EngineClient, + model_config: ModelConfig, + state: State, + args: Namespace, +) -> None: + if args.served_model_name is not None: + served_model_names = args.served_model_name + else: + served_model_names = [args.model] + + if args.disable_log_requests: + request_logger = None + else: + request_logger = RequestLogger(max_log_len=args.max_log_len) + + base_model_paths = [ + BaseModelPath(name=name, model_path=args.model) + for name in served_model_names + ] + + state.engine_client = engine_client + state.log_stats = not args.disable_log_stats + + state.openai_serving_chat = OpenAIServingChat( + engine_client, + model_config, + base_model_paths, + args.response_role, + lora_modules=args.lora_modules, + prompt_adapters=args.prompt_adapters, + request_logger=request_logger, + chat_template=args.chat_template, + return_tokens_as_token_ids=args.return_tokens_as_token_ids, + enable_auto_tools=args.enable_auto_tool_choice, + tool_parser=args.tool_call_parser) + state.openai_serving_completion = OpenAIServingCompletion( + engine_client, + model_config, + base_model_paths, + lora_modules=args.lora_modules, + prompt_adapters=args.prompt_adapters, + request_logger=request_logger, + return_tokens_as_token_ids=args.return_tokens_as_token_ids, + ) + state.openai_serving_embedding = OpenAIServingEmbedding( + engine_client, + model_config, + base_model_paths, + request_logger=request_logger, + ) + state.openai_serving_tokenization = OpenAIServingTokenization( + engine_client, + model_config, + base_model_paths, + lora_modules=args.lora_modules, + request_logger=request_logger, + chat_template=args.chat_template, + ) + + +async def run_server(args, **uvicorn_kwargs) -> None: + logger.info("vLLM API server version %s", VLLM_VERSION) + logger.info("args: %s", args) + + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: + ToolParserManager.import_tool_parser(args.tool_parser_plugin) + + valide_tool_parses = ToolParserManager.tool_parsers.keys() + if args.enable_auto_tool_choice \ + and args.tool_call_parser not in valide_tool_parses: + raise KeyError(f"invalid tool call parser: {args.tool_call_parser} " + f"(chose from {{ {','.join(valide_tool_parses)} }})") + + # workaround to make sure that we bind the port before the engine is set up. + # This avoids race conditions with ray. + # see https://github.com/vllm-project/vllm/issues/8204 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("", args.port)) + + def signal_handler(*_) -> None: + # Interrupt server on sigterm while initializing + raise KeyboardInterrupt("terminated") + + signal.signal(signal.SIGTERM, signal_handler) + + async with build_async_engine_client(args) as engine_client: + app = build_app(args) + + model_config = await engine_client.get_model_config() + init_app_state(engine_client, model_config, app.state, args) + + shutdown_task = await serve_http( + app, + host=args.host, + port=args.port, + log_level=args.uvicorn_log_level, + timeout_keep_alive=TIMEOUT_KEEP_ALIVE, + ssl_keyfile=args.ssl_keyfile, + ssl_certfile=args.ssl_certfile, + ssl_ca_certs=args.ssl_ca_certs, + ssl_cert_reqs=args.ssl_cert_reqs, + fd=sock.fileno(), + **uvicorn_kwargs, + ) + + # NB: Await server shutdown only after the backend context is exited + await shutdown_task + + +if __name__ == "__main__": + # NOTE(simon): + # This section should be in sync with vllm/scripts.py for CLI entrypoints. + parser = FlexibleArgumentParser( + description="vLLM OpenAI-Compatible RESTful API server.") + parser = make_arg_parser(parser) + args = parser.parse_args() + validate_parsed_serve_args(args) + + uvloop.run(run_server(args)) diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py new file mode 100644 index 00000000..a089985a --- /dev/null +++ b/vllm/entrypoints/openai/cli_args.py @@ -0,0 +1,252 @@ +""" +This file contains the command line arguments for the vLLM's +OpenAI-compatible server. It is kept in a separate file for documentation +purposes. +""" + +import argparse +import json +import ssl +from typing import List, Optional, Sequence, Union + +from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str +from vllm.entrypoints.chat_utils import validate_chat_template +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath, + PromptAdapterPath) +from vllm.entrypoints.openai.tool_parsers import ToolParserManager +from vllm.utils import FlexibleArgumentParser + + +class LoRAParserAction(argparse.Action): + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Optional[Union[str, Sequence[str]]], + option_string: Optional[str] = None, + ): + if values is None: + values = [] + if isinstance(values, str): + raise TypeError("Expected values to be a list") + + lora_list: List[LoRAModulePath] = [] + for item in values: + if item in [None, '']: # Skip if item is None or empty string + continue + if '=' in item and ',' not in item: # Old format: name=path + name, path = item.split('=') + lora_list.append(LoRAModulePath(name, path)) + else: # Assume JSON format + try: + lora_dict = json.loads(item) + lora = LoRAModulePath(**lora_dict) + lora_list.append(lora) + except json.JSONDecodeError: + parser.error( + f"Invalid JSON format for --lora-modules: {item}") + except TypeError as e: + parser.error( + f"Invalid fields for --lora-modules: {item} - {str(e)}" + ) + setattr(namespace, self.dest, lora_list) + + +class PromptAdapterParserAction(argparse.Action): + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Optional[Union[str, Sequence[str]]], + option_string: Optional[str] = None, + ): + if values is None: + values = [] + if isinstance(values, str): + raise TypeError("Expected values to be a list") + + adapter_list: List[PromptAdapterPath] = [] + for item in values: + name, path = item.split('=') + adapter_list.append(PromptAdapterPath(name, path)) + setattr(namespace, self.dest, adapter_list) + + +def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + parser.add_argument("--host", + type=nullable_str, + default=None, + help="host name") + parser.add_argument("--port", type=int, default=8000, help="port number") + parser.add_argument( + "--uvicorn-log-level", + type=str, + default="info", + choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'], + help="log level for uvicorn") + parser.add_argument("--allow-credentials", + action="store_true", + help="allow credentials") + parser.add_argument("--allowed-origins", + type=json.loads, + default=["*"], + help="allowed origins") + parser.add_argument("--allowed-methods", + type=json.loads, + default=["*"], + help="allowed methods") + parser.add_argument("--allowed-headers", + type=json.loads, + default=["*"], + help="allowed headers") + parser.add_argument("--api-key", + type=nullable_str, + default=None, + help="If provided, the server will require this key " + "to be presented in the header.") + parser.add_argument( + "--lora-modules", + type=nullable_str, + default=None, + nargs='+', + action=LoRAParserAction, + help="LoRA module configurations in either 'name=path' format" + "or JSON format. " + "Example (old format): 'name=path' " + "Example (new format): " + "'{\"name\": \"name\", \"local_path\": \"path\", " + "\"base_model_name\": \"id\"}'") + parser.add_argument( + "--prompt-adapters", + type=nullable_str, + default=None, + nargs='+', + action=PromptAdapterParserAction, + help="Prompt adapter configurations in the format name=path. " + "Multiple adapters can be specified.") + parser.add_argument("--chat-template", + type=nullable_str, + default=None, + help="The file path to the chat template, " + "or the template in single-line form " + "for the specified model") + parser.add_argument("--response-role", + type=nullable_str, + default="assistant", + help="The role name to return if " + "`request.add_generation_prompt=true`.") + parser.add_argument("--ssl-keyfile", + type=nullable_str, + default=None, + help="The file path to the SSL key file") + parser.add_argument("--ssl-certfile", + type=nullable_str, + default=None, + help="The file path to the SSL cert file") + parser.add_argument("--ssl-ca-certs", + type=nullable_str, + default=None, + help="The CA certificates file") + parser.add_argument( + "--ssl-cert-reqs", + type=int, + default=int(ssl.CERT_NONE), + help="Whether client certificate is required (see stdlib ssl module's)" + ) + parser.add_argument( + "--root-path", + type=nullable_str, + default=None, + help="FastAPI root_path when app is behind a path based routing proxy") + parser.add_argument( + "--middleware", + type=nullable_str, + action="append", + default=[], + help="Additional ASGI middleware to apply to the app. " + "We accept multiple --middleware arguments. " + "The value should be an import path. " + "If a function is provided, vLLM will add it to the server " + "using @app.middleware('http'). " + "If a class is provided, vLLM will add it to the server " + "using app.add_middleware(). ") + parser.add_argument( + "--return-tokens-as-token-ids", + action="store_true", + help="When --max-logprobs is specified, represents single tokens as " + "strings of the form 'token_id:{token_id}' so that tokens that " + "are not JSON-encodable can be identified.") + parser.add_argument( + "--disable-frontend-multiprocessing", + action="store_true", + help="If specified, will run the OpenAI frontend server in the same " + "process as the model serving engine.") + + parser.add_argument( + "--enable-auto-tool-choice", + action="store_true", + default=False, + help= + "Enable auto tool choice for supported models. Use --tool-call-parser" + "to specify which parser to use") + + valid_tool_parsers = ToolParserManager.tool_parsers.keys() + parser.add_argument( + "--tool-call-parser", + type=str, + metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in " + "--tool-parser-plugin", + default=None, + help= + "Select the tool call parser depending on the model that you're using." + " This is used to parse the model-generated tool call into OpenAI API " + "format. Required for --enable-auto-tool-choice.") + + parser.add_argument( + "--tool-parser-plugin", + type=str, + default="", + help= + "Special the tool parser plugin write to parse the model-generated tool" + " into OpenAI API format, the name register in this plugin can be used " + "in --tool-call-parser.") + + parser = AsyncEngineArgs.add_cli_args(parser) + + parser.add_argument('--max-log-len', + type=int, + default=None, + help='Max number of prompt characters or prompt ' + 'ID numbers being printed in log.' + '\n\nDefault: Unlimited') + + parser.add_argument( + "--disable-fastapi-docs", + action='store_true', + default=False, + help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint" + ) + + return parser + + +def validate_parsed_serve_args(args: argparse.Namespace): + """Quick checks for model serve args that raise prior to loading.""" + if hasattr(args, "subparser") and args.subparser != "serve": + return + + # Ensure that the chat template is valid; raises if it likely isn't + validate_chat_template(args.chat_template) + + # Enable auto tool needs a tool call parser to be valid + if args.enable_auto_tool_choice and not args.tool_call_parser: + raise TypeError("Error: --enable-auto-tool-choice requires " + "--tool-call-parser") + + +def create_parser_for_docs() -> FlexibleArgumentParser: + parser_for_docs = FlexibleArgumentParser( + prog="-m vllm.entrypoints.openai.api_server") + return make_arg_parser(parser_for_docs) diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py new file mode 100644 index 00000000..7913f872 --- /dev/null +++ b/vllm/entrypoints/openai/logits_processors.py @@ -0,0 +1,86 @@ +from functools import lru_cache, partial +from typing import Dict, FrozenSet, Iterable, List, Optional, Union + +import torch + +from vllm.sampling_params import LogitsProcessor +from vllm.transformers_utils.tokenizer import AnyTokenizer + + +class AllowedTokenIdsLogitsProcessor: + """Logits processor for constraining generated tokens to a + specific set of token ids.""" + + def __init__(self, allowed_ids: Iterable[int]): + self.allowed_ids: Optional[List[int]] = list(allowed_ids) + self.mask: Optional[torch.Tensor] = None + + def __call__(self, token_ids: List[int], + logits: torch.Tensor) -> torch.Tensor: + if self.mask is None: + self.mask = torch.ones((logits.shape[-1], ), + dtype=torch.bool, + device=logits.device) + self.mask[self.allowed_ids] = False + self.allowed_ids = None + logits.masked_fill_(self.mask, float("-inf")) + return logits + + +@lru_cache(maxsize=32) +def _get_allowed_token_ids_logits_processor( + allowed_token_ids: FrozenSet[int], + vocab_size: int, +) -> LogitsProcessor: + if not allowed_token_ids: + raise ValueError("Empty allowed_token_ids provided") + if not all(0 <= tid < vocab_size for tid in allowed_token_ids): + raise ValueError("allowed_token_ids contains " + "out-of-vocab token id") + return AllowedTokenIdsLogitsProcessor(allowed_token_ids) + + +def logit_bias_logits_processor( + logit_bias: Dict[int, float], + token_ids: List[int], + logits: torch.Tensor, +) -> torch.Tensor: + for token_id, bias in logit_bias.items(): + logits[token_id] += bias + return logits + + +def get_logits_processors( + logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]], + allowed_token_ids: Optional[List[int]], + tokenizer: AnyTokenizer, +) -> List[LogitsProcessor]: + logits_processors: List[LogitsProcessor] = [] + if logit_bias: + try: + # Convert token_id to integer + # Clamp the bias between -100 and 100 per OpenAI API spec + clamped_logit_bias: Dict[int, float] = { + int(token_id): min(100.0, max(-100.0, bias)) + for token_id, bias in logit_bias.items() + } + except ValueError as exc: + raise ValueError( + "Found token_id in logit_bias that is not " + "an integer or string representing an integer") from exc + + # Check if token_id is within the vocab size + for token_id, bias in clamped_logit_bias.items(): + if token_id < 0 or token_id >= tokenizer.vocab_size: + raise ValueError(f"token_id {token_id} in logit_bias contains " + "out-of-vocab token id") + + logits_processors.append( + partial(logit_bias_logits_processor, clamped_logit_bias)) + + if allowed_token_ids is not None: + logits_processors.append( + _get_allowed_token_ids_logits_processor( + frozenset(allowed_token_ids), tokenizer.vocab_size)) + + return logits_processors diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py new file mode 100644 index 00000000..6f1135f8 --- /dev/null +++ b/vllm/entrypoints/openai/protocol.py @@ -0,0 +1,992 @@ +# Adapted from +# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py +import time +from argparse import Namespace +from typing import Any, Dict, List, Literal, Optional, Union + +import torch +from openai.types.chat import ChatCompletionContentPartParam +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Annotated, Required, TypedDict + +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +from vllm.pooling_params import PoolingParams +from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams, + RequestOutputKind, SamplingParams) +from vllm.sequence import Logprob +from vllm.utils import random_uuid + +# torch is mocked during docs generation, +# so we have to provide the values as literals +_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807) +_LONG_INFO: Union["torch.iinfo", Namespace] + +try: + from sphinx.ext.autodoc.mock import _MockModule + + if isinstance(torch, _MockModule): + _LONG_INFO = _MOCK_LONG_INFO + else: + _LONG_INFO = torch.iinfo(torch.long) +except ModuleNotFoundError: + _LONG_INFO = torch.iinfo(torch.long) + +assert _LONG_INFO.min == _MOCK_LONG_INFO.min +assert _LONG_INFO.max == _MOCK_LONG_INFO.max + + +class CustomChatCompletionMessageParam(TypedDict, total=False): + """Enables custom roles in the Chat Completion API.""" + role: Required[str] + """The role of the message's author.""" + + content: Union[str, List[ChatCompletionContentPartParam]] + """The contents of the message.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the + same role. + """ + + tool_call_id: Optional[str] + + tool_calls: Optional[List[dict]] + + +class OpenAIBaseModel(BaseModel): + # OpenAI API does not allow extra fields + model_config = ConfigDict(extra="forbid") + + +class ErrorResponse(OpenAIBaseModel): + object: str = "error" + message: str + type: str + param: Optional[str] = None + code: int + + +class ModelPermission(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}") + object: str = "model_permission" + created: int = Field(default_factory=lambda: int(time.time())) + allow_create_engine: bool = False + allow_sampling: bool = True + allow_logprobs: bool = True + allow_search_indices: bool = False + allow_view: bool = True + allow_fine_tuning: bool = False + organization: str = "*" + group: Optional[str] = None + is_blocking: bool = False + + +class ModelCard(OpenAIBaseModel): + id: str + object: str = "model" + created: int = Field(default_factory=lambda: int(time.time())) + owned_by: str = "vllm" + root: Optional[str] = None + parent: Optional[str] = None + max_model_len: Optional[int] = None + permission: List[ModelPermission] = Field(default_factory=list) + + +class ModelList(OpenAIBaseModel): + object: str = "list" + data: List[ModelCard] = Field(default_factory=list) + + +class UsageInfo(OpenAIBaseModel): + prompt_tokens: int = 0 + total_tokens: int = 0 + completion_tokens: Optional[int] = 0 + + +class RequestResponseMetadata(BaseModel): + request_id: str + final_usage_info: Optional[UsageInfo] = None + + +class JsonSchemaResponseFormat(OpenAIBaseModel): + name: str + description: Optional[str] = None + # schema is the field in openai but that causes conflicts with pydantic so + # instead use json_schema with an alias + json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema') + strict: Optional[bool] = None + + +class ResponseFormat(OpenAIBaseModel): + # type must be "json_schema", "json_object" or "text" + type: Literal["text", "json_object", "json_schema"] + json_schema: Optional[JsonSchemaResponseFormat] = None + + +class StreamOptions(OpenAIBaseModel): + include_usage: Optional[bool] = True + continuous_usage_stats: Optional[bool] = True + + +class FunctionDefinition(OpenAIBaseModel): + name: str + description: Optional[str] = None + parameters: Optional[Dict[str, Any]] = None + + +class ChatCompletionToolsParam(OpenAIBaseModel): + type: Literal["function"] = "function" + function: FunctionDefinition + + +class ChatCompletionNamedFunction(OpenAIBaseModel): + name: str + + +class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel): + function: ChatCompletionNamedFunction + type: Literal["function"] = "function" + + +class ChatCompletionRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/chat/create + messages: List[ChatCompletionMessageParam] + model: str + frequency_penalty: Optional[float] = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: Optional[bool] = False + top_logprobs: Optional[int] = 0 + max_tokens: Optional[int] = None + n: Optional[int] = 1 + presence_penalty: Optional[float] = 0.0 + response_format: Optional[ResponseFormat] = None + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max) + stop: Optional[Union[str, List[str]]] = Field(default_factory=list) + stream: Optional[bool] = False + stream_options: Optional[StreamOptions] = None + temperature: Optional[float] = 0.7 + top_p: Optional[float] = 1.0 + tools: Optional[List[ChatCompletionToolsParam]] = None + tool_choice: Optional[Union[Literal["none"], Literal["auto"], + ChatCompletionNamedToolChoiceParam]] = "none" + + # NOTE this will be ignored by VLLM -- the model determines the behavior + parallel_tool_calls: Optional[bool] = False + user: Optional[str] = None + + # doc: begin-chat-completion-sampling-params + best_of: Optional[int] = None + use_beam_search: bool = False + top_k: int = -1 + min_p: float = 0.0 + repetition_penalty: float = 1.0 + length_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = Field(default_factory=list) + include_stop_str_in_output: bool = False + ignore_eos: bool = False + min_tokens: int = 0 + skip_special_tokens: bool = True + spaces_between_special_tokens: bool = True + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + prompt_logprobs: Optional[int] = None + # doc: end-chat-completion-sampling-params + + # doc: begin-chat-completion-extra-params + echo: bool = Field( + default=False, + description=( + "If true, the new message will be prepended with the last message " + "if they belong to the same role."), + ) + add_generation_prompt: bool = Field( + default=True, + description= + ("If true, the generation prompt will be added to the chat template. " + "This is a parameter used by chat template in tokenizer config of the " + "model."), + ) + continue_final_message: bool = Field( + default=False, + description= + ("If this is set, the chat will be formatted so that the final " + "message in the chat is open-ended, without any EOS tokens. The " + "model will continue this message rather than starting a new one. " + "This allows you to \"prefill\" part of the model's response for it. " + "Cannot be used at the same time as `add_generation_prompt`."), + ) + add_special_tokens: bool = Field( + default=False, + description=( + "If true, special tokens (e.g. BOS) will be added to the prompt " + "on top of what is added by the chat template. " + "For most models, the chat template takes care of adding the " + "special tokens so this should be set to false (as is the " + "default)."), + ) + documents: Optional[List[Dict[str, str]]] = Field( + default=None, + description= + ("A list of dicts representing documents that will be accessible to " + "the model if it is performing RAG (retrieval-augmented generation)." + " If the template does not support RAG, this argument will have no " + "effect. We recommend that each document should be a dict containing " + "\"title\" and \"text\" keys."), + ) + chat_template: Optional[str] = Field( + default=None, + description=( + "A Jinja template to use for this conversion. " + "As of transformers v4.44, default chat template is no longer " + "allowed, so you must provide a chat template if the tokenizer " + "does not define one."), + ) + chat_template_kwargs: Optional[Dict[str, Any]] = Field( + default=None, + description=("Additional kwargs to pass to the template renderer. " + "Will be accessible by the chat template."), + ) + guided_json: Optional[Union[str, dict, BaseModel]] = Field( + default=None, + description=("If specified, the output will follow the JSON schema."), + ) + guided_regex: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the regex pattern."), + ) + guided_choice: Optional[List[str]] = Field( + default=None, + description=( + "If specified, the output will be exactly one of the choices."), + ) + guided_grammar: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the context free grammar."), + ) + guided_decoding_backend: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default guided decoding backend " + "of the server for this specific request. If set, must be either " + "'outlines' / 'lm-format-enforcer'")) + guided_whitespace_pattern: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default whitespace pattern " + "for guided json decoding.")) + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-chat-completion-extra-params + + def to_beam_search_params(self, + default_max_tokens: int) -> BeamSearchParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + n = self.n if self.n is not None else 1 + temperature = self.temperature if self.temperature is not None else 0.0 + + return BeamSearchParams( + beam_width=n, + max_tokens=max_tokens, + ignore_eos=self.ignore_eos, + temperature=temperature, + length_penalty=self.length_penalty, + ) + + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + prompt_logprobs = self.prompt_logprobs + if prompt_logprobs is None and self.echo: + prompt_logprobs = self.top_logprobs + + guided_json_object = None + if (self.response_format is not None + and self.response_format.type == "json_object"): + guided_json_object = True + + guided_decoding = GuidedDecodingParams.from_optional( + json=self._get_guided_json_from_tool() or self.guided_json, + regex=self.guided_regex, + choice=self.guided_choice, + grammar=self.guided_grammar, + json_object=guided_json_object, + backend=self.guided_decoding_backend, + whitespace_pattern=self.guided_whitespace_pattern) + + return SamplingParams.from_optional( + n=self.n, + best_of=self.best_of, + presence_penalty=self.presence_penalty, + frequency_penalty=self.frequency_penalty, + repetition_penalty=self.repetition_penalty, + temperature=self.temperature, + top_p=self.top_p, + top_k=self.top_k, + min_p=self.min_p, + seed=self.seed, + stop=self.stop, + stop_token_ids=self.stop_token_ids, + logprobs=self.top_logprobs if self.logprobs else None, + prompt_logprobs=prompt_logprobs, + ignore_eos=self.ignore_eos, + max_tokens=max_tokens, + min_tokens=self.min_tokens, + skip_special_tokens=self.skip_special_tokens, + spaces_between_special_tokens=self.spaces_between_special_tokens, + include_stop_str_in_output=self.include_stop_str_in_output, + truncate_prompt_tokens=self.truncate_prompt_tokens, + output_kind=RequestOutputKind.DELTA if self.stream \ + else RequestOutputKind.FINAL_ONLY, + guided_decoding=guided_decoding, + logit_bias=self.logit_bias) + + def _get_guided_json_from_tool( + self) -> Optional[Union[str, dict, BaseModel]]: + # user has chosen to not use any tool + if self.tool_choice == "none" or self.tools is None: + return None + + # user has chosen to use a named tool + if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam: + tool_name = self.tool_choice.function.name + tools = {tool.function.name: tool.function for tool in self.tools} + if tool_name not in tools: + raise ValueError( + f"Tool '{tool_name}' has not been passed in `tools`.") + tool = tools[tool_name] + return tool.parameters + + return None + + @model_validator(mode="before") + @classmethod + def validate_stream_options(cls, data): + if data.get("stream_options") and not data.get("stream"): + raise ValueError( + "Stream options can only be defined when `stream=True`.") + + return data + + @model_validator(mode="before") + @classmethod + def check_logprobs(cls, data): + if (prompt_logprobs := data.get("prompt_logprobs")) is not None: + if data.get("stream") and prompt_logprobs > 0: + raise ValueError( + "`prompt_logprobs` are not available when `stream=True`.") + + if prompt_logprobs < 0: + raise ValueError("`prompt_logprobs` must be a positive value.") + + if (top_logprobs := data.get("top_logprobs")) is not None: + if top_logprobs < 0: + raise ValueError("`top_logprobs` must be a positive value.") + + if not data.get("logprobs"): + raise ValueError( + "when using `top_logprobs`, `logprobs` must be set to true." + ) + + return data + + @model_validator(mode="before") + @classmethod + def check_guided_decoding_count(cls, data): + if isinstance(data, ValueError): + raise data + + guide_count = sum([ + "guided_json" in data and data["guided_json"] is not None, + "guided_regex" in data and data["guided_regex"] is not None, + "guided_choice" in data and data["guided_choice"] is not None + ]) + # you can only use one kind of guided decoding + if guide_count > 1: + raise ValueError( + "You can only use one kind of guided decoding " + "('guided_json', 'guided_regex' or 'guided_choice').") + # you can only either use guided decoding or tools, not both + if guide_count > 1 and data.get("tool_choice", + "none") not in ("none", "auto"): + raise ValueError( + "You can only either use guided decoding or tools, not both.") + return data + + @model_validator(mode="before") + @classmethod + def check_tool_usage(cls, data): + + # if "tool_choice" is not specified but tools are provided, + # default to "auto" tool_choice + if "tool_choice" not in data and data.get("tools"): + data["tool_choice"] = "auto" + + # if "tool_choice" is specified -- validation + if "tool_choice" in data: + + # ensure that if "tool choice" is specified, tools are present + if "tools" not in data or data["tools"] is None: + raise ValueError( + "When using `tool_choice`, `tools` must be set.") + + # make sure that tool choice is either a named tool + # OR that it's set to "auto" + if data["tool_choice"] != "auto" and not isinstance( + data["tool_choice"], dict): + raise ValueError( + "`tool_choice` must either be a named tool or \"auto\". " + "`tool_choice=\"none\" is not supported.") + + # ensure that if "tool_choice" is specified as an object, + # it matches a valid tool + if isinstance(data["tool_choice"], dict): + valid_tool = False + specified_function = data["tool_choice"]["function"] + if not specified_function: + raise ValueError( + "Incorrectly formatted `tool_choice`. Should be like " + "`{\"type\": \"function\"," + " \"function\": {\"name\": \"my_function\"}}`") + specified_function_name = specified_function["name"] + if not specified_function_name: + raise ValueError( + "Incorrectly formatted `tool_choice`. Should be like " + "`{\"type\": \"function\", " + "\"function\": {\"name\": \"my_function\"}}`") + for tool in data["tools"]: + if tool["function"]["name"] == specified_function_name: + valid_tool = True + break + if not valid_tool: + raise ValueError( + "The tool specified in `tool_choice` does not match any" + " of the specified `tools`") + return data + + @model_validator(mode="before") + @classmethod + def check_generation_prompt(cls, data): + if data.get("continue_final_message") and data.get( + "add_generation_prompt"): + raise ValueError("Cannot set both `continue_final_message` and " + "`add_generation_prompt` to True.") + return data + + +class CompletionRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/completions/create + model: str + prompt: Union[List[int], List[List[int]], str, List[str]] + best_of: Optional[int] = None + echo: Optional[bool] = False + frequency_penalty: Optional[float] = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: Optional[int] = None + max_tokens: Optional[int] = 16 + n: int = 1 + presence_penalty: Optional[float] = 0.0 + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max) + stop: Optional[Union[str, List[str]]] = Field(default_factory=list) + stream: Optional[bool] = False + stream_options: Optional[StreamOptions] = None + suffix: Optional[str] = None + temperature: Optional[float] = 1.0 + top_p: Optional[float] = 1.0 + user: Optional[str] = None + + # doc: begin-completion-sampling-params + use_beam_search: bool = False + top_k: int = -1 + min_p: float = 0.0 + repetition_penalty: float = 1.0 + length_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = Field(default_factory=list) + include_stop_str_in_output: bool = False + ignore_eos: bool = False + min_tokens: int = 0 + skip_special_tokens: bool = True + spaces_between_special_tokens: bool = True + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + allowed_token_ids: Optional[List[int]] = None + prompt_logprobs: Optional[int] = None + # doc: end-completion-sampling-params + + # doc: begin-completion-extra-params + add_special_tokens: bool = Field( + default=True, + description=( + "If true (the default), special tokens (e.g. BOS) will be added to " + "the prompt."), + ) + response_format: Optional[ResponseFormat] = Field( + default=None, + description= + ("Similar to chat completion, this parameter specifies the format of " + "output. Only {'type': 'json_object'} or {'type': 'text' } is " + "supported."), + ) + guided_json: Optional[Union[str, dict, BaseModel]] = Field( + default=None, + description="If specified, the output will follow the JSON schema.", + ) + guided_regex: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the regex pattern."), + ) + guided_choice: Optional[List[str]] = Field( + default=None, + description=( + "If specified, the output will be exactly one of the choices."), + ) + guided_grammar: Optional[str] = Field( + default=None, + description=( + "If specified, the output will follow the context free grammar."), + ) + guided_decoding_backend: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default guided decoding backend " + "of the server for this specific request. If set, must be one of " + "'outlines' / 'lm-format-enforcer'")) + guided_whitespace_pattern: Optional[str] = Field( + default=None, + description=( + "If specified, will override the default whitespace pattern " + "for guided json decoding.")) + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-completion-extra-params + + def to_beam_search_params(self, + default_max_tokens: int) -> BeamSearchParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + n = self.n if self.n is not None else 1 + temperature = self.temperature if self.temperature is not None else 0.0 + + return BeamSearchParams( + beam_width=n, + max_tokens=max_tokens, + ignore_eos=self.ignore_eos, + temperature=temperature, + length_penalty=self.length_penalty, + ) + + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams: + max_tokens = self.max_tokens + if max_tokens is None: + max_tokens = default_max_tokens + + prompt_logprobs = self.prompt_logprobs + if prompt_logprobs is None and self.echo: + prompt_logprobs = self.logprobs + + echo_without_generation = self.echo and self.max_tokens == 0 + + guided_json_object = None + if (self.response_format is not None + and self.response_format.type == "json_object"): + guided_json_object = True + + guided_decoding = GuidedDecodingParams.from_optional( + json=self.guided_json, + regex=self.guided_regex, + choice=self.guided_choice, + grammar=self.guided_grammar, + json_object=guided_json_object, + backend=self.guided_decoding_backend, + whitespace_pattern=self.guided_whitespace_pattern) + + return SamplingParams.from_optional( + n=self.n, + best_of=self.best_of, + presence_penalty=self.presence_penalty, + frequency_penalty=self.frequency_penalty, + repetition_penalty=self.repetition_penalty, + temperature=self.temperature, + top_p=self.top_p, + top_k=self.top_k, + min_p=self.min_p, + seed=self.seed, + stop=self.stop, + stop_token_ids=self.stop_token_ids, + logprobs=self.logprobs, + ignore_eos=self.ignore_eos, + max_tokens=max_tokens if not echo_without_generation else 1, + min_tokens=self.min_tokens, + prompt_logprobs=prompt_logprobs, + skip_special_tokens=self.skip_special_tokens, + spaces_between_special_tokens=self.spaces_between_special_tokens, + include_stop_str_in_output=self.include_stop_str_in_output, + truncate_prompt_tokens=self.truncate_prompt_tokens, + output_kind=RequestOutputKind.DELTA if self.stream \ + else RequestOutputKind.FINAL_ONLY, + guided_decoding=guided_decoding, + logit_bias=self.logit_bias, + allowed_token_ids=self.allowed_token_ids) + + @model_validator(mode="before") + @classmethod + def check_guided_decoding_count(cls, data): + guide_count = sum([ + "guided_json" in data and data["guided_json"] is not None, + "guided_regex" in data and data["guided_regex"] is not None, + "guided_choice" in data and data["guided_choice"] is not None + ]) + if guide_count > 1: + raise ValueError( + "You can only use one kind of guided decoding " + "('guided_json', 'guided_regex' or 'guided_choice').") + return data + + @model_validator(mode="before") + @classmethod + def check_logprobs(cls, data): + if (prompt_logprobs := data.get("prompt_logprobs")) is not None: + if data.get("stream") and prompt_logprobs > 0: + raise ValueError( + "`prompt_logprobs` are not available when `stream=True`.") + + if prompt_logprobs < 0: + raise ValueError("`prompt_logprobs` must be a positive value.") + + if (logprobs := data.get("logprobs")) is not None and logprobs < 0: + raise ValueError("`logprobs` must be a positive value.") + + return data + + @model_validator(mode="before") + @classmethod + def validate_stream_options(cls, data): + if data.get("stream_options") and not data.get("stream"): + raise ValueError( + "Stream options can only be defined when `stream=True`.") + + return data + + +class EmbeddingRequest(OpenAIBaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/embeddings + model: str + input: Union[List[int], List[List[int]], str, List[str]] + encoding_format: Literal["float", "base64"] = "float" + dimensions: Optional[int] = None + user: Optional[str] = None + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None + + # doc: begin-embedding-pooling-params + additional_data: Optional[Any] = None + + # doc: end-embedding-pooling-params + + # doc: begin-embedding-extra-params + priority: int = Field( + default=0, + description=( + "The priority of the request (lower means earlier handling; " + "default: 0). Any priority other than 0 will raise an error " + "if the served model does not use priority scheduling.")) + + # doc: end-embedding-extra-params + + def to_pooling_params(self): + return PoolingParams(additional_data=self.additional_data) + + +class CompletionLogProbs(OpenAIBaseModel): + text_offset: List[int] = Field(default_factory=list) + token_logprobs: List[Optional[float]] = Field(default_factory=list) + tokens: List[str] = Field(default_factory=list) + top_logprobs: List[Optional[Dict[str, + float]]] = Field(default_factory=list) + + +class CompletionResponseChoice(OpenAIBaseModel): + index: int + text: str + logprobs: Optional[CompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = Field( + default=None, + description=( + "The stop string or token id that caused the completion " + "to stop, None if the completion finished for some other reason " + "including encountering the EOS token"), + ) + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None + + +class CompletionResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseChoice] + usage: UsageInfo + + +class CompletionResponseStreamChoice(OpenAIBaseModel): + index: int + text: str + logprobs: Optional[CompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = Field( + default=None, + description=( + "The stop string or token id that caused the completion " + "to stop, None if the completion finished for some other reason " + "including encountering the EOS token"), + ) + + +class CompletionStreamResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseStreamChoice] + usage: Optional[UsageInfo] = Field(default=None) + + +class EmbeddingResponseData(OpenAIBaseModel): + index: int + object: str = "embedding" + embedding: Union[List[float], str] + + +class EmbeddingResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}") + object: str = "list" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + data: List[EmbeddingResponseData] + usage: UsageInfo + + +class FunctionCall(OpenAIBaseModel): + name: str + arguments: str + + +class ToolCall(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}") + type: Literal["function"] = "function" + function: FunctionCall + + +class DeltaFunctionCall(BaseModel): + name: Optional[str] = None + arguments: Optional[str] = None + + +# a tool call delta where everything is optional +class DeltaToolCall(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}") + type: Literal["function"] = "function" + index: int + function: Optional[DeltaFunctionCall] = None + + +class ExtractedToolCallInformation(BaseModel): + # indicate if tools were called + tools_called: bool + + # extracted tool calls + tool_calls: List[ToolCall] + + # content - per OpenAI spec, content AND tool calls can be returned rarely + # But some models will do this intentionally + content: Optional[str] = None + + +class ChatMessage(OpenAIBaseModel): + role: str + content: Optional[str] = None + tool_calls: List[ToolCall] = Field(default_factory=list) + + +class ChatCompletionLogProb(OpenAIBaseModel): + token: str + logprob: float = -9999.0 + bytes: Optional[List[int]] = None + + +class ChatCompletionLogProbsContent(ChatCompletionLogProb): + top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list) + + +class ChatCompletionLogProbs(OpenAIBaseModel): + content: Optional[List[ChatCompletionLogProbsContent]] = None + + +class ChatCompletionResponseChoice(OpenAIBaseModel): + index: int + message: ChatMessage + logprobs: Optional[ChatCompletionLogProbs] = None + # per OpenAI spec this is the default + finish_reason: Optional[str] = "stop" + # not part of the OpenAI spec but included in vLLM for legacy reasons + stop_reason: Optional[Union[int, str]] = None + + +class ChatCompletionResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") + object: Literal["chat.completion"] = "chat.completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseChoice] + usage: UsageInfo + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None + + +class DeltaMessage(OpenAIBaseModel): + role: Optional[str] = None + content: Optional[str] = None + tool_calls: List[DeltaToolCall] = Field(default_factory=list) + + +class ChatCompletionResponseStreamChoice(OpenAIBaseModel): + index: int + delta: DeltaMessage + logprobs: Optional[ChatCompletionLogProbs] = None + finish_reason: Optional[str] = None + stop_reason: Optional[Union[int, str]] = None + + +class ChatCompletionStreamResponse(OpenAIBaseModel): + id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") + object: Literal["chat.completion.chunk"] = "chat.completion.chunk" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseStreamChoice] + usage: Optional[UsageInfo] = Field(default=None) + + +class BatchRequestInput(OpenAIBaseModel): + """ + The per-line object of the batch input file. + + NOTE: Currently only the `/v1/chat/completions` endpoint is supported. + """ + + # A developer-provided per-request id that will be used to match outputs to + # inputs. Must be unique for each request in a batch. + custom_id: str + + # The HTTP method to be used for the request. Currently only POST is + # supported. + method: str + + # The OpenAI API relative URL to be used for the request. Currently + # /v1/chat/completions is supported. + url: str + + # The parameters of the request. + body: Union[ChatCompletionRequest, EmbeddingRequest] + + +class BatchResponseData(OpenAIBaseModel): + # HTTP status code of the response. + status_code: int = 200 + + # An unique identifier for the API request. + request_id: str + + # The body of the response. + body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None + + +class BatchRequestOutput(OpenAIBaseModel): + """ + The per-line object of the batch output and error files + """ + + id: str + + # A developer-provided per-request id that will be used to match outputs to + # inputs. + custom_id: str + + response: Optional[BatchResponseData] + + # For requests that failed with a non-HTTP error, this will contain more + # information on the cause of the failure. + error: Optional[Any] + + +class TokenizeCompletionRequest(OpenAIBaseModel): + model: str + prompt: str + + add_special_tokens: bool = Field(default=True) + + +class TokenizeChatRequest(OpenAIBaseModel): + model: str + messages: List[ChatCompletionMessageParam] + + add_generation_prompt: bool = Field(default=True) + continue_final_message: bool = Field(default=False) + add_special_tokens: bool = Field(default=False) + + @model_validator(mode="before") + @classmethod + def check_generation_prompt(cls, data): + if data.get("continue_final_message") and data.get( + "add_generation_prompt"): + raise ValueError("Cannot set both `continue_final_message` and " + "`add_generation_prompt` to True.") + return data + + +TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest] + + +class TokenizeResponse(OpenAIBaseModel): + count: int + max_model_len: int + tokens: List[int] + + +class DetokenizeRequest(OpenAIBaseModel): + model: str + tokens: List[int] + + +class DetokenizeResponse(OpenAIBaseModel): + prompt: str + + +class LoadLoraAdapterRequest(BaseModel): + lora_name: str + lora_path: str + + +class UnloadLoraAdapterRequest(BaseModel): + lora_name: str + lora_int_id: Optional[int] = Field(default=None) diff --git a/vllm/entrypoints/openai/run_batch.py b/vllm/entrypoints/openai/run_batch.py new file mode 100644 index 00000000..f5249a0c --- /dev/null +++ b/vllm/entrypoints/openai/run_batch.py @@ -0,0 +1,285 @@ +import asyncio +from http import HTTPStatus +from io import StringIO +from typing import Awaitable, Callable, List, Optional + +import aiohttp +import torch +from prometheus_client import start_http_server +from tqdm import tqdm + +from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.entrypoints.logger import RequestLogger, logger +# yapf: disable +from vllm.entrypoints.openai.protocol import (BatchRequestInput, + BatchRequestOutput, + BatchResponseData, + ChatCompletionResponse, + EmbeddingResponse, ErrorResponse) +# yapf: enable +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat +from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from vllm.entrypoints.openai.serving_engine import BaseModelPath +from vllm.usage.usage_lib import UsageContext +from vllm.utils import FlexibleArgumentParser, random_uuid +from vllm.version import __version__ as VLLM_VERSION + + +def parse_args(): + parser = FlexibleArgumentParser( + description="vLLM OpenAI-Compatible batch runner.") + parser.add_argument( + "-i", + "--input-file", + required=True, + type=str, + help= + "The path or url to a single input file. Currently supports local file " + "paths, or the http protocol (http or https). If a URL is specified, " + "the file should be available via HTTP GET.") + parser.add_argument( + "-o", + "--output-file", + required=True, + type=str, + help="The path or url to a single output file. Currently supports " + "local file paths, or web (http or https) urls. If a URL is specified," + " the file should be available via HTTP PUT.") + parser.add_argument("--response-role", + type=nullable_str, + default="assistant", + help="The role name to return if " + "`request.add_generation_prompt=True`.") + + parser = AsyncEngineArgs.add_cli_args(parser) + + parser.add_argument('--max-log-len', + type=int, + default=None, + help='Max number of prompt characters or prompt ' + 'ID numbers being printed in log.' + '\n\nDefault: Unlimited') + + parser.add_argument("--enable-metrics", + action="store_true", + help="Enable Prometheus metrics") + parser.add_argument( + "--url", + type=str, + default="0.0.0.0", + help="URL to the Prometheus metrics server " + "(only needed if enable-metrics is set).", + ) + parser.add_argument( + "--port", + type=int, + default=8000, + help="Port number for the Prometheus metrics server " + "(only needed if enable-metrics is set).", + ) + + return parser.parse_args() + + +# explicitly use pure text format, with a newline at the end +# this makes it impossible to see the animation in the progress bar +# but will avoid messing up with ray or multiprocessing, which wraps +# each line of output with some prefix. +_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501 + + +class BatchProgressTracker: + + def __init__(self): + self._total = 0 + self._pbar: Optional[tqdm] = None + + def submitted(self): + self._total += 1 + + def completed(self): + if self._pbar: + self._pbar.update() + + def pbar(self) -> tqdm: + enable_tqdm = not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0 + self._pbar = tqdm(total=self._total, + unit="req", + desc="Running batch", + mininterval=5, + disable=not enable_tqdm, + bar_format=_BAR_FORMAT) + return self._pbar + + +async def read_file(path_or_url: str) -> str: + if path_or_url.startswith("http://") or path_or_url.startswith("https://"): + async with aiohttp.ClientSession() as session, \ + session.get(path_or_url) as resp: + return await resp.text() + else: + with open(path_or_url, "r", encoding="utf-8") as f: + return f.read() + + +async def write_file(path_or_url: str, data: str) -> None: + if path_or_url.startswith("http://") or path_or_url.startswith("https://"): + async with aiohttp.ClientSession() as session, \ + session.put(path_or_url, data=data.encode("utf-8")): + pass + else: + # We should make this async, but as long as this is always run as a + # standalone program, blocking the event loop won't effect performance + # in this particular case. + with open(path_or_url, "w", encoding="utf-8") as f: + f.write(data) + + +def make_error_request_output(request: BatchRequestInput, + error_msg: str) -> BatchRequestOutput: + batch_output = BatchRequestOutput( + id=f"vllm-{random_uuid()}", + custom_id=request.custom_id, + response=BatchResponseData( + status_code=HTTPStatus.BAD_REQUEST, + request_id=f"vllm-batch-{random_uuid()}", + ), + error=error_msg, + ) + return batch_output + + +async def make_async_error_request_output( + request: BatchRequestInput, error_msg: str) -> BatchRequestOutput: + return make_error_request_output(request, error_msg) + + +async def run_request(serving_engine_func: Callable, + request: BatchRequestInput, + tracker: BatchProgressTracker) -> BatchRequestOutput: + response = await serving_engine_func(request.body) + + if isinstance(response, (ChatCompletionResponse, EmbeddingResponse)): + batch_output = BatchRequestOutput( + id=f"vllm-{random_uuid()}", + custom_id=request.custom_id, + response=BatchResponseData( + body=response, request_id=f"vllm-batch-{random_uuid()}"), + error=None, + ) + elif isinstance(response, ErrorResponse): + batch_output = BatchRequestOutput( + id=f"vllm-{random_uuid()}", + custom_id=request.custom_id, + response=BatchResponseData( + status_code=response.code, + request_id=f"vllm-batch-{random_uuid()}"), + error=response, + ) + else: + batch_output = make_error_request_output( + request, error_msg="Request must not be sent in stream mode") + + tracker.completed() + return batch_output + + +async def main(args): + if args.served_model_name is not None: + served_model_names = args.served_model_name + else: + served_model_names = [args.model] + + engine_args = AsyncEngineArgs.from_cli_args(args) + engine = AsyncLLMEngine.from_engine_args( + engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER) + + model_config = await engine.get_model_config() + base_model_paths = [ + BaseModelPath(name=name, model_path=args.model) + for name in served_model_names + ] + + if args.disable_log_requests: + request_logger = None + else: + request_logger = RequestLogger(max_log_len=args.max_log_len) + + # Create the openai serving objects. + openai_serving_chat = OpenAIServingChat( + engine, + model_config, + base_model_paths, + args.response_role, + lora_modules=None, + prompt_adapters=None, + request_logger=request_logger, + chat_template=None, + ) + openai_serving_embedding = OpenAIServingEmbedding( + engine, + model_config, + base_model_paths, + request_logger=request_logger, + ) + + tracker = BatchProgressTracker() + logger.info("Reading batch from %s...", args.input_file) + + # Submit all requests in the file to the engine "concurrently". + response_futures: List[Awaitable[BatchRequestOutput]] = [] + for request_json in (await read_file(args.input_file)).strip().split("\n"): + # Skip empty lines. + request_json = request_json.strip() + if not request_json: + continue + + request = BatchRequestInput.model_validate_json(request_json) + + # Determine the type of request and run it. + if request.url == "/v1/chat/completions": + response_futures.append( + run_request(openai_serving_chat.create_chat_completion, + request, tracker)) + tracker.submitted() + elif request.url == "/v1/embeddings": + response_futures.append( + run_request(openai_serving_embedding.create_embedding, request, + tracker)) + tracker.submitted() + else: + response_futures.append( + make_async_error_request_output( + request, + error_msg="Only /v1/chat/completions and " + "/v1/embeddings are supported in the batch endpoint.", + )) + + with tracker.pbar(): + responses = await asyncio.gather(*response_futures) + + output_buffer = StringIO() + for response in responses: + print(response.model_dump_json(), file=output_buffer) + + output_buffer.seek(0) + await write_file(args.output_file, output_buffer.read().strip()) + + +if __name__ == "__main__": + args = parse_args() + + logger.info("vLLM batch processing API version %s", VLLM_VERSION) + logger.info("args: %s", args) + + # Start the Prometheus metrics server. LLMEngine uses the Prometheus client + # to publish metrics at the /metrics endpoint. + if args.enable_metrics: + logger.info("Prometheus metrics enabled") + start_http_server(port=args.port, addr=args.url) + else: + logger.info("Prometheus metrics disabled") + + asyncio.run(main(args)) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py new file mode 100644 index 00000000..4931195a --- /dev/null +++ b/vllm/entrypoints/openai/serving_chat.py @@ -0,0 +1,891 @@ +import asyncio +import json +import time +from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List, + Optional) +from typing import Sequence as GenericSequence +from typing import Union + +from fastapi import Request + +from vllm.config import ModelConfig +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.multiprocessing.client import MQLLMEngineClient +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.chat_utils import (ConversationMessage, + apply_hf_chat_template, + apply_mistral_chat_template, + load_chat_template, + parse_chat_messages_futures) +from vllm.entrypoints.logger import RequestLogger +from vllm.entrypoints.openai.protocol import ( + ChatCompletionLogProb, ChatCompletionLogProbs, + ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam, + ChatCompletionRequest, ChatCompletionResponse, + ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage, + DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata, + ToolCall, UsageInfo) +from vllm.entrypoints.openai.serving_engine import (BaseModelPath, + LoRAModulePath, + OpenAIServing, + PromptAdapterPath, + TextTokensPrompt) +from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager +from vllm.inputs import TokensPrompt +from vllm.logger import init_logger +from vllm.outputs import CompletionOutput, RequestOutput +from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.sequence import Logprob +from vllm.tracing import (contains_trace_headers, extract_trace_headers, + log_tracing_disabled_warning) +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.utils import iterate_with_cancellation, random_uuid + +logger = init_logger(__name__) + + +class OpenAIServingChat(OpenAIServing): + + def __init__(self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + response_role: str, + *, + lora_modules: Optional[List[LoRAModulePath]], + prompt_adapters: Optional[List[PromptAdapterPath]], + request_logger: Optional[RequestLogger], + chat_template: Optional[str], + return_tokens_as_token_ids: bool = False, + enable_auto_tools: bool = False, + tool_parser: Optional[str] = None): + super().__init__(engine_client=engine_client, + model_config=model_config, + base_model_paths=base_model_paths, + lora_modules=lora_modules, + prompt_adapters=prompt_adapters, + request_logger=request_logger, + return_tokens_as_token_ids=return_tokens_as_token_ids) + + self.response_role = response_role + self.use_tool_use_model_template = False + self.chat_template = load_chat_template(chat_template) + + # set up tool use + self.enable_auto_tools: bool = enable_auto_tools + if self.enable_auto_tools: + logger.info( + "\"auto\" tool choice has been enabled please note that while" + " the parallel_tool_calls client option is preset for " + "compatibility reasons, it will be ignored.") + + self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None + if self.enable_auto_tools: + try: + self.tool_parser = ToolParserManager.get_tool_parser( + tool_parser) + except Exception as e: + raise TypeError("Error: --enable-auto-tool-choice requires " + f"tool_parser:'{tool_parser}' which has not " + "been registered") from e + + async def create_chat_completion( + self, + request: ChatCompletionRequest, + raw_request: Optional[Request] = None, + ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, + ErrorResponse]: + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/chat/create + for the API specification. This API mimics the OpenAI + ChatCompletion API. + + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + logger.error("Error with model %s", error_check_ret) + return error_check_ret + + # If the engine is dead, raise the engine's DEAD_ERROR. + # This is required for the streaming case, where we return a + # success status before we actually start generating text :). + if self.engine_client.errored: + raise self.engine_client.dead_error + + try: + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + model_config = self.model_config + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + conversation, mm_data_future = parse_chat_messages_futures( + request.messages, model_config, tokenizer) + + tool_dicts = None if request.tools is None else [ + tool.model_dump() for tool in request.tools + ] + + prompt: Union[str, List[int]] + is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer) + if is_mistral_tokenizer: + prompt = apply_mistral_chat_template( + tokenizer, + messages=request.messages, + chat_template=request.chat_template or self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + tools=tool_dicts, + documents=request.documents, + **(request.chat_template_kwargs or {}), + ) + else: + prompt = apply_hf_chat_template( + tokenizer, + conversation=conversation, + chat_template=request.chat_template or self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + tools=tool_dicts, + documents=request.documents, + **(request.chat_template_kwargs or {}), + ) + except Exception as e: + logger.exception("Error in applying chat template from request") + return self.create_error_response(str(e)) + + try: + mm_data = await mm_data_future + except Exception as e: + logger.exception("Error in loading multi-modal data") + return self.create_error_response(str(e)) + + # validation for OpenAI tools + # tool_choice = "required" is not supported + if request.tool_choice == "required": + return self.create_error_response( + "tool_choice = \"required\" is not supported!") + + if not is_mistral_tokenizer and request.tool_choice == "auto" and not ( + self.enable_auto_tools and self.tool_parser is not None): + # for hf tokenizers, "auto" tools requires + # --enable-auto-tool-choice and --tool-call-parser + return self.create_error_response( + "\"auto\" tool choice requires " + "--enable-auto-tool-choice and --tool-call-parser to be set") + + request_id = f"chat-{random_uuid()}" + + request_metadata = RequestResponseMetadata(request_id=request_id) + if raw_request: + raw_request.state.request_metadata = request_metadata + + try: + if self.enable_auto_tools and self.tool_parser: + request = self.tool_parser(tokenizer).adjust_request( + request=request) + + if isinstance(prompt, str): + prompt_inputs = self._tokenize_prompt_input( + request, + tokenizer, + prompt, + truncate_prompt_tokens=request.truncate_prompt_tokens, + add_special_tokens=request.add_special_tokens, + ) + else: + assert isinstance(prompt, list) and isinstance( + prompt[0], int + ), "Prompt has to be either a string or a list of token ids" + prompt_inputs = TextTokensPrompt( + prompt=tokenizer.decode(prompt), prompt_token_ids=prompt) + + assert prompt_inputs is not None + + sampling_params: Union[SamplingParams, BeamSearchParams] + default_max_tokens = self.max_model_len - len( + prompt_inputs["prompt_token_ids"]) + if request.use_beam_search: + sampling_params = request.to_beam_search_params( + default_max_tokens) + else: + sampling_params = request.to_sampling_params( + default_max_tokens) + + self._log_inputs(request_id, + prompt_inputs, + params=sampling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + engine_inputs = TokensPrompt( + prompt_token_ids=prompt_inputs["prompt_token_ids"]) + if mm_data is not None: + engine_inputs["multi_modal_data"] = mm_data + + is_tracing_enabled = (await + self.engine_client.is_tracing_enabled()) + trace_headers = None + if is_tracing_enabled and raw_request: + trace_headers = extract_trace_headers(raw_request.headers) + if (not is_tracing_enabled and raw_request + and contains_trace_headers(raw_request.headers)): + log_tracing_disabled_warning() + + if isinstance(sampling_params, BeamSearchParams): + assert isinstance(self.engine_client, + (AsyncLLMEngine, + MQLLMEngineClient)), \ + "Beam search is only supported with" \ + "AsyncLLMEngine and MQLLMEngineClient." + result_generator = self.engine_client.beam_search( + engine_inputs['prompt_token_ids'], + request_id, + sampling_params, + ) + else: + result_generator = self.engine_client.generate( + engine_inputs, + sampling_params, + request_id, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=request.priority, + ) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + if raw_request: + result_generator = iterate_with_cancellation( + result_generator, raw_request.is_disconnected) + + # Streaming response + if request.stream: + return self.chat_completion_stream_generator( + request, result_generator, request_id, conversation, tokenizer, + request_metadata) + + try: + return await self.chat_completion_full_generator( + request, result_generator, request_id, conversation, tokenizer, + request_metadata) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + def get_chat_request_role(self, request: ChatCompletionRequest) -> str: + if request.add_generation_prompt: + return self.response_role + return request.messages[-1]["role"] + + async def chat_completion_stream_generator( + self, + request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], + request_id: str, + conversation: List[ConversationMessage], + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + ) -> AsyncGenerator[str, None]: + model_name = self.base_model_paths[0].name + created_time = int(time.time()) + chunk_object_type: Final = "chat.completion.chunk" + first_iteration = True + + # Send response for each token for each request.n (index) + num_choices = 1 if request.n is None else request.n + previous_num_tokens = [0] * num_choices + finish_reason_sent = [False] * num_choices + num_prompt_tokens = 0 + + if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): + tool_choice_function_name = request.tool_choice.function.name + else: + tool_choice_function_name = None + + # Determine whether tools are in use with "auto" tool choice + tool_choice_auto = ( + not tool_choice_function_name + and self._should_stream_with_auto_tool_parsing(request)) + + all_previous_token_ids: Optional[List[List[int]]] + if tool_choice_auto: + # These are only required in "auto" tool choice case + previous_texts = [""] * num_choices + all_previous_token_ids = [[]] * num_choices + else: + previous_texts, all_previous_token_ids = None, None + + # Prepare the tool parser if it's needed + try: + if tool_choice_auto and self.tool_parser: + tool_parsers: List[Optional[ToolParser]] = [ + self.tool_parser(tokenizer) + ] * num_choices + else: + tool_parsers = [None] * num_choices + except RuntimeError as e: + logger.error("Error in tool parser creation: %s", e) + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + yield "data: [DONE]\n\n" + return + + try: + async for res in result_generator: + if res.prompt_token_ids is not None: + num_prompt_tokens = len(res.prompt_token_ids) + if res.encoder_prompt_token_ids is not None: + num_prompt_tokens += len(res.encoder_prompt_token_ids) + + # We need to do it here, because if there are exceptions in + # the result_generator, it needs to be sent as the FIRST + # response (by the try...catch). + if first_iteration: + # Send first response for each request.n (index) with + # the role + role = self.get_chat_request_role(request) + + # NOTE num_choices defaults to 1 so this usually executes + # once per request + for i in range(num_choices): + tool_parser = tool_parsers[i] + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage( + role=role, + content="", + ), + logprobs=None, + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + + # if usage should be included + if (request.stream_options + and request.stream_options.include_usage): + # if continuous usage stats are requested, add it + if request.stream_options.continuous_usage_stats: + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=0, + total_tokens=num_prompt_tokens) + chunk.usage = usage + # otherwise don't + else: + chunk.usage = None + + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + + # Send response to echo the input portion of the + # last message + if request.echo or request.continue_final_message: + last_msg_content: str = "" + if conversation and "content" in conversation[ + -1] and conversation[-1].get("role") == role: + last_msg_content = conversation[-1]["content"] or "" + + if last_msg_content: + for i in range(num_choices): + choice_data = ( + ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage( + content=last_msg_content), + logprobs=None, + finish_reason=None)) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if (request.stream_options and + request.stream_options.include_usage): + if (request.stream_options. + continuous_usage_stats): + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=0, + total_tokens=num_prompt_tokens) + chunk.usage = usage + else: + chunk.usage = None + + data = chunk.model_dump_json( + exclude_unset=True) + yield f"data: {data}\n\n" + first_iteration = False + + for output in res.outputs: + i = output.index + tool_parser = tool_parsers[i] + + if finish_reason_sent[i]: + continue + + if request.logprobs and request.top_logprobs is not None: + assert output.logprobs is not None, ( + "Did not output logprobs") + logprobs = self._create_chat_logprobs( + token_ids=output.token_ids, + top_logprobs=output.logprobs, + tokenizer=tokenizer, + num_output_top_logprobs=request.top_logprobs, + ) + else: + logprobs = None + + delta_text = output.text + delta_message: Optional[DeltaMessage] + + # handle streaming deltas for tools with named tool_choice + if tool_choice_function_name: + delta_message = DeltaMessage(tool_calls=[ + DeltaToolCall(function=DeltaFunctionCall( + name=tool_choice_function_name, + arguments=delta_text), + index=i) + ]) + + # handle streaming deltas for tools with "auto" tool choice + elif tool_choice_auto: + assert previous_texts is not None + assert all_previous_token_ids is not None + assert tool_parser is not None + #TODO optimize manipulation of these lists + previous_text = previous_texts[i] + previous_token_ids = all_previous_token_ids[i] + current_text = previous_text + delta_text + current_token_ids = previous_token_ids + list( + output.token_ids) + + delta_message = ( + tool_parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=output.token_ids, + request=request)) + + # update the previous values for the next iteration + previous_texts[i] = current_text + all_previous_token_ids[i] = current_token_ids + + # handle streaming just a content delta + else: + delta_message = DeltaMessage(content=delta_text) + + # set the previous values for the next iteration + previous_num_tokens[i] += len(output.token_ids) + + # if the message delta is None (e.g. because it was a + # "control token" for tool calls or the parser otherwise + # wasn't ready to send a token, then + # get the next token without streaming a chunk + if delta_message is None: + continue + + if output.finish_reason is None: + # Send token-by-token response for each request.n + + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=delta_message, + logprobs=logprobs, + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + + # handle usage stats if requested & if continuous + if (request.stream_options + and request.stream_options.include_usage): + if request.stream_options.continuous_usage_stats: + completion_tokens = len(output.token_ids) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + + completion_tokens, + ) + chunk.usage = usage + else: + chunk.usage = None + + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + + # if the model is finished generating + else: + # check to make sure we haven't "forgotten" to stream + # any tokens that were generated but previously + # matched by partial json parsing + # only happens if we are NOT using guided decoding + auto_tools_called = False + if tool_parser: + auto_tools_called = len( + tool_parser.prev_tool_call_arr) > 0 + index = len(tool_parser.prev_tool_call_arr + ) - 1 if auto_tools_called else 0 + else: + index = 0 + + if self._should_check_for_unstreamed_tool_arg_tokens( + delta_message, output) and tool_parser: + # get the expected call based on partial JSON + # parsing which "autocompletes" the JSON + expected_call = json.dumps( + tool_parser.prev_tool_call_arr[index].get( + "arguments", {})) + + # get what we've streamed so far for arguments + # for the current tool + actual_call = tool_parser.streamed_args_for_tool[ + index] + + # check to see if there's anything left to stream + remaining_call = expected_call.replace( + actual_call, "", 1) + + # set that as a delta message + delta_message = DeltaMessage(tool_calls=[ + DeltaToolCall(index=index, + function=DeltaFunctionCall( + arguments=remaining_call). + model_dump(exclude_none=True)) + ]) + + # Send the finish response for each request.n only once + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=delta_message, + logprobs=logprobs, + finish_reason=output.finish_reason + if not auto_tools_called else "tool_calls", + stop_reason=output.stop_reason) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if (request.stream_options + and request.stream_options.include_usage): + if request.stream_options.continuous_usage_stats: + completion_tokens = len(output.token_ids) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + + completion_tokens, + ) + chunk.usage = usage + else: + chunk.usage = None + data = chunk.model_dump_json(exclude_unset=True) + yield f"data: {data}\n\n" + finish_reason_sent[i] = True + + # once the final token is handled, if stream_options.include_usage + # is sent, send the usage + if (request.stream_options + and request.stream_options.include_usage): + completion_tokens = previous_num_tokens[i] + final_usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + completion_tokens, + ) + + final_usage_chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[], + model=model_name, + usage=final_usage) + final_usage_data = (final_usage_chunk.model_dump_json( + exclude_unset=True, exclude_none=True)) + yield f"data: {final_usage_data}\n\n" + + # report to FastAPI middleware aggregate usage across all choices + num_completion_tokens = sum(previous_num_tokens) + request_metadata.final_usage_info = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_completion_tokens, + total_tokens=num_prompt_tokens + num_completion_tokens) + + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + logger.error("error in chat completion stream generator: %s", e) + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + # Send the final done message after all response.n are finished + yield "data: [DONE]\n\n" + + async def chat_completion_full_generator( + self, + request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], + request_id: str, + conversation: List[ConversationMessage], + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + ) -> Union[ErrorResponse, ChatCompletionResponse]: + + model_name = self.base_model_paths[0].name + created_time = int(time.time()) + final_res: Optional[RequestOutput] = None + + try: + async for res in result_generator: + final_res = res + except asyncio.CancelledError: + return self.create_error_response("Client disconnected") + + assert final_res is not None + + choices: List[ChatCompletionResponseChoice] = [] + + role = self.get_chat_request_role(request) + for output in final_res.outputs: + token_ids = output.token_ids + out_logprobs = output.logprobs + + if request.logprobs and request.top_logprobs is not None: + assert out_logprobs is not None, "Did not output logprobs" + logprobs = self._create_chat_logprobs( + token_ids=token_ids, + top_logprobs=out_logprobs, + num_output_top_logprobs=request.top_logprobs, + tokenizer=tokenizer, + ) + else: + logprobs = None + + # In the OpenAI API the finish_reason is "tools_called" + # if the tool choice is auto and the model produced a tool + # call. The same is not true for named function calls + auto_tools_called = False + + # if auto tools are not enabled, and a named tool choice using + # outlines is not being used + if (not self.enable_auto_tools + or not self.tool_parser) and not isinstance( + request.tool_choice, + ChatCompletionNamedToolChoiceParam): + message = ChatMessage(role=role, content=output.text) + + # if the request uses tools and specified a tool choice + elif request.tool_choice and type( + request.tool_choice) is ChatCompletionNamedToolChoiceParam: + + message = ChatMessage( + role=role, + content="", + tool_calls=[ + ToolCall(function=FunctionCall( + name=request.tool_choice.function.name, + arguments=output.text)) + ]) + + # if the request doesn't use tool choice + # OR specifies to not use a tool + elif not request.tool_choice or request.tool_choice == "none": + + message = ChatMessage(role=role, content=output.text) + + # handle when there are tools and tool choice is auto + elif request.tools and ( + request.tool_choice == "auto" + or request.tool_choice is None) and self.enable_auto_tools \ + and self.tool_parser: + + try: + tool_parser = self.tool_parser(tokenizer) + except RuntimeError as e: + logger.error("Error in tool parser creation: %s", e) + return self.create_error_response(str(e)) + + tool_call_info = tool_parser.extract_tool_calls( + output.text, request=request) + # In the OpenAI API the finish_reason is "tools_called" + # if the tool choice is auto and the model produced a tool + # call. The same is not true for named function calls + auto_tools_called = tool_call_info.tools_called + if tool_call_info.tools_called: + message = ChatMessage(role=role, + content=tool_call_info.content, + tool_calls=tool_call_info.tool_calls) + + else: + # FOR NOW make it a chat message; we will have to detect + # the type to make it later. + message = ChatMessage(role=role, content=output.text) + + # undetermined case that is still important to handle + else: + logger.error( + "Error in chat_completion_full_generator - cannot determine" + " if tools should be extracted. Returning a standard chat " + "completion.") + message = ChatMessage(role=role, content=output.text) + + choice_data = ChatCompletionResponseChoice( + index=output.index, + message=message, + logprobs=logprobs, + finish_reason="tool_calls" if auto_tools_called else + output.finish_reason if output.finish_reason else "stop", + stop_reason=output.stop_reason) + choices.append(choice_data) + + if request.echo or request.continue_final_message: + last_msg_content = "" + if conversation and "content" in conversation[-1] and conversation[ + -1].get("role") == role: + last_msg_content = conversation[-1]["content"] or "" + + for choice in choices: + full_message = last_msg_content + (choice.message.content + or "") + choice.message.content = full_message + + assert final_res.prompt_token_ids is not None + num_prompt_tokens = len(final_res.prompt_token_ids) + if final_res.encoder_prompt_token_ids is not None: + num_prompt_tokens += len(final_res.encoder_prompt_token_ids) + num_generated_tokens = sum( + len(output.token_ids) for output in final_res.outputs) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + ) + + request_metadata.final_usage_info = usage + + response = ChatCompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=usage, + prompt_logprobs=final_res.prompt_logprobs, + ) + + return response + + def _get_top_logprobs( + self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int], + tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]: + return [ + ChatCompletionLogProb(token=(token := self._get_decoded_token( + p[1], + p[0], + tokenizer, + return_as_token_id=self.return_tokens_as_token_ids)), + logprob=max(p[1].logprob, -9999.0), + bytes=list( + token.encode("utf-8", errors="replace"))) + for i, p in enumerate(logprobs.items()) + if top_logprobs and i < top_logprobs + ] + + def _create_chat_logprobs( + self, + token_ids: GenericSequence[int], + top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]], + tokenizer: AnyTokenizer, + num_output_top_logprobs: Optional[int] = None, + ) -> ChatCompletionLogProbs: + """Create OpenAI-style logprobs.""" + logprobs_content: List[ChatCompletionLogProbsContent] = [] + + for i, token_id in enumerate(token_ids): + step_top_logprobs = top_logprobs[i] + if step_top_logprobs is None: + token = tokenizer.decode(token_id) + if self.return_tokens_as_token_ids: + token = f"token_id:{token_id}" + + logprobs_content.append( + ChatCompletionLogProbsContent( + token=token, + bytes=list(token.encode("utf-8", errors="replace")), + )) + else: + step_token = step_top_logprobs[token_id] + step_decoded = step_token.decoded_token + + logprobs_content.append( + ChatCompletionLogProbsContent( + token=self._get_decoded_token( + step_token, + token_id, + tokenizer, + self.return_tokens_as_token_ids, + ), + logprob=max(step_token.logprob, -9999.0), + bytes=None if step_decoded is None else list( + step_decoded.encode("utf-8", errors="replace")), + top_logprobs=self._get_top_logprobs( + step_top_logprobs, + num_output_top_logprobs, + tokenizer, + ), + )) + + return ChatCompletionLogProbs(content=logprobs_content) + + def _should_stream_with_auto_tool_parsing(self, + request: ChatCompletionRequest): + """ + Utility function to check if streamed tokens should go through the tool + call parser that was configured. + + We only want to do this IF user-provided tools are set, a tool parser + is configured, "auto" tool choice is enabled, and the request's tool + choice field indicates that "auto" tool choice should be used. + """ + return (request.tools and self.tool_parser and self.enable_auto_tools + and request.tool_choice in ['auto', None]) + + def _should_check_for_unstreamed_tool_arg_tokens( + self, + delta_message: Optional[DeltaMessage], + output: CompletionOutput, + ) -> bool: + """ + Check to see if we should check for unstreamed tool arguments tokens. + This is only applicable when auto tool parsing is enabled, the delta + is a tool call with arguments. + """ + + # yapf: disable + return bool( + # if there is a delta message that includes tool calls which + # include a function that has arguments + output.finish_reason is not None + and self.enable_auto_tools and self.tool_parser and delta_message + and delta_message.tool_calls and delta_message.tool_calls[0] + and delta_message.tool_calls[0].function + and delta_message.tool_calls[0].function.arguments is not None + ) diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py new file mode 100644 index 00000000..077312dd --- /dev/null +++ b/vllm/entrypoints/openai/serving_completion.py @@ -0,0 +1,554 @@ +import asyncio +import time +from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List, + Optional) +from typing import Sequence as GenericSequence +from typing import Tuple, Union, cast + +from fastapi import Request + +from vllm.config import ModelConfig +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.engine.multiprocessing.client import MQLLMEngineClient +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.logger import RequestLogger +# yapf conflicts with isort for this block +# yapf: disable +from vllm.entrypoints.openai.protocol import (CompletionLogProbs, + CompletionRequest, + CompletionResponse, + CompletionResponseChoice, + CompletionResponseStreamChoice, + CompletionStreamResponse, + ErrorResponse, + RequestResponseMetadata, + UsageInfo) +# yapf: enable +from vllm.entrypoints.openai.serving_engine import (BaseModelPath, + LoRAModulePath, + OpenAIServing, + PromptAdapterPath) +from vllm.logger import init_logger +from vllm.outputs import RequestOutput +from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.sequence import Logprob +from vllm.tracing import (contains_trace_headers, extract_trace_headers, + log_tracing_disabled_warning) +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import merge_async_iterators, random_uuid + +logger = init_logger(__name__) + +TypeTokenIDs = List[int] +TypeTopLogProbs = List[Optional[Dict[int, float]]] +TypeCreateLogProbsFn = Callable[ + [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs] + + +class OpenAIServingCompletion(OpenAIServing): + + def __init__( + self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + *, + lora_modules: Optional[List[LoRAModulePath]], + prompt_adapters: Optional[List[PromptAdapterPath]], + request_logger: Optional[RequestLogger], + return_tokens_as_token_ids: bool = False, + ): + super().__init__(engine_client=engine_client, + model_config=model_config, + base_model_paths=base_model_paths, + lora_modules=lora_modules, + prompt_adapters=prompt_adapters, + request_logger=request_logger, + return_tokens_as_token_ids=return_tokens_as_token_ids) + + async def create_completion( + self, + request: CompletionRequest, + raw_request: Request, + ) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]: + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/completions/create + for the API specification. This API mimics the OpenAI Completion API. + + NOTE: Currently we do not support the following feature: + - suffix (the language models we currently support do not support + suffix) + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + # If the engine is dead, raise the engine's DEAD_ERROR. + # This is required for the streaming case, where we return a + # success status before we actually start generating text :). + if self.engine_client.errored: + raise self.engine_client.dead_error + + # Return error for unsupported features. + if request.suffix is not None: + return self.create_error_response( + "suffix is not currently supported") + + model_name = self.base_model_paths[0].name + request_id = f"cmpl-{random_uuid()}" + created_time = int(time.time()) + + request_metadata = RequestResponseMetadata(request_id=request_id) + if raw_request: + raw_request.state.request_metadata = request_metadata + + # Schedule the request and get the result generator. + generators: List[AsyncGenerator[RequestOutput, None]] = [] + try: + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + prompts = list( + self._tokenize_prompt_input_or_inputs( + request, + tokenizer, + request.prompt, + truncate_prompt_tokens=request.truncate_prompt_tokens, + add_special_tokens=request.add_special_tokens, + )) + + for i, prompt_inputs in enumerate(prompts): + sampling_params: Union[SamplingParams, BeamSearchParams] + default_max_tokens = self.max_model_len - len( + prompt_inputs["prompt_token_ids"]) + if request.use_beam_search: + sampling_params = request.to_beam_search_params( + default_max_tokens) + else: + sampling_params = request.to_sampling_params( + default_max_tokens) + + request_id_item = f"{request_id}-{i}" + + self._log_inputs(request_id_item, + prompt_inputs, + params=sampling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + is_tracing_enabled = (await + self.engine_client.is_tracing_enabled()) + trace_headers = None + if is_tracing_enabled: + trace_headers = extract_trace_headers(raw_request.headers) + if not is_tracing_enabled and contains_trace_headers( + raw_request.headers): + log_tracing_disabled_warning() + + if isinstance(sampling_params, BeamSearchParams): + assert isinstance(self.engine_client, + (AsyncLLMEngine, + MQLLMEngineClient)), \ + "Beam search is only supported with" \ + "AsyncLLMEngine and MQLLMEngineClient." + generator = self.engine_client.beam_search( + prompt_inputs["prompt_token_ids"], + request_id_item, + sampling_params, + ) + else: + generator = self.engine_client.generate( + { + "prompt_token_ids": + prompt_inputs["prompt_token_ids"] + }, + sampling_params, + request_id_item, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + trace_headers=trace_headers, + priority=request.priority, + ) + + generators.append(generator) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + result_generator = merge_async_iterators( + *generators, is_cancelled=raw_request.is_disconnected) + + # Similar to the OpenAI API, when n != best_of, we do not stream the + # results. In addition, we do not stream the results when use + # beam search. + stream = (request.stream + and (request.best_of is None or request.n == request.best_of) + and not request.use_beam_search) + + # Streaming response + if stream: + return self.completion_stream_generator( + request, + result_generator, + request_id, + created_time, + model_name, + num_prompts=len(prompts), + tokenizer=tokenizer, + request_metadata=request_metadata) + + # Non-streaming response + final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts) + try: + async for i, res in result_generator: + final_res_batch[i] = res + + for i, final_res in enumerate(final_res_batch): + assert final_res is not None + + # The output should contain the input text + # We did not pass it into vLLM engine to avoid being redundant + # with the inputs token IDs + if final_res.prompt is None: + final_res.prompt = prompts[i]["prompt"] + + final_res_batch_checked = cast(List[RequestOutput], + final_res_batch) + + response = self.request_output_to_completion_response( + final_res_batch_checked, + request, + request_id, + created_time, + model_name, + tokenizer, + request_metadata, + ) + except asyncio.CancelledError: + return self.create_error_response("Client disconnected") + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + # When user requests streaming but we don't stream, we still need to + # return a streaming response with a single event. + if request.stream: + response_json = response.model_dump_json() + + async def fake_stream_generator() -> AsyncGenerator[str, None]: + yield f"data: {response_json}\n\n" + yield "data: [DONE]\n\n" + + return fake_stream_generator() + + return response + + async def completion_stream_generator( + self, + request: CompletionRequest, + result_generator: AsyncIterator[Tuple[int, RequestOutput]], + request_id: str, + created_time: int, + model_name: str, + num_prompts: int, + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + ) -> AsyncGenerator[str, None]: + num_choices = 1 if request.n is None else request.n + previous_text_lens = [0] * num_choices * num_prompts + previous_num_tokens = [0] * num_choices * num_prompts + has_echoed = [False] * num_choices * num_prompts + num_prompt_tokens = [0] * num_prompts + + try: + async for prompt_idx, res in result_generator: + prompt_token_ids = res.prompt_token_ids + prompt_logprobs = res.prompt_logprobs + prompt_text = res.prompt + + # Prompt details are excluded from later streamed outputs + if res.prompt_token_ids is not None: + num_prompt_tokens[prompt_idx] = len(res.prompt_token_ids) + + delta_token_ids: GenericSequence[int] + out_logprobs: Optional[GenericSequence[Optional[Dict[ + int, Logprob]]]] + + for output in res.outputs: + i = output.index + prompt_idx * num_choices + # TODO(simon): optimize the performance by avoiding full + # text O(n^2) sending. + + assert request.max_tokens is not None + if request.echo and request.max_tokens == 0: + assert prompt_token_ids is not None + assert prompt_text is not None + # only return the prompt + delta_text = prompt_text + delta_token_ids = prompt_token_ids + out_logprobs = prompt_logprobs + has_echoed[i] = True + elif (request.echo and request.max_tokens > 0 + and not has_echoed[i]): + assert prompt_token_ids is not None + assert prompt_text is not None + assert prompt_logprobs is not None + # echo the prompt and first token + delta_text = prompt_text + output.text + delta_token_ids = [ + *prompt_token_ids, *output.token_ids + ] + out_logprobs = [ + *prompt_logprobs, + *(output.logprobs or []), + ] + has_echoed[i] = True + else: + # return just the delta + delta_text = output.text + delta_token_ids = output.token_ids + out_logprobs = output.logprobs + + if request.logprobs is not None: + assert out_logprobs is not None, ( + "Did not output logprobs") + logprobs = self._create_completion_logprobs( + token_ids=delta_token_ids, + top_logprobs=out_logprobs, + num_output_top_logprobs=request.logprobs, + tokenizer=tokenizer, + initial_text_offset=previous_text_lens[i], + ) + else: + logprobs = None + + previous_text_lens[i] += len(output.text) + previous_num_tokens[i] += len(output.token_ids) + finish_reason = output.finish_reason + stop_reason = output.stop_reason + + chunk = CompletionStreamResponse( + id=request_id, + created=created_time, + model=model_name, + choices=[ + CompletionResponseStreamChoice( + index=i, + text=delta_text, + logprobs=logprobs, + finish_reason=finish_reason, + stop_reason=stop_reason, + ) + ]) + if (request.stream_options + and request.stream_options.include_usage): + if (request.stream_options.continuous_usage_stats + or output.finish_reason is not None): + prompt_tokens = num_prompt_tokens[prompt_idx] + completion_tokens = previous_num_tokens[i] + usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + if request.stream_options.continuous_usage_stats: + chunk.usage = usage + else: + chunk.usage = None + + response_json = chunk.model_dump_json(exclude_unset=False) + yield f"data: {response_json}\n\n" + + if (request.stream_options + and request.stream_options.include_usage): + final_usage_chunk = CompletionStreamResponse( + id=request_id, + created=created_time, + model=model_name, + choices=[], + usage=usage, + ) + final_usage_data = (final_usage_chunk.model_dump_json( + exclude_unset=False, exclude_none=True)) + yield f"data: {final_usage_data}\n\n" + + # report to FastAPI middleware aggregate usage across all choices + total_prompt_tokens = sum(num_prompt_tokens) + total_completion_tokens = sum(previous_num_tokens) + request_metadata.final_usage_info = UsageInfo( + prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + total_tokens=total_prompt_tokens + total_completion_tokens) + + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + data = self.create_streaming_error_response(str(e)) + yield f"data: {data}\n\n" + yield "data: [DONE]\n\n" + + def request_output_to_completion_response( + self, + final_res_batch: List[RequestOutput], + request: CompletionRequest, + request_id: str, + created_time: int, + model_name: str, + tokenizer: AnyTokenizer, + request_metadata: RequestResponseMetadata, + ) -> CompletionResponse: + choices: List[CompletionResponseChoice] = [] + num_prompt_tokens = 0 + num_generated_tokens = 0 + + for final_res in final_res_batch: + prompt_token_ids = final_res.prompt_token_ids + assert prompt_token_ids is not None + prompt_logprobs = final_res.prompt_logprobs + prompt_text = final_res.prompt + + token_ids: GenericSequence[int] + out_logprobs: Optional[GenericSequence[Optional[Dict[int, + Logprob]]]] + + for output in final_res.outputs: + assert request.max_tokens is not None + if request.echo and request.max_tokens == 0: + assert prompt_text is not None + token_ids = prompt_token_ids + out_logprobs = prompt_logprobs + output_text = prompt_text + elif request.echo and request.max_tokens > 0: + assert prompt_text is not None + token_ids = [*prompt_token_ids, *output.token_ids] + + if request.logprobs is None: + out_logprobs = None + else: + assert prompt_logprobs is not None + assert output.logprobs is not None + out_logprobs = [ + *prompt_logprobs, + *output.logprobs, + ] + + output_text = prompt_text + output.text + else: + token_ids = output.token_ids + out_logprobs = output.logprobs + output_text = output.text + + if request.logprobs is not None: + assert out_logprobs is not None, "Did not output logprobs" + logprobs = self._create_completion_logprobs( + token_ids=token_ids, + top_logprobs=out_logprobs, + tokenizer=tokenizer, + num_output_top_logprobs=request.logprobs, + ) + else: + logprobs = None + + choice_data = CompletionResponseChoice( + index=len(choices), + text=output_text, + logprobs=logprobs, + finish_reason=output.finish_reason, + stop_reason=output.stop_reason, + prompt_logprobs=final_res.prompt_logprobs, + ) + choices.append(choice_data) + + num_generated_tokens += len(output.token_ids) + + num_prompt_tokens += len(prompt_token_ids) + + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + ) + + request_metadata.final_usage_info = usage + + return CompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=usage, + ) + + def _create_completion_logprobs( + self, + token_ids: GenericSequence[int], + top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]], + num_output_top_logprobs: int, + tokenizer: AnyTokenizer, + initial_text_offset: int = 0, + ) -> CompletionLogProbs: + """Create logprobs for OpenAI Completion API.""" + out_text_offset: List[int] = [] + out_token_logprobs: List[Optional[float]] = [] + out_tokens: List[str] = [] + out_top_logprobs: List[Optional[Dict[str, float]]] = [] + + last_token_len = 0 + + for i, token_id in enumerate(token_ids): + step_top_logprobs = top_logprobs[i] + if step_top_logprobs is None: + token = tokenizer.decode(token_id) + if self.return_tokens_as_token_ids: + token = f"token_id:{token_id}" + + out_tokens.append(token) + out_token_logprobs.append(None) + out_top_logprobs.append(None) + else: + step_token = step_top_logprobs[token_id] + + token = self._get_decoded_token( + step_token, + token_id, + tokenizer, + return_as_token_id=self.return_tokens_as_token_ids, + ) + token_logprob = max(step_token.logprob, -9999.0) + + out_tokens.append(token) + out_token_logprobs.append(token_logprob) + + # makes sure to add the top num_output_top_logprobs + 1 + # logprobs, as defined in the openai API + # (cf. https://github.com/openai/openai-openapi/blob/ + # 893ba52242dbd5387a97b96444ee1c742cfce9bd/openapi.yaml#L7153) + out_top_logprobs.append({ + # Convert float("-inf") to the + # JSON-serializable float that OpenAI uses + self._get_decoded_token( + top_lp[1], + top_lp[0], + tokenizer, + return_as_token_id=self.return_tokens_as_token_ids): + max(top_lp[1].logprob, -9999.0) + for i, top_lp in enumerate(step_top_logprobs.items()) + if num_output_top_logprobs >= i + }) + + if len(out_text_offset) == 0: + out_text_offset.append(initial_text_offset) + else: + out_text_offset.append(out_text_offset[-1] + last_token_len) + last_token_len = len(token) + + return CompletionLogProbs( + text_offset=out_text_offset, + token_logprobs=out_token_logprobs, + tokens=out_tokens, + top_logprobs=out_top_logprobs, + ) diff --git a/vllm/entrypoints/openai/serving_embedding.py b/vllm/entrypoints/openai/serving_embedding.py new file mode 100644 index 00000000..e9504cfa --- /dev/null +++ b/vllm/entrypoints/openai/serving_embedding.py @@ -0,0 +1,203 @@ +import asyncio +import base64 +import time +from typing import AsyncGenerator, List, Literal, Optional, Union, cast + +import numpy as np +from fastapi import Request +from typing_extensions import assert_never + +from vllm.config import ModelConfig +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.logger import RequestLogger +from vllm.entrypoints.openai.protocol import (EmbeddingRequest, + EmbeddingResponse, + EmbeddingResponseData, + ErrorResponse, UsageInfo) +from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing +from vllm.logger import init_logger +from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput +from vllm.utils import merge_async_iterators, random_uuid + +logger = init_logger(__name__) + +TypeTokenIDs = List[int] + + +def _get_embedding( + output: EmbeddingOutput, + encoding_format: Literal["float", "base64"], +) -> Union[List[float], str]: + if encoding_format == "float": + return output.embedding + elif encoding_format == "base64": + # Force to use float32 for base64 encoding + # to match the OpenAI python client behavior + embedding_bytes = np.array(output.embedding, dtype="float32").tobytes() + return base64.b64encode(embedding_bytes).decode("utf-8") + + assert_never(encoding_format) + + +def request_output_to_embedding_response( + final_res_batch: List[EmbeddingRequestOutput], request_id: str, + created_time: int, model_name: str, + encoding_format: Literal["float", "base64"]) -> EmbeddingResponse: + data: List[EmbeddingResponseData] = [] + num_prompt_tokens = 0 + for idx, final_res in enumerate(final_res_batch): + prompt_token_ids = final_res.prompt_token_ids + embedding = _get_embedding(final_res.outputs, encoding_format) + embedding_data = EmbeddingResponseData(index=idx, embedding=embedding) + data.append(embedding_data) + + num_prompt_tokens += len(prompt_token_ids) + + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + total_tokens=num_prompt_tokens, + ) + + return EmbeddingResponse( + id=request_id, + created=created_time, + model=model_name, + data=data, + usage=usage, + ) + + +class OpenAIServingEmbedding(OpenAIServing): + + def __init__( + self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + *, + request_logger: Optional[RequestLogger], + ): + super().__init__(engine_client=engine_client, + model_config=model_config, + base_model_paths=base_model_paths, + lora_modules=None, + prompt_adapters=None, + request_logger=request_logger) + self._enabled = self._check_embedding_mode(model_config.embedding_mode) + + async def create_embedding( + self, + request: EmbeddingRequest, + raw_request: Optional[Request] = None, + ) -> Union[EmbeddingResponse, ErrorResponse]: + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/embeddings/create + for the API specification. This API mimics the OpenAI Embedding API. + """ + if not self._enabled: + return self.create_error_response("Embedding API disabled") + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + encoding_format = request.encoding_format + if request.dimensions is not None: + return self.create_error_response( + "dimensions is currently not supported") + + model_name = request.model + request_id = f"embd-{random_uuid()}" + created_time = int(time.monotonic()) + + truncate_prompt_tokens = None + + if request.truncate_prompt_tokens is not None: + if request.truncate_prompt_tokens <= self.max_model_len: + truncate_prompt_tokens = request.truncate_prompt_tokens + else: + return self.create_error_response( + "truncate_prompt_tokens value is " + "greater than max_model_len." + " Please, select a smaller truncation size.") + + # Schedule the request and get the result generator. + generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = [] + try: + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + pooling_params = request.to_pooling_params() + + prompts = list( + self._tokenize_prompt_input_or_inputs(request, tokenizer, + request.input, + truncate_prompt_tokens)) + + for i, prompt_inputs in enumerate(prompts): + request_id_item = f"{request_id}-{i}" + + self._log_inputs(request_id_item, + prompt_inputs, + params=pooling_params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + if prompt_adapter_request is not None: + raise NotImplementedError( + "Prompt adapter is not supported " + "for embedding models") + + generator = self.engine_client.encode( + {"prompt_token_ids": prompt_inputs["prompt_token_ids"]}, + pooling_params, + request_id_item, + lora_request=lora_request, + priority=request.priority, + ) + + generators.append(generator) + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + result_generator = merge_async_iterators( + *generators, + is_cancelled=raw_request.is_disconnected if raw_request else None, + ) + + # Non-streaming response + final_res_batch: List[Optional[EmbeddingRequestOutput]] + final_res_batch = [None] * len(prompts) + try: + async for i, res in result_generator: + final_res_batch[i] = res + + for final_res in final_res_batch: + assert final_res is not None + + final_res_batch_checked = cast(List[EmbeddingRequestOutput], + final_res_batch) + + response = request_output_to_embedding_response( + final_res_batch_checked, request_id, created_time, model_name, + encoding_format) + except asyncio.CancelledError: + return self.create_error_response("Client disconnected") + except ValueError as e: + # TODO: Use a vllm-specific Validation Error + return self.create_error_response(str(e)) + + return response + + def _check_embedding_mode(self, embedding_mode: bool) -> bool: + if not embedding_mode: + logger.warning( + "embedding_mode is False. Embedding API will not work.") + else: + logger.info("Activating the server engine with embedding enabled.") + return embedding_mode diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py new file mode 100644 index 00000000..e6d2ab93 --- /dev/null +++ b/vllm/entrypoints/openai/serving_engine.py @@ -0,0 +1,487 @@ +import json +import pathlib +from dataclasses import dataclass +from http import HTTPStatus +from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union + +from pydantic import Field +from typing_extensions import Annotated + +from vllm.config import ModelConfig +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.logger import RequestLogger +# yapf conflicts with isort for this block +# yapf: disable +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + CompletionRequest, + DetokenizeRequest, + EmbeddingRequest, ErrorResponse, + LoadLoraAdapterRequest, + ModelCard, ModelList, + ModelPermission, + TokenizeChatRequest, + TokenizeCompletionRequest, + TokenizeRequest, + UnloadLoraAdapterRequest) +# yapf: enable +from vllm.inputs.parse import parse_and_batch_prompt +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.sequence import Logprob +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import AtomicCounter + +logger = init_logger(__name__) + + +@dataclass +class BaseModelPath: + name: str + model_path: str + + +@dataclass +class PromptAdapterPath: + name: str + local_path: str + + +@dataclass +class LoRAModulePath: + name: str + path: str + base_model_name: Optional[str] = None + + +AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest, + EmbeddingRequest, TokenizeRequest] + + +class TextTokensPrompt(TypedDict): + prompt: str + prompt_token_ids: List[int] + + +class OpenAIServing: + + def __init__( + self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + *, + lora_modules: Optional[List[LoRAModulePath]], + prompt_adapters: Optional[List[PromptAdapterPath]], + request_logger: Optional[RequestLogger], + return_tokens_as_token_ids: bool = False, + ): + super().__init__() + + self.engine_client = engine_client + self.model_config = model_config + self.max_model_len = model_config.max_model_len + + self.base_model_paths = base_model_paths + + self.lora_id_counter = AtomicCounter(0) + self.lora_requests = [] + if lora_modules is not None: + self.lora_requests = [ + LoRARequest(lora_name=lora.name, + lora_int_id=i, + lora_path=lora.path, + base_model_name=lora.base_model_name + if lora.base_model_name + and self._is_model_supported(lora.base_model_name) + else self.base_model_paths[0].name) + for i, lora in enumerate(lora_modules, start=1) + ] + + self.prompt_adapter_requests = [] + if prompt_adapters is not None: + for i, prompt_adapter in enumerate(prompt_adapters, start=1): + with pathlib.Path(prompt_adapter.local_path, + "adapter_config.json").open() as f: + adapter_config = json.load(f) + num_virtual_tokens = adapter_config["num_virtual_tokens"] + self.prompt_adapter_requests.append( + PromptAdapterRequest( + prompt_adapter_name=prompt_adapter.name, + prompt_adapter_id=i, + prompt_adapter_local_path=prompt_adapter.local_path, + prompt_adapter_num_virtual_tokens=num_virtual_tokens)) + + self.request_logger = request_logger + self.return_tokens_as_token_ids = return_tokens_as_token_ids + + async def show_available_models(self) -> ModelList: + """Show available models. Right now we only have one model.""" + model_cards = [ + ModelCard(id=base_model.name, + max_model_len=self.max_model_len, + root=base_model.model_path, + permission=[ModelPermission()]) + for base_model in self.base_model_paths + ] + lora_cards = [ + ModelCard(id=lora.lora_name, + root=lora.local_path, + parent=lora.base_model_name if lora.base_model_name else + self.base_model_paths[0].name, + permission=[ModelPermission()]) + for lora in self.lora_requests + ] + prompt_adapter_cards = [ + ModelCard(id=prompt_adapter.prompt_adapter_name, + root=self.base_model_paths[0].name, + permission=[ModelPermission()]) + for prompt_adapter in self.prompt_adapter_requests + ] + model_cards.extend(lora_cards) + model_cards.extend(prompt_adapter_cards) + return ModelList(data=model_cards) + + def create_error_response( + self, + message: str, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse: + return ErrorResponse(message=message, + type=err_type, + code=status_code.value) + + def create_streaming_error_response( + self, + message: str, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str: + json_str = json.dumps({ + "error": + self.create_error_response(message=message, + err_type=err_type, + status_code=status_code).model_dump() + }) + return json_str + + async def _check_model( + self, + request: AnyRequest, + ) -> Optional[ErrorResponse]: + if self._is_model_supported(request.model): + return None + if request.model in [lora.lora_name for lora in self.lora_requests]: + return None + if request.model in [ + prompt_adapter.prompt_adapter_name + for prompt_adapter in self.prompt_adapter_requests + ]: + return None + return self.create_error_response( + message=f"The model `{request.model}` does not exist.", + err_type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND) + + def _maybe_get_adapters( + self, request: AnyRequest + ) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[ + None, PromptAdapterRequest]]: + if self._is_model_supported(request.model): + return None, None + for lora in self.lora_requests: + if request.model == lora.lora_name: + return lora, None + for prompt_adapter in self.prompt_adapter_requests: + if request.model == prompt_adapter.prompt_adapter_name: + return None, prompt_adapter + # if _check_model has been called earlier, this will be unreachable + raise ValueError(f"The model `{request.model}` does not exist.") + + def _normalize_prompt_text_to_input( + self, + request: AnyRequest, + tokenizer: AnyTokenizer, + prompt: str, + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]], + add_special_tokens: bool, + ) -> TextTokensPrompt: + if truncate_prompt_tokens is None: + encoded = tokenizer(prompt, add_special_tokens=add_special_tokens) + else: + encoded = tokenizer(prompt, + add_special_tokens=add_special_tokens, + truncation=True, + max_length=truncate_prompt_tokens) + + input_ids = encoded.input_ids + + input_text = prompt + + return self._validate_input(request, input_ids, input_text) + + def _normalize_prompt_tokens_to_input( + self, + request: AnyRequest, + tokenizer: AnyTokenizer, + prompt_ids: List[int], + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]], + ) -> TextTokensPrompt: + if truncate_prompt_tokens is None: + input_ids = prompt_ids + else: + input_ids = prompt_ids[-truncate_prompt_tokens:] + + input_text = tokenizer.decode(input_ids) + + return self._validate_input(request, input_ids, input_text) + + def _validate_input( + self, + request: AnyRequest, + input_ids: List[int], + input_text: str, + ) -> TextTokensPrompt: + token_num = len(input_ids) + + # Note: EmbeddingRequest doesn't have max_tokens + if isinstance(request, EmbeddingRequest): + if token_num > self.max_model_len: + raise ValueError( + f"This model's maximum context length is " + f"{self.max_model_len} tokens. However, you requested " + f"{token_num} tokens in the input for embedding " + f"generation. Please reduce the length of the input.") + return TextTokensPrompt(prompt=input_text, + prompt_token_ids=input_ids) + + # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens + # and does not require model context length validation + if isinstance(request, (TokenizeCompletionRequest, TokenizeChatRequest, + DetokenizeRequest)): + return TextTokensPrompt(prompt=input_text, + prompt_token_ids=input_ids) + + if request.max_tokens is None: + if token_num >= self.max_model_len: + raise ValueError( + f"This model's maximum context length is " + f"{self.max_model_len} tokens. However, you requested " + f"{token_num} tokens in the messages, " + f"Please reduce the length of the messages.") + elif token_num + request.max_tokens > self.max_model_len: + raise ValueError( + f"This model's maximum context length is " + f"{self.max_model_len} tokens. However, you requested " + f"{request.max_tokens + token_num} tokens " + f"({token_num} in the messages, " + f"{request.max_tokens} in the completion). " + f"Please reduce the length of the messages or completion.") + + return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids) + + def _tokenize_prompt_input( + self, + request: AnyRequest, + tokenizer: AnyTokenizer, + prompt_input: Union[str, List[int]], + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None, + add_special_tokens: bool = True, + ) -> TextTokensPrompt: + """ + A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs` + that assumes single input. + """ + return next( + self._tokenize_prompt_inputs( + request, + tokenizer, + [prompt_input], + truncate_prompt_tokens=truncate_prompt_tokens, + add_special_tokens=add_special_tokens, + )) + + def _tokenize_prompt_inputs( + self, + request: AnyRequest, + tokenizer: AnyTokenizer, + prompt_inputs: Iterable[Union[str, List[int]]], + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None, + add_special_tokens: bool = True, + ) -> Iterator[TextTokensPrompt]: + """ + A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs` + that assumes multiple inputs. + """ + for text in prompt_inputs: + if isinstance(text, str): + yield self._normalize_prompt_text_to_input( + request, + tokenizer, + prompt=text, + truncate_prompt_tokens=truncate_prompt_tokens, + add_special_tokens=add_special_tokens, + ) + else: + yield self._normalize_prompt_tokens_to_input( + request, + tokenizer, + prompt_ids=text, + truncate_prompt_tokens=truncate_prompt_tokens, + ) + + def _tokenize_prompt_input_or_inputs( + self, + request: AnyRequest, + tokenizer: AnyTokenizer, + input_or_inputs: Union[str, List[str], List[int], List[List[int]]], + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None, + add_special_tokens: bool = True, + ) -> Iterator[TextTokensPrompt]: + """ + Tokenize/detokenize depending on the input format. + + According to `OpenAI API `_ + , each input can be a string or array of tokens. Note that each request + can pass one or more inputs. + """ + for prompt_input in parse_and_batch_prompt(input_or_inputs): + # Although our type checking is based on mypy, + # VSCode Pyright extension should still work properly + # "is True" is required for Pyright to perform type narrowing + # See: https://github.com/microsoft/pyright/issues/7672 + if prompt_input["is_tokens"] is False: + yield self._normalize_prompt_text_to_input( + request, + tokenizer, + prompt=prompt_input["content"], + truncate_prompt_tokens=truncate_prompt_tokens, + add_special_tokens=add_special_tokens, + ) + else: + yield self._normalize_prompt_tokens_to_input( + request, + tokenizer, + prompt_ids=prompt_input["content"], + truncate_prompt_tokens=truncate_prompt_tokens, + ) + + def _log_inputs( + self, + request_id: str, + inputs: Union[str, List[int], TextTokensPrompt], + params: Optional[Union[SamplingParams, PoolingParams, + BeamSearchParams]], + lora_request: Optional[LoRARequest], + prompt_adapter_request: Optional[PromptAdapterRequest], + ) -> None: + if self.request_logger is None: + return + + if isinstance(inputs, str): + prompt = inputs + prompt_token_ids = None + elif isinstance(inputs, list): + prompt = None + prompt_token_ids = inputs + else: + prompt = inputs["prompt"] + prompt_token_ids = inputs["prompt_token_ids"] + + self.request_logger.log_inputs( + request_id, + prompt, + prompt_token_ids, + params=params, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + + @staticmethod + def _get_decoded_token(logprob: Logprob, + token_id: int, + tokenizer: AnyTokenizer, + return_as_token_id: bool = False) -> str: + if return_as_token_id: + return f"token_id:{token_id}" + + if logprob.decoded_token is not None: + return logprob.decoded_token + return tokenizer.decode(token_id) + + async def _check_load_lora_adapter_request( + self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]: + # Check if both 'lora_name' and 'lora_path' are provided + if not request.lora_name or not request.lora_path: + return self.create_error_response( + message="Both 'lora_name' and 'lora_path' must be provided.", + err_type="InvalidUserInput", + status_code=HTTPStatus.BAD_REQUEST) + + # Check if the lora adapter with the given name already exists + if any(lora_request.lora_name == request.lora_name + for lora_request in self.lora_requests): + return self.create_error_response( + message= + f"The lora adapter '{request.lora_name}' has already been" + "loaded.", + err_type="InvalidUserInput", + status_code=HTTPStatus.BAD_REQUEST) + + return None + + async def _check_unload_lora_adapter_request( + self, + request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]: + # Check if either 'lora_name' or 'lora_int_id' is provided + if not request.lora_name and not request.lora_int_id: + return self.create_error_response( + message= + "either 'lora_name' and 'lora_int_id' needs to be provided.", + err_type="InvalidUserInput", + status_code=HTTPStatus.BAD_REQUEST) + + # Check if the lora adapter with the given name exists + if not any(lora_request.lora_name == request.lora_name + for lora_request in self.lora_requests): + return self.create_error_response( + message= + f"The lora adapter '{request.lora_name}' cannot be found.", + err_type="InvalidUserInput", + status_code=HTTPStatus.BAD_REQUEST) + + return None + + async def load_lora_adapter( + self, + request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]: + error_check_ret = await self._check_load_lora_adapter_request(request) + if error_check_ret is not None: + return error_check_ret + + lora_name, lora_path = request.lora_name, request.lora_path + unique_id = self.lora_id_counter.inc(1) + self.lora_requests.append( + LoRARequest(lora_name=lora_name, + lora_int_id=unique_id, + lora_path=lora_path)) + return f"Success: LoRA adapter '{lora_name}' added successfully." + + async def unload_lora_adapter( + self, + request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]: + error_check_ret = await self._check_unload_lora_adapter_request(request + ) + if error_check_ret is not None: + return error_check_ret + + lora_name = request.lora_name + self.lora_requests = [ + lora_request for lora_request in self.lora_requests + if lora_request.lora_name != lora_name + ] + return f"Success: LoRA adapter '{lora_name}' removed successfully." + + def _is_model_supported(self, model_name): + return any(model.name == model_name for model in self.base_model_paths) diff --git a/vllm/entrypoints/openai/serving_tokenization.py b/vllm/entrypoints/openai/serving_tokenization.py new file mode 100644 index 00000000..a269c94c --- /dev/null +++ b/vllm/entrypoints/openai/serving_tokenization.py @@ -0,0 +1,157 @@ +from typing import List, Optional, Union + +from vllm.config import ModelConfig +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.chat_utils import (apply_hf_chat_template, + apply_mistral_chat_template, + load_chat_template, + parse_chat_messages_futures) +from vllm.entrypoints.logger import RequestLogger +# yapf conflicts with isort for this block +# yapf: disable +from vllm.entrypoints.openai.protocol import (DetokenizeRequest, + DetokenizeResponse, + ErrorResponse, + TokenizeChatRequest, + TokenizeRequest, + TokenizeResponse) +# yapf: enable +from vllm.entrypoints.openai.serving_engine import (BaseModelPath, + LoRAModulePath, + OpenAIServing) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import MistralTokenizer +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +class OpenAIServingTokenization(OpenAIServing): + + def __init__( + self, + engine_client: EngineClient, + model_config: ModelConfig, + base_model_paths: List[BaseModelPath], + *, + lora_modules: Optional[List[LoRAModulePath]], + request_logger: Optional[RequestLogger], + chat_template: Optional[str], + ): + super().__init__(engine_client=engine_client, + model_config=model_config, + base_model_paths=base_model_paths, + lora_modules=lora_modules, + prompt_adapters=None, + request_logger=request_logger) + + # If this is None we use the tokenizer's default chat template + # the list of commonly-used chat template names for HF named templates + hf_chat_templates: List[str] = ['default', 'tool_use'] + self.chat_template = chat_template \ + if chat_template in hf_chat_templates \ + else load_chat_template(chat_template) + + async def create_tokenize( + self, + request: TokenizeRequest, + ) -> Union[TokenizeResponse, ErrorResponse]: + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + request_id = f"tokn-{random_uuid()}" + + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + prompt: Union[str, List[int]] + if isinstance(request, TokenizeChatRequest): + model_config = self.model_config + + conversation, mm_data_future = parse_chat_messages_futures( + request.messages, model_config, tokenizer) + + mm_data = await mm_data_future + if mm_data: + logger.warning( + "Multi-modal inputs are ignored during tokenization") + + if isinstance(tokenizer, MistralTokenizer): + prompt = apply_mistral_chat_template( + tokenizer, + messages=request.messages, + chat_template=self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + ) + else: + prompt = apply_hf_chat_template( + tokenizer, + conversation=conversation, + chat_template=self.chat_template, + add_generation_prompt=request.add_generation_prompt, + continue_final_message=request.continue_final_message, + ) + else: + prompt = request.prompt + + self._log_inputs(request_id, + prompt, + params=None, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + # Silently ignore prompt adapter since it does not affect tokenization + + prompt_input = self._tokenize_prompt_input( + request, + tokenizer, + prompt, + add_special_tokens=request.add_special_tokens, + ) + input_ids = prompt_input["prompt_token_ids"] + + return TokenizeResponse(tokens=input_ids, + count=len(input_ids), + max_model_len=self.max_model_len) + + async def create_detokenize( + self, + request: DetokenizeRequest, + ) -> Union[DetokenizeResponse, ErrorResponse]: + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + request_id = f"tokn-{random_uuid()}" + + ( + lora_request, + prompt_adapter_request, + ) = self._maybe_get_adapters(request) + + tokenizer = await self.engine_client.get_tokenizer(lora_request) + + self._log_inputs(request_id, + request.tokens, + params=None, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request) + + if prompt_adapter_request is not None: + raise NotImplementedError("Prompt adapter is not supported " + "for tokenization") + + prompt_input = self._tokenize_prompt_input( + request, + tokenizer, + request.tokens, + ) + input_text = prompt_input["prompt"] + + return DetokenizeResponse(prompt=input_text) diff --git a/vllm/entrypoints/openai/tool_parsers/__init__.py b/vllm/entrypoints/openai/tool_parsers/__init__.py new file mode 100644 index 00000000..309d9bed --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/__init__.py @@ -0,0 +1,10 @@ +from .abstract_tool_parser import ToolParser, ToolParserManager +from .hermes_tool_parser import Hermes2ProToolParser +from .internlm2_tool_parser import Internlm2ToolParser +from .llama_tool_parser import Llama3JsonToolParser +from .mistral_tool_parser import MistralToolParser + +__all__ = [ + "ToolParser", "ToolParserManager", "Hermes2ProToolParser", + "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser" +] diff --git a/vllm/entrypoints/openai/tool_parsers/__pycache__/__init__.cpython-310.pyc b/vllm/entrypoints/openai/tool_parsers/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b6e806d0bf7afde6e0bd52f436c91171dc302f9 GIT binary patch literal 562 zcmY*Wu};G<5Vg}ZNkc19T0SB}v04Tegct#}s1lvLST3=eNOo-5E~vl51Ovaohh$~q z7nraEsEsW@-My!~dne1mAO-GB-#?012*9UL4o~2cYxm6DK@M`rfq)o_00+?L0q=+q zhfUn!T@m4^i9;TX1Sd`0TZ@k8hY3%*>Wr?q$c1ZY#@l#l@^ekLR2UdVXhQ)#8VX(pXLT0U2q(pYC`FJ;WZm z9wG<#Ss#z-+E+bVVx3K*8lU?zd{Q*#TB>h_Ps@LIEzLK_L+&lMcs~!1#IMDc;yu+d zA(V4M-hl48yn*i(vO*zMs)?m~!|dcjYzr|J&xn_6q;|HD6?)=aAXZtuQ_@*7NGev* j5*@5#5Fuq%T0+j$u1+q6%(t9f`TsF}2gjff|8DRDj^m!U literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/tool_parsers/__pycache__/abstract_tool_parser.cpython-310.pyc b/vllm/entrypoints/openai/tool_parsers/__pycache__/abstract_tool_parser.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77c6782979962b0f399cd83302ea12afe95bf789 GIT binary patch literal 5959 zcmb7IOOxDI6_&2l-Rj5m^z@7$lh_~!8>cC|hmb%ZF~Px3f(e;~*x9IxQZ%}^XEdm# z&Xszc>1wmUu8KDlY+yr$vUg;`h9AI!;#X+PDr*)%UgkSjkLj5)Dd;X;-RHUIp7Wi@ zRlKy+((rrcPk-6@>nTn9J2e)61{zoKB!5J~HLi!6lfU)i%^Ze)78+yIF?AXZ^q-7k=5Rb zFfcH(6W4aU^jbWggo38NP1CTlb6td~_l8IkZ(C$%UU@&2o}UVSD~?0j>P9q*<=CSo zV5l8LLF$I__O_5{c3zJ5Z^b_sQ7}{OR*<+M_;KTrPiI*C_0hPBC;0;kp*cF&AbZ_0 zI1}cGaf6%pOs6L5ye1mL63r1?*U(?*je92K*5a0EW2`+gd6Ty=x5L}QDtebjb>87i z=I@6|!g=})BCb5k7L^`Uv!lhRrCr&5Y2&AS8coIKPKd1K)w z7Xm?r$Xwgi9_SBAkox9; z$yM|@O`O2pi+yivpvyJ%s6*z(Ni?&X8a#(;LeyI3thc~{8q69pH_&4WCLWZ6O;qOO zFZ9{#3KoRYn<-4yFF=(eQ(-ZUZIACxlN5czE@<134^);4SpbVDC_Zr%#rPB0<0$vt zoRC`(xriz17-ns@zHp4gv$9T(gLs-wrm0Vkx%^SR6zV@LG*Te;uKr7Xz`RR1?U$Xc zusRq`K#pw;c)(ZyK3n9DEQq#k0*FnBPVAA4#}z%#_H%bAY{lH=Z4n9S5e0x5D|2fj zp}a%(Ok|@tx|}1OC`*w{;wXV7-`){Y*tdnfk0_DcxEkIY; zFe08at5*AA00IXL6nfnfz6X8UL3$A$1G*L=Jomx0*$+xLwV;VY6uC*{a@g^b!c~cA zQFP}%(zDggIK6Qo%~zz1<-o}Ca0cp;j{`;j_$EWlP3olF7Sqi~v$J2yQnp0iCy1ws zn}Sx>sr(GJTJRyMmyaknl}{w6;s!j)R;t-m*K)C>?AY`PTjh`HCFtZ?Gyt}}MX=q7 zAYO#gL#Fh9!mZLa!4A&e4x$QiC3Id?up*rTLJ{B-@$d*}(N$5Zus5`S1RcVxfqg~! zFuNsCfD~ozfMNj{`*KB$TtOi?P=YLz4IYd}$t8Phn%Z8N#0rbSK?-1dTd-iTiiigc zRg_BIh5O(K$54)!tN2y`1NQTDBCsFqC6!Y@1*od!vm5^tY~AYgwE5!!EWbj^v~swT zijtY7e^jh=vzF_^+e|~@ zx>?6{-dkPup;CE8fg5j9E)*EJ+`O|irouZ=8~KG(1YX&Zj-Frl3MJ10lMcd>(zM_)PKCdCru4O9+8h^u!kTJ`w zlTyM|23GNnxm!n{QiqxjKkwDcaEox24gyruHILiWQWD>iRs_1w0yc41?* zQC35Q^@UGrD1St^hDWL87pSCAMXKLrNjJrNpAU?#kOHNGaQpV{Al*^6wvC%zl-n|8 zmSod(c^*>C_7T&80hPNW)Oi&(E___5@8TF@S+3EZO&d%e3lp`3YQM$9{?*N)g@xZLsFZ6i?GXH$}0%F z9~nmwb?ll&nxxxQLiZ5J9ak+ScU{SyE?jI=UcngdSQMA>B8-Bt!fS=n9F@w!G%F1K z8Z#4e2R&eJ_S|u_9YOB!N{H)Rn7lNsNDLbC1rW$gl9Bu_s`7hOC>vU)n&PUKvqSch zTaGfMm#C=%^7pBhyT2oQrWi>mnQ68Adf%`pHZulmF(1KlYQ%ZZFwMug#Uqrm_9UEc z2NB$Ix^pNvl_sOS;0SxA+OGb9sS+OWA^scz3O>)=tv#sE8T!~AV@~FvC1>WwZi8y_ zLTWu|azl;7EI30*qZO^W@lYqvxZ8Npo|_N#d~9IOwkr_gsu4?Mn}l^~b1@1c1Tu;P z3jvQZRpOq2MD@ktd`U%b6oldc(~_>so3m44>~Z@-egRZ(T)z<42Azz>h(|;_vyw>| zsLO}7<>5X?p`(d=CD_Afh|HX(L71INCc<|qp?7H~d<*l{$FZzVTOc36o4$@^H9;4r ztXBjgwSv}WZSg)RK>dy$>&H z!msx9+4x{kQL;XL4H|YrQ%I04o+7cQPP$1r+5-Fke0TlGuG2t8@Z=$ zeoi@Qzsy99&gX2E`Z4lZ+1o;X+A=$zG5fTxXbql}uYyw6t+Y5KCCVRCXH)q|;4D#T zc2-AF#LY`8KFK*qbqaMTm0FV+)bSjfzC{fezC{i5Z&5>h+e>2~4ngJTDh;PJ*lDW; zIWpuD^_){Z_@Wpk@Vdx`5|>y@hSircbq#P@YMvUB%B@h;mOrL~@_2R3$H_9YuH+DV di7X(YywuVx1lf*$USDG!ql22ox<>!%e*iFYO`8A! literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/tool_parsers/__pycache__/hermes_tool_parser.cpython-310.pyc b/vllm/entrypoints/openai/tool_parsers/__pycache__/hermes_tool_parser.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f621c0262abbbd7a7c72f37e799857b9a847b218 GIT binary patch literal 6839 zcma)B&2t>bb)V^(ot<6GE*6W$#}Xj9q%2ubiGZb&qc}z=sYp?lBho5?GObCRt>Jb9 zSa3d;J+lO{8K_)=QaYi+9CAvf3h+&bTyltS{sa6c%r#|K$^SrX$prFyJ^Mvkt`b=D zw)=JW>-W{K-)lydN?F6-Yyb7@-Fusw_P^Ab{wtvKHoo{b09@m`uetfJ?lSb5Zv>`m z1_ieeSgsWm-J+h4+pdjq!!HG8x2$lcKNC2v6I9$vP<5-ptUDXj+*&Z_&INV1uI3c{ z`Jmx8f(3U$^(}ufSaO$CyXY?mO}DAqwtpd5aaV#>cUARE{>9*udr8+sQ(QjM-IiE5 zxx&k*`gRj%47`^2~XLkUg5R#PZFcC5kyz3>R&AZ#%7@X;b-K6bDJ2aPN&O713-ROY`y91D`-G=Pa^T!xkAia=B zLA&4Ya@X1=f^mmbHvLy0cH&292+h^G2B*-4vCTf$wsp?9@tNbA+!O^~5Ei#YaogfW zZhvONIZD8m@y+0KL`jsJ8m|n0tHa-X)r@fX>^5s^+cjR}b68#ZTr)Md3PaYj#*Y+t zuYV|`iC(f&GLo;urg)~diO;|n{|`W-9cd?es-Nn*matUYGZOQN$;G5_VsV8f++H!& z+6KpX+uqRtOWXh~tM&{x(RO$Nu#!|yX1SGA_iCp!drtL><9?l}#cw6^+&jv+u2wGzyln6K4|Lil9AlSHz``;r71 zqi|5x2R|K3^?D0-S1Un`P6y`oeclSAq~%8)oB>de{1%DYiW5&J8zQ6~;r(%|8>)@R zio3Gb%nEV8hf`sTqN9Lr!S=l z6U>FCLoX2R7~QPilj2d1B%a!yl-cYvbGtlSgzy4OIEHuPHUflNku8i3)mF0PjAZi@ zG=v)>nG{*EgMiibMP}Xn$=!E8_#m5|%#>nBJa*@%jMY}Cto&2Y?~8Xy6u0mQI=r75 zJ0i)JCu^rRZxg$dk@5b7cRuQeNjJ!8A?}#5@VAwEzRNP^x* zuiLu@6JfJ!ZwCF{;I&|X!|&eT=p7`xQMmEQ_XAijkq5m9Q7+zydLs0?x%cMQk2msI zl`A;Mt@jRO6DC=~7heZ(bcdNb)6MfTvzVo;e}<)-)IQIlWiU0vGECrV%t4P2%>C#Dbt|Pi2 zh#nhT+Ta>S*oi@L>d@c@H$T%-CMmjuYA_F~+d1wZn5j4oUMURJ|-p_P?4 zeOz;N9K2`bfQEw=TW?_A?`~e1q0}A0X4kbK+(7Ker-jF}?{< zE>akPtY^$BTkPB_FlL=uCM#Nhw8}OPK4ZBq?5E7IpH>SNW+MnFZ2ips1$#R9?zi-v ztU{5$8};LM0-iEw9Q3JI#1x&Rc^hhF|A$YHnJ<^ zaC@^SHC|E@k_O7aR7nS6INa%%Y9}Ut`8;)^8UDSR#pLhOnY08ReuwpQM5C#l)cFi# zucbBl+hqQvk=9OimDzz48pp2eu%{krL{Mal5T2=kx&j;O_tIcuN>=p z%e=~Gk9A%G{QUmAJW= zXC$kiu98PW?`Wx<Li~24X&iY{ByxlAurrP;W56(hBd)<)DQWUHCxM{`XJVDUa6^VRcd{YkCKOGL{Kcte0s-e+U4o$sGrIQGw{?0B2RGEbO zOr_E-WZhBg$6G(SL;L612qhqcY6qNl@HTRdNL>S0bjKnuxs4zB7C^?jT=6n^^{za~ z1s)O|-fUA`?01vh*84ZfZcS2zQXq=DaVn&&*YI{cD$JBxA#<#-m$$}tA#=SLx2~ZW zoGcw|#k;CPMEThbdQlvA@B0FqL(z}KWF93nB54I(evRt1^@+W68xT)kh{6Po^+ApD z8bV<|q)A}>rCVJI-i8_kZp=I=cSQG*;ELaiTf&naap9SnNF+)W}dn*}24rwMVj!mn)(=#v2X$mC=0lswgReem>b= zg$VSznX8G;O^k;Mddh%*KE#6fQ`BwFjH8!XSudmFHce9{)XziPv@u@RQL7hGd!JL^ z(J>2cB<$xV>T;7+QMEgIO}7nOCk&05=N6+rRqT|i6OB>IZk{=)>nW|LM4rYKJi|-C zIr~F(u8O%BVa&A8Y#Kj9`>bX(zzO|BRB@_;h3Pr0GZkc^5OmA+yPWvL9SL^&w2Csd~f@Na4wOM)&Q*T&!4r{k@eEYt@)0oq4KkR#cJ|gKo zSiQ7l28=F?)KxX$yVRm**UZ2TBI!RBRNRsOK)A~AA*dtp9M+z4scSTkig&pHu#7LJ z`^(nq2D?gzVFr83%<`Yu8)&EaXSbJeu9jkXP`2uKV z7OH6Yrlc1MX%HYja~A2Hz@k!d|r6>OY`$re3Mtxr8pVVEAL)_XEaqE6)J z>Y(tKi>{)f(vB+IsepHX@Rx|FK$6GCc-^}X?T!!GFk)7T@GKE|SX{k(6-Nj9Hk19uk(q<}e0jO|1w$$@+Z!w585Z};BKc|JBh zvvzKo1*9{IgmjX5Mo4>)c|eN9D?&WxiKje`c;kiq1%MO#sOs5WuN{NDr>W_#?yjz` zs;*znMU{$0;CJSaf7u(pK*&F^Gyf|<=S_IxKLg=}Q=d5aWKpz-u;?tR zcF|u7mYrqQHvN@g)mc^TlD`(5a!yeqmc>(t)HyAhgqOK}n{Zn+4mB8i zhSMF=u6zb3Alq#%(=U46Br|u!`vVbnMOOGx=tW_hW`*;v5;BgLHoT<2%AVybWP-fT`x?83%m~z3%$hk zquqS2eLg(68r=|~H&koOvJ>(sa0dgAJEjs4`au_sGyiK&I`JK72;xvq9LA~8dhA2e zqnvT=w(aO#7X@Arg`Un0Uc9Y41~+->HgSsF5+*OhW5ZJsC1I@*UY)#alXt$lENouy zu{F}G^9FB%lnP&fRrzSOXYfV71l-z(L?=!ioVlDWT*jk==C406P*zH&2ko-C!(dyR zJn=F#5^_k!G=*S+V4^8;87Cl>ZA%KH5F&+m&1Jj>iam52xJQqL-;;|l z^COy-T(|3Y;@EXRB6mBRNzmWyd;RsKBX>oz{zfp^3(f?$Hhu5yv+oh?)H_}Ysuq)`8g6qQldVvBbtq@uHjr zuOUS;kl~jMgT}dl0#Ry8_lYb6rJUZ-WSO&+y6}K;wCzj}qfoqs=0T-mr>u1A?nPc# zI1BL&ukXfv(e*k$Xblk;x0zf8sq$$gPtC|_Xl2z7-yg(@JJIhM;Kc@zIx}>g4xfMA zsUBYPrK6ya)lpu72KbfUB(jv!F|?222zVIaG!=&_4tgC9W`Wr|fq;ODvl4i25P*MT zxJw#1adL=Ei>~iKdI%fFU09&ze+7?#>C^%|Rf07jU&Coy!PpL-U zBqMwH&$K+M49&zEm(xmWAG1;Q8VMJ4l34rper3OUFrozilG?b=i+JIGO0EG~?T}k| zJ|q59S{*mIc}z#Ov?f1RXD%t-w1g#1)wg&#SvVxWCx2kcVp`+&5#3+n6<$4}ymqAT zQeHozqdKp^i40h=JYL~C?iK!)lTkhVFeT$v;IE;yH8|I(p*U4&HPdGNN!o}%Q8Oj< zK200)Q_%MmZ>R)?5_)O#4uxQD+@WzPd1`z*ZNl1xbb*s}Vf=Ke9ivqN?N*deMvGJ3 zLElB*oYQyVh)wlHO;Mu)StIG<8;~QnNO(r=bQX7#XJk6 zW95nVSjqR8o;|Jd#VwLncQNkP()u;<3V80>JM^GO$q0~s*i?2{N>`8B#D~jhlWX8b zu-ED@SJ9Js>lVF5+AG_``c?{<3#rSjTL+-lp;l=y0`79b0_R^#@$Im{!|*R`5bDKFIXspf57!art$a2+CV| znuT&B9&TOU{_*)Mm(OosZLPPw08lCRuKNNt+UwjDty%JFNfC>r6?lAo#)G|IK#@#- z171qJVhlGfNg2r&K#~{k=CVQ#QsYF5PLT5=ztsg5&bD5Phu_(OT;E52cW)5hXm#RN z2R9t}Nh|7s0GRw>?W9-%kF*5t({5#&FG9J3N?>wwY7f#FV4%kVfBU|dz$=kn-&uZW z7>(@4=u9k59qs(P?>gGf)hpQo_#{E_*^j~bIg(}tKk7iD)Vq-snFSUXeTF(%nHf`nT|TbW*{rhy-|Y|S?SVL!S-4Xod_xt@l3Xnyo%(jNWO*y zfz>gg{zzO11*a5>TdvCg5JiY4Mt&2rLeKB)#*U3f=|H@oDo$kCD5X6L%h*2MYvOys0rna)=gY(~YSD0_NZ?#>jB;cibt za|28e{~d^FS%A5gW>JIH0efv$g{O{K%$n3TY|2iUrc(yxg-*?rCPS2F_YDJ)8b+xB zj|C5-`iY|V^?T-tVOBw+uEVSoi&qfTuRXb!}1{8O-ZMg|2l&>LCnVJ4V zC1CkHaxfiskv;ukHH@~}fhWciAM_zVH%3rQ?nCABJN7=?$xB(tqL`nRF!(578?R+F zc{FmZE#HfiE__Lz`zuOB@rz`5_Q|}tvE@gQIO22An2Le&+p+N;#4hF(!Q~sM#X1ns z2%#m<@l)?&*bmbolarrfLpQTs_x(Y~&qrhfR8n7B z62C~~b4W0wXBwm(iSdzX0R2+O0YotQ4iaU+D>z!2d=+*DqzO`Qy~g>_<g?iWUZeQp@+u2E$FVJ7>xRkHp?eg&hmIn6JWy!Nxo zv6%rS1fa9Ta8v5$ntmI)4puIYqT~jm<3^MtV-Bk={7l_Il%0I0!Z3cOZuDi8MBT{G zO@qLgTR?%OvVe*nj3eh;|4WEUJteWB%U8j1W~7~ z7UXy?>&SUdabS}$R*)}aSIys;{l7rD+_BWWT$IB79n|8xD556CF5E#OCo!Y!J?B^z l?C99c+{&Zs9Of1ERc(?*zzmzd$m#`~!5_r8$t-P|{ulbKM796` literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/tool_parsers/__pycache__/llama_tool_parser.cpython-310.pyc b/vllm/entrypoints/openai/tool_parsers/__pycache__/llama_tool_parser.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc102bf3f30c111c979a5512bacc610ff0e3ef8d GIT binary patch literal 6039 zcmbVQTW=)Cb?&b2>FMd|8Ir^MUZj?FiAM@aD?+TuTCb&OcV)?|m9nI;aYsUysV-GLV7T# zt4`OcI(6z?D{ZwL4bO}J_V3$IeNWT=i#ivd8al7xpZzxg*SH>Oe)ZOUhCYk5*zk?m z^i7@cMpTO}--_#gJ+^&2Zukv_*CWSw@ZF4>vFp1ESBqM4+i$D36-~v{{&YO!&%_{bkj+qf7B+|FUW~qAT%L|Eg*`(G&5L{*$^U zt_tr^_n+cT?mp1`@9-9HG>WS+3G7)Y_%-v_qYPx^TfrCZgYrVJCF#=&sd=RHQ!n>;4o zq+oC0pZzt!sF`cWdZ88iLzc5bJJdJXJ#AexwcI#(LD$Ai*G6+(FW8Rxu_m9(Ylqs{ zD2!vP&>rgdG)wy_)NIT?W$W6{C#^kwPwTQfBS*==<6#~`u(N|u=KU}Vb~56;+F~tA zL!OmJBoaw{mPVXyNrPIB4EF-A&Kf7%=qH0=9%Q*JYnxHHm6f_E4SfEUCZELcFFst$ zQYqeuF!}Ht~1yZwW^yge-RUPgp?8X0brjkA z87AD@7KyjA0u8z!iIs4erz@m2E4}Ts-xK&!^}yiDguxh&NrsUZO5qXTnaI_2Wg|az zLyws&nFN_E8AP@LkZXrzt~zPWmR4v1P6ZI>0%$5sjT>lNg}zfSv>l5xZXPjSTc*)= z-Z*x+l{a>p4~fF%F!gTz&gUn1ZjIX4ipo%eJlra-cXhuLavo#@(IdlEJZ6$qdDQse zZ$4Rl{zu~vKJ2#2S~eWOU)e#R+)w~(k>caZgy?*NP|9+uH7!@`#%2J5&*2mv|u^-FCu^v-13*`6#6mFYX|3?b*Rl% zD`%YHjA466#=16oGdIW?Z)s!me7t-*H;M94PjqgMYefxx^maA*7Tk8N&Q>OJPR$pGnK*747-1CWV*czLYnh;=y(k?YIY)G-;Lj`%nS3`Cn4bVwxP%J`1h z-?NY4(7RS?Z%%TE(j=8H8&L9L3=Xo=ftLU(7c%QI`9rKGX-U%9`~U`ajXI{BrVmpD zz5`I!_%I%Tpk%Ax^Jge%p01$w`}G*%T#Mj@NO0FfkY`r_ER8kT?-+F3(oN{NePS{8 z$o$HDWSxv&{YEVpReE!7u8*=urXRwc^aIsH=*b!TjvK^o#W}T{lI%R zD{D9J-hKCOw^=$F0(qX1x0Wlk+cp8R8c9|uDg2gJI6wsFW#i_4PYhIUWmQ>FSraNl ztPLb0K({IXjEMf609m$t0>GbnOjOEEHPo_jC;O$5-rp(fn4Ws_6%dpLj8|SI`mcrF z&i@q1O$^K425{?e8z&}H0)E6y%P@7*H1#$d2|IDwfl553QvvTHa^ciYu zLUM|@)1z$W^jSvZ%zym@&ZuB#{%e}r%70BG!Gxh4A=iP2GUP&T?$i>CApX{r_L?@f zKhsBl2andk-x=LKpGU6c6wfNZW{jIf6Md%qTAtelE1Fy*uU6QH41UcWw+iRDkvsGS zzt*DqdZ;2N_47J6CmyfKYlREr?Q;unJ?p)#@%o|ubG>j?+;q8hO)DD5jlxx4@oeSn z<$GE(i&VL}`y7TgZJ3ez-{9YPGQ%S|5!AzlNq=Py|0bi zXJ457mumDG#{5i|GgXf$cw#kPo?ZD&Kll$s*d;_*I>oDesaR5{NE)|Hvc3nM!!DLST_U}! zS}Lkux+DJrW{T3`2Y9J?Cts!3j|fnp8Lg;{#oH2zkVt^JhxBTjlIuxS@=#SGcb&wm z(RY(H_e7Eow~$YhW61ETa&gMKQ;&*s??B|AtCVqJ3&YQ-J>K8kB##ekXo37aJ zMMExvBuxY&S`DSH*stOqR0E@KmD*@gd1#6wY2qbfuL>v6@+x9dkQ}usk?$dmuTpR< zLD3Yk^B!LEHGq=!xte&iT16<-gjHzrwz11dcvTkqShyl~M?YNO?GFZo?`;p0T`$a# zh#-Uwqufh3D`w7VCt(3azo+6+_X$aHp^9i!naLLjkTdlS%2$o`jk{HVdx}2XBJebU z1%T4*rATre$f=5UncsRRjOb!PaFVwCF^#HmaeufapQpBgT396t@^zZ!PYJXLP|oQ) zRgOw2gq$aAm%uE6HwnB2&|R)#=X-?sD*|^3P)a1JCXw$GP(hWF3BPrgUQs~xn-_{w z`4bv?o4`9X^g^JSBDknPYW~xx7)IG`l*6H_vl!&&AaOStpf;lBtep}Tb zrz5lT{$whvx;**DXntjKY!}iSx_UrWf^+`_oM%^{(r%ODy2D({Rq-8B9VwKF=zbd6 zU5fc9R4`Cl#gJH;C)0*)I5ZBCA5s6;=Fd7)j*hR7T+694f|{kb%{kq%5dDz>Q1MQq zX*Lw_iq>T2Bl86FSy&ZRNZAaNwV9>6xQnoK+obylhCWeOGzucEvrlZ>VouFOK~v4` zFva@`T@RQ>HG@NRgeN}#kCiQ~Z0Xa;Rn+cJ+6LW#j9&bfnaa6byVF!bsyZ+zogjc} z4kHl+r5gkv4Z~>ir5*%4?V((2kQz~?RvI`PN%=>WBfd*5OMQSFl6C4TQws^#niPCJ zfx>iB^r>X-B^u`ucm{w{&Wx1W)=bu^vne`BhP0S5=Mnq9Y8mV+X0k7lxBbDg*cVov z{hrzEcTS`8TQx`bxk-wyDo?pab5li8U2s`Q>Y+}O3Vc~3Ks;CMlCk^bO22+%wn`3Q zHC3r@R=EuqR|-j{e1Oc9RD6#vx>l*InOt;%f-bsN2QtmmUK&->A{S6eMtKC4V5`D{ zN(}#%|7{Xw-_?yqwi@1t@Nv;unM#F~Gc++>BKgmMt4W4=KgyKb@tcaE%Hic4P2ng^ zHAMNS`!qW_I(5xfU1v=}#Hc!ioT!bG|Ai!7ov;U*(!N$H)^%90mQe|1Yc?|QCEbP& VyZSS1x&~WA%Vu`XZrb*O{znY9QabYHO^c>Tn+gdj1ZhSisV0N%1~A}U zRz0%>v0k`bfn0t9yz*D|G}N_7 zj}6aIiKj=E*!0Y}>Q!akh^)Bg)nePT<0)?{c05P&)u`^((Y_Kj;-=S>JTq#=)84c! zSEF`36ZZKI@&8^;&c;e#Lu5mhI@( z*!5glo{G-LuX(S<-|)U6>rV7~e8Ib*D%|C7?5p0J+~n1H#e3^V3ahimmkMj}#eLmd z;%5)vX3ZlN{1%%Af101!*U{sfOkGo!+Fz5ER+rSG+TBXSF3*;Pr&z6kB_#o`&g>f^?^1@oOlpWR06(A~uyj4n)R9G5xfBD@ZVifTD}M6zqoyrW$h=208axxUqqcQ(+S3 zew1!fTdi=eB)fOg`#cE;5ZAf|BZwfHaHxX_5|+k(zaO$L+4bb3qw+r9>|GG9cq&so z?XJRgrZNrdq93T9!7JQk#wCSUS%sNjDp)6rRk2Q1(VPre8s&unG|0V516Uxra;P3D z=%}tPX<`waqP@uzE@*xuHAD6rMV!GWNgnwtdE8s+g}voG5Su(-z8Lqn;`id6l_*?a z>FwrQX|nPlisBWXMa(Z(!|Hv0G?=#I7df$Z5AxVCtcgtGUKB*i$wi| zV9iJx3ZkAhx2ZrrJ2x^oA-|-K&lk;{|sibJ3FxK*s*aR8MCbH#W3iDjO=( znenCLRTz0C@P$>mwP7-g)xI>m8neNj!t3CzbDK{s66%z{&Cze!%i(p_+Rzr24PX#! zqj%$%ilKN-tj|m_eZr>YW*ro^i-&fs+XXtfrDwU=^X{mzd+~?vjRc14cXRi_yPa>n zBYD_yk@-OD9t;!9bBuFE4`M>5?&aF?c#@mz{4YWWyMy5xl8hsp+C z9d9N;a%CTzc&Hw!s*-C%Wn0gUeJz{LD~BeN+)ZVu_*I5)zQuIVn&jBbKzRy1oWnZk z)RBsMBX1tISS4?6Pajc_Hoi2ni}?&Q_tn1!8euJ}tAk6cX|g=F@kCRX@thO>mWJqO zVY2B4leK_{^QhD5e0_}P`oQ`0&PN~J^sipId2{XaLG80of7*Hb2Ya7=KKO7fSgvfj z)^0cLN6dxiyHVQ3YS4-wqds~1OTpNhpd=;h9UQEhVR83e2-&~ zGj!X`e7L6UQqWpE%$^%Sjz0d)sg)?FM5UL4|fQz^oM!YaT_~;HyRWxbRzSKnU zP5!WOZugTsjQKSoQi?!E#-k1K4kq#&wW#@iHwrQ!#R<4t!MFGDX0L!as-xMeSup_# z_|sZy3%?rvjAP??aC)>Dv=-%3bS9Q`O6#Tlyw}f*su;y>Q}Dbmk}mn^_@QV+P<9n> z_OBrDD`luYP=%ALOoc}xz8&am%HZ4hq8;i(^$`BKjfi&ygl5K~vZs?z5A`E;&nV;G z(D(qcZ?A&3RcbqGXF_6iSjnv+e3lva4RL8`lJtAdz2i%ikS!M_IB5{~>oOLC`4ic-AK~cSEEx{GV13nGJWP zxm>}sUYcaQ<5Hp%qB@bY5hiRI*}<~lfSfLEAKF6^cfaVTInPS0Vrd`nGG-CleYp0~ zszg6DerF;r4(@X|>jPPK)4n*dS>THO7~Yw=EKT0bUH*Uz?454ZXJfJFCsM92>7GMA zmr1ANtgt5@n2~!Dp60^EnhIpiN|(s^A-JMtU&EVqKnQw1pV?D$q|iBFi>Zr%En02{ z;Ku?YAT#4Y`<3>E2B@hN*2XCRD-7CIMQx%yGL|1hqPPJO{z-cwX3-A|xlYZkGepvr{0ZqE)p=v@cBT7~`?= z#5^8+=f&MI&J~{;3CRj}c$8g6VJ!B}>R?4~Si-DP7g}!WRU(>np0Tfd&%KZ>IpQMp z5KSWA2PvxLFMuT4dgL!${5i2JQ5vuez}JhyywD?_6t!y)ySztvu%L}!RJa5SOZoOUI!^u{=C`RFV3CTBfAV@JB9=m^NFULhNEi71TZ#W zN2{8@YfX`1HMOByk2T$T+^9IJp%JJ3#QM4Rc<}nm7`mtvREKFlL(qN*G@XDl)oetc zJSe|NOfMS}1_NxfVtUeg!a`~(yNnsDUBa>Pvw*?`(|!)LlbEIf(`>jlGWG*xmV{3nn5F~!Y6B~;9ooa% z&^prg?0ZTwL-@5lwe4)zcWYfS3RZxEcZW74wicB{ z4VAsA$rnxhRoQyNF~o1nGPP&gyn^1pmF3F)iun8C)ZsKUfy2(w5r3Dr4`+tXk$QLv zEoRFRSQT0V+4DK7=kr5{SqJL&0z*D{pt7k0ZBu2=fx1VThwM{P7L}oM5sA$uC8;oF zub$wu<&?rYh?qQuU9yKHTFR3NYw0n(VL;UAh z7dqeOr}ve=QT|qgUJcehq1ViTG1d#Q03#*;BsKX*NIXMhEGna!Hzzh|4fVrQ>=c{D zjHidKuhh&QHi^ofzml8<;)n(4dX~+R*7DmveG{H+?MW-Pplw^uXnr_7Y#-^EQJdjC zJU5&^QelhfeRW>>1)y?oMz&iR&d|Kav%`#LUdqxlBTHYAI{aGd_v?ucr(}t2H!DY3 z9G)7^k~aA(RNB|}<|P00aPGO$=U!^`xzXryg~==ZGv&&`r(gX_-TfCt<#|M9vc*|; zW;icLp_M*M*1Ja$87+P$_fc7rG5H*>C?q(2O+iR3gACrHoRKGKe-l|gZR89;YlBNSR)2Ek_Khp6 zcid$+jC*O8g%tat_Ezu!M}|oxN2K7n8;8gr#=Jp`;sdF4A)_#JAsf7LJrr3!wgY&R z#z;%LJbb_zg_^~es`;%+;@WvDHCRc6TO>mlxBR|cN{8V z;l4|RQVoI2Jw?!Q-qVpYd-~d)+u}B=#SG3}c^E`Kr3Is_2=N#6#ptF;w{0wS87MQa zek+JDdnE7VBBQ{`_B$ zKN3D_l$GBn!bkP`xV^kd6F;GeP41b5%)`#Um{9l=U)V*5>@93`yKbv?6$yjxn3Px; zTC1WNhH9wRaT{kFOVyq@RT~M1cI+VGaE!KUn>G>*^iz?BP)dSdN=PJqVjNq?h6xF% zTgRquXblbhke=X#L_Jjvbxrm@(d)5wY+0s8J+)(eKejaof2IcAq;ykv&{|Q31<6=?XOs-fL$dfDScO?$X_mKoCL9Ttw|>sF-=2%_x}`LccI=@p9f4Z271Y}vS!Cs9wj6CFUr>qO{M(raRYa4O_foj`T7G_l&&KQY`Gsmb#!?@O4Lh9Xp?PTnl9j?$FY$sX6t3 E0WWST_W%F@ literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/tool_parsers/__pycache__/utils.cpython-310.pyc b/vllm/entrypoints/openai/tool_parsers/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31b9a8141df05d0d2cc864c8c3b9cfef6ca943f2 GIT binary patch literal 2986 zcmeHJO>Z1E7@qOWZZ_K{lx=D00p#YAF3M(Agj5Mt6%wdbp-Kb^LR3Pl$&Po|WM(|I zJxK$bQ_3IU7c@r>`~`lG&jGAeb#?Rv?;hEM(+C(c$=U~@5ww+JgXpPcK z>E-9alQ>=22)c_kGk5TQSU0t>l`e)1lN>z_RARiJ;x&&^q~SiZfeoj@L5MOu2oKnP zWLe6l(ERwyl;QBYICOWd+W|e{Er< z+XCB8iq!czX%=XN(?Z!3$2f27rqUwYo1KeT9k22cqB8Iy8TgRgUjA`uq$_EcKIThF zxwOB$!cRAqnLulo_wKuSPj@c6wmNu;R0_IL9bqSRl!>Bl=%g5nZbKs@0!=wsH~o}# z8%6`zjP0Zj4g_7%3@w{|zZA zyAe3Fyyww!De!oqe;}quc%k}>=rAzHUsH?%f;y)VGiqug@%(UV4E>ok|FA{4UykLbN(z zCtN~GxP<7CeLSqRt|OJD=W>JA<5=`(RBV6<6LYd@n9`5^6d~H-l%0GLvmZt$-}|8@ z9(8;>4fbOjP1!-H?@i+qoB!}YVjWPlh`8L9kCa1}9Z+VTp$@M`XEllh%I%MctaqO;!))0z(%Q>ccpR1Jw*Xa8 ziG%i>7G;(Ug=2n4ETEFBFwobic$gPj@b`bH;BQNGRQFx zIU_~oL-KZXO;UyH(C=WKoBk|W9h1b?&7_15r`@=2dUsFekLLna`!&=JRK&BhM|s7G zG$umq-CDnZe|Lg$BMuq(xP>xi?N{tC{XY8Ex{W%*G(K{4)b3zi-)qiKg4Md0tF+3* SeJB1TAJ>EHH?DUs-})OA(L`4O literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py new file mode 100644 index 00000000..5ce31bd4 --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py @@ -0,0 +1,161 @@ +import importlib +import importlib.util +import os +from functools import cached_property +from typing import Callable, Dict, List, Optional, Sequence, Type, Union + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaMessage, + ExtractedToolCallInformation) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import is_list_of + +logger = init_logger(__name__) + + +class ToolParser: + """ + Abstract ToolParser class that should not be used directly. Provided + properties and methods should be used in + derived classes. + """ + + def __init__(self, tokenizer: AnyTokenizer): + self.prev_tool_call_arr: List[Dict] = [] + # the index of the tool call that is currently being parsed + self.current_tool_id: int = -1 + self.current_tool_name_sent: bool = False + self.streamed_args_for_tool: List[str] = [] + + self.model_tokenizer = tokenizer + + @cached_property + def vocab(self) -> Dict[str, int]: + # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab + # whereas all tokenizers have .get_vocab() + return self.model_tokenizer.get_vocab() + + def adjust_request( + self, request: ChatCompletionRequest) -> ChatCompletionRequest: + """ + Static method that used to adjust the request parameters. + """ + return request + + def extract_tool_calls( + self, model_output: str, + request: ChatCompletionRequest) -> ExtractedToolCallInformation: + """ + Static method that should be implemented for extracting tool calls from + a complete model-generated string. + Used for non-streaming responses where we have the entire model response + available before sending to the client. + Static because it's stateless. + """ + raise NotImplementedError( + "AbstractToolParser.extract_tool_calls has not been implemented!") + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + """ + Instance method that should be implemented for extracting tool calls + from an incomplete response; for use when handling tool calls and + streaming. Has to be an instance method because it requires state - + the current tokens/diffs, but also the information about what has + previously been parsed and extracted (see constructor) + """ + raise NotImplementedError( + "AbstractToolParser.extract_tool_calls_streaming has not been " + "implemented!") + + +class ToolParserManager: + tool_parsers: Dict[str, Type] = {} + + @classmethod + def get_tool_parser(cls, name) -> Type: + """ + Get tool parser by name which is registered by `register_module`. + + Raise a KeyError exception if the name is not registered. + """ + if name in cls.tool_parsers: + return cls.tool_parsers[name] + + raise KeyError(f"tool helper: '{name}' not found in tool_parsers") + + @classmethod + def _register_module(cls, + module: Type, + module_name: Optional[Union[str, List[str]]] = None, + force: bool = True) -> None: + if not issubclass(module, ToolParser): + raise TypeError( + f'module must be subclass of ToolParser, but got {type(module)}' + ) + if module_name is None: + module_name = module.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in cls.tool_parsers: + existed_module = cls.tool_parsers[name] + raise KeyError(f'{name} is already registered ' + f'at {existed_module.__module__}') + cls.tool_parsers[name] = module + + @classmethod + def register_module( + cls, + name: Optional[Union[str, List[str]]] = None, + force: bool = True, + module: Union[Type, None] = None) -> Union[type, Callable]: + """ + Register module with the given name or name list. it can be used as a + decoder(with module as None) or normal function(with module as not + None). + """ + if not isinstance(force, bool): + raise TypeError(f'force must be a boolean, but got {type(force)}') + + # raise the error ahead of time + if not (name is None or isinstance(name, str) + or is_list_of(name, str)): + raise TypeError( + 'name must be None, an instance of str, or a sequence of str, ' + f'but got {type(name)}') + + # use it as a normal method: x.register_module(module=SomeClass) + if module is not None: + cls._register_module(module=module, module_name=name, force=force) + return module + + # use it as a decorator: @x.register_module() + def _register(module): + cls._register_module(module=module, module_name=name, force=force) + return module + + return _register + + @classmethod + def import_tool_parser(cls, plugin_path: str) -> None: + """ + Import a user defined tool parser by the path of the tool parser define + file. + """ + module_name = os.path.splitext(os.path.basename(plugin_path))[0] + spec = importlib.util.spec_from_file_location(module_name, plugin_path) + if spec is None or spec.loader is None: + logger.error("load %s from %s failed.", module_name, plugin_path) + return + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py new file mode 100644 index 00000000..f3cc1d9c --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -0,0 +1,338 @@ +import json +import regex as re +from typing import Dict, List, Sequence, Union + +import partial_json_parser +from partial_json_parser.core.options import Allow + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.entrypoints.openai.tool_parsers.utils import ( + extract_intermediate_diff) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +@ToolParserManager.register_module("hermes") +class Hermes2ProToolParser(ToolParser): + + def __init__(self, tokenizer: AnyTokenizer): + super().__init__(tokenizer) + + if isinstance(self.model_tokenizer, MistralTokenizer): + logger.error( + "Detected Mistral tokenizer when using a Hermes model") + self.model_tokenizer = self.model_tokenizer.tokenizer + + self.current_tool_name_sent: bool = False + self.prev_tool_call_arr: List[Dict] = [] + self.current_tool_id: int = -1 + self.streamed_args_for_tool: List[str] = [ + ] # map what has been streamed for each tool so far to a list + + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + + self.tool_call_regex = re.compile( + r"(.*?)|(.*)", re.DOTALL) + self.scratch_pad_regex = re.compile( + r"(.*?)", re.DOTALL) + + if not self.model_tokenizer: + raise ValueError( + "The model tokenizer must be passed to the ToolParser " + "constructor during construction.") + self.tool_call_start_token_id = self.vocab.get( + self.tool_call_start_token) + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) + if not self.tool_call_start_token_id or not self.tool_call_end_token_id: + raise RuntimeError( + "Hermes 2 Pro Tool parser could not locate tool call start/end " + "tokens in the tokenizer!") + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + + # sanity check; avoid unnecessary processing + if self.tool_call_start_token not in model_output: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + else: + + try: + # there are two possible captures - between tags, or between a + # tag and end-of-string so the result of + # findall is an array of tuples where one is a function call and + # the other is None + function_call_tuples = ( + self.tool_call_regex.findall(model_output)) + + # load the JSON, and then use it to build the Function and + # Tool Call + raw_function_calls = [ + json.loads(match[0] if match[0] else match[1]) + for match in function_call_tuples + ] + tool_calls = [ + ToolCall( + type="function", + function=FunctionCall( + name=function_call["name"], + # function call args are JSON but as a string + arguments=json.dumps(function_call["arguments"]))) + for function_call in raw_function_calls + ] + + content = model_output[:model_output. + find(self.tool_call_start_token)] + return ExtractedToolCallInformation( + tools_called=True, + tool_calls=tool_calls, + content=content if content else None) + + except Exception as e: + logger.error("Error in extracting tool call from response %s", + e) + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + + logger.debug("delta_text: %s", delta_text) + logger.debug("delta_token_ids: %s", delta_token_ids) + # check to see if we should be streaming a tool call - is there a + if self.tool_call_start_token_id not in current_token_ids: + logger.debug("No tool call tokens found!") + return DeltaMessage(content=delta_text) + + try: + + # figure out where we are in the parsing by counting tool call + # start & end tags + prev_tool_start_count = previous_token_ids.count( + self.tool_call_start_token_id) + prev_tool_end_count = previous_token_ids.count( + self.tool_call_end_token_id) + cur_tool_start_count = current_token_ids.count( + self.tool_call_start_token_id) + cur_tool_end_count = current_token_ids.count( + self.tool_call_end_token_id) + + # case: if we're generating text, OR rounding out a tool call + if (cur_tool_start_count == cur_tool_end_count + and prev_tool_end_count == cur_tool_end_count): + logger.debug("Generating text content! skipping tool parsing.") + if delta_text != self.tool_call_end_token: + return DeltaMessage(content=delta_text) + + # case: if tool open & close tag counts don't match, we're doing + # imaginary "else" block here + # something with tools with this diff. + # flags for partial JSON parting. exported constants from + # "Allow" are handled via BIT MASK + flags = Allow.ALL if self.current_tool_name_sent \ + else Allow.ALL & ~Allow.STR + + # case -- we're starting a new tool call + if (cur_tool_start_count > cur_tool_end_count + and cur_tool_start_count > prev_tool_start_count): + if len(delta_token_ids) > 1: + tool_call_portion = current_text.split( + self.tool_call_start_token)[-1] + else: + tool_call_portion = None + delta = None + + text_portion = None + + # set cursors and state appropriately + self.current_tool_id += 1 + self.current_tool_name_sent = False + self.streamed_args_for_tool.append("") + logger.debug("Starting on a new tool %s", self.current_tool_id) + + # case -- we're updating an existing tool call + elif (cur_tool_start_count > cur_tool_end_count + and cur_tool_start_count == prev_tool_start_count): + + # get the portion of the text that's the tool call + tool_call_portion = current_text.split( + self.tool_call_start_token)[-1] + text_portion = None + + # case -- the current tool call is being closed. + elif (cur_tool_start_count == cur_tool_end_count + and cur_tool_end_count > prev_tool_end_count): + diff = self.prev_tool_call_arr[self.current_tool_id].get( + "arguments") + if diff: + diff = json.dumps(diff).replace( + self.streamed_args_for_tool[self.current_tool_id], "") + logger.debug( + "Finishing tool and found diff that had not " + "been streamed yet: %s", diff) + self.streamed_args_for_tool[self.current_tool_id] \ + += diff + return DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=diff).model_dump( + exclude_none=True)) + ]) + + # case -- otherwise we're just generating text + else: + text = delta_text.replace(self.tool_call_start_token, "") + text = text.replace(self.tool_call_end_token, "") + delta = DeltaMessage(tool_calls=[], content=text) + return delta + + try: + + current_tool_call = partial_json_parser.loads( + tool_call_portion or "{}", + flags) if tool_call_portion else None + logger.debug("Parsed tool call %s", current_tool_call) + except partial_json_parser.core.exceptions.MalformedJSON: + logger.debug('not enough tokens to parse into JSON yet') + return None + + # case - we haven't sent the tool name yet. If it's available, send + # it. otherwise, wait until it's available. + if not self.current_tool_name_sent: + function_name: Union[str, None] = current_tool_call.get("name") + if function_name: + self.current_tool_name_sent = True + return DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + type="function", + id=f"chatcmpl-tool-{random_uuid()}", + function=DeltaFunctionCall( + name=function_name).model_dump( + exclude_none=True)) + ]) + else: + return None + # case -- otherwise, send the tool call delta + + # if the tool call portion is None, send the delta as text + if tool_call_portion is None: + # if there's text but not tool calls, send that - + # otherwise None to skip chunk + delta = DeltaMessage(content=delta_text) \ + if text_portion is not None else None + return delta + + # now, the nitty-gritty of tool calls + # now we have the portion to parse as tool call. + + logger.debug("Trying to parse current tool call with ID %s", + self.current_tool_id) + + # if we're starting a new tool call, push an empty object in as + # a placeholder for the arguments + if len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + + # main logic for tool parsing here - compare prev. partially-parsed + # JSON to the current partially-parsed JSON + prev_arguments = ( + self.prev_tool_call_arr[self.current_tool_id].get("arguments")) + cur_arguments = current_tool_call.get("arguments") + + logger.debug("diffing old arguments: %s", prev_arguments) + logger.debug("against new ones: %s", cur_arguments) + + # case -- no arguments have been created yet. skip sending a delta. + if not cur_arguments and not prev_arguments: + logger.debug("Skipping text %s - no arguments", delta_text) + delta = None + + # case -- prev arguments are defined, but non are now. + # probably impossible, but not a fatal error - just keep going + elif not cur_arguments and prev_arguments: + logger.error("should be impossible to have arguments reset " + "mid-call. skipping streaming anything.") + delta = None + + # case -- we now have the first info about arguments available from + # autocompleting the JSON + elif cur_arguments and not prev_arguments: + + cur_arguments_json = json.dumps(cur_arguments) + logger.debug("finding %s in %s", delta_text, + cur_arguments_json) + + # get the location where previous args differ from current + args_delta_start_loc = cur_arguments_json.index(delta_text) \ + + len(delta_text) + + # use that to find the actual delta + arguments_delta = cur_arguments_json[:args_delta_start_loc] + logger.debug("First tokens in arguments received: %s", + arguments_delta) + + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=arguments_delta).model_dump( + exclude_none=True)) + ]) + self.streamed_args_for_tool[self.current_tool_id] \ + += arguments_delta + + # last case -- we have an update to existing arguments. + elif cur_arguments and prev_arguments: + + cur_args_json = json.dumps(cur_arguments) + prev_args_json = json.dumps(prev_arguments) + logger.debug("Searching for diff between\n%s", cur_args_json) + logger.debug("and\n%s", prev_args_json) + argument_diff = extract_intermediate_diff( + cur_args_json, prev_args_json) + logger.debug("got argument diff %s", argument_diff) + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=argument_diff).model_dump( + exclude_none=True)) + ]) + self.streamed_args_for_tool[self.current_tool_id] \ + += argument_diff + + # handle saving the state for the current tool into + # the "prev" list for use in diffing for the next iteration + if self.current_tool_id == len(self.prev_tool_call_arr) - 1: + self.prev_tool_call_arr[self.current_tool_id] = \ + current_tool_call + else: + self.prev_tool_call_arr.append(current_tool_call) + + return delta + + except Exception as e: + logger.error("Error trying to handle streaming tool call: %s", e) + return None # do not stream a delta. skip this token ID. diff --git a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py new file mode 100644 index 00000000..905ab7db --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py @@ -0,0 +1,208 @@ +import json +from typing import Dict, Sequence, Union + +import partial_json_parser +from partial_json_parser.core.options import Allow + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.entrypoints.openai.tool_parsers.utils import ( + extract_intermediate_diff) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +@ToolParserManager.register_module(["internlm"]) +class Internlm2ToolParser(ToolParser): + + def __init__(self, tokenizer: AnyTokenizer): + super().__init__(tokenizer) + self.position = 0 + + def adjust_request( + self, request: ChatCompletionRequest) -> ChatCompletionRequest: + if request.tools and request.tool_choice != 'none': + # do not skip special tokens because internlm use the special + # tokens to indicated the start and end of the tool calls + # information. + request.skip_special_tokens = False + return request + + def get_argments(self, obj): + if "parameters" in obj: + return obj.get("parameters") + elif "arguments" in obj: + return obj.get("arguments") + return None + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + if '<|action_start|>' not in current_text: + self.position = len(current_text) + return DeltaMessage(content=delta_text) + # if the tool call is sended, return a empty delta message + # to make sure the finish_reason will be send correctly. + if self.current_tool_id > 0: + return DeltaMessage(content='') + + last_pos = self.position + if '<|action_start|><|plugin|>' not in current_text[last_pos:]: + return None + + new_delta = current_text[last_pos:] + text, action = new_delta.split('<|action_start|><|plugin|>') + + if len(text) > 0: + self.position = self.position + len(text) + return DeltaMessage(content=text) + + action = action.strip() + action = action.split('<|action_end|>'.strip())[0] + + # bit mask flags for partial JSON parsing. If the name hasn't been + # sent yet, don't allow sending + # an incomplete string since OpenAI only ever (as far as I have + # seen) allows sending the entire tool/ function name at once. + flags = Allow.ALL if self.current_tool_name_sent \ + else Allow.ALL & ~Allow.STR + + try: + parsable_arr = action + + # tool calls are generated in an object in inernlm2 + # it's not support parallel tool calls + try: + tool_call_arr: Dict = partial_json_parser.loads( + parsable_arr, flags) + except partial_json_parser.core.exceptions.MalformedJSON: + logger.debug('not enough tokens to parse into JSON yet') + return None + + # if the current tool name hasn't been sent, send if available + # - otherwise send nothing + if not self.current_tool_name_sent: + function_name = tool_call_arr.get("name") + if function_name: + self.current_tool_id = self.current_tool_id + 1 + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + type="function", + id=f"chatcmpl-tool-{random_uuid()}", + function=DeltaFunctionCall( + name=function_name).model_dump( + exclude_none=True)) + ]) + self.current_tool_name_sent = True + self.streamed_args_for_tool.append("") + else: + delta = None + # now we know we're on the same tool call and we're streaming + # arguments + else: + prev_arguments = self.get_argments( + self.prev_tool_call_arr[self.current_tool_id]) + cur_arguments = self.get_argments(tool_call_arr) + + # not arguments generated + if not cur_arguments and not prev_arguments: + delta = None + # will never happen + elif not cur_arguments and prev_arguments: + logger.error( + "INVARIANT - impossible to have arguments reset " + "mid-arguments") + delta = None + # first time to get parameters + elif cur_arguments and not prev_arguments: + cur_arguments_json = json.dumps(cur_arguments) + + arguments_delta = cur_arguments_json[:cur_arguments_json. + index(delta_text) + + len(delta_text)] + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=arguments_delta). + model_dump(exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += arguments_delta + # both prev and cur parameters, send the increase parameters + elif cur_arguments and prev_arguments: + cur_args_json = json.dumps(cur_arguments) + prev_args_json = json.dumps(prev_arguments) + + argument_diff = extract_intermediate_diff( + cur_args_json, prev_args_json) + + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=argument_diff).model_dump( + exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += argument_diff + + # check to see if the name is defined and has been sent. if so, + # stream the name - otherwise keep waiting + # finish by setting old and returning None as base case + tool_call_arr["arguments"] = self.get_argments(tool_call_arr) + self.prev_tool_call_arr = [tool_call_arr] + return delta + except Exception as e: + logger.error("Error trying to handle streaming tool call: %s", e) + logger.debug( + "Skipping chunk as a result of tool streaming extraction " + "error") + return None + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + text = model_output + tools = request.tools + if '<|action_start|><|plugin|>' in text: + text, action = text.split('<|action_start|><|plugin|>') + action = action.split('<|action_end|>'.strip())[0] + action = action[action.find('{'):] + action_dict = json.loads(action) + name, parameters = action_dict['name'], json.dumps( + action_dict.get('parameters', action_dict.get('arguments', + {}))) + + if not tools or name not in [t.function.name for t in tools]: + ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=text) + + tool_calls = [ + ToolCall( + function=FunctionCall(name=name, arguments=parameters)) + ] + return ExtractedToolCallInformation( + tools_called=True, + tool_calls=tool_calls, + content=text if len(text) > 0 else None) + + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=text) diff --git a/vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py new file mode 100644 index 00000000..f950ab16 --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py @@ -0,0 +1,277 @@ +import json +import regex as re +from json import JSONDecodeError, JSONDecoder +from typing import Dict, List, Sequence, Union + +import partial_json_parser +from partial_json_parser.core.options import Allow +from transformers import PreTrainedTokenizerBase + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.entrypoints.openai.tool_parsers.utils import find_common_prefix +from vllm.logger import init_logger +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +# partial_json_parser doesn't support extra data and +# JSONDecorder.raw_decode doesn't support partial JSON +def partial_json_loads(input_str, flags): + try: + return (partial_json_parser.loads(input_str, flags), len(input_str)) + except JSONDecodeError as e: + if "Extra data" in e.msg: + dec = JSONDecoder() + return dec.raw_decode(input_str) + else: + raise + + +def is_complete_json(input_str): + try: + json.loads(input_str) + return True + except JSONDecodeError: + return False + + +@ToolParserManager.register_module("llama3_json") +class Llama3JsonToolParser(ToolParser): + """ + Tool call parser for Llama 3.1 models intended for use with the + examples/tool_chat_template_llama.jinja template. + + Used when --enable-auto-tool-choice --tool-call-parser mistral are all set + """ + + def __init__(self, tokenizer: PreTrainedTokenizerBase): + super().__init__(tokenizer) + + # initialize properties used for state when parsing tool calls in + # streaming mode + self.prev_tool_call_arr: List[Dict] = [] + self.current_tool_id: int = -1 + self.current_tool_name_sent: bool = False + self.streamed_args_for_tool: List[str] = [ + ] # map what has been streamed for each tool so far to a list + self.bot_token = "<|python_tag|>" + self.bot_token_id = tokenizer.encode(self.bot_token, + add_special_tokens=False)[0] + self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL) + + def extract_tool_calls( + self, model_output: str, + request: ChatCompletionRequest) -> ExtractedToolCallInformation: + """ + Extract the tool calls from a complete model response. + """ + # case -- if a tool call token is not present, return a text response + if not (model_output.startswith(self.bot_token) + or model_output.startswith('{')): + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + try: + # load the JSON, and then use it to build the Function and + # Tool Call + dec = JSONDecoder() + function_call_arr = [] + + # depending on the prompt format the Llama model may or may not + # prefix the output with the <|python_tag|> token + start_idx = len(self.bot_token) if model_output.startswith( + self.bot_token) else 0 + while start_idx < len(model_output): + (obj, end_idx) = dec.raw_decode(model_output[start_idx:]) + start_idx += end_idx + len('; ') + function_call_arr.append(obj) + + tool_calls: List[ToolCall] = [ + ToolCall( + type="function", + function=FunctionCall( + name=raw_function_call["name"], + # function call args are JSON but as a string + arguments=json.dumps(raw_function_call["arguments"] \ + if "arguments" in raw_function_call \ + else raw_function_call["parameters"]))) + for raw_function_call in function_call_arr + ] + + # get any content before the tool call + ret = ExtractedToolCallInformation(tools_called=True, + tool_calls=tool_calls, + content=None) + return ret + + except Exception as e: + logger.error("Error in extracting tool call from response: %s", e) + print("ERROR", e) + # return information to just treat the tool call as regular JSON + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + + if not (current_text.startswith(self.bot_token) + or current_text.startswith('{')): + return DeltaMessage(content=delta_text) + + # bit mask flags for partial JSON parsing. If the name hasn't been + # sent yet, don't allow sending + # an incomplete string since OpenAI only ever (as far as I have + # seen) allows sending the entire tool/ function name at once. + flags = Allow.ALL if self.current_tool_name_sent \ + else Allow.ALL & ~Allow.STR + try: + tool_call_arr = [] + is_complete = [] + try: + # depending on the prompt format the Llama model may or may not + # prefix the output with the <|python_tag|> token + start_idx = len(self.bot_token) if current_text.startswith( + self.bot_token) else 0 + while start_idx < len(current_text): + (obj, + end_idx) = partial_json_loads(current_text[start_idx:], + flags) + is_complete.append( + is_complete_json(current_text[start_idx:start_idx + + end_idx])) + start_idx += end_idx + len('; ') + # depending on the prompt Llama can use + # either arguments or parameters + if "parameters" in obj: + assert "arguments" not in obj, \ + "model generated both parameters and arguments" + obj["arguments"] = obj["parameters"] + tool_call_arr.append(obj) + except partial_json_parser.core.exceptions.MalformedJSON: + logger.debug('not enough tokens to parse into JSON yet') + return None + + # select as the current tool call the one we're on the state at + current_tool_call: Dict = tool_call_arr[self.current_tool_id] \ + if len(tool_call_arr) > 0 else {} + + # case -- if no tokens have been streamed for the tool, e.g. + # only the array brackets, stream nothing + if len(tool_call_arr) == 0: + return None + + # case: we are starting a new tool in the array + # -> array has > 0 length AND length has moved past cursor + elif (len(tool_call_arr) > 0 + and len(tool_call_arr) > self.current_tool_id + 1): + + # if we're moving on to a new call, first make sure we + # haven't missed anything in the previous one that was + # auto-generated due to JSON completions, but wasn't + # streamed to the client yet. + if self.current_tool_id >= 0: + cur_arguments = current_tool_call.get("arguments") + if cur_arguments: + cur_args_json = json.dumps(cur_arguments) + sent = len( + self.streamed_args_for_tool[self.current_tool_id]) + argument_diff = cur_args_json[sent:] + + logger.debug("got arguments diff: %s", argument_diff) + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=argument_diff). + model_dump(exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += argument_diff + else: + delta = None + else: + delta = None + # re-set stuff pertaining to progress in the current tool + self.current_tool_id = len(tool_call_arr) - 1 + self.current_tool_name_sent = False + self.streamed_args_for_tool.append("") + logger.debug("starting on new tool %d", self.current_tool_id) + return delta + + # if the current tool name hasn't been sent, send if available + # - otherwise send nothing + elif not self.current_tool_name_sent: + function_name = current_tool_call.get("name") + if function_name: + + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + type="function", + id=f"chatcmpl-tool-{random_uuid()}", + function=DeltaFunctionCall( + name=function_name).model_dump( + exclude_none=True)) + ]) + self.current_tool_name_sent = True + else: + delta = None + + # now we know we're on the same tool call and we're streaming + # arguments + else: + cur_arguments = current_tool_call.get("arguments") + delta = None + + if cur_arguments: + sent = len( + self.streamed_args_for_tool[self.current_tool_id]) + cur_args_json = json.dumps(cur_arguments) + prev_arguments = self.prev_tool_call_arr[ + self.current_tool_id].get("arguments") + + argument_diff = None + if is_complete[self.current_tool_id]: + argument_diff = cur_args_json[sent:] + elif prev_arguments: + prev_args_json = json.dumps(prev_arguments) + if cur_args_json != prev_args_json: + + prefix = find_common_prefix( + prev_args_json, cur_args_json) + argument_diff = prefix[sent:] + + if argument_diff is not None: + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=argument_diff). + model_dump(exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += argument_diff + + self.prev_tool_call_arr = tool_call_arr + return delta + + except Exception as e: + logger.error("Error trying to handle streaming tool call: %s", e) + logger.debug( + "Skipping chunk as a result of tool streaming extraction " + "error") + return None diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py new file mode 100644 index 00000000..b7d5d0aa --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -0,0 +1,306 @@ +import json +import regex as re +from random import choices +from string import ascii_letters, digits +from typing import Dict, List, Sequence, Union + +import partial_json_parser +from partial_json_parser.core.options import Allow +from pydantic import Field + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.entrypoints.openai.tool_parsers.utils import ( + extract_intermediate_diff) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.utils import random_uuid + +logger = init_logger(__name__) + +ALPHANUMERIC = ascii_letters + digits + + +class MistralToolCall(ToolCall): + id: str = Field( + default_factory=lambda: MistralToolCall.generate_random_id()) + + @staticmethod + def generate_random_id(): + # Mistral Tool Call Ids must be alphanumeric with a maximum length of 9. + # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299 + return "".join(choices(ALPHANUMERIC, k=9)) + + +@ToolParserManager.register_module("mistral") +class MistralToolParser(ToolParser): + """ + Tool call parser for Mistral 7B Instruct v0.3, intended for use with the + examples/tool_chat_template_mistral.jinja template. + + Used when --enable-auto-tool-choice --tool-call-parser mistral are all set + """ + + def __init__(self, tokenizer: AnyTokenizer): + super().__init__(tokenizer) + + if not isinstance(self.model_tokenizer, MistralTokenizer): + logger.info("Non-Mistral tokenizer detected when using a Mistral " + "model...") + + # initialize properties used for state when parsing tool calls in + # streaming mode + self.prev_tool_call_arr: List[Dict] = [] + self.current_tool_id: int = -1 + self.current_tool_name_sent: bool = False + self.streamed_args_for_tool: List[str] = [ + ] # map what has been streamed for each tool so far to a list + self.bot_token = "[TOOL_CALLS]" + self.bot_token_id = self.vocab.get(self.bot_token) + self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL) + if not self.bot_token_id: + raise RuntimeError( + "Mistral Tool Parser could not locate the tool call token in " + "the tokenizer!") + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + """ + Extract the tool calls from a complete model response. Requires + find-and-replacing single quotes with double quotes for JSON parsing, + make sure your tool call arguments don't ever include quotes! + """ + + # case -- if a tool call token is not present, return a text response + if self.bot_token not in model_output: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + try: + + # use a regex to find the tool call. remove the BOT token + # and make sure to replace single quotes with double quotes + raw_tool_call = self.tool_call_regex.findall( + model_output.replace(self.bot_token, ""))[0] + + # load the JSON, and then use it to build the Function and + # Tool Call + function_call_arr = json.loads(raw_tool_call) + tool_calls: List[MistralToolCall] = [ + MistralToolCall( + type="function", + function=FunctionCall( + name=raw_function_call["name"], + # function call args are JSON but as a string + arguments=json.dumps(raw_function_call["arguments"]))) + for raw_function_call in function_call_arr + ] + + # get any content before the tool call + content = model_output.split(self.bot_token)[0] + return ExtractedToolCallInformation( + tools_called=True, + tool_calls=tool_calls, + content=content if len(content) > 0 else None) + + except Exception as e: + logger.error("Error in extracting tool call from response: %s", e) + # return information to just treat the tool call as regular JSON + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + + # if the tool call token is not in the tokens generated so far, append + # output to contents since it's not a tool + if self.bot_token not in current_text: + return DeltaMessage(content=delta_text) + + # if the tool call token ID IS in the tokens generated so far, that + # means we're parsing as tool calls now + + # handle if we detected the BOT token which means the start of tool + # calling + if (self.bot_token_id in delta_token_ids + and len(delta_token_ids) == 1): + # if it's the only token, return None, so we don't send a chat + # completion any don't send a control token + return None + + # bit mask flags for partial JSON parsing. If the name hasn't been + # sent yet, don't allow sending + # an incomplete string since OpenAI only ever (as far as I have + # seen) allows sending the entire tool/ function name at once. + flags = Allow.ALL if self.current_tool_name_sent \ + else Allow.ALL & ~Allow.STR + try: + + # replace BOT token with empty string, and convert single quotes + # to double to allow parsing as JSON since mistral uses single + # quotes instead of double for tool calls + parsable_arr = current_text.split(self.bot_token)[-1] + + # tool calls are generated in an array, so do partial JSON + # parsing on the entire array + try: + tool_call_arr: List[Dict] = partial_json_parser.loads( + parsable_arr, flags) + except partial_json_parser.core.exceptions.MalformedJSON: + logger.debug('not enough tokens to parse into JSON yet') + return None + + # select as the current tool call the one we're on the state at + + current_tool_call: Dict = tool_call_arr[self.current_tool_id] \ + if len(tool_call_arr) > 0 else {} + + # case -- if no tokens have been streamed for the tool, e.g. + # only the array brackets, stream nothing + if len(tool_call_arr) == 0: + return None + + # case: we are starting a new tool in the array + # -> array has > 0 length AND length has moved past cursor + elif (len(tool_call_arr) > 0 + and len(tool_call_arr) > self.current_tool_id + 1): + + # if we're moving on to a new call, first make sure we + # haven't missed anything in the previous one that was + # auto-generated due to JSON completions, but wasn't + # streamed to the client yet. + if self.current_tool_id >= 0: + diff: Union[str, None] = current_tool_call.get("arguments") + + if diff: + diff = json.dumps(diff).replace( + self.streamed_args_for_tool[self.current_tool_id], + "") + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=diff).model_dump( + exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += diff + else: + delta = None + else: + delta = None + # re-set stuff pertaining to progress in the current tool + self.current_tool_id = len(tool_call_arr) - 1 + self.current_tool_name_sent = False + self.streamed_args_for_tool.append("") + logger.debug("starting on new tool %d", self.current_tool_id) + return delta + + # case: update an existing tool - this is handled below + + # if the current tool name hasn't been sent, send if available + # - otherwise send nothing + if not self.current_tool_name_sent: + function_name = current_tool_call.get("name") + if function_name: + + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + type="function", + id=f"chatcmpl-tool-{random_uuid()}", + function=DeltaFunctionCall( + name=function_name).model_dump( + exclude_none=True)) + ]) + self.current_tool_name_sent = True + else: + delta = None + + # now we know we're on the same tool call and we're streaming + # arguments + else: + + prev_arguments = self.prev_tool_call_arr[ + self.current_tool_id].get("arguments") + cur_arguments = current_tool_call.get("arguments") + + new_text = delta_text.replace("\'", "\"") + + if not cur_arguments and not prev_arguments: + + delta = None + elif not cur_arguments and prev_arguments: + logger.error( + "INVARIANT - impossible to have arguments reset " + "mid-arguments") + delta = None + elif cur_arguments and not prev_arguments: + cur_arguments_json = json.dumps(cur_arguments) + logger.debug("finding %s in %s", new_text, + cur_arguments_json) + + arguments_delta = cur_arguments_json[:cur_arguments_json. + index(new_text) + + len(new_text)] + logger.debug("First tokens in arguments received: %s", + arguments_delta) + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=arguments_delta). + model_dump(exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += arguments_delta + + elif cur_arguments and prev_arguments: + cur_args_json = json.dumps(cur_arguments) + prev_args_json = json.dumps(prev_arguments) + logger.debug("Searching for diff between \n%s\n%s", + cur_args_json, prev_args_json) + + argument_diff = extract_intermediate_diff( + cur_args_json, prev_args_json) + logger.debug("got arguments diff: %s", argument_diff) + delta = DeltaMessage(tool_calls=[ + DeltaToolCall(index=self.current_tool_id, + function=DeltaFunctionCall( + arguments=argument_diff).model_dump( + exclude_none=True)) + ]) + self.streamed_args_for_tool[ + self.current_tool_id] += argument_diff + else: + # try parsing it with regular JSON - if it works we're + # at the end, and we need to send the difference between + # tokens streamed so far and the valid JSON + delta = None + + # check to see if the name is defined and has been sent. if so, + # stream the name - otherwise keep waiting + # finish by setting old and returning None as base case + self.prev_tool_call_arr = tool_call_arr + return delta + + except Exception as e: + logger.error("Error trying to handle streaming tool call: %s", e) + logger.debug( + "Skipping chunk as a result of tool streaming extraction " + "error") + return None diff --git a/vllm/entrypoints/openai/tool_parsers/utils.py b/vllm/entrypoints/openai/tool_parsers/utils.py new file mode 100644 index 00000000..db7fc525 --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/utils.py @@ -0,0 +1,87 @@ +def find_common_prefix(s1: str, s2: str) -> str: + """ + Finds a common prefix that is shared between two strings, if there is one. + Order of arguments is NOT important. + + This function is provided as a UTILITY for extracting information from JSON + generated by partial_json_parser, to help in ensuring that the right tokens + are returned in streaming, so that close-quotes, close-brackets and + close-braces are not returned prematurely. + + e.g. find_common_prefix('{"fruit": "ap"}', '{"fruit": "apple"}') -> + '{"fruit": "ap' + """ + prefix = '' + min_length = min(len(s1), len(s2)) + for i in range(0, min_length): + if s1[i] == s2[i]: + prefix += s1[i] + else: + break + return prefix + + +def find_common_suffix(s1: str, s2: str) -> str: + """ + Finds a common suffix shared between two strings, if there is one. Order of + arguments is NOT important. + Stops when the suffix ends OR it hits an alphanumeric character + + e.g. find_common_suffix('{"fruit": "ap"}', '{"fruit": "apple"}') -> '"}' + """ + suffix = '' + min_length = min(len(s1), len(s2)) + for i in range(1, min_length + 1): + if s1[-i] == s2[-i] and not s1[-i].isalnum(): + suffix = s1[-i] + suffix + else: + break + return suffix + + +def extract_intermediate_diff(curr: str, old: str) -> str: + """ + Given two strings, extract the difference in the middle between two strings + that are known to have a common prefix and/or suffix. + + This function is provided as a UTILITY for extracting information from JSON + generated by partial_json_parser, to help in ensuring that the right tokens + are returned in streaming, so that close-quotes, close-brackets and + close-braces are not returned prematurely. The order of arguments IS + important - the new version of the partially-parsed JSON must be the first + argument, and the secnod argument must be from the previous generation. + + What it returns, is tokens that should be streamed to the client. + + e.g. extract_intermediate_diff('{"fruit": "apple"}', '{"fruit": "ap"}') + -> 'ple' + + """ + suffix = find_common_suffix(curr, old) + + old = old[::-1].replace(suffix[::-1], '', 1)[::-1] + prefix = find_common_prefix(curr, old) + diff = curr + if len(suffix): + diff = diff[::-1].replace(suffix[::-1], '', 1)[::-1] + + if len(prefix): + # replace the prefix only once in case it's mirrored + diff = diff.replace(prefix, '', 1) + + return diff + + +def find_all_indices(string, substring): + """ + Find all (starting) indices of a substring in a given string. Useful for + tool call extraction + """ + indices = [] + index = -1 + while True: + index = string.find(substring, index + 1) + if index == -1: + break + indices.append(index) + return indices diff --git a/vllm/envs.py b/vllm/envs.py new file mode 100644 index 00000000..3361afb7 --- /dev/null +++ b/vllm/envs.py @@ -0,0 +1,453 @@ +import os +import tempfile +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional + +if TYPE_CHECKING: + VLLM_HOST_IP: str = "" + VLLM_PORT: Optional[int] = None + VLLM_RPC_BASE_PATH: str = tempfile.gettempdir() + VLLM_USE_MODELSCOPE: bool = False + VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60 + VLLM_INSTANCE_ID: Optional[str] = None + VLLM_NCCL_SO_PATH: Optional[str] = None + LD_LIBRARY_PATH: Optional[str] = None + VLLM_USE_TRITON_FLASH_ATTN: bool = False + LOCAL_RANK: int = 0 + CUDA_VISIBLE_DEVICES: Optional[str] = None + VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60 + VLLM_API_KEY: Optional[str] = None + S3_ACCESS_KEY_ID: Optional[str] = None + S3_SECRET_ACCESS_KEY: Optional[str] = None + S3_ENDPOINT_URL: Optional[str] = None + VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm") + VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm") + VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai" + VLLM_NO_USAGE_STATS: bool = False + VLLM_DO_NOT_TRACK: bool = False + VLLM_USAGE_SOURCE: str = "" + VLLM_CONFIGURE_LOGGING: int = 1 + VLLM_LOGGING_LEVEL: str = "INFO" + VLLM_LOGGING_CONFIG_PATH: Optional[str] = None + VLLM_TRACE_FUNCTION: int = 0 + VLLM_ATTENTION_BACKEND: Optional[str] = None + VLLM_USE_FLASHINFER_SAMPLER: bool = False + VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False + VLLM_PP_LAYER_PARTITION: Optional[str] = None + VLLM_CPU_KVCACHE_SPACE: int = 0 + VLLM_CPU_OMP_THREADS_BIND: str = "" + VLLM_OPENVINO_DEVICE: str = "CPU" + VLLM_OPENVINO_KVCACHE_SPACE: int = 0 + VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None + VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False + VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache") + VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024 + VLLM_USE_RAY_SPMD_WORKER: bool = False + VLLM_USE_RAY_COMPILED_DAG: bool = False + VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True + VLLM_WORKER_MULTIPROC_METHOD: str = "spawn" + VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets") + VLLM_IMAGE_FETCH_TIMEOUT: int = 5 + VLLM_AUDIO_FETCH_TIMEOUT: int = 5 + VLLM_TARGET_DEVICE: str = "cuda" + MAX_JOBS: Optional[str] = None + NVCC_THREADS: Optional[str] = None + VLLM_USE_PRECOMPILED: bool = False + VLLM_NO_DEPRECATION_WARNING: bool = False + VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False + CMAKE_BUILD_TYPE: Optional[str] = None + VERBOSE: bool = False + VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False + VLLM_TEST_FORCE_FP8_MARLIN: bool = False + VLLM_RPC_TIMEOUT: int = 10000 # ms + VLLM_PLUGINS: Optional[List[str]] = None + VLLM_TORCH_PROFILER_DIR: Optional[str] = None + VLLM_USE_TRITON_AWQ: bool = False + VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False + VLLM_SKIP_P2P_CHECK: bool = False + VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False + VLLM_TORCH_COMPILE_LEVEL: int = 0 + VLLM_V0_USE_OUTLINES_CACHE: bool = False + + +def get_default_cache_root(): + return os.getenv( + "XDG_CACHE_HOME", + os.path.join(os.path.expanduser("~"), ".cache"), + ) + + +def get_default_config_root(): + return os.getenv( + "XDG_CONFIG_HOME", + os.path.join(os.path.expanduser("~"), ".config"), + ) + + +# The begin-* and end* here are used by the documentation generator +# to extract the used env vars. + +# begin-env-vars-definition + +environment_variables: Dict[str, Callable[[], Any]] = { + + # ================== Installation Time Env Vars ================== + + # Target device of vLLM, supporting [cuda (by default), + # rocm, neuron, cpu, openvino] + "VLLM_TARGET_DEVICE": + lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda"), + + # Maximum number of compilation jobs to run in parallel. + # By default this is the number of CPUs + "MAX_JOBS": + lambda: os.getenv("MAX_JOBS", None), + + # Number of threads to use for nvcc + # By default this is 1. + # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. + "NVCC_THREADS": + lambda: os.getenv("NVCC_THREADS", None), + + # If set, vllm will use precompiled binaries (*.so) + "VLLM_USE_PRECOMPILED": + lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")), + + # CMake build type + # If not set, defaults to "Debug" or "RelWithDebInfo" + # Available options: "Debug", "Release", "RelWithDebInfo" + "CMAKE_BUILD_TYPE": + lambda: os.getenv("CMAKE_BUILD_TYPE"), + + # If set, vllm will print verbose logs during installation + "VERBOSE": + lambda: bool(int(os.getenv('VERBOSE', '0'))), + + # Root directory for VLLM configuration files + # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set + # Note that this not only affects how vllm finds its configuration files + # during runtime, but also affects how vllm installs its configuration + # files during **installation**. + "VLLM_CONFIG_ROOT": + lambda: os.path.expanduser( + os.getenv( + "VLLM_CONFIG_ROOT", + os.path.join(get_default_config_root(), "vllm"), + )), + + # ================== Runtime Env Vars ================== + + # Root directory for VLLM cache files + # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set + "VLLM_CACHE_ROOT": + lambda: os.path.expanduser( + os.getenv( + "VLLM_CACHE_ROOT", + os.path.join(get_default_cache_root(), "vllm"), + )), + + # used in distributed environment to determine the ip address + # of the current node, when the node has multiple network interfaces. + # If you are using multi-node inference, you should set this differently + # on each node. + 'VLLM_HOST_IP': + lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""), + + # used in distributed environment to manually set the communication port + # Note: if VLLM_PORT is set, and some code asks for multiple ports, the + # VLLM_PORT will be used as the first port, and the rest will be generated + # by incrementing the VLLM_PORT value. + # '0' is used to make mypy happy + 'VLLM_PORT': + lambda: int(os.getenv('VLLM_PORT', '0')) + if 'VLLM_PORT' in os.environ else None, + + # path used for ipc when the frontend api server is running in + # multi-processing mode to communicate with the backend engine process. + 'VLLM_RPC_BASE_PATH': + lambda: os.getenv('VLLM_RPC_BASE_PATH', tempfile.gettempdir()), + + # If true, will load models from ModelScope instead of Hugging Face Hub. + # note that the value is true or false, not numbers + "VLLM_USE_MODELSCOPE": + lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true", + + # Instance id represents an instance of the VLLM. All processes in the same + # instance should have the same instance id. + "VLLM_INSTANCE_ID": + lambda: os.environ.get("VLLM_INSTANCE_ID", None), + + # Interval in seconds to log a warning message when the ring buffer is full + "VLLM_RINGBUFFER_WARNING_INTERVAL": + lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")), + + # path to cudatoolkit home directory, under which should be bin, include, + # and lib directories. + "CUDA_HOME": + lambda: os.environ.get("CUDA_HOME", None), + + # Path to the NCCL library file. It is needed because nccl>=2.19 brought + # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 + "VLLM_NCCL_SO_PATH": + lambda: os.environ.get("VLLM_NCCL_SO_PATH", None), + + # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl + # library file in the locations specified by `LD_LIBRARY_PATH` + "LD_LIBRARY_PATH": + lambda: os.environ.get("LD_LIBRARY_PATH", None), + + # flag to control if vllm should use triton flash attention + "VLLM_USE_TRITON_FLASH_ATTN": + lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in + ("true", "1")), + + # Internal flag to enable Dynamo fullgraph capture + "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE": + lambda: bool( + os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"), + "VLLM_TORCH_COMPILE_LEVEL": + lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")), + + # local rank of the process in the distributed setting, used to determine + # the GPU device id + "LOCAL_RANK": + lambda: int(os.environ.get("LOCAL_RANK", "0")), + + # used to control the visible devices in the distributed setting + "CUDA_VISIBLE_DEVICES": + lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), + + # timeout for each iteration in the engine + "VLLM_ENGINE_ITERATION_TIMEOUT_S": + lambda: int(os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")), + + # API key for VLLM API server + "VLLM_API_KEY": + lambda: os.environ.get("VLLM_API_KEY", None), + + # S3 access information, used for tensorizer to load model from S3 + "S3_ACCESS_KEY_ID": + lambda: os.environ.get("S3_ACCESS_KEY_ID", None), + "S3_SECRET_ACCESS_KEY": + lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None), + "S3_ENDPOINT_URL": + lambda: os.environ.get("S3_ENDPOINT_URL", None), + + # Usage stats collection + "VLLM_USAGE_STATS_SERVER": + lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"), + "VLLM_NO_USAGE_STATS": + lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1", + "VLLM_DO_NOT_TRACK": + lambda: (os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get( + "DO_NOT_TRACK", None) or "0") == "1", + "VLLM_USAGE_SOURCE": + lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"), + + # Logging configuration + # If set to 0, vllm will not configure logging + # If set to 1, vllm will configure logging using the default configuration + # or the configuration file specified by VLLM_LOGGING_CONFIG_PATH + "VLLM_CONFIGURE_LOGGING": + lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")), + "VLLM_LOGGING_CONFIG_PATH": + lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"), + + # this is used for configuring the default logging level + "VLLM_LOGGING_LEVEL": + lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO"), + + # Trace function calls + # If set to 1, vllm will trace function calls + # Useful for debugging + "VLLM_TRACE_FUNCTION": + lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")), + + # Backend for attention computation + # Available options: + # - "TORCH_SDPA": use torch.nn.MultiheadAttention + # - "FLASH_ATTN": use FlashAttention + # - "XFORMERS": use XFormers + # - "ROCM_FLASH": use ROCmFlashAttention + # - "FLASHINFER": use flashinfer + "VLLM_ATTENTION_BACKEND": + lambda: os.getenv("VLLM_ATTENTION_BACKEND", None), + + # If set, vllm will use flashinfer sampler + "VLLM_USE_FLASHINFER_SAMPLER": + lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))), + + # Pipeline stage partition strategy + "VLLM_PP_LAYER_PARTITION": + lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None), + + # (CPU backend only) CPU key-value cache space. + # default is 4GB + "VLLM_CPU_KVCACHE_SPACE": + lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")), + + # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31", + # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'. + "VLLM_CPU_OMP_THREADS_BIND": + lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "all"), + + # OpenVINO device selection + # default is CPU + "VLLM_OPENVINO_DEVICE": + lambda: os.getenv("VLLM_OPENVINO_DEVICE", "CPU").upper(), + + # OpenVINO key-value cache space + # default is 4GB + "VLLM_OPENVINO_KVCACHE_SPACE": + lambda: int(os.getenv("VLLM_OPENVINO_KVCACHE_SPACE", "0")), + + # OpenVINO KV cache precision + # default is bf16 if natively supported by platform, otherwise f16 + # To enable KV cache compression, please, explicitly specify u8 + "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION": + lambda: os.getenv("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION", None), + + # Enables weights compression during model export via HF Optimum + # default is False + "VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS": + lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)), + + # If the env var is set, then all workers will execute as separate + # processes from the engine, and we use the same mechanism to trigger + # execution on all workers. + # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it. + "VLLM_USE_RAY_SPMD_WORKER": + lambda: bool(int(os.getenv("VLLM_USE_RAY_SPMD_WORKER", "0"))), + + # If the env var is set, it uses the Ray's compiled DAG API + # which optimizes the control plane overhead. + # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it. + "VLLM_USE_RAY_COMPILED_DAG": + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG", "0"))), + + # If the env var is set, it uses NCCL for communication in + # Ray's compiled DAG. This flag is ignored if + # VLLM_USE_RAY_COMPILED_DAG is not set. + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1")) + ), + + # Use dedicated multiprocess context for workers. + # Both spawn and fork work + "VLLM_WORKER_MULTIPROC_METHOD": + lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn"), + + # Path to the cache for storing downloaded assets + "VLLM_ASSETS_CACHE": + lambda: os.path.expanduser( + os.getenv( + "VLLM_ASSETS_CACHE", + os.path.join(get_default_cache_root(), "vllm", "assets"), + )), + + # Timeout for fetching images when serving multimodal models + # Default is 5 seconds + "VLLM_IMAGE_FETCH_TIMEOUT": + lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")), + + # Timeout for fetching audio when serving multimodal models + # Default is 5 seconds + "VLLM_AUDIO_FETCH_TIMEOUT": + lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")), + + # Path to the XLA persistent cache directory. + # Only used for XLA devices such as TPUs. + "VLLM_XLA_CACHE_PATH": + lambda: os.path.expanduser( + os.getenv( + "VLLM_XLA_CACHE_PATH", + os.path.join(get_default_cache_root(), "vllm", "xla_cache"), + )), + "VLLM_FUSED_MOE_CHUNK_SIZE": + lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "32768")), + + # If set, vllm will skip the deprecation warnings. + "VLLM_NO_DEPRECATION_WARNING": + lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))), + + # If set, the OpenAI API server will stay alive even after the underlying + # AsyncLLMEngine errors and stops serving requests + "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": + lambda: bool(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", 0)), + + # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows + # the user to specify a max sequence length greater than + # the max length derived from the model's config.json. + # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. + "VLLM_ALLOW_LONG_MAX_MODEL_LEN": + lambda: + (os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in + ("1", "true")), + + # If set, forces FP8 Marlin to be used for FP8 quantization regardless + # of the hardware support for FP8 compute. + "VLLM_TEST_FORCE_FP8_MARLIN": + lambda: + (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in + ("1", "true")), + "VLLM_TEST_FORCE_LOAD_FORMAT": + lambda: os.getenv("VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"), + + # Time in ms for the zmq client to wait for a response from the backend + # server for simple data operations + "VLLM_RPC_TIMEOUT": + lambda: int(os.getenv("VLLM_RPC_TIMEOUT", "10000")), + + # a list of plugin names to load, separated by commas. + # if this is not set, it means all plugins will be loaded + # if this is set to an empty string, no plugins will be loaded + "VLLM_PLUGINS": + lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[ + "VLLM_PLUGINS"].split(","), + + # Enables torch profiler if set. Path to the directory where torch profiler + # traces are saved. Note that it must be an absolute path. + "VLLM_TORCH_PROFILER_DIR": + lambda: (None if os.getenv("VLLM_TORCH_PROFILER_DIR", None) is None else os + .path.expanduser(os.getenv("VLLM_TORCH_PROFILER_DIR", "."))), + + # If set, vLLM will use Triton implementations of AWQ. + "VLLM_USE_TRITON_AWQ": + lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))), + + # If set, allow loading or unloading lora adapters in runtime, + "VLLM_ALLOW_RUNTIME_LORA_UPDATING": + lambda: + (os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in + ("1", "true")), + + # By default, vLLM will check the peer-to-peer capability itself, + # in case of broken drivers. See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa + # If this env var is set to 1, vLLM will skip the peer-to-peer check, + # and trust the driver's peer-to-peer capability report. + "VLLM_SKIP_P2P_CHECK": + lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1", + + # If set, allowing the use of deprecated block manager V1 + "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1": + lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0" + ) == "1", + + # Whether to turn on the outlines cache for V0 + # This cache is unbounded and on disk, so it's not safe to use in + # an environment with potentially malicious users. + "VLLM_V0_USE_OUTLINES_CACHE": + lambda: os.environ.get("VLLM_V0_USE_OUTLINES_CACHE", "0") == "1", +} + +# end-env-vars-definition + + +def __getattr__(name: str): + # lazy evaluation of environment variables + if name in environment_variables: + return environment_variables[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return list(environment_variables.keys()) diff --git a/vllm/executor/__init__.py b/vllm/executor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/executor/__pycache__/__init__.cpython-310.pyc b/vllm/executor/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4435721563df6af49f9e31c1ddd53d1204119c3 GIT binary patch literal 154 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hmeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_eriQ(a%o9^k$!x9W?p7Ve7s&k X*5(7Xz z1Ik3eUE9=_wY8;9HpwB$?q;d2Ty)4Whg@>VR&IODX)2XVPCo09TJKuS_xB7yfDgMu zRyVr;e*OLT|LVV|nocJr{9XR%e_Q_kRZ03Es&xLvPT0Iu#>|+jo4Ok}<8H!CxIJc%n>3T6k6i9Gd&T&an>Nz|rj;{ppV{YT&8*vR z_PaSVC*ZO2A$PzWa0ksnQP;~u?yx!Rj+i6vs5vTN@$zAJ%p4QtM0wmjVjdCYp7K%m zn0ZW;lV!s_ZXOrq-tr0eqt;vU%YHiDg*dV~O=SgPW>ZKt0R)QSWyO&cLQ-USv5Z%MLl&MU4%x!N;0;36LBh zIY5TkFd&x!=>wz>kP$Wt$ZLS~1JV!3VKxTH>ulV)vZ$~l>?mq)M73kgKc#xiRyn5NcX;&EL3({fhooc0o z$*`;55=vINy0qko>h0>>#GLc6?sz_0)AP2AP4L;eU#k;<7IAC-1hZ?t!*@C!3XyTd zwp!p=(DOm*y0x(GJ6AIHoDlbOa+zfxG*>0ud115YNM_$VaM>0L|+ zzSPYWYH25qamuI!egY?A&I}vCY=cND(-!rJFZYYBXxGqolpO!;9iuL=0w6~_`dnm}067NCF0}ojZP=S7wShoc3QFB@IbM^8OPo zPvjM8MP~A<;w3(mK9^Bfm{ga%{tuxLH|5V|U)_{^Z6(H39H%Bqv1@>A=}c$wXUc}m z5>0tc;_pVSJ?NJd{d!SKiBh^<%Ct*;m?e(&5-UBMvgnlsBne3GCh)L&{S?Y+QBM0A zl>0=v&(G2*QBJV_4aM)@lr~g9*HoJ7rh<`&{DCL3Ke(xEXw-A1XCvkhZAw3dVn^%n z66GUJjsNBg=?ghxr-VPs2AZ)=<%T3?9>K^FHZV$k4~z03lYRpBqp|)N^6?7bTSuOW z9{y2+G-GUVRptL0%{v4(rFjQC=M^{{3wwJ14lMo~>G!k((%~g}SlZAlD%Cf1-)QPj zB(bMsPo%Bm&mk6=txi<5kZ3 zhF7nV5->;;nc0tS-@ao_&&=POn3Nt0ynVyEJ2!QEcH%}LHG0dXg%Whu8wIawrf=Q3YhAxRJ9+Eg^qr|dzGNl| za_{cM{rRaz52>f~l$qlMet&lE)`zpxGuHKqxw+}7IqUAky?ax0Gv+9@-n~7sPty@< znwuijrf2t#%wp|{+t!2GIUqSdZDP^Ouf0{cg#XK3f}lVENPitddT|2TlGKzwPH)J* z^hEN77&PT3G}=?J)Gw5W%DnVkhM~~7U0Ds1{ZGxPn!`(r>lV($BC9CyXpKpA)%Nr) zlEcyN+f_4m&+D~Zj#mHd9>}rXW7?>O#NdaCfTOhU;e!l)vgEZ)dXVinY2nuJei}R2 zo;GAs%eTE%kTI-zK@42IkMjcB%uJbCHHcTa$}Bo!rBn+v;m;&WUWlHV#H8p( zXKVI{7)Q&HfP@SwSd+LC;wqe^7XKds%&Q>F$~iR^PpT<3At%+Of`2(FE2=Ihw^M3L zh?KsqD4MKqYq~C{knus8JG6hl0$~wj=7@441m5kgt z+;LF5L{f+!KL?zI7Acb5ZvBxh??akI@_b}W-cT8Ao-Z}krpDB#%0{e7)?KR0WY1yC z71UKE4T;)xq&N~RIIJi7BJ~h#C5gPZsW5F-VX>#_4ZRs#Nin@7HT8dze-2B7dK$JS zKGPT?w!>&P@K8w93WEgyse7uIzfK~E0%4~6CknQe42j`+u^vo zNqmB`E0n#9jC26BMI;byx^SWae*>ABSzWV2QDj1C4Az84ZP}qq9St?JV3{rs7rc?r zaAE>6U7)d2(LWFq{!x3$-j4P7141HHaN+I(J^YQ>c;_%VscL>78H6voPUx$TU<166 zQFqiQ3DO3VM{KLbz&{G9bp2%}MqIe_AF6xsC1Bvj7uh~t*~MR7777P*Boq@8IPSRB zw(;_;YNfm$47v6vM6_a^bEo3Bgfh@W6~Mm_jQGdMiev%lkA4$KM z3_he$!{qP8>{(c-xq8Jj{AI^jg5s(eA$~@+V!#d<;Td>^WYUQKCP3$_?HU2Ukq}pn z=|$rnuRCIiX!1uTOugV3ARZPfuXjyHQw2VM<8TPJP_sZ)A?3NKa`i5yBU8;~nTYbaJk|Q^g3}gn0ZpL4H8lj*O6t@h?ao@gAHXWoR4j&iO9g zN=)A3gngWyJ)0mFo~Jnnkcmy`w23Y0nbd*wb2(7qJ4BcUdR5%!cgry0_dA-4}UTIi2;H7c5R0##CQg;`?qF`Ek%`TNG++*DO$PnD=84K%>Ah;OB zr(n`HZ`Lc!;pKG#hsU|Yw!u->TEN2OR(o12ob$P|L*3_cNL|51JDjuxy#E8qBOXp@ zaGkdsuXJwzn^y=jwC)n@V7OvfC%YKdyNtMsC1OZKFwa2dOIsjT6Xb%4g(74MnPwPs z6_-1@@y8mI(}*b{QU)hSh%V?QEYNOp8Q(yrM)stEEk%UBYMf?4mYkt*+R%c6p;v$r zUqv>92$&v*u>!4BS*&tmqP!Xsu-$Dp)0ffd_efq6St_Xuvz2_=81E$K?sbFywhEwI z0nF0FwsFBUB6!iJ8951{lVGt`jUN&;LjXgRgo$C=Qe_ zSOe_8kc4(gsmbV~mUx-aV9~cVr7_w`rmn@iq>ePo9)eI5hsP-%&_hy4o}GeVoAeBf zL$WE+Ql!OUGF}<)nl^X+{=(Z*c;sUD@I}jmW$8bWR zmz;z(N8rF80SHYcsMKqs@~TE*zbMd0Y#}o+ z14)daOKe6sIIUOW*(c?nx{v=hC(GY9?MDh5e@ccGBdC+q#hubm5!(5acI-9VQ%vmnqInf7yLvd#Bq&2~rut;-5l+ zlrfX*GS5(HOMWJKSs!+vu1fSqCf>=^#@RW$gxEeje#0S8(I_n%PGyN4DR|`d$}%D& z{_=WZreQn~mu>@lq6aG)!yVzX!~dc?yGc**!MU9RqNQeFtSZ=X|2h9Uj@y{{CWy~q8W=`(*aJuF=5DWekbCB*& z38&y&m;5Vqzk((CU!u^>s+|+S+#2XG1Gb_)hmok$LBbM&Da#6y7Db0Ccx53#X+5ml zWr|{17C(mmfwoYsmV-VEp=4a|ZSspew9)l3P)ikGL__&^Xll}5oY*S#7)h0Ix^!@I zm-#SdYp7GXKJo@;H^qzH#DC{wZaov^gnDPTGn;q3mar5l?VB z0ZugO;22>qFA?l0G72$!3|Uf2A@->(h{pp(d8w$$uaz^9qdy}4^b+^PKj|^$k4ilG zhoLii9M%i+mE2ac)T%4Vm&tT4lT+~@SJYN(E|H7pw4|QUbFrkJQN@ZyEY|f$+kX$q zJCCgOw9I4&4m~|Bi-%o2`r;W>X9;?TL@CZUJ=KLtVak7>m9NE)q_T@@WFQPmw== zszU%L=%Kx^&%o{%TjE7hoC0N1WS_y44|Na8-gB9d6i2+D*3NY*;}EY2!i&_f@@YzL zoZN%a-Vyv4=#5CkU*m`W4P}2z*)Bf)JA(ZKWjn;#;VU|nw`pbCLIVCyi$^kXabSTS z#dHm{?-6~Hl}RM;ZDeF6eLPdID!k8Ew={$naCrd)0d2a@tmxih0awJO2H|cCVQ#R7 zHq&@x5%FXH@|$~uSO|lG6z6f1_Iwd;Km;!8Txh&IdH==)oL;-OeDKIc+#2Yyl@vBb z)Epb%d>HhERJfOqo3St+9m97a#bq-d&COlx2=QN=lVLU!tu@WGLzirPnIWcmF7sca zV+U!4v|a;jil|D88Ltp{anM!rB(-h&<c?)(-kc+O$fP!4=OykPr za=o$&6NYye==Ji!o5zEch~1R@!uxsE94>|f>A{8uT26(UnbH5TUoNZAgni2b|= z*gWlrRHCjWDUPjhf_CkK{8Z)k6?y4L&78lElB5ukIA`33D(! z!tg}#B!I_HAh2_wfrdprb7=ql@DWe|Y=J1)>Kcu8EU#vRy~Wo{{0p=~%My=wyx@o) zyLsdt^!Wkth=GC*#|Gg7@C=On$hu83o?ganY9kI08tt5=+J)D zx1z5$cF4Q0yj%zV^zoIKMM3PJWM9@^3hAd-5jpO%fy9cGW|2hXi7a4K-h@kw$32yv zS>`4F_h^M-`6+&$laUosq~gV1yo^VShY?tQh^FZm(@@Z*5CtlPhiIM< z@bJ@x77B&3z3ys@0=lAe`9OZ6$iNU2h5cdXP+S0=J$2)>?>L)MHQo(-kF|$eDCk~ zwbHq{x`ylYAO5nv?rPdU=w;N@l9*&op`jp>o*(XSEdvEdmyJ)4mkTb>nHyh?0) zc3kzUvEw;$&8x+AuO2tNh8$-_&3Mk6lh2iCK3?z^eFXTwfjSZ{?{UO4Y`^Ds?< zs4(BtaOZhY32d*y3x~c&C@AVlCTAIg?&EiUNw zt@Og$1^!WwXE|Qy-VI_bDcsTwrQ{mv-d2o%BtLE4sMIpHfz8ujB>(j_T#*<*VaDIOzE zuo`>(*9Ln6t3AmV*i+?7d}Y&NPqSzCY>YgHkuAO=*SFYL*t2r}@n2|`<~vg&4L_wCh>82a^qC7l>kuD#o-wjELF;DV<%q44gpqJrg z;>p6jfIB;ZhH1N+uZ7g=xp70gWbW#CGk&Z8QYSNiSG%pRx2)oDHxNM-@yNfLipyO1 zd5~Ssil_KBSg@})=YC9<;)`A~7?t^#uLfc(D=L!g!ts4+f_}>^tc*vSg(W}RYS?YP zsQJER&i8L?|9*QlkGreguzM=UFrJ@!A?|I*r{k-uQMj?%-GSF6t5>2ZUM;y_Wh43d zTixD(vQ)IYJ7O99E)nB48r!tZWuyP-{sM28gci2oc~2x0QX~nIBwwY)zJX>}yNN(_ zOTVt?%I39Q{j7FFzoB0@&|@)kZ`5P%8oTDE0ex43??O3edCo-~Cfv=pxtsLj4K7Hx z!IdD4C=9sYf6v{B($3|qU9Y=?>*98pxnXi8z05QBLsqWgW4=F}>8oiUx@qF_ptJ2N z<+(w^+&sX{;c~P!XrO)RQSc0wm7R3AdeD24a~>vJ_!8vuGgQYrFj2RIMY)@WeGm}{ zK3uMo#$DJS5q~wzw_TXdVDE7rE=bX4XNJ>=w^MFTk?nnAgfysFI7x!>JHz)6hI5KX z9?blN=8S>nGW_0o2+b$Z-3Qg{`Yz(uQSCiV*FHk@f^G~!ezPu)qrGhuM*F!!-;i&r zc^Lgg?Zqvg@M~T1(n-ADA-ga>Ue@|=&akQnQ`&wJN)DpzrS@>ynZyehd$o*zj))YG zp=XLf@dWx#QU8;2c+nhp&ccMoTb4M2Z(g0`t7KiwPs(1sk7Pf+$)lK0V#*ve^~O?V zNq3C?*Z(IOh!)lmUqQ3pAr~tz5-nwZ>2g2DL$2M_8Pv9G+%k1I*RFQSxMUuJIxkt5 zD#x_zrW|3Y_g!-|%D3Tw80}ZlIt&4C*I?GCScO%fc)S1HIi$%jh+x&y+Tni6&fy;u zhFtn$|HR`i#b71AGwze@r2Dx0>O@cfYi|eF!nhZ^oiyq6gy2bM$K6bY`()VO6i2&H9E zI-^8~Kn;B(P#e&>*3+|xzo-2i`ln#LUL4DiXYw+`sqK+Nftb>6@l-&-O8QR@Wu?NO zN{N}DCehBtmM)$HH4x%K{78X{LwG6<`%HvakUA&$YMIji#43h(^Metogkaf}n2WiI znQvf5Mmeo+SrDyt-!P1Omg(HB+fCil`%lae^u!|j1{f*QuVLk(NUMYdNUQ+Y`lmV^ z4#K{Uw}DU%5~{)rEd$tAvIEwn2%g7E#fnd;)`!6>=m2vfph6hswxL8I2U8$Xe9H*? zSeB<K`nmU+n!s={&-t2#K2J>IIT+xhDAew<2PFNgS-_YV*yzJ_* zIQ_Df)xL-NYDyn?Ebev?wZW_)EYRkxn3+~0o0gn2ZC|tBQ)k59(@*b#B0>NP7O)0 z%okIirbJ5NY6rJ8QQIa1MLxLOZ_VKB^Sgge(IjeOL;|lIfxM*$DCV^bJ-}UA4-HBZ z0A-oURd^jB(J?T69_17P-1QVF{AviW+6j7@%Kn2ndFoQ^kjuzaI@kg6%ccA<#G|Nw z5?TveO2|D#)D*&oAlN-e6am6_P^q6(kLnm;w2(r_lBl-(4q&;s8cq!cT%hU%2!ilC zRbQ^k9qr*VD5?&wr}??EEadDpAyQE~BjufG;^>nHFEi{?nWh z@c9jH#hKMVJ&Usk0UFAh>xlR3tw-q8BMD;e`$gR+(A|sZx#{~K^@3>drRw`E?fAYt zq!2aCF3d2=1=Uu9;I%kF4S9!HpysR8e1n>KG=;U1rjdjJf>NnWV1nu~L7Aj5ajGP$ z^h#k^kk|!zxiG0AT?%5mm>V3wba)hjn32v`Xb2%Pv51B!$QT+&Bh0m2w+|zl-$y*Z zZ!8lGwaxn7`odCUsY-taqP_JW)o^QL)AyH(aC7QyFp`2HqfwxBbrRz*5XQI!@Qre32XA)^G(#c@i{(-{c~6^4sl;I&dX<(Rhn7RjuGIu3c8tfGrVwXri??87g#ln+O#_#$>*JAaNg z3K@^PfQ(7*J~O0Q!Hv_3I*{VxKsHEr0R8C1zct8euRVAgN=^drm*uJlxABhxsmM&IyzLzQzk)~ z7%9mLD$u-R^go0__`<&mR2;)W#RjmsXwZn2$r0oKSr8N?y@nj1&f>%z^(+p0M9>}_ zCdf?fosj+NG_KsK9R)jFWbI7-2To>qZ%z)(=sYm#a3xtOZAvIIDN6SDlVah#sBi}T bQ%=iP&^TICe+Id_sh59_alq7mwtW1*Lx;7n literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/executor_base.cpython-310.pyc b/vllm/executor/__pycache__/executor_base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c483d25cdd135233e4e807c43cd026b044bfbdb GIT binary patch literal 6174 zcmbst%W~Yt5x`=x%U$g9P4S@zM7_8YwUqpllsL91N>LQcluESXD!5W8EM~|>2`>$> zly-}iifxrEw;XcwDLJYt`Gs8a7r5q>TdJI=lGy1UJh&iNvYcQGy)!*M-90_sJ&o}j zjhX<@<=_3jyD}pPf5gu4BSYspyy0&F5P^t};EKBFO1dOsUvf&Wtjlg$FT0AaxD~y^ z@g=9~j_G4=O|Q9iz3!^I>NfO-+ti!xxIXSq=o9XwKIu;BQ=DFQrrjBRhPTVktUIUA z@wVa|ap(1U-mW+c?xMaZ3bzGPC1cM7GDgMEOZpP@YoreSI$eVPQKD`Mt;U}~W1%I* zrS%&dantOC5i_mGrBT->EjgC&+F=w|?*)V3TK&_r*M)q#1LQ9P6DEI+=+@oOx-Nu&bg5vC6FA92ym?^N{ zC!D_Z6qrSn>zbl1G}Q1j2c7HihF^+;Km}bSf-XIT&DTp*rez|%E07XZh)gS_Oshnp zW28cBq)O|sB`R$!z=pto6aMS)KMwyY{7=AtgEY72$T*pJChL=Qid-d=Wa^pxxu{P= zuR&+%ES=q!7KCkyOp_TtG6y4oC3EoNe1#k#^L*q8j4aT3n5WVOy0{>~Jee$#B|f?Y z+>erDAn7R39jC{DPKNOlMC0Sf0ecd8a_llW#jz&}*wf?;$7+CWP@v|Toh9cub{ViM z^i;~BMb2~VX~15{`L2?S9DAmKZId;QJzL;=iCpH`bAY`{-XL%C-U_teB5%VwTI3ya z<(Z(LCrxniYq6Sar_XMfAsze~p=LQ|7;4)-)4%JPZ`+necr0*UQ9q#C3T^MSSGA4%4_CF@2sUYt z`QEBVBdgtNYxkIDN1A1NT89E30#V284!9u66-k(N&4esbfff~TdroA7ho|CUHMkhP z+6!rDFtZ=4Tvx+NBi6$3&3fZO8h06DBFthe!!(j*W!<_1F+S8`X4aWPL!{A|;bsFK z$A*~%a)zGqhjT!NRTzzW%-gi;Lm`K43EmBix*q}96ZZr|*pmQ?dli7vo&vD6R{|*S z$pFiHWq`_F6<}p=3}AJy1`t}muKh;9l({857dKmVb_6)Gc?8E097V8zU=hI*g5wBI zAkYBBO~dx=$jH2>RgdM6I@?2>4$S8~Irk!`=K?ze3}4(>i`-x>u!D<{$#!UT@rv8) zx|iK2YmVJn3;K|iy|rD(an~}LdBf;{TepLLTs0D{3|vZxSxRXr;=!LP4rXtqUXKCM z&SqJV&GHA^_;QQmDTmf#%h2HZJ{Q^Ptj!VZEOt);xD4-CxLL9^SSkuC*zBd^zCv-4 z@R`^W%?)65lS1})Z7*?oujh6s)BJ7C+%;{-#K`P8zV#@HX$(sUGwp+)Kum!bYoVAE$VWQG1JL>EMV*!6o3(Lx&zk3NK!LwC#w)bM%m^*lnE z)5m1+vCke+*4DOsm*&!j*RHu{ACk%Ap3QjLz*GSz7NibX2h2Rb1|OY^rJgi1h~`# zBi>@nXKV#LEpcBALhi%yagzC)ftuxBmKy49APl*CHwPDnx48Z4d2zF7$)vw;aH|B2 zvj^K*^s)2MX^DgP??A?}O$WA}TQ~P`zUOG~9WcHJ-Ujw zVFb>T9jswQgUpxGwP8i)Su||()jXGy)w}_m4=$upQ06XDU}hRD7S>Q8dUZx3A{-ae z;QRv!Lr9yUmYtef8f)4%9CK++;P`&f-b{2FHRg)0ViJ2egIu%clY&@=T(KdNq zso4g8Z-mr3T4$GX!Jp)U=L^OdB6(er?E52RRRbHX4ra>*6<3&zztlx|l-wOmkZB=KuI!Hp5}r*M;t!h>9R(I$r}|1Zaw;oDvm9Y(qAlZAzkqJ6dW zmiXfSMweoVWGB@IP3u4W#OYGgiMsu^c4u3Ivvd_dN7BNFnKpbdnO=Ul(x(>X7XnHC z9qHvNR9)61BVqd#7LmA>JXk0!W0>mY>f>qKk_K~I-CjieCrMqAY5%GOTMg40Kg-Qn zjOc9qkZe4rHc)L90DLP5*|!3Z?v}{#`(xzFD~16zqG5c7vylyy+B1bkrv#ApPQvhJ z>jKv7p6OD`pLB75PXu0G2PCRZHrUyI4>&MVA7%IxD2$~2c z5R4(fyDL^kfFDq762Ulv27+@4u+rdVJF7q|F4{JujVz4)@ZfTBmC@P-(|RRJPP#OP9ZWHe=l*smpMN9mPj1TXKlyq z!`ERuOzv}eTGE#|Ve$CPSz#{H8=N9czaA2+VNmncjxtkReK$$Q@1fUnTTKY_;w4Fy MRS|y^rJ1My1&h)h3;+NC literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/gpu_executor.cpython-310.pyc b/vllm/executor/__pycache__/gpu_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d7cc79cd36f9345bb8ee1c5e93e9163d6f0f45b GIT binary patch literal 7010 zcmb7J%WoUU8Q&L|)QY4iN|r23mee~}|MS+hMMe2HHHJUxJ&zLo7l0{D^_4(%)If8zKzH=OaEu`5|^tZH)#pvg+C{F&h#eM?~ztn!({D!jO*JIBzTWL31Q{1_kKGMwXV z>bg>!{v5kkYGa9hA>2#~7d_wiR(zfqmzs?@G2U-RaZ>o89XDH{=O_Ag9w)gEJ8c^A z;bxmBxf>z6YFaXR`BUEL#4T|l+6)`t^&*}W2PkUQ)MUIFHe=Ust*voE&G%c^E?ncE zba)h_S-S28pesJ;#O)3trmu=t(2g%KuN`wS7^o#vDOrA{#kk)`C&jC+*3M)jnP20v z%fP*8u5@C~T)B3@<6AA3nAF*9CnfsZYV*)-w?v#+fw#e3kNC%iteY*d!G+w(WLm!| zymq^fs@{>^ZqQ;KpSz(K@MN-&YxrIirKpJvYBg-=bl&|hhrd+bLy3+7#7a-;sXdLU z_w>6)&sbNP)>L}>AJku|s)Du#k?G41buow0;sAkpfQP_}{irzU8K;XnX8O058yTLM z8+Sah7A5&R>5jxfqD0_-7nb6nz0_{D&%_wQ<1?=Yom;`#;LeiYTv=*w;sC>?+rA$x z@xH*9*4iC+(5SaJlTt%)FXnE#dIeOYGJvLQD*epIm7J-H!$37qGT%5_=TV}6sEWcB zM`enm-GFgAIyZQZX|F5nWtL+mRNNfP<7co}Sb^u+7>Wf`7@Jd=&MiL1$H^pC3v8Sf zKg&5qU`xP`VN4lDGr=p{Mx#@$Z#a|Bq6%tcR5_=**C-eunOV-6lI!JpUUH$aDK-tB zr`bO6sWW5MU^8qMn0;(NpTXDypXK}c96yj6GB`5W96Nv!^K70kU^Pi8I_m^`4w$3t{JBQjy)LvvSp;luCboA9^{H?1uQbTQtHW0W%s|&Y4z^$A%5*v=x^Flr%i4l zvZH3W=5reZ>dQ4VnP__gZj$?MqZO_;*BlFr_{@#4W>R)D%Ab^&Xx`?cZwr0%Wc+rs z!5xb@A_v0#hiYO(++W=>L}7y~kduCEU|ocZiU4KUlJUgSEw!w64-Bzh&rT4uw*(6{ zJRip&ZX||;vwOJ4hJo>ekLqN-QYtH`^mL|n6fqtvvAU(i+Lp4V-PL1~hk=qqF;Vg; z1(Y#zWj*S(P{vV;D7m82J`Fr->!lgUc7y6=rr#?|3D&Mj1sO<=_HUE3vR7Ne_Uttr zXK2$V>dSI>wQ^FFXV@2qH~~(RO4PW;$uo;S8j_HwK4=yj9!3mW<5CoSQKbUif0DH@jD z)To|ST?&P%vAAh)pji=tf~lzmXq%=P>dr6qfHOtgK-*wB)XclO)GT$CtUkZoJxCF$p7LK0I)2=A zqnNh`W;bz#P}hOEDWi7xs2oqSbQy2J0_#Kd!K`ZElx0^+yIUPzD8)HuN)#cOB;}MK z_Z=3bkZOG)P^=z`#oJLgHTH#JA@uVKI4)PF6PGbHA{{X_2tzgTBWOS4PUbvFCFxA$c~Ei{!#yGYf=Z^YjCM}B75=XKeG{ZFfg`Hr9 z3%j*yd$+x&OzrLWKC)L35H=!vWz%kkw_6*CZxk4}BVs3HT=<)WPNlUIEU~?lSTk8< zgLPc2;cioRQ_`Y_L8CkW4san#M4{d?x~GPf_Sy83GOa?1W=cPkMw8gK)%Z;`Vw?w) zB6~oJN?t`<*Qe>EjP=~Sf|x}kDDI$5$}Ter^M0Ph{6O2zCN7R-6F521^}gXz9>#rw1wbOIY8sL>IEhmB1c@Qz1z+}eP1w{h{XMS5ET zk@~sSPnTsV5lmhS5G|g~*9;WiEP+U#(Bv)=};G)ih zgd^;P5_|*k*%t zWbjd#pd#&(GLlr~d+Gy(i5~7m57p&?VwN(b9_pzRx{OJQ*^*bwQRd%5-xf+ldQ{X6 zwL3S=Y=&M^l*vF}4jCqi;Q^D6@ZbQwTgSy^!-&RqGT3$XzD8_=Wj#$_x`!miCbNT1 z6x%D@UX!;c0gLoP`-OT?w2TXV^y`N zcBh9oHXxDY`yA}O9rE!#O2>_;FtiyMQMwuNJ_ZaVFfMq|LUu${ANJW>M*k>#SAn~S z@`%p8LhQ{Bvo}O#PsYRtM0a@cLc1BtmBsg{Yve>IYW$k;*8sAoF_^cfuN805L~<{K zjaok57vy5muKw;rjQtJDBkAwd?#%G^cIGBCBl+W3X_uh)f^Ge&jm&V9g zvgH8xJCsMnmW5C?+Jv3Z6TY3D@<)uodGr+6p7|Jx_f)0);ST4bPu~ z#3$$!(iELcL+Av?2+R_ov^`B&DLECCr-@4hCJ9h#B`DDp69lRRC}s+hlAs_X$dwCn zt|Cu>9G{>#Eyz&_GD|@QAxI~MOMv7f=(L7xSUPK)a7O?r=0rUJ3m(SOv?687#uu8V zeTkRE$EKluVHwsJ+WdlQJ<%#p_iEHTCGn2|aevjmGluB=N=h|SDLn>>#qug8N|N~Ut1Y4l3T&GgkzzTx4m53gbcmP2}O zCY2#ukPqd8G83r`87D*nqVg|VWf9=#D!mP_tLu0Qqey&DeFsMQwvy&QxQEqLNeUV@ zgW=Mvr&4;Gn%Ov*Ol6Xq^vV10v=oV1`x)Izicbu!du){5`YscfYh!}ZBAmb~fj#xc zr`{cAb|k)mj=!PE*R6sASwUK5z5G~E&vNZVV#b^BoNK8n)7hk&Ewz)hJykua-cqw^ zE6%*UuBS}5M4x-SldRxB5@MWY6=~SG9G0fiqC}XKq4W6DE8JAEX^r_$ta!8R4a9XKoMWki)tEl_cidN7py$TDd K6qTROEc_qhPO@hJ literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/msgspec_utils.cpython-310.pyc b/vllm/executor/__pycache__/msgspec_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9dd439091a312a11e555d2ee336ce16470acf72f GIT binary patch literal 1072 zcmcgr!EV$r5Vf6TH{I^i0un+M2MZ3>9-62p5JHGjDg=}TwJW6VWtGO>CT*Mq+bQg7 zIY9Xfgw!56@C$rMuAKM<1S(^9iz;zvB#%8KXXed&anb3t5ZLP1Z^_6(=(`CnS_6Uy zuv8x$Lkwp~paH_>OfoFUfCy*c*x1Qj;SD@EyPPnOlab3BtohL$G?>qwzu5+xV=c%w z+59%@wtqln)Wx+INg0j1q;}TwaqafUC5PxSgIl6j0xk(rsZ!MzSk-2tc36lr`!N2NbYgHa$8i1SQT%=EZq&uu}+FSJj^l?^0z#$ zbRo}&99BBbR8WrVdEgGf3=MrvD5@60wY)a=v5On0?y2v%zvLxI)~*F*@SPj{o5(Dy z=}kC*qNqZ0K^tZ9J2EB5WQtEPMpFXnVNj3!qaFhV|F3$zu48$Ymfg9LEW{bVj$D8# zEXjj5vuEey-C{q909BeS#({d=NdRmSg|cVxS@(IRe>E6!5;4`V_a)Bpeg literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/multiproc_gpu_executor.cpython-310.pyc b/vllm/executor/__pycache__/multiproc_gpu_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9f62cd7459d5d7cc3fc7c2952d3881f7d9f8ee8 GIT binary patch literal 8073 zcmbtZO>i7ZR<5k7t}eA&EywOS=IKaqZP2*ChW^h1i^&}*a#U20!P4&69+C3oVY=7D2^P6xp3jc?2g3uvbtMp z`DZqQ?x^gn{C)ZIz3=^WJT_L+@VWfA|FHINzocpZNiX|91-x9vll(Udu5lJi$Uwriu$3`W9|TM9?rQPo}u%Hfzhrs`Hu3CG>> zaKfDkC*4WaRtye>Q|^?i+re~r*gdT3Bf(5~#66_hjg}PI$^a6`poaGfhm1ce>0y^RdRueC&zF$3&@XxbNbR*u=owt+d%n(J+S{FB|?cD5%q&?f;D6#ivZ!s5mijUVF{FFrBdBDeX-6Ae;Z;-mN-;bl=0qemgb_$`aEquNnz z#pGkWg3%SO^8z3LGo4R}aXu*u{18S>sL_)vHlO0tPb~KkRzJ*VFn)?35tH2H1+4HE zKZ=%VUVyC4X2)*fh8xZN`Ww;eJ}GKnyt4^bMNT+8etxPQ?sk3g>{QFut--m zVwSmO@B+O(V)ED0;*{mbXXm$y z=}gB;kECy{um+vtXs3v8f!kjgf62bYs4W|<6Whl+w%Wbe(PuP1vZ2dAMEhjF{h^1q zLwj%OEY0dzU7dFBSlh>YT$&oxPGFbP7vPxpHn@0_kE->x^^?4exoq9xV_)b5Qn=>I zqC4^F&6~Hpg~hw~t}R}l_ZH?dcG*2~@AlI58w-nb_phVoUB7+n&ce-k@7|52`D=4` zzoM&i$I<95-oMpvZU4oR;H^4D#km*D`kDif@`FGGjU;qdWE?v0awlB_!0^OLWAzq0 z{-e0TomLbHf*D_KI@cjM^*BnQMnJQgb62F!fvucIgp~y3D-{xe=j>BGS~_h-0s|U} z^T-cc!toO)5`qg}YmZ!h|AX2kd@lptOzB4(ZV4#j2N2F^BRldMRFHJ!DwA*1$#Zzi z0f%P$+zScG0AVws|2Z%ys<)&h9LRw*sh-M?d4RI|hS%2#kJtmq5i-ff`>dj=D~;8x zgn0q?5};K!4NfGnRO5-As#P8aA(wV_!zN zoy>q`aK{N{_YITdSWr$;F-=Psdz5FBY18Af@kq%2VX{LJp!q;sB_9t)C?g=rjAsS# z6u?c|Ep;kv6z)-vNF1^>-Uf(UO)`t@OT;B%K~@0V%G9k456#AgcFxMmB=o3-wFUnN z+@!0)iTi|CM=~=J!HPUhw7f&K9N}Wc2a0)d*xb(!7j`4R!Tw^9;{5h2QW-yQEw_I1 z1kq~c1iaAXck%l5Cl}MOd9m4OUceD>+6y0qt+nuS`1oSbSiaa)=y{Qh%Ei7oTpVDj zx7utCq@>o|%qqEg>W?H5OqwWcW*asu8zsZEOU%;EU7eXbrh(Ehe_kn+&~GulU@>#o zGAz?FZQaJW3e$UEbJx`Ky7>)e?vzK$timdK1>;K0Ea=SME$Nt#@ui)1_4Ov9HcaCX zxSK>P9tG}yg$iH@K%_uj1Gwv)rN+7m=$=groN;~IN{gwTjsQ#nl+RfMzkuXYSCf}$ zMR)r8{kd!2rwex%J_7vB&40RZeg5v(?7Vwiq2`0zOP|aydAIK0 zyti;?>GpN+*8II2x98kK()1rk?WZ)`c?|SvN_Z!6D+x9MT?0R)<~$IvmPzWsIx0Z> zk(&6UgND^nn&9LPibBMcVD?Qk5DcDGtDUX2Pr>_wX?CFV&a2SGIoCc@r5WThfe;_2 z-T&NCxadF}fyeNwoO55ZYWrkwEr3!>64id{K=^xTg_WvR2f(|x((9M2j>}UJhdp(a zXXy1^DvqGY%=NetWk%>XWtAFhR2-rn14<>&((5@YCQ(#P`5mh7Dfi?{j-k@dy!cfb z>!3*JL99Xg#7fYfX=d$O<^(jLWLSC$x}c-Q+%1`T&E9Q)_f4AcTAUx!3c^Fu45bxx z?i6|f>DRgjTo?Sdn4IXaZAJ#3INQlo-!(= z<=fD+=M=lJ_TMbEqMk$8TcU5pb37*@Ld;Psm7Lz^j-MnX=8G|``Qt_qICa0TD7`rl z*(8mdN_P}ly&d2<43-IE3_#dGwvV3gN~Ra!%Qt*2Uco+JUXA9S&}n z!QC}5#(bv3m4$2jOlxS*7`e7^Z0%KsD8zW8bu1ctgj`L_{~h>H5CQ@i;jF@=0LrM4 z%RwjsPjT-W2KR6cPQmvEZzLDL7(>42EUY;9BwRn*tGA01p=B|&94!gi00jz~_Y1|n zJAH!AJ5dba#7YN5m1gh%EC32gPW>Q>on@g`Rs^phF3jN{PJ@$2K#uRgHBLzsS6T>( zMDj9aJ~oxu8GyV@u`Y=)m_?-IUdTR~ZJ&R&?@4G(^e3$*1+)mzHbq){35zU<#1x+Y zAuOUW=d%hdG+Wo6>8tFS_UCMAaKnG3Kh*E$8t{U-m4U>o%Y30{dhL?@6|C|C9(L~5 zpOL|E6PkIAEMGu2>i5`ip z46I5Up-`8A(C<5->%a|W<=oci`}{KoB&0^uIvnktH?m^?yzFp4#^}kkN^m6u^u~(H zvKshR9GXF;19bk(+R{6Eo+x<;FB&MTx5Wd$kvc12Mh{hba09(pLHPb~>0A|fm97nN zXAfL+!`Rm`Wn{=dBwY;PeLU1Gm1~O6=(|ApV+Q3Vjy>D3gi}%2Sk2R6jjEM@{0xJ74!V! z$WyonZV56pnw@;T5?`Z~=aBN!$%!q>u^^aAwJw~84&}XcatIK*AzQVM;H*nose;Oy zS>(>XV2jlv;`|mRm2r70ZR|;sZv}Jm8fw{O54*_DkH8c?wYCxSVjQgi59}y3yo)Gzl_BXl zLaS1i_6zu4pB?^Peak>{Q{OK2W|`}wG^>MIh;>=ld=9UhX%F8|t@Uz8|5D58*eW13 zh`Yxf+r#-+0$;jAlHDQ8mf|2omW?701rv=Je>=b>>DxHgq>*U-ZHawdbpBy~@nV^6^gmG1scsd396P3^@;SRYB2-`nt01Ri zG7GURvNk(9SUyp(jW@YdN~LRW*m+3ZNmS@jGJhJCALlruY_^IFSc;T=hxMfn(q_CY zZ3L-G+DO@zA63l#D6TzcxhGX+?yO3D^vz<H!wH%;r5_bX*-GD4xWMN z*(7>*zE$+zu-&U<59p(uGZGpZpy)-2ZlNB_I5~K`$LW75*-r}ha(w;cd_&|CnBaOD zIaL#8`^x`^VCfM-qw)cMio*5nx_W!u*i&iUbfHI1aRpweH zpxse5DNmovGA(mjwd4tq0eykLi6nk4$^Yb3-hn%&1}5~^CjK#0zJoid+Is(N<{OgD zbW5r|Zy>Ny*({aj+RMkQe1C2jDKnN|>RGChOQ6`jPud0;oY_}$H|Rr#VF_{9MxLvT e9G9(6=w*Ex{zn<_Wp<9)g$YCZ{qpxI>-&GOg0xrw literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/multiproc_worker_utils.cpython-310.pyc b/vllm/executor/__pycache__/multiproc_worker_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d6a61117e879ecd8e1f995107356f608cf62955 GIT binary patch literal 8637 zcma)B+mjPldhfehm(gf03=Ci_gR#*r46t6C-8J@N!`Q}#0mA@s+q+#t=ZMiDwZ!Qb z!>BZl;~@pEls9{++Lu(ShDu&|NTn*3hvX%dmptdaU-RTYAXSNX*W~woN17Q8v$bi} z`TE@YT)zAFsXaE9Gw_`Lhksl8r^ANvU)0$C_-LF%iGOVxhA@QLGy?rL1C!fUVDZ}y zY<@d|!*4fm`RxTB-c~cy@&n(bZ|$bp$_81kJI!1xALP02HVdt>V2tZtv)CFB##^PJ z)S3t;T9d(KYbuy(O$XDhJ;9#VOfb{h8|>w|W^-R_Hkjplrn$d$AUI$e@_;wrQE4o{QnzLeF)5GK9Wf=QA3DLS_-2oo!8bGb=GeL&RM4|m>_gAKLC^QlGb{F^ zXaBktd|w=pr8|Zk=d}*Xtat(CkTC1Mcu|zG=5g_o+#?Rl6XJ-R8;raxj$-7bctyPW z&Mp<Ujq~J@w|P);AUoTBFl$ ze{3|)p~Pnaq(M9tSU13Dfep;L(i7HMa7>VqzRb=VXgl1_p`GWpOKo7nOTB9{?lzO& z5&bT=>*b^x-z|$q+^HtDB`L~DyS&m?cX8MZ4TTM{kh=G)&8|$d@dupgdG(_Ork4`47&dE-rb0Vp#J2E+Orqx|)oo=n1Y&LG6?5rkB z?datFX0vrtK9IF;(pD#1xO1Ze(uVqKVK-?s_<;xqnwm zV44x10~k_*=o5B+ILm!guNYG@~s9VAn9=I_hGSWlK7g@CYM}}hrSx_UF7Bm&! z>n2?#dq=OYMztl?j@sS0T<5-Wy{*boJ33K~F;=btCp9RU&4=p5GsGe>p)oPmjScgO zL4w{lLa3eV##3{lVyAB0?SQ7B8uMeAdWu=$DFh&v%{t!MFl6BXNf_IN7j*>S`?S0R zkTY{;Z_oBq&kfg8FJrt$hZ#Opr*a-8COr2|90|u79`L%MUP5=p;%`$smI>;wpo=@U zNUrE4X%6F|KJ-bn`9PjoFnfDP4zx9kt_aLVyP}Lw^#iPf^B9D%zOf9+>Y8!>L*q-F zblF@6`nsmtqw9k{$dNv^qX;AVq{OnCjgL}olUMb@Z7}I;=w}}D%_7e1uW!!Zliq%-3v z>Q6n*WqgMb(%BfL+cA=Uj!FXbB<8xM3P6|DH=mfqVb=+H_7h{x>RTJm6C*y+H?zzp-FGCblxF)(lZGi%M(qe-@JCplrR z6DBO4d0=q&Z}Zw%z&J_)#WaNtneRX-r;$jd-k=_q#9X*()zumFrgjHB#$OPtq%P*S zViw|b;`~yhDF%+lyi#pdakgnaW>o%iry5b+=qeVANk>ew8kw{#(*kdLrt{4E&Ux0G z7{wyK7(pP}3nCRGkc=&m0(G2NLwVMcX!Ff*hi;;Nf?1k~JC5VBBz}tFn6!M!>`jes z&f`xl&5afRkE^z~X^V3NP7v7IXG@KdM16p=sm6+y@0Rk4`fDg7&!!xnL-&XrTGA0V zTvM3ny2CO_Cb|IkdrV|Sj*WF*6d;{jX1bV;-C}>}avP3ZTlJ0)Dg!AjLt!Pb*t^SO zn$b)Yr_%$9>&0%VYKQD`aUe5GTI6d)o#8|EVcA7!*_=Z2LR@WOljt(zItz>HHGKJA z#oZEEbsjxx9w7C!3WZ8!OS?Isb`jFs_#meErf7L!G7{4fTx^QzE-Gh4SpNlUByd@U z^^isOg0T*fV{x_O1wK9Qv1~#(UnXm^Y_HjUd)eum_YHNpZ*RB>1R6%#Ue2(6Im0#Q zuC2}_{zg`~Xy;J!)c?eRz)vN<`YFZ=LVu}O_t;jAk-@+BE_N(#j1P@J_V)GO!`<$E z%oxp#>Qdr%BFl9YcDnB1NM$L%J(l_x;$j&!Et3=DQW+P`QtsTt==a< zLYO+s?M9RqKfH2rAzZw8dH%|c#qfIS(>J6eD7do=7Qiq}eXcLM68d1HS=I>(v zfh*2T$&w)F=vASF3k0?~nMP6dtj*)!r!K{ZFXucQGYI;oW_Bo;et+;0NLIy)jK+%`Pq?p2cLb&jFrqu;(-jkO|z{&$*D6eQ`&CwVImUEgT=QIwF=gN2} zQe!1fB!(M%rzuZF5QZ|xo+sYFdZV1kD8{T#Rl$do%{i@)i+ZuJUai5ptPV6ODVX6y zbyC92k2g^v=MDOvgOm-6-8LJ_#~~Ylw~IFl93wBu2s81?Ml8cZEIV*=U@L_4HO1z$l73{<a{Lht{0b!| zN$@QlXL)du^v{6|IDfEQ|9^_P-V1}h^=$|Cc*~%X8P9d z5)NqB;B*3?Rv*ch-WcI~!|A)gyxn)|4%x0-tPdO`aaRwS&<-Yq6D@D`2d1&+nMRN3 zf$!fkh-&L*-}@2)=$iMrw_>iqmMo|rVK+EZ`|(O$%BcL1(WpK{hx#e?=B}^C@HFS4 ziPa0#`xZbYqfI?~mQ|YOFr~*lqh}z%STWYI<_6FA%IQfAOz)`syoX&lmEF|4auW z6=8i*z}?`dRsJ7$)8CGmK5`;9RSo4Edp?+QGeo#$L`ATyy*i^A7-NXGNP&D}L|{^$ zyJ`A#Fp=3}PYx#1+)YX*a^Rzd+aY&ATa%;Ma>tDoX}o_$i3jd04j2VWRdwnpE5El#3YncN9)b!Ni)q@~)@b z=K547ca4cCrctD26!EIIw`9S!2#~jvdOBr@lv$!Qt1QukWdejQMW@+@N0tkAn8>_+ zjS?ROAgk=bIkKKvX6{*Ol3=k=bbYswVMC~u?4QwMZxEow!bpfxLPs8j(Rmi!GfY4( zBf?Ig0!c0sIPXXVK{nQH$>#` z$QvEU_{3@qwOfSR-IGjiOyQj4IKwnT5xbx37yIK+T(CrO&A$cS#_q+>i;=6}q>ho^ z+LO%mv;9ophdH%L!QVTEy=Sm?Y4xk&ZY(c*SAVP9*kPY=X|9$RzBPv@0x?4+)0@@S zXw&nM_u&S^O>cL42M0rx=URQUT%~_4$k*UB%R=wRo5}Jrl0qx&GL!$mf@Fgnu1E8= zF2oRp0>@EcRi}u!(*!7!*&77B<$4w0ie7QBGx_&ep)#wD`*^LZ6pj~mbcmzf0ILm; z9omh&{d`yz0>iT2cmQ5q#Ymipb~oWSf&;bX=>C{?r(B9w5*aK}O~BnXbfps?$<^EK zsuCBGZc?h-(bg@^CaPMKlmSa~aKKtsPW-8>v<%rpop8x+qDH50N;}v(C=71Ee`u&{ z^dbAxqySS7=cPmlVMZfr)bdc=qyJ`5lIB~D0uV@oxe8(5)0v;dJ=B=&||OT8AJB&LV6evw*C$; zv%sNuYUaLqhUXhHp^NX~AD1HgtYIWqLP7?SDH5sOlo2Fu9kWuqAsz&Ngaj)7Wqa_{ zR-aREj>s9VD5MNiPvc566!B6z!2=;-b|6{5Lw8&TAU^V#Z>9msPEycEd^G*PB_B?~ zE8FC8D~a&jf{P0kH+9Z`@(%v^yEcD)elc}dCdh;Qa5xLK!#AH$u1TSmIN^c-&d4m6N2gW#cWOP3c z4&FqqPHgQ|*@oE)$ppNZoldLaIz1Edrd;_7?g6IwwH! zymlB!Au*qTx$zN^nz6VI*UTl%%l=vYzwz6|0P$4--=KfShQ9(>n1092f{j?t@-4@5 qY_Pdy*)!%ozcfW2%jDSP-r}6~Blcq~>pRP~{$M$|-|s(FIPiZ#?XSD2f0@ks>wHBL0J$w8dT)40j{b+9lat z(vA)FQW?mtPti*beT6Vu2{_+X= z8=2;60rM2LEr19js3Mv&O7ZL|Py5W*0Sg@OtCnuFwmS!^qeB+Db4zt~kM$^#q3kax zi=GqF7M%+sI?|WzMZg9i2}KtqT{(~eNbU&wJsI`>0k33l4QG9X_l5;<F>(;FwfN4&{jo}n{w#1 zhp`>%VyGoVDMo%B$n3?D8M?4_|B_^4(WEFTXoD+<(S*w_kp{|4khoR#};9 zxou2tph1C^>J97af-lXyC#Da<{`+E5>S9u)#a0=csVuj?(AANCtWPH@eLX2$VJ0U^ z>B&0#$yM0=ZBg;HcwC&-n>;y^$?QtTi`c~APa1m%%M$@<(~x@b1((6)`RD(GMx!g^ z+imSZKSRzlFwJ@0<6P&WQh1Iy|GA3Q>ZZfF$P>=Z0Q|ai8w6u99Uxc++XfPnfJUB^ z-GG@7_qgmsAZwVtBX6L~!n+8W|BeIzHtg{Yyf>0^c)tc8G#tRbi^d-TDair=G^Z(< zd%x4)V3bR5PLBQKU_s}Ew;Yh@-l$!-03_02CEU1hgGsP~FzJaj{4_T+Y1m-dABi=R zirrt`qI4-5`39euI)L`ux;;C^|9e_@IZsq%1t7bTDx-oK1LPibOTNw!FEnV^1wGJ8Qo=F#N`-=!Gs`T zkCyyuO2;2?I&I+8;IJAjJKMaj-$b*)1X?NmZuGx^lam>h?+n;pBpA j;Owe+b@=5Hr(faoVKMuN+ylf#G@_xm;YU4>hL`^V`6`%W literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/neuron_executor.cpython-310.pyc b/vllm/executor/__pycache__/neuron_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..778b23770d213e96af67faca5250c0260dce6208 GIT binary patch literal 4893 zcmb7ITXWmS6$Ta{1cD$%Qg_Q2_!7HpYN_rmZR(_U>|C5CY$r<745l+E#9c|SaPtC` zOO7&=RDR7L=#%xRhyI1W^k3k$Pkr`Fr!%p8&Vrywh>EA+?9uMo+u3uzbIz`!dfkxV zyYu&d?3pb|`WH4PzX~+&K}r4wgi2KQr9igjfY>BZY(+*M@ztPWR|3t}L|gIa0^Qa{ zUG=MhVH=`e@oRx;o1(7y^`K!lM19Vm51Mu}XxXixZMTC3dqI|%&K8ekd+9@o>a_Yy zqE)6HDfTk74O)YCjV-gvk!r6{b4%*f{|)j<9VJsg@sc!Cwpf}~KJUjq>yWJV{$E(H zpGN$CG6;L`xe3e6f`zS)oYlP0OC3Mj-DMn`pG2SC|BOB9vm}LPeaj7CDgL{D8uyX0 zkWXiyMwI!*I5k_}WvN3!0Pbz~Q$`)JYQWOHh-MmgdU00AzfsIWCyscU8G*ac92X_) z;Ta}B4I1~LB&sY)OtNJv+2jvk1zTY%t59-RqPJ*;YR^<#qjOY$CfRdT0o_$ub%(4; zJA_uL0llCzR;4Bsg{nImt+3c7dv3=O1K^F6;9sEbR9|ruP$;INTj%8fR&rhut1%Yy4rIKzRTq`4Nyq z=`oeHK7*eO~!0WN1Y4%(gF>2X{ zAN5=xFcTMsBo?k;?NHUj<=#4T*`Qj${qV1tjZ8E+V-EB&$;n zu;NZW*UlqWE+q=&Qb(MzqwqzXBi6o*HH1e_ma&HB#HAAphzWMp!JdSua`7X;1MYu} zYC|5b&$Mzb!<3CD{pW$FTjcHDxZOkW7a!JDD8doY5V?{7^yHX8 z$dN%=Wcc6_BqPuDy*Gi)KUOH&C&@B! zlu>L}pH~j$V;N!t0rjZe;V&MbGg^>CIfU9MOe~iJrd}7GMQj+p=||DNyT@Gm`WYR~ zZsovn(#T1kx-kSfyIo>M{nT!p@_JD`uon;%e-N z$zuTHi9E)U?otXT;w}J71egwtEsKiz^&ylOxT+}+S4u(^Yh-gm9*>?Y3jZN4k8W82 zxyD%#9WWuo$w-C`98`kuCh$H){%s(`3nd9A7WqH+)?+Ue8}lE}*b$xOnAYbQz&`Fl80-hL!M4Qb4D}_I^qi*ND2`Lo$Q2khwp4f zJE;}(D2P+M?ezMbvoQ4sB{zZKj1fHFaeVcG+sHyUN+01hIe=-5zR!8YM_93!^K~7U zx-sMd3=s&#lIgNG;$wQ~?@?B~2V+iBUk$I8j(T=R)|y_ynSruG3;d_BGXEKpp93ji z^($md!zx=EA!mB=&vE20fedey1e-bhTyvBJ;cw57pp8hH-u}z+_GP`C9eY0hrj;-Y zaoM7P)b!@bpT@giFCBi?=rjA{jX~rD%!qi~7Cw;ceMX+nObGhDeFs9fpYHt`<_P$! z%9%UL zNH8h#caXe`

?LCT?x>(9wXBU;r{ClNdy=E;ls|`~vRf#w(%`LSAabc-gME75p`n zwu--o%5ioN%G5W0+oOcV>ypY0GBCa_#fuPLi}3c+;0+G%OQ_A!3gqA_IQ!HqOgyYP zCNZJyDVXDdvHTNCf)~&Nx}DWbO9v1yc60cmtpvjDb@6h^brkWDvLfzm!~aY;qKBhK?3x&s|L=s) zA*VIsaCf7K^ymuqHNnrHV&i3fcffeEk>vkk5#D3h#lQr!MA&2Ez_uZH{nYc5@|#J( rIPqQ_g>@vaU3hPlV4x>?xOf|0$ky|Q=LGEK|0WlAerax#@6 z8Bit)N_kCJDtXw-Q=Yb5B~_mEuzz7+@-OB!PkQ#-R%)5=^k6`Mpf*{cy3u|5cKYd-JcC=b}gveby+w3Mlfg31x>pt>ji&4Sg;pl+4N5Z zr|r|SJma4U&e~^Xx#*t@&fDi@x#TYf7wikcMf+lK$-Wd^wl4>kZK(=h;BOqL_7zs< zS2s0QVb$*q``WNJ%W9}yzo)P|YkaS;25%nd_6^kMSQGUozrp7Y4Es$s&+8ad-!#|) zJB2Yf*=b(bX@On?y|!txGwdwrKVjOs(mMBwq}S3@+`jimR6SC z&Ng3;!cA{0t$YzN?q?OW&8>XHJG~?lcjNuA^N}0#w3?Hs)>6}o7kY`~M_XH5Q1yQF z=Ac8GytJvEQvm1^E2Fi{D!jz1NIEkB z#thgMR_9gppXD|5HCThqp>G|qZh&flGS3!3nPaDDq@e+pvL63%9~@F&H4T7~k_c zJZ{aVvsuyUNR44vhSg-h%hQS<33u2cAuN%WYSCV5EJuQ;GtnNJQMlmjap7(5J6OP> z%N&s684gXW6Sb+n9nx5%zLT{~T|AA%E*CG=)QGviId(hb#i(VS^1Eo!X`M#aP@8H^ z)l}oKtXJ_{RL#Sx(Z8I_DF;D&a{L4Y==keM5(58>0#orA;emRlsssmZM`t?V(7<6E z^01AhaG)gSfpVZd(*^_)I%GO{9P>^Ii4Gpey;H?p>VT-TMA@m)ZTua|P9637^9I2| zmgYvlm{}8qN~qpizKyS>a9$C)|9(!)_m&;~2newkZuwk7A&!8h#Om~f;9=tLx4&%7 zimRAJTt}AHyRLA3|L71KF;0o~p#5`1T&B)%QfC0gWka%E_M?vL<4T3Qwn2G%<~90X zSQu>Ab^#&@0Y<2^8_=UncmR}>oqMZkFOvRkXVrF;Yu~IcZV?}CnyL(n&z#unb5SEI z9i3=UHrkn3qUwlu@i$bpkfoI|E^sf#S+KKrF}obJLu^-qx}}Ss&;*i)+f*7oEJ7;i zo1hVIF|AWoYdYak(~FdAhC2S$fKjxnCSmo*a=oVZFCIV7Ay%`UPau`ff8tRDa+K%l z3sNU)-}#s)Tm)Xotz?^9VK3O=!iqL6chB{FcLSjK{E4-JquY(GjeW}t_o7{Zg?g8* z;infeF8qBW507Hh)Y8P;U?9E3@^u`n;f!W?vZ+N2osNs>Z$Vo{isfTz^p~bK{g)?A zYqS(z#Ni@Y9WU^(-l0pJe+nqj(hpGKn|HpvoA^u2HLyLGm5y@-8wv*YVjI*cDnLKagW^ zy@UP_km4#bqeNgNs5Z6!8#(j2YH1%|IGxK>Fdu4v!N2a%k>(#d21-)&bSX204%8Pa zG$q9QQiTPZ?P_*ns+YrC_C5xsW+bi8<0$$i+WrYCHjq_yL+xLh;&@`xv?`xBa7otf z1gFGx&gqXNi6c(e74ZQGIiE$BF__E31wkr;57Bs>wU0n!NYWats{M0QjODYWGmcU$J_z>xFV@@yQ8m-UDrC zwC41_^TDsxWa8&EdVITOAI^kaF+Ne7e2M`94IMB*;-g2?3Zsz~#u@1V(yqC;k=tu+ST6HxQL;+TQ-If&vlSsC zF~p30I-A*XncIbyc7~8V2BWk|D4FcqC2mQb$(Q4>e|2iR(__-P$vK=8(3VR=kX{y_ zQT92q9A2a$j&pZvh@Z*H6Y9B!tbb#QwG#)t_E3(oA|9T=LzD0{$-p-w2Bu{*-TU=8 zoD{@K$V(R_v?iC22cz{zX8cHjp45Me1A5n`#w5K)&VjrQne^S15XcJ`j2x2+ay)Y5<#tV8}1gTR&_!1S$NNtNUWuzqqDJel(Ly-Oz zbeY8$l>LUXN0hBo_9bOUZqKR;-;~G)>%)LrpCoYIRN1{%mUH zKR3;$PT!i|H0WD11dZ$ z3Y|lD`E+Iy{S{S@3$oe8_3 zpMOBZzaYirJ`@!|WeAqqjG(sE%pB#ZYf2zlrhx=wG8Yn@+O94Z68*UWHxHEG!-o7p ze*wpl+Ti9d=syfm8$Lj{pH6W-?g@c+uf#NlP}fS(EOFUWo^RYqY#6YMDiWuE~0=I`8jY7 zO$1=F!1T1c}&#ul3oMtKaixGUWLnp zbXe3)l*m0YVZ90JBfYkXD?(x>WJpm{z?cGm6p#^@5)DC@c_~ay3c0Xj!uM94Vz_i9 z;)JLrNxeoR#4j;J85c@RAf1uvv5?F^L#Z>ck0Z* zkw1~E!ExBB|6ch)@`8g^oq9MY`V#YR#^tTJOF7`eY;&;UMSpC(iQS z#67a)&{o_H)HMo`pEy0 zjlda9Q6W6Wct|kny=U!tuJ!*~8ZlLC&lT-g77n=oVGeB>t%$mb7r0oMJXDJue8TkG zZ{5n*Bj}G}8CHzQ#w&6$#-^dT*U8h|e&;VtBjYWv2y^|N6-0eS6p{%?oO;r4p4{8# zPcn(FKfJrVV%@y=QA-MPw3$4Fm)}U9+=3{H`=68f$oQt)^OJV}7pvhOyi8_Mr1`H} zvw!85#lPwDPGY%MH;UmM?m=3xI_WlJ(jITOPDwY?zBbAH$(YDOe{{{oFDQEhS!#Sq zu^JuN5N{E&Lm7Fcc9r6Pu!6c=B>QP$8!N}@JX}|fKob;0SMUx{7KOI4f#GRco~aDl z*o%|=ObAoV((D&7Zx(ys-RsDxQQUPgtNgb?Y@kb8ebftq7M|JIg)H<#YRE1jUYW&k zgh%zG(370xxSG0wKL3dnleJOSOtq|%JyF$6l*u(V4y#&K(qK=3f&}G-rxcN$tZZsU z#2~4S6q`{l|H&}5{~>GLJTwjktA2 zY+61dqy|yMHjQ$T*(Dh@gSg`Cv@5cnJz#<~sW7Qq7F^4o_ec!i`A9e$_)AF4P`4s= z%SLCR2vH)+oSct5)^_f)kSSqbk@PI+JDS#a_qmAMarO_AH2&;a*)b*$9LSI_wOy99 zUgG(&Jn`d!JZb32Ctf1*?}g@fvLJcJ3v7;S5_G$5h%)0#*SuHU| m=@yit)f8#_^ zR$4OOIk#sp19B+=HFvsi-@cD?&pG!y-@VfuAJ1s`yZnPc*)02-_Lp=s{FlJZm+=Ws z>6#`qq1QB5ee14{d!uI5P1n>Z&#YPXgqx__uFdyWEm=>wse0N?*E4RWKIV?qvu?IN z?vC?wy_R!xT<1hR@8&r#QJbt6+(LcIo#K1DHeEmB9^va`ZKgi!&erGLIlfQT=Ick@ zqkNsN9jhOAkMnh=cB1~A`y5}7)t;|Au2Vnho~)m8Pt{Mmr|W0jGxf9XSzYtzgx=Pr zL?5!XbM^D?dE}*J2lwN(7wSc~i2I}di@Un}(lt%w#Kb*KO!$Re)4hQEyqLuOq<_Jm z+O^z^LSNO2g&&dhR*HHwCj9ksyB3OSB`g||T@7m0iXYk1Z~5i0Xh-8~@7!GS7GGOh ze0_Q4T4XLXb|UL4N=Mf9Y7j=L8?CU~Y?NzJVy)e(`KV;(Djvw{?RMykYd7Cqdf%_K z!={XmJw1OR*lAR#D}JNW6u!6FY;L2L{H^lN+fBLc%eQ5@)$(Oz%JL4Ct2U~kS8Hx; z_!76{tK~Y{l{ea9s~sX^N;T-efy!#P{5x$wpn*(zvfc2)a17$Zd81)DxyriyzP7Ccr4D% zTDjua{YH3AHruV$%BC;cwQ6HyHI(Jh-=J~iM$1%aN{0U?;+x=QD7CMVBD=<$P;S@s zEk7ZQE1I|{62iV`xi+qnB897@F#VLD_A~3oytZzLw8$WL>`?BQ$Ral@#zpR)=8lV! zm=p!%n#U8)U$^dS>$;c`v-h&@w3rhX{wbaR zFd>fmX>rWA#Br|Wh@TNBs6X^Cp5wehA35)Nly#6l!!2jlbK;~pb@I>x|*kj`j&xj6Wu-2(t}c1E7T)m|Z@h7H z`TEjT@9M&}FJ5*e44^9gz!@pDdh?B|-rF~By}oqoi6w%(L?%Tjovyb)P`U8646DgT-kP$OYFF$9H(Z+s}1-ePdPK` zCa?PB)uc@0p-fRSNyTza_E(-zkBeqx1%7Q^p2I^oO#^NRnrkC!fm_12|U zwRJHpOZ4pG=j-jw`sMoFOSS6lORXI^ti~mBN|*fD-Cm+PHd<}3pHXVDa80;z83)??dD8T6g8QhLrz=@~s^<_(*^A9qgN8ZoQVXpK);l!e;aqE>Jb zLdRh-hCB^hfj`5tc9yR8jfWZ*LPKaZvq#osUDdvxgpKI?iLOBwaR=HCoyR+?uuJtR znk&*lt&*%59zMBQ4$5IDInzyrK9sC^w;{=qM#*>%BWt@_s})UoiQeQ#cB~~NXqL9T zOmC!u&2~r?T*F=BBPOd~_Sg^us-G?r{=h}^+y0IB{CM#`GzJR3BKzh4Y4mwlk)d0Mf zY*zdr@Xof*M^*z`2d&b>NN;xX^wz}w_Sna%`$7_a`3-EIl2OR-p@NzxtZ z*PEg5ZNpYK0{I1WT?vQ9#1c9gTT)vp6YZM`nV{TOpTSI6%}G()tgw6 z(a&<-k0#^13d9+xtcj7V^+q%q%61U;r9w7cwA|w!fq~H)%MIaiMUh3Q3Q&@ONIW`8 z{ViHb`@W28xj@NRC^@8mbHjDtRT#*lx%k^C%tEyd`!PfcTcv+nw|~|-e^{tP{Tr@h zM3!Vf^kG?&P$4zg*he}c#CcLb@4d1NJUDI-9yA_r@R6>Daj4(NxZi`9!lrlI_mqB# zE$TX%hpl&Bd|FqZ{u+Blc@*!B*cQ2U3Jk{Z38tYDd-fLj6@A-~lYDIquFWm0XEt!( zGX=_di496^PLo_VCC~2J`)TqwJtL%*+CYBxp+W2MuJ!@$_Hz%lo)u1P<@YCHwPeGI zs@Ai+3HD{zgxO7ePpj%(TUb}Lu+Y`FrgnAsvbC=ny<{)dP2oLQ`a>gl`_A=nx~q$1 z7r5<#*-M8fP`}x3H^I zsVmxd^gA=*>27w<*g$VFmZ|Ve7cY%xSQ=xBc1C`A)WvIG|4AC5AD(JYxfDS9!1|$-3jj7Om_z3eSyb&4BAzEsEOkb zNbYC$tX)q)zI z6DMiZoPP?bwsk?AR?oJwsQZ707daP*6%x79o5Njle{KUhgkGp;K)85zp*P>9r?9VS ztvf%f^^T&KDBYc#$H=?$+s1eF;LjNE&Z#H6`ZgpQ{CCRhm)xg1?Cw#*usX^62l)IU zTVaXQOJRXZaGMTh=vXh)Jx03IJK8;pCuo006E<2iB#o=ow*l1iSGd14e9s8^?8=YG zxpiirf(&1A&IWE_WpVMkw|Zmo^`$j$`PG$$HZuBbU z7QzYNt87+lf zyqz(%#01pe&a9Bt_k8JzcD=q6J7a$EEqGK%Gy(P-&Cr3TZdOo-Q*P`y7)0PS8&0_C zJL+|m5q`;8Y+|JbK6T1s3+o#~R4zj`4MyNFJa(GW0U~Mii^SES(lS-A>Kj>C*wI7u ztK^@_S#Mxv`FaMCkhYKYdCS;0$y@!7amQFy%il*9Lo!)J;b@H3F6t!(;Rx9Xj(lW9 zM(JhwOL%^Gz06#xA&{pq;Y&ZH1_^81nqlM~+mAb+JG|I%pdU+>;)~K1l2$GF5-$4H zJTVvo9aB^{0S+Dn;Ds6lV5`2TZJT@eo(ITVHvu^G@*hC;GS1N7Dkub&aNM?YyBzp} zrslH_@EdrJu<8e|jJ$KXu;iU`Fg&?0P%Pm1jg4xs1+qrIDhaDLHGKe&d2ueM<9X)h>}K&bogWDijY1#rxHd)J6?I zv$@ZfoL8%$l4?66EylwgZZD`)BqyEao6uW<@HU`EpJ?qD8KZmyb6GT4r6Qx+0y0j= z1_rfDS6e`Cz~RAAWN;$DBzm`Zyto{Us?i65)z*I^1tCqzX9{}f+Apqn&#VF*yh zGkFWye}NCfxM{8P@)CT}j?#r#-kat3;B}mKqYCnEXpTzGHTb2@5iT4!tT{9k2Sac= z2{NvMa0@(a5FmwQ@hgn*;8z<+4s3RE5fVMp+FRc|0T{73qF zJSj!%WBo$Wbf*{Jyt?4MwY<6vV#T|<^w#p?(rT3F;VrL#by-&waTNP-(&Q9{7!>Dp(w9H;`O?eyzdRyuvMma!S>7dFu)L>8-uNK++cUN_$Qy(s zEi)j?AakKTFqN7Yb%-;=-X4y>u^m~Q&VC9FTr}Lma3rSU;V2O=|6nGk1pkO&trd1U(g2Yt;^0Tr?Fs!eK-rKSv@MZ?(Lz)$fQqgO#>_1A?nh z6UbkovbN&%BdgkgwVgqhJVMD5CFd!5ffB|BX&?B>Cy3-`V`N37Tp;JE2+=x{xCcqJ zO=Q#lK|@HQprY}(uS&L2Lio4a8&T%1a;@zzNr`<5n;fw(qAc$i^rw%&Ok@!@h;09T z2vI->LcbosZom|pHNO#MZz;w`)nmb52XdCWagCB?O33+2vJUq+P1ax*XdZ#A`of74 zP|6K36A<1=hl8llXf_Vz6^|?JB6uplM9J&a!V%SXuOIlw3Y2eDDm5>_{s-YJ00_pI zZn}?9g=?!zNzRNHMmHT-Rc*PM!DS$6J&|uwTeER4tNhT?6zNqq3^y6CRc?}Wx!MZc zM1#MQ4F-9E`$FIFJ#YrkauV*$5PD-qA~wn-w2h3}O+svRp$v1AdPyurG0DD*ty8X1 z?wgbdN~)AxqJ+(pSkXE@0d@(stO3f;uuR(m<&uCuBQt%RGeMNv_Jvk#)I@OUFMk!q>GeCYf zLEuq%c3#aZk);}!rN2G!J>D9{gf&?K+?O$CBgAkqiB z2Y~tGUmh{kgFwuGA5Oy|0Kuzpf1_q-*W!zpqw$epVg+IU7eGc)Zt#frC9wdrn%=`R zwe&IpM_L;l={ttOE5to6&@z+7^HhrFSCRXjGlyq0?*7QDBh!~a74#v%6A%cX=?l`K zzN_uiq=C0a4tO!}O|(O139#F|gGD@SV&A4R;MS7+DJrM9HssJ#@NHdFn8XY8tS+&& z*2=3NVVBfz5H3?YtJS9B`m-#ZC=s*~FUj9PPo#qc+ckq-B3fv$!6YwIz8&wz@Vdb3 zM=~CzB>WKBY8R%MIFM{K5%|C)SmJogDJ(}xwPO=Rwz@#)(w<^^Jx@&pdk*T(%3xtP z$#dlNEL)MKca9&D`iQh&EheIKyoands}HTzO4|dk*{JPEAI(VOQ{~^JWE)9jtDS54 zdz3>3N{31erjw?TczGKil1gwBJ;pR>5(KDB4=mD_6Coy%H#Ci4Cd5|gmbt4pEWQGU z?WOrGpx1^jERZ#(NMQP33KQLAH@#~#5+PBV8IjmEK}%wz15aPU)70Rp6+g{V-Mw+N z{~E5W-Z*bC+ad{)E7=|Ij&-xUCKG2llzFF{TLtp#O;G#6hjF{2n{1d$k$U;KwMI9E zJd2d8H`z@DyLcnDmEO}f^ltti>bEpdtCK{nZs@&<#yp;)w{1Q6O(L$sF>J~rXDhum z{^T1E`5inr<@dQ{4(glhPIPm#U_FYdavm-ZPE5t-8=!3(j+!Hi__5v%+=#PSZcua# zXC?YhZoLT#=PumYz%Sh`?|>T^+%5nmA5aAs!2#eX#m`lpSBOnmlL*a-4T`(NzVfy| zsM;a-hfdaZ`n`$$-rCK6pD!R5Qe1}e9C7}EM|7O!P&r``D@__5)ybXQMBvqI2dJHb zKD8b1lpTUs4*cwTn+UaFxHn~VR-(rwCOW%8ULG=}A)q9VumlGULlTh8C4~xJaTX5b z4BDfN(2m|>>SU8UQFi9MCMc6cbEpsav$|gO#h@baEDce8uPWN8 zXJ?SzdG!~YCyG8v!tio*@C156YY6RYJG5?!ZgWXgww` zz|qtNTyQj1=x?NfrUT{5ns3CmVv?;4Q3f9w@;=^Uz>Te~U!`jz=-imKsg-89Oy`0qy7vVGa@nO8 zDJ+rSq=ZH;X)TM)PPHXztyKdd+;_l3s&N(6(!iY_xYtA8HL^%YB0EN-2_}x*StizE zV9&BpB(OU{+7(NMUUVmiGE^Pz7_DQm-**cKH(tCL;pB}XgX4NsHqQH~T&}hs@u(`5 z$o~Nik(aBxxcVB&UEso=B|vIDwg3d_+pw|(7ary80zjtmxL{CfV>6de1DN(FmTeh; ztJbtWty^Y+&kX=@0>aW+0t5S#v`2}W5dPaq?3){%W6zk==*S=bkT+y4u_K8+44IH@)}3K3dZEZf)WA9qY%|^fkEPG6 zt^@E$5ToXYerW|rY=9{GsO%5Wsl%x3sCO6r;q6l%4WkrP0mS()Nb)qGoY^TJmdmHU zHA07wz0h*b;`lBufV#Ar)&sqSgO7S0l!&f%K~oEpu@`kwatSlqF<5`4j^K^(>8Nw3|uKmg!6r z_Q(<Bt1V|$>FMajX^ z?7R!=1CAO;`LIl&=D$@&C0~D+rn;Y74NI<7X}5Md-=_45inw$At~v-nD*hN0g>BNoDZ<8*(4Y z5G7zxfgj~*Gi5+=LGZFoPjl5ett~i2R3btrvN%nzW8v~>L1R8JIyVxyeCl9|Ddi*| zqGoQ~&G&`k#nT#1^tq{iMtaZ#H%*e~X5BOmRrP?Eh`~A$9mh5s2w>W?@wj|ENi2^a zk~z^LmO@O6h@B(jlm(}S=r+aCHpM@9YUap2Q8e_p^AeP0=%irjSuM!lMC~K)i0qF( zUAOJ-p6UMM0XKHEgLXx6QsQBU+ta#T2oY}bVgA~i=R z6s)ejtx1nsk*gvNu%7wA#GW+m!;jIsdxm=N&gZ%yPyy_dZ^LFS{O_W_jnri=XxxR1 zrT=&E=_a=(aDE~CfKDro$99dqlVv*owgP+-O}qhAR=vUJ+$aMC<-}T3RyNl-u36h@ z`JI=3G0k$mh>gz?uh(A$7`Eggb%rfg4^U+Yny@DZK4DB!K`HbhqYmidyi@rQ{P@eh zGtO&IZWs<+3@IM;^l9j0q=%#r17PL!dze+*0$otl`{*le7LM$I65Tu!6T%4UBC>A7 zB4ajQehZZ`!YbiiSbNCkS{Gq!89>tl5tk`?(g8e*h{tkv-B|l4lYh zmM2j&EKls!(s{K#(CHx15xoy6bOgWHp>71^J#fG?$7iZTOsThduze)|E-J#x_)qaA z|0N}(J^eq_Qz9m?Pm7)cF#z+)59p=5Vylxx5<}whpCHps;;IDM%Ri#Jen43-Qr75N zfDobK*Gf2YUfJ|wwwrpUy3wRpTa=untwa7(6~V-xKmmeD=cpz+zg0XD zr938`Juk|5#1qmfJP%~McL!k-(a~U2NCBbq&1Nl{@I1_CGi3iB;Mr%WF76-ct1MGO zcE+0sYO{cu_7^EHO$iYl2S;z^9^L*1B^gTIq2w+le@F=_8qbORH@J$%V~Phz0;3+1 z#_v)&p1@WC{7yx&rTUXF*2DS7v4%Ga>w!)CAC97Y~+tdPJayF`@a%k;sh(s zy_5^ohqLwgy?}4iNq&Apt4h{Qrjz)w2>n{X!m0Z~36T=%dkGi68xUhSQJ)d`l>olS z_+)(+zJ4N_8lCS2VjV_Qj9ec@SIjJ3!9}R;VoB^`XHy+&BW1&JUfu`AHyx_CgX3U% zn3=(GBF)S6kNL7n7RTh@K|Q2A%>8>&3PwgeK%ngSG@dHX^b}pqQ4&x>p=neY;x@5? zML+L=GrdI5zemZ3l<i71dkdQO)=Yzf{-) zs!*gC@v>l3*VwTt{GCyvVv$abE;k(iNC&-BRzL)*e1OCqP}HhOi5`3#$uMmyze6Q6 zgyU4`rM%rC(f%n4jgx4}GL3K8CY)SMl4juk4xeIG!hK3NS4PB3ra%_`XZQsF0f`c? zObysff@RZickv#>6T&`=L!H(=EhOBkB(||X7Qbh0Ww!{nVS*0cqOQhzBJ4~<0 zU&k2OQH>ZEL|&=+EXTh7a2WmIC`*#(s@lY_wO}jqpHYph;s<(u3wcGF z3<9xqV9F-qW$`ICUV7^8f}-Z128USJ7G!p+r(LAf50_EoKLc_7H!c;82^_XLbZpGNuU+jXyf} zj8PpUxzL8H@r@MkjbZ-}J|F{EZYMHE>XBvVXdg|{8wo?tJ+`oN)6N`n9}%3%+bQ!f zjinU&8SOqX;WX*2*EDa3^_qS{fCI14L4xo&)Pp1MUA?bUZ2X;HQZ$CcJ6i}k5em}2 zisBSUK={bM)Ds;cAI6$T9+ou^zic9s5Wi#?AXxyyV?5n`dP~_ZqE5K%W=sBkyen@~ z@)TWM9M;7Q>EhL=>f$pRc$zW-f}Nojzex#+QvNv-U=vyeTyS-=8J}bsmSrD=LpL~*1ZKnjYu-Bi%Z7NipHbC_lmV;;_)PvQ-0%V@|217b1(7jKSw>&v4^iSU zxfZvr0GV1z$K}7Js{S1%|DF$#6Fe|U zBm{+0KtVY7(?>da3y24($IN_^I-5^RDqAe6SvZgDXsm**4!=T-SRhdeju+u%Hp2l9Nu^y zv|6f_cR+`guq8o1W#wOpx#L_gpdYY;PvQx3Gn}UyVQ)zOU!9Vfp{rR+$THpM&>N<7 zN>3gt#sO14C*#fy3JuQssyK$Ry3&&-RY}m3XKiTe56mQWMZx$lk?IDNpJDY}0L(#G` zEHt|t-GBf6_y2CROC?*w@9aPP>*{}ePt*R38pFQ~8gJl<4(po6HLi!6tN!b*jy4Nf z({K%)`i#(QX538EaxK|5!)!C>=9+mo-?Ux3S#S%@qFZd1+)}gbmYZYlm>j2vmFBoR zF6)`F+MIAFWZeoUn}^&(%_(1%KeKJPFQHxG<7kijFZmPOrhA&}i(0Mv3!>UWO-~BkU#@q< zmhUVu<<%;k0z+T7LpHgH{lGVS8o87ig9i>rDU`A9UkRH%7;V8tcE`=6-n9 zztQ!hxRyzdtoX6VftLubcVnM>5=_&NSKB>PTeQ2K#m1`7yJ65;S&T(J_E%^J<qW17_ zJZ(f^56#zzH(hoO{OTIM>1Q}Qr`^?-b#8F;PR_M>jc0l8j_GFki@d;#sO9n&e~Fj; zyl>BHJmVMq;;go8%G1xdC7im2KPqwlUs$~Am-&Qm@=44n_+x&BA6jO!8vXGp**ET& z{mP(k8e?bBU&Z@b)H8Tj;fMK=JC-}a=P*CREv$Ex9|Ja%ct7S>{VL60X8alMp!X0z z?pOHTzWbwgGGjzVJXRVBR}%M zCJhj7@H0>|@r>b#PN3jgPxG{ZNnZ5+#|aOKh9-6mNMCjKqxma5*vM zYfBA#peK3HYlQVE^1KJyZ!evRo1HVAp!0H!VSfDbtIh6e^KA3hnJ~D1rn3PCZJi+| zJfje3i!<1;2X0R>YK!^KhB%IWK0_O}P*{d#OtH=BQF_hy5hHnyyP~ylAlWVul($gy zv^8x_=lVK}azD^M(`B1&>9Mh9?t&}1ai49m9^>Y^A?CP2{lu$1ZP)7Q_fyi=G7ExE z!kxZ){)gVR#q)1p@)noQFD-hP7S6x<-ldC){z{^st!0z30r?y4)^f1o&JD|JfA`+? zdSl&haSZQEp0`4xm#}t}Wc}97K(t$l0a7O8>Lk=@N=B&};v|p}^Mp~J2-%J5&?Kj* zQx-MUlCqlEUz7TxB8rNh)5}Iqw^42$-A`VrQu=e`65=o>H5iKiZxWR(o&uieHr{Y; zC*x@=R7&RjoW)?zHFI0PsQvh-KVnzv(Z(d>D2W@cA*vjl=*0Nmd zYS9aHMrh^M^5PYq*@DDxWm_iCqL$mr2HIAR=VN=Va8nbPdb!=AYVrK67O2y*)3;0V zt!#1V6ZJMOt(Ex+oVv~nf!@pT;unSzp?GXtUn}>lUKXoY*2Z}Wc;$OJ5%=K3<(I|i^S5@mT| zTH7&t#dF$?f4R%<8qix4eDY3is}#@D++K+vy2t30&^pYgppB>Rv8{4^B%a&WKR0n| zWqD>LoY@$j%GUUf*{jTIzrdNBcRpgVX1Q zb@rFKsPZG=h-Y?K>}+e>>~sCDe%HL4SvL3_LFliFeN9LxM|%_e7(v;|>>lRN(7LkU zL3w=b1V2viNafW|F1QA?sUX$41i6qRadMwfd5wAps>1ggt3k+>LUiZ$54zrM@eszb z#r@lx$4ObuE18>b!kGxXDE2#2W2s)Z`aX3pq8A8!Bz5iP)bKRYZt2+kV2C@5DtwR2 zR~Ef1m#%v6e6YCWz4^hlg^P>s7`+ePPTJ@%%jHi7{U#qTIbfef4ezP5eF0clh1vM&q&3P``oaQ39=?jl{ zo6U`M7xKkt&=(GG`;pUX#|{ily8#cuskb&9U>`Z{mJ_e~j+%}!+@E(Yv|CZYec{yk z8dim0A;%D;!67(O{W)#nz$mr)LsBpRS|{)d-iX2{wM5op5l=LSVg;)E$2JtawyQ%0 zv0Z~i^{?0swy2cj15>JUaTbG<0(nP-C3#M;C`672EMfDnB$lGbeywVs3*mq@+Re`E zpU|2)6qd$V`LXqQ^E(G>)0D7e_U+zQe?_&#l+R@gH z9sHlwdit^fA<*l84NkG0q2D50KS|_tZRa{1R8FV!ZKosP9f~;cqt`~}oSk6Cnd>pZii;xpvP``Ue{}Tw7`YD1+8|V#;!Q*hL%)>1xcadZeHf z#}8a|X`*VT+`o7qSR`4c!nk-BEttUt>MY96R9ckBcnbZ396?ES+5VtI7?k9Fm^z<~nOjWlan$h}kyO@V7DL=Bb#%je4Escjk&j(TZ6OR++EWT5 zZ_jLtGngq}p%pB}0?GuOqwX?6EvkS>%@%(^UE^&SKmof&>_Ti33FK$Aq?B$$A)Z(i z=fjh?R5YJtxPQI7lGq>C!>)fxh_*;95;>nI#o>TUkmX8DvI1h7sz&Iy1bNnCii(#} zxN}6&K}12shGp#2wKh2v28_eF^v!vCQu)3|!(B^N#qSg3SE=|efiDhkQK}|4ueJpR zEi!^6C?cJ{&559-m2Ru31X5iJirp zR8U+e{)h@00ZCT=7Mj0@C;BH8mT57|sOpNDiJ_TpJuF)}-PA4JWTtLrY}6ToAnTEB zniy$5whWAt%+E~oOX!Q=;h~wj^=s2E>J?ViX^iT#AC|LaRz?6tZOqP?%z9*+Iekhm zvr0umdz>>S^@?sW+h7^%ca{ZckIROcC#dG*vT32WoV6KRfMP$|biPi9C*vwE8-(xv zjghd7IEpkB5h)_JreJp$!7nrsyq_A|{QIljmNNpx2}T??bq@3Pz{Xuch)>?wLc8TV zw}LQq8g*Ip}y%HOrf;LzY}@LXUyR&%*UJ0?x%Yy;3JdX zs#(cVnJ<$si8zkzS+VFs8Hyj4J|CgENGolL)DDJ$swAGO>w1 z^llqlg&tBHczY6W^Mki$`nD+3b4ZIJAj=TEUZGdqHe_&D>M`-XUTINF;h*p2S`@rn zv)WcUh5BwUk3LiOjrB5-2N>2`c1K&$d*#2;uTt)6jIvc*mDUX2<$#MGEmOcB=XY)D zS<9{!o*a)ryF&PgkL9=$+NE9vP)pMk0%q&~3vz(5m1nvFyU=nJfyv*6N*S=0bD`cM z-w=nw`;5QbMhtTcEI#1XTlEci#)Ia|@OhCY{iZ2>GNoP_l$~zC$KHSPpzX07P`-!Oeho0cVe#o2L(UImnkM ziXxdwNpPI@h9A!_i0_bCfm;SYQkNuF(xE$zigf6zKQgv-a_100B6}!VlBFzc5DO27 zlT2{T6XZ+ueKK{&+OMsIP|{t zKPZxvX^>hmnOJDYm+3}PVfGDk;ZwOj}-mL~bEfz;aIC-Acz# zj(A_5nxY7~PU}mL;7?HdPdpJh4(5!q(l)#TSnS7DMx|{Q4eOz4nQW3x>ZU=~pFJiU z4bOn=H2j0T9`iA*ylLw;+dT4=GZ7T?4AeI28kuCG^uQ#mv;<|eFLJgMBGk!Aa*5Mjk%&Y<{|l0;cbrf}RA zeW)QHi<~gINK%9rBvn8r6=7cAPo(_06~E=*?ugfM=c)aY0%xjshidQw=9UNDL4On&~9!u1&O36mrW{rzE9nC$fz+ zW7^&i`v&II9dtx0%GRs+%jugh|G&1Z)}VBPT2tHxtOjvJ|3`I7AzZ){k!Mc=Lmq68 z-8Xb_2YCV>aRU7}QQ{JZ-g4s}-pi!0xv+WSq|6tpRgk#kr=63|1!NHSdXfr79zi_} zHW5g|AH@Z(*uaVsDTE#_>WLY&mfLPozE6Y8y+MNL+rNz&Ox;hoO$>5~a8*=tjC{@< zL-axVh5Y_!_K|QuT2Ckx;x|d3c~n3F>>6HO6*WCiG6P!Ras%`sQ$@FikTRIE`^4ao zoFljfMq(*dy2{Iic;h;{-H|To(gR)e02)jqRRrx)(7l{kroyK}nyx+j)j;2i{XJ4t zMZ08-Ju;2lL)+S%+qcIN-}*qVmm8#zeXv2?5WxWjJKTr7QtuX|H{BjsKsQ1Y%w%6y5o%X0 z_FI-^zp--c*DTL|#R~TSRKGu!p^%wOvi+FZG@hVTOM+G+^hSQPCHh4a2KK)`kO2+M z3cfc$2!tDde23s8WWgOjZtrm~pT)NYxKBham(TJsjLG{Zz7E(MXY<0HjBE2EFWrGl zC-3o#5Ti;mG0FqyDMlD!WCNIxq{@hme4|*ytvxi^-mQI1S_~J?+a#qKVsFJ1mT)j$ zoh+HFJr^yh4h5%P8b+R=A@Vr(1oAlgE)nw*3Z$#|i=ae!aGNfOZjwh2)HeTyw-LF= zpORit`lcM~yTD)FucTsVk{Djb`*#L=$w)y=veH)x+HRpC@D+|Py^CZHS65%Di;^P; zhVSaDm4mcUGBKi7)WFMq31*+z2J$rU@G9o!3kRIu`~kB^zb+s@f*whZ?~alqx2K&H z0ru&rlq||bsbgkJNBtCl4pdJse63zlTU??oQoaXUgqkY%)ztS6+T=Hc`PFPdDN14k za@;7f)IaVTTaao4>G&Pm(LrW{w(1eKqY@V+74`@@`+Glv)*-#VpVtVk{z5wnpq zIU#YQuMFTef>CAi02$4WlOBA$rkO$LEEat~$xZ14VX3opeQ6@N&D z6t_K@djowd{oT=>piM#C#e+`aX?#EA7Y^zH+&JzBke+84N4pDmgK;Qz7@nbe)3RUTlOg|NNmL+DyFEQD+fWm?j8fC z^E77u$uA0UN33pLW(I=>H}>j-CFRO|LeR)xmeyGw($Gl{m@>(oQ{N@Lj?-2KC8y<( gfv@PN^rAuki|OB-o&Z(uD9L8s^RoZ zvq@DqM;f9B>-88}=fMf`7GPmu1Oy2DkiW360rC(ac?$Y9PY!}0c}bAOmhzpd=4D25 z5DwU^uFI+G`ObGvO?P^_s^N3)uYb4p-W!_sPxLbRq0)EoP5xHbG^Q~<(meI6dph2Y z$mp1!sZ*XAS)Gzs>e!y$DSPFP<2jv*SCM6URQ0N;V?|S)npcx~rD(cS_v)P)Z$`e` z(QIeVo0I8sG~Ze97G&Cq7CQ&L1D%83LHS;Z4s{NDhdWE&Qs;Ye;Zi}eTEHI2=(xz9B=$B*rp-YKSgr_uJXZ2MQb=AD7$ z8ggBJ#ygGQQ}?uwuy)>=pPz|K>h}TUuzB8GHP`}M{M_=+vIFek=bHB#JH!sNC0=7k z__oAo&D4 ziF)UYx0l%~c>6Ka?`Vxv-_S1KZ0PwEQU3g5TWwAWYKS`B^UwyKxZZ zrR9Dv;(6(AjCVAu-wH$!MLfFPjaS2V!yqxOF5~`Mx4V%~@p$w?U4(I%`cb#t<^r#? zt-cUEPW@gKq^n)gLGy*H4|%Jf@*9v5-RAfE9Q`j`k5eu>oP|NkmwB9YMba>HyUkNS z?B#Zt_-n=Q!(Jbf=k9gI1{e24(CcyWZjkWMDQg-^4xq0V?@+I8(e3yAG+2o^3hK9p z1=lDaImd#{AdINk7d+|qMT;js#Q8DChVoL9VmK{YzsV2!k)Py`@xwLJ4$r_|n{4(s z+N#bhR{Ff=SW9}aU%k`JZNbyNh;O!lC8Ljtl!fmMzR5*AQf)_D*E2oTGUG4xKh>d! zdZzhCYOY)BrPNN#%=pqkjw5p_X_c8@QjIC5Vb0dg_ZqtA;5f>&NNJo#m9Am)`q;|c z@qKwF{rp&~+v$Z7XFdzseyh&{pYhGG#eGa9FL#o5(&H_e=mo6}uY5(Gb0MZNR8her zpAO?*KSeV-^TI1%m4{yl!p^+j?Wa_#G(zC`S_zLj>6!DL{#xf;=fT-1Tshm@O4qvaS=zU=T&hi1oE5>A-%rCR zY4*0n5%hc<-=v1eyl$5q!_b{4i^gDS|9NN*S7Ohm&<4@>G6}KqpiQ}>eXRpA1nm@l zT>OX=qsK|4@+neH(i7-s6B&tthpi8eJ=N_*#XUXKaiB)B=VM5u+OGac+ch3(+d9%F zevM4ej8)@~CSFE`hLxL`eZD{&Y@qqE;Zf3-{8de&u^ktUK9Mj-xYpJXxgG_4nKq*qy=)?pCOTI znV#x9hM3B9rW?k#v2DgyrvE~wGHu=1H8bOpz70fyRkpTE$SWl;XQg#}w~Vy2UU@{i zV?r~i%-OAGR%Sjjwr$9^(<#XQNoHjBDsnRWj)pq5-RaDRO!?lh0@yOG0A{yC$PAr# z@7%g^#a#`OHJE}6Za;zccjE|{vc`)-1tuvDElS2|Fm+yjmiQdUjxInSMm5Yi_;xvG61H00-F=SEe;b+(iS!yTmqHb;HOdMO zNawx!c(|~j_r1l*MfQtD_U87=MMd$tl&Ao6zZ%Z(txm)E5vnIH9@RS3(5M+s$<}9e z8)-wgo>Waox2)P@%NV@)lzKibEw?rUk?c`QQc%Jpsg$!wq%cgoKp&}Cz(a*UW^Qzo zyv*axP;}$m?)Ml7Lc>r?avtsWPiC&W$%VlAfLYk|so#%LOM>OBJ~;l=XrHPM;fK-U zW@CX&SsVcI{JiSZ9`(Un{JiG-_xnLql$3p+bz8nK4MT1wsSvNwun5@&!A@>skHi^z zB_tBB(W68Uxz)7Nf>uKOJ|6jWVXIm^iXe1s174&mWC+9|Jm}OV^l&swubcJS5z{DH zx`iFGo>)(eCx%u1_PLj3T{)c=>hAf_2FS&CkVwG;`ovw0!3(z~kx1NS(T8fSO5y=M z88Npe*{KE^O$<&BQ(CE6mX0?1qGh7S@IxZ$E4T6QkWltKao)#v?&zOk z{W5Jw-!?Oofy8dZL@^V;rPNF<|y5UA&i8n3Yw2t1&WuRs6!J;aQe>&Q}%cJF^SfM-(&VU7s5K^M^p^)h#!0+u1nuYLE^6T zV+I4^M%@;mfy@qBCs%~+wNy1I>>q$1hLMcp>#l$*Cko}yxc34P(_x}Bp&-=l6$O# z`V;g&0wwLY*IWj_QG_e-i;KKUD!{XFhq(n&o4Vxc) za05eVg8?*y%P`t%H~rym`UxDWw@%2_S_#Qz#!*4L0E7T)yNL~o`<)eJcURpL;2f`@ zE`A$o2D0Z92`GdlGtWFE7?zv#1P{R6`YFxn^;2@n2}oSnTBC#HGTs&eWRKP<_HJ6$=0yK~^NUB8q5~ z!jyO5c2RsFX5v*UK1Ic~+sbiJC~AZ(HzOWTlLq)QKO#1BC5`-8HPoM#=sk>&dKBM~E?!cu&?$mm1>z68`mrlbB=)W~sIW=JnoUf?Q%4fy{tx+BP*}VYBN#`Jfw(SEz z30}~ctpcd81Gfc^eIgYkjHQ00099ao+wH&^0FjptfmEPjkg2cD0H4T*jg#tKIf<%|61`UXy@I@l)L5UK?C}IY5 zkhyOWPW=BsuE5MuK&&~qa9LTK!qa)iilOt-A{J3M)$PC;&USEpJfu6o)%{pI1_fG0 zz<)wvCbFnc4d^CZpIV_ovYXtZb<0bHb6fe$t!w_hOV^kEE^o;9s+XaCEX zUPM0l8~&i`x-Dt8@vZSha5UYc!9@IRe5D3|ibUbwNK%!WGSq_D!!{ysl+JYMuLb?R zDC>cgNz$hE;R;gZEi;$_oi$R#ADEdUIyY>!qd$U1;=Mu<3z?dbmn>52B6cS}#5lws z;W2o7@{9}KfkRCebbO`?UHZN=4pf%(*rKqQcTh^;6my=0G~rQyzrZ6uN`ap5K|Xmp z#4o9?JRGGY;=I?e?<5e87Irss?UMjME1?~T0U;7yX{?Bw7 zn5U~8C!MFdJNgy!MXKq-j+@(yM2d$7(h zfqgysm-jXJNQV)+)4tGluz!^D;oHFXI2L?`O}{=0!%I^%!N5m=DIG{q_rkEi$~_3- zM+^xY2q3{LDvVlCY)o0Er!`KRE;2mSsPH{b7tFD6B9v7m3=CI8xI73I5?!SRBNH+b zBYE89u$9bWuzMqcBbKb(&I>Y9rqX?!CiJ6h-LM`!ak^JR{y1DVp-!rOpmb|E!ijZ2 zC~#E<_)b%xQyFc9{N#rwQ{0{HT{y zbcNQ@&Nb z7?4tr#Isexs~7aJIELOqQoZ5@oD3Iml3$7dlDz70CBvRKGfWpwn|EMO#&G!Rm z__pzB6*-iPck=5@`#XxP&X$MyRI_R!sQ1_|*U$rzLseh;fwGPDooY85 z77Z6q<`K|7T-Qq6gPU^zICrUqm@>yn?1~#f3=c|(7`-%XZwaW~^x<-C;+QsralJ%r zp@uxAVv|yr@bF%kBy)1tXs;)mfxO9v)W)ky_@mRBc+>P8UHI>(b8{q^WTBRH%##ZQ zBXr&(aall<<{HN0U-)8w;jVa_NX|XS#R;p-*Dha`8Z(j>1Q#8$K*7 z-R?coHVpAv2118GD3%vBDyU(DVwt;6R+d50llKp#l?qK@h+|e0o-|E8pi+aA!mOiiT$>03aiw zu4R5AeoSd1`1w?;i~m+>Nq(T>L3X;;gFm5eHRxEY>kjgV)D;MEb8kxibA;j-f_Bh5 zDszUR&1P{Ir>@@RzXQl?|DvIa6NxEmSeFf9LFK=^Btk;uw9HKC8WXWK^*;r~=)I~e zkQe$23`cnE9g;-_M^Zoj3a$*qJQ|8O=|R4%bZVqpcn;|f$&&Ff@9^-oIkM+!rDbT} zo8O~p6!_#4fb*&A@?JqXzX#DpZu_#>`@#4gVT|Qt6{5=^ag`eAJhxW5-ALZDC`0oO z6;jwq0X%s;5_hB?OEr@FJ8^7IP!$QV&YSAO>mrVbmXI6fXrzP3Hfpet52$Dswa&Fz$)Htqx!hMGGF;$ D(o67u literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/ray_xpu_executor.cpython-310.pyc b/vllm/executor/__pycache__/ray_xpu_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..686dd9b594d15b7a9b79d263df885ea61e3a48ca GIT binary patch literal 1765 zcma)6&2Jk;6rY)0uQwapah=dYsQ_`pm1@%yLW-hBZK)y!7quxuBhYB$nK+y5S7+8q z9XTgQDoB9Dogk_0@k=UTj) zG`sYm}&lYR@RY_K3*3^e>3?Rb2%$mv_LjA_K4ls-t|ctV{ZobXR|Y zjL4wNN_RIhMd?4zizrLO7%Z(P;pEXL&mO)|!*P-6a{a2lZ6@ikOUu>IUfn@8N~1!= z*=VFRIN2DfLY%~LBBIn3VLDVIl4X#D$4Z3gVTcJVF9ymz*k%KSBAiOXnWU0k5bj6^ zl6GxsOJRo;t}d1X2NyPe&2112l#t^jnFy()6Q!?2#jJ76>lpQ25F?2DMe}orcuv10 z1u5ten?Wt#vjh4adydg4^r!vl?(WB8XYh1ydvO1u*m+Q{Tg%>)ZBRdYHn_jH^LWs8 zO2-tsbX9s{bOYSOzp~!@B^8%cuY!I_etp<4lDwZs`JE!v(DpkYB;&*6ZgSd>qy2t9 zDGsx=kE7_T`AGXZoQM}t;=<6&CuOi10}_T=l5gF>Vi4LM@oMZ3pEZ6z+X56UwZ12C zz>t}|%`FpSL>92PFSj5mZHr{Xlz$4VRRV=+$Y8WkV#8~9mcLD2f7 z!K~#Uv_LrHGPgi1XklTD)bP3U5>U@wfR$sJx-=up)GKIRaKv^1IZ}`rIigbm z=0Br(Ve|7G9*g0z={ou~@KL%r<K-LpNz9w5n#2ckn;G@xg< zuFlLB-CN|>t!V_aejUXdD9}aOoI4*XfY&^L9naf%#fG{Gr*~k_$FspSTDlv&ZlA9x zE4^ZpN9jo86QHrPd<9w7vs`VV{#K>WS=U3>;lr|EoyN&)d_DNht2dXqsj*hRYA;mJ ztFEIZu$FNV#TA+CC~UQ*H@}AvvuXbu084+fg6tN)qW0DGNt-qpZ1`(apLLw;>wf`N Ce#e9W literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/tpu_executor.cpython-310.pyc b/vllm/executor/__pycache__/tpu_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59a4791e18ae79cdc9d9563880ec080e70a4f71f GIT binary patch literal 5401 zcmb7I-E$np5#OEN`&!-U<0R|T*y}II0Y1e*AaMw>0RskG5DQ3ZAF{Q&8A&Vc$1%Gn zqKlkL5XDQ1{0&l7Ui^^!33Q=X~HLt;aIJ$rY$JIg@!YWik+db@kNdwO~{ zuGK6J&$Yk&ZTlZ*H0_@>7=8*E+(1kI3cxk42U@5*ddM6W8jcZ~ju{r5LRfT)I?c17 zeR6@(K6y6Lb!m3kM{X$R+C!I;vF9uWLv@@;xrC=tkJ9X7B2eaXvGZ&t5 z&V=*Me7N8&=$fdC#Y5d$dRyZaZhfwCOIU}7vyAZsuVP#k%c62sG(;Qe&nZa5N~Y>Ny9twy_@&MCmoTb7}nOk5WM93owVH{#BBb$xEpg3^y$oO zWlN+khc43J=%j+Xifbs+?U-jpn)KUQjsD}dh}?E8)65FJUEzA9S&LRS{7hhQ11-6u zYnsp;oof#J2>x;mVTuB0uWI}%FYw~$rcAXN#`r9s19qC95i^*ni&@Ap z`8;0$c8)KKGr*d_Eb(Pv=J^WoQ<$^-DPR_cFz5JrU=}gIBue1JxXmvBv#j1={35@E z(F(sj;BkdN4b0gg%rpE4z&tg?qrsmA<{a$w9Dg3Y^V~!ft!5J+e(+I02;R!dokS$A z^!Br&6lq6BYb}_3@KE1Gdjluv{7QBNE zK_jQ!S1FrxTT*zba6gUZu8>)cOPEFWhEARXxY+Ys!m+Rn?kdR-^vp~| zu=zlf=ivJP++0n=_G;U2UrjxU&3yHhu(KUr3qM^A{EgN2K6ZPwx)%iDs_5mNRiq8K zKWMi1p^3^2ebgL<74f!orkmfC8NoSbVpP=4uPp;VeNyi(55;VAl!c@a&7gJ53lPWA zZe}XD@Pu^KqHyj%)UTqc@O~Yg1B2@wO-`g*svl}8JJb%@eIuptMY!%1(28g!v@%-d zKtEsy#sTAOmvQ4ui$b>BxTkiKom(3L;ionXmZ%{0>@CU*P@&d@^MFN)qoM}ICM;r|~lQcU;U>YDZIZhEJ zTwW$536xg|JWb#kfQBibqW&mz6%aW%kJ+s5QgY81?Cv50w-REQ*Z}1c)64KHgI`a6 z3_d0=7ysQoH{#vL7>#(WL=4fC$9{zlyg&(Qk?T0Eaf;tJkwi>x`j|N~bN?J0f7bt` z-!B{#){(Ywn&X7Oj}!i&_++|9q30YutKaPeslT3zHWdA(+Cn%AD}#ZuL!+du=hR^J zkmy~cfwpsj_Cqr_O!EZq(#FWK@>#4xHUOOYVd5c*=P(KWr}=?ic8WbtL^tLcxeVM- z(Gm)=a*lv=kaceMc8DQIgs81!9a*mBo zsvYTECnp>mDz~#ELyd3_sG`Q`UcZg);d_DK6?Tfm``*X4ig0^lA3J9+-lgP96P<_) z8SK-rXU^`$OL?AFnq9ZA@aOGI_8W8x3@~em-$iJWnSO*bVuiZXv?Ftou9DR-a@#(S zS$UNJ<)oAFma7=vMN263$`#5|b=EyQ9Jk{DJX|LXS zz}C7iQjtux#7SX51<6S@B8>Nh zlDG4#dn4 zFXx2d+gC&WzrG%CrYJ|_u$}&&VW2GEbGkF1vv57`wUGe&fjI7rI=Oh1bZ@YTk+bu0 z!fa;rMfW@7<0i2ljt|jm{-9Yw; zMII;ojyO6Id`eo(=2V!BN=`l-h^8HXt5^CqPa}OyxQ=&KBBXRhHy-%{aZ}szGzF@t z{aU;3w(x><`wNh&1Z}eJ;;`spA_N;%*=tCiBus&|hMkpNH}XQ^x|!utrp7G={i^GJ z((!_PM$Y1e%-o3MAe(YsFG7~Xg({8`*OfNS8Ge*XI`K0DcYjG~T~gXq<%}vl)X^!a zYLj%P$Ql7!tfabIQo5FO+n3h~yhh;X1Ss9f8w6e_aEriO1l}g_4uO-LrFv9{$~pj@ zDao$^ERAX$R#$2BA!F=oX0S&^lRdNw)_>X3INo{)+8z}%t%vh9R-kdwupZS*b(4OR z3evxl)q)g@jV7xnBS#eF1m(Yx z{X%voGyk0gQJDNePw5!dcW?_oz`gu7V(A?%KTcId!TU3yQ>UsjdjBl_1z}$zaAHnX z(a(f%gcgKu|GokA^4GPn`Jy zCjN+)5LsD+E0l94A4!^NoXv{35u+Z=Lrw*siYyAqJb+Z(Enw)ZsM0!DEtWkUW9e?- zGnk|UURH=~LXqV+X-O+}W_lRuHG74EidW~N!sj>i0o%aa7cyxk`ByU);!aHwCRAiZ zEuq%!SPIuk{U8~;lI3nvwxY6C1({mzX%@=LujV`VHQL&$!nd_Dn`Tu#|1IieW7?Si G?f(FSsYFZw literal 0 HcmV?d00001 diff --git a/vllm/executor/__pycache__/xpu_executor.cpython-310.pyc b/vllm/executor/__pycache__/xpu_executor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bbd8076346da16a5b7d0e1732ad9fce2f287539f GIT binary patch literal 3505 zcmaJ@TXP$?6<%PM%jI53>S{@rZQ5}zY-=i>wn^K#Z4^08lP0n!w%ZO~c2)!^(b@}* zUCN<`o@uST_CGj}=}i7YU-~chwNHKN^r6?ZiF*!qS0p`|Vh7&>2fzVvu1Gx3)}Y+{ z;txlE@igsE*qJXi=)4Uz{VM?1I0?0gFcMKlBc17y!HlTJYLUrIg5xx-vpW3s(25$Y z5!uX+9OgtWb0d#=QIj>J7HdUq){d6gQnbvL)jA_wiB{RFYS+TGXq~OAwi#ZCHrR%0 z*Tc-e^T~hr<4x!{#dWcIX0VreOL)93yn(@&`0}R)dxg_`T6g8oaBHn==GI;ihQUE7a^u5c zn&sA=Q8rBCAk1s`$0H=)pNvFadk_P4zia2tUeG@hdr3SP9_H?CaWd>zJ^RBX;N=Lk zKi)0d&PNFsVcA)^bC3#o5*!S}VKym=&AWkwbIc~)l}R+pc6l($ge)hu?!m@<9G(-F z?u|r$90u9&q&z~GmtlyXhqEzZ0Yah|YU=A5?Vg4&!0whsoigkkIw+1rc0T z-Wg}3an_~zN-zkfRu46?Ce}B#0o~L9b!?R;aDlIg4VdeSO|c~|itSa;Y5bS2IBZI25Hd1!r2ZbS2ig%-dB(HKoYLGT!eyAo{9cn?l`#nmz=W z)*ga3j%ix|KzmB24e5OV4u3|Tl8j8r8F>OY)qFaofcn$~G^TaH+EfQLrv_ksS_8DE z7GQ&GfcDe|bf(%f_cIM07-XvNGyRj%`1FlA+wE}CPxvrC?8HgdnTV{Djz^RF{cz>hJLy2p9nb|O#C3`{-MbHdBw$PA{yVF<#{KxhA{7@XOtuEE?^MhNBxGeF96XVn9QOs%+pjXmAa*tbrY9VpfP%p@%msnydjVijdKzO8)uEev-Y9aVQcoX#QO( z6WKK+o{hDNRenDV($pWsY`G+Wx*;I?aS#c%v@i)tN(BXc3)q0QpouIlMZ-~Se+VZ7Ao1qFUCbleIB{tTS!Gx7wS3xrGd zA^dt=LPVG4L33|`>F2rt03!LuvVb zZu@w0l&GJ(zW-<(gvE%wh}#(lkOK0S58eiz3VA1qQ{R^^j7bZjiGUZA4Fr6FNIZkY zkRw|NO9=2F(d4%g@EMZpX(lBKN}33p2saQgb4Zk!tRt);TtL7Z&fOYaO`?kB7Qiah z^c?_ObEzs{Q=NWA4fCnGZ*Zv#5x=rlo)B*iJ zU#88k6~8{PS}5mlCyXdnLgPf}oMJ*#iA3iHBodR?K&xino|C^y%u0Xq6mSkupoDCyk9mJ-albn=x2)fdV8 zBbb+>rs!{MgA(_-K~Jy0c--DMEZpzb<#%zD?;(62VWuyosvjY3c7*eDNc6Q|Kuytf zEUnOysW-n;Be_{5)R^iM-S5^}jjJ41gU4F`h*{NfafZtx%t=_ez^g#}CDatP;S!@r6_6;3mI?7d;+*cQxd{h=4GP~= zex|$??YnSA<#xBA%N!NROT{Q>Z{RHD;>x>K6i|+>qJVOC6$;Sf*+sRyu%bkk=9EbP z03M8T8E0%Q&N!UQf^ubcfVxLZ-d6c+;ww!8|oF#g}Z{7loN( z9AUN^7-z#URsO>+C_<4U%CuFSQ3WMs?mvcwDlRBvRid0z!MTy=)~zCz{sgTy#kWXD Wv*_#8HC%#UTcJhU`tqCW&i?@WfrhF8 literal 0 HcmV?d00001 diff --git a/vllm/executor/cpu_executor.py b/vllm/executor/cpu_executor.py new file mode 100644 index 00000000..e32993e0 --- /dev/null +++ b/vllm/executor/cpu_executor.py @@ -0,0 +1,389 @@ +import os +from functools import partial +from typing import Any, Awaitable, List, Optional, Set, Tuple, Union + +import torch + +import vllm.envs as envs +from vllm.config import (CacheConfig, ModelConfig, ParallelConfig, + SchedulerConfig) +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase +from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper, + ResultHandler, WorkerMonitor) +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port, + get_vllm_instance_id, make_async) +from vllm.worker.worker_base import WorkerWrapperBase + +logger = init_logger(__name__) + + +class CPUExecutor(ExecutorBase): + + uses_ray: bool = False + + def _init_executor(self) -> None: + assert self.device_config.device_type == "cpu" + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + assert self.lora_config is None, "cpu backend doesn't support LoRA" + + # + # Environment variables for CPU executor + # + + # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers + os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id() + + # Disable torch async compiling which won't work with daemonic processes + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" + + # Intel OpenMP setting + ld_prealod_str = os.getenv("LD_PRELOAD", "") + if "libiomp5.so" in ld_prealod_str: + # The time(milliseconds) that a thread should wait after + # completing the execution of a parallel region, before sleeping. + os.environ['KMP_BLOCKTIME'] = "1" + # Prevents the CPU to run into low performance state + os.environ['KMP_TPAUSE'] = "0" + # Provides fine granularity parallelism + os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist" + os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist" + os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist" + + # To hint IPEX uses shared memory based AllReduce + os.environ["LOCAL_WORLD_SIZE"] = str( + self.parallel_config.tensor_parallel_size) + + self.model_config = _verify_and_get_model_config(self.model_config) + self.cache_config = _verify_and_get_cache_config(self.cache_config) + self.scheduler_config = _verify_and_get_scheduler_config( + self.scheduler_config) + self.parallel_config = _verify_and_get_parallel_config( + self.parallel_config) + + # Multiprocessing-based executor does not support multi-node setting. + # Since it only works for single node, we can use the loopback address + # 127.0.0.1 for communication. + ip = "127.0.0.1" + port = get_open_port() + self.distributed_init_method = get_distributed_init_method(ip, port) + + is_async = isinstance(self, CPUExecutorAsync) + + world_size = self.parallel_config.tensor_parallel_size + result_handler = ResultHandler() + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None + self.workers = [] + + if is_async: + self.workers = [ + ProcessWorkerWrapper( + result_handler, + partial( + self._create_worker, + rank=rank, + local_rank=rank, + )) for rank in range(0, world_size) + ] + self.driver_worker = self.workers[0] + self.workers = self.workers[1:] + self.driver_method_invoker = _async_driver_method_invoker + else: + self.driver_worker = self._create_worker() + self.driver_method_invoker = _driver_method_invoker + + if world_size != 1: + self.workers = [ + ProcessWorkerWrapper( + result_handler, + partial( + self._create_worker, + rank=rank, + local_rank=rank, + )) for rank in range(1, world_size) + ] + + self.worker_monitor = None + if world_size != 1 or is_async: + if is_async: + async_worker_list = self.workers + [self.driver_worker] + else: + async_worker_list = self.workers + self.worker_monitor = WorkerMonitor(async_worker_list, + result_handler) + result_handler.start() + self.worker_monitor.start() + + self._run_workers("init_device") + self._run_workers("load_model") + + def _create_worker( + self, + local_rank: int = 0, + rank: int = 0, + ): + worker_module_name = "vllm.worker.cpu_worker" + worker_class_name = "CPUWorker" + + wrapper = WorkerWrapperBase( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + ) + + assert self.distributed_init_method is not None + + kwargs = dict( + model_config=self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + device_config=self.device_config, + cache_config=self.cache_config, + load_config=self.load_config, + local_rank=local_rank, + rank=rank, + distributed_init_method=self.distributed_init_method, + lora_config=self.lora_config, + kv_cache_dtype=self.cache_config.cache_dtype, + prompt_adapter_config=self.prompt_adapter_config, + is_driver_worker=rank == 0, + ) + wrapper.init_worker(**kwargs) + + return wrapper.worker + + def _run_workers( + self, + method: str, + *args, + async_run_remote_workers_only: bool = False, + max_concurrent_workers: Optional[int] = None, + **kwargs, + ) -> Any: + """Runs the given method on all workers. + + Args: + async_run_remote_workers_only: If True the method will be run only + in the remote workers, not the driver worker. It will also be + run asynchronously and return a list of futures rather than + blocking on the results. + """ + + if max_concurrent_workers: + raise NotImplementedError( + "max_concurrent_workers is not supported yet.") + + # Start the workers first. + worker_outputs = [ + worker.execute_method(method, *args, **kwargs) + for worker in self.workers + ] + + if async_run_remote_workers_only: + # Just return futures + return worker_outputs + + driver_worker_output = self.driver_method_invoker( + self.driver_worker, method, *args, **kwargs) + + # Get the results of the workers. + return [driver_worker_output + ] + [output.get() for output in worker_outputs] + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks by invoking the + underlying worker. + """ + return self.driver_method_invoker(self.driver_worker, + "determine_num_available_blocks") + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache by invoking the underlying worker. + """ + # NOTE: We log here to avoid multiple logs when number of workers is + # greater than one. We could log in the engine, but not all executors + # have GPUs. + # NOTE: `cpu block` for CPU backend is located on CPU memory but is + # referred as `gpu block`. Because we want to reuse the existing block + # management procedure. + logger.info("# CPU blocks: %d", num_gpu_blocks) + + self._run_workers("initialize_cache", + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + if (self.parallel_config.tensor_parallel_size > 1 + and self.parallel_worker_tasks is None): + self.parallel_worker_tasks = self._run_workers( + "start_worker_execution_loop", + async_run_remote_workers_only=True, + ) + output = self.driver_method_invoker(self.driver_worker, + "execute_model", execute_model_req) + return output + + def stop_remote_worker_execution_loop(self) -> None: + if self.parallel_worker_tasks is None: + return + """ + Passing None will cause the driver to stop the model execution + loop running in each of the remote workers. + """ + self.driver_method_invoker(self.driver_worker, "execute_model", None) + parallel_worker_tasks = self.parallel_worker_tasks + self.parallel_worker_tasks = None + # Ensure that workers exit model loop cleanly + # (this will raise otherwise) + self._wait_for_tasks_completion(parallel_worker_tasks) + + def add_lora(self, lora_request: LoRARequest) -> bool: + return all(self._run_workers("add_lora", lora_request)) + + def remove_lora(self, lora_id: int) -> bool: + return all(self._run_workers("remove_lora", lora_id)) + + def pin_lora(self, lora_id: int) -> bool: + assert lora_id > 0, "lora_id must be greater than 0." + return all(self._run_workers( + "pin_lora", + lora_id=lora_id, + )) + + def list_loras(self) -> Set[int]: + return self.driver_method_invoker(self.driver_worker, "list_loras") + + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + return all( + self._run_workers( + "add_prompt_adapter", + prompt_adapter_request, + )) + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + return all( + self._run_workers( + "remove_prompt_adapter", + prompt_adapter_id, + )) + + def list_prompt_adapters(self) -> Set[int]: + return self.driver_method_invoker(self.driver_worker, + "list_prompt_adapters") + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + return all(self._run_workers( + "pin_prompt_adapter", + prompt_adapter_id, + )) + + def check_health(self) -> None: + """Raises an error if engine is unhealthy.""" + if self.worker_monitor is not None and not self.worker_monitor.is_alive( + ): + raise RuntimeError("Worker processes are not running") + + def shutdown(self): + if (worker_monitor := getattr(self, "worker_monitor", + None)) is not None: + worker_monitor.close() + + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None: + """Wait for futures returned from _run_workers() with + async_run_remote_workers_only to complete.""" + for result in parallel_worker_tasks: + result.get() + + def start_profile(self) -> None: + self.driver_method_invoker(self.driver_worker, "start_profile") + + def stop_profile(self) -> None: + self.driver_method_invoker(self.driver_worker, "stop_profile") + + +class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + output = await make_async(self.execute_model + )(execute_model_req=execute_model_req, ) + return output + + async def check_health_async(self) -> None: + self.check_health() + + +def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig: + if config.dtype == torch.float16: + logger.warning("float16 is not supported on CPU, casting to bfloat16.") + config.dtype = torch.bfloat16 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if not config.enforce_eager: + logger.warning( + "CUDA graph is not supported on CPU, fallback to the eager " + "mode.") + config.enforce_eager = True + return config + + +def _verify_and_get_scheduler_config( + config: SchedulerConfig) -> SchedulerConfig: + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if config.chunked_prefill_enabled: + logger.warning("Chunked prefill is not supported on CPU, disable it.") + config.chunked_prefill_enabled = False + + return config + + +def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig: + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if config.enable_prefix_caching: + logger.warning("Prefix caching is not supported on CPU, disable it.") + config.enable_prefix_caching = False + + kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE + + if kv_cache_space >= 0: + if kv_cache_space == 0: + config.cpu_kvcache_space_bytes = 4 * GiB_bytes # type: ignore + logger.warning("Environment variable VLLM_CPU_KVCACHE_SPACE (GB) " + "for CPU backend is not set, using 4 by default.") + else: + config.cpu_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore + else: + raise RuntimeError( + "Invalid environment variable VLLM_CPU_KVCACHE_SPACE" + f" {kv_cache_space}, expect a positive integer value.") + + return config + + +def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig: + if (config.distributed_executor_backend is not None + and config.distributed_executor_backend != "mp"): + logger.warning( + "%s is not supported on CPU, fallback to mp distributed executor " + "backend.", config.distributed_executor_backend) + config.distributed_executor_backend = "mp" + return config + + +def _driver_method_invoker(driver, method: str, *args, **kwargs): + return getattr(driver, method)(*args, **kwargs) + + +def _async_driver_method_invoker(driver, method: str, *args, **kwargs): + return driver.execute_method(method, *args, **kwargs).get() diff --git a/vllm/executor/distributed_gpu_executor.py b/vllm/executor/distributed_gpu_executor.py new file mode 100644 index 00000000..deb7cb1c --- /dev/null +++ b/vllm/executor/distributed_gpu_executor.py @@ -0,0 +1,212 @@ +import asyncio +from abc import abstractmethod +from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union + +from vllm.executor.executor_base import ExecutorAsyncBase +from vllm.executor.gpu_executor import GPUExecutor +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest + +logger = init_logger(__name__) + + +class DistributedGPUExecutor(GPUExecutor): + """Abstract superclass of multi-GPU executor implementations.""" + + def __init__(self, *args, **kwargs): + # This is non-None when the execute model loop is running + # in the parallel workers. It's a coroutine in the AsyncLLMEngine case. + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None + # Updated by implementations that require additional args to be passed + # to the _run_workers execute_model call + self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {} + + super().__init__(*args, **kwargs) + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks. + + This invokes `determine_num_available_blocks` on each worker and takes + the min of the results, guaranteeing that the selected cache sizes are + compatible with all workers. + + Returns: + - tuple[num_gpu_blocks, num_cpu_blocks] + """ + # Get the maximum number of blocks that can be allocated on GPU and CPU. + num_blocks = self._run_workers("determine_num_available_blocks", ) + + # Since we use a shared centralized controller, we take the minimum + # number of blocks across all workers to make sure all the memory + # operators can be applied to all workers. + num_gpu_blocks = min(b[0] for b in num_blocks) + num_cpu_blocks = min(b[1] for b in num_blocks) + + return num_gpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache in all workers. + """ + + # NOTE: We log here to avoid multiple logs when number of workers is + # greater than one. We could log in the engine, but not all executors + # have GPUs. + logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, + num_cpu_blocks) + max_concurrency = (num_gpu_blocks * self.cache_config.block_size / + self.model_config.max_model_len) + logger.info("Maximum concurrency for %s tokens per request: %.2fx", + self.model_config.max_model_len, max_concurrency) + + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + + self._run_workers("initialize_cache", + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[SamplerOutput]: + if self.parallel_worker_tasks is None: + self.parallel_worker_tasks = self._run_workers( + "start_worker_execution_loop", + async_run_tensor_parallel_workers_only=True, + **self.extra_execute_model_run_workers_kwargs) + + # Only the driver worker returns the sampling results. + driver_outputs = self._driver_execute_model(execute_model_req) + assert driver_outputs is not None + return driver_outputs + + def stop_remote_worker_execution_loop(self) -> None: + if self.parallel_worker_tasks is None: + return + + self._driver_execute_model(execute_model_req=None) + parallel_worker_tasks = self.parallel_worker_tasks + self.parallel_worker_tasks = None + # Ensure that workers exit model loop cleanly + # (this will raise otherwise) + self._wait_for_tasks_completion(parallel_worker_tasks) + + def add_lora(self, lora_request: LoRARequest) -> bool: + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." + return self._run_workers( + "add_lora", + lora_request=lora_request, + ) + + def remove_lora(self, lora_id: int) -> bool: + assert lora_id > 0, "lora_id must be greater than 0." + return self._run_workers( + "remove_lora", + lora_id=lora_id, + ) + + def pin_lora(self, lora_id: int) -> bool: + assert lora_id > 0, "lora_id must be greater than 0." + return self._run_workers( + "pin_lora", + lora_id=lora_id, + ) + + def list_loras(self) -> Set[int]: + return self._run_workers("list_loras") + + def save_sharded_state( + self, + path: str, + pattern: Optional[str] = None, + max_size: Optional[int] = None, + ) -> None: + self._run_workers("save_sharded_state", + path=path, + pattern=pattern, + max_size=max_size) + + @abstractmethod + def _driver_execute_model( + self, execute_model_req: Optional[ExecuteModelRequest] + ) -> Optional[List[SamplerOutput]]: + """Run execute_model in the driver worker. + + Passing None will cause the driver to stop the model execution loop + running in each of the remote workers. In this case, this method + returns None. Otherwise, this method returns the model output. + """ + raise NotImplementedError + + @abstractmethod + def _run_workers( + self, + method: str, + *args, + async_run_tensor_parallel_workers_only: bool = False, + max_concurrent_workers: Optional[int] = None, + **kwargs, + ) -> Any: + """Runs the given method on all workers. + + Args: + async_run_tensor_parallel_workers_only: If True the method will be + run only in the remote TP workers, not the driver worker. + It will also be run asynchronously and return a list of futures + rather than blocking on the results. + """ + raise NotImplementedError + + @abstractmethod + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None: + """Wait for futures returned from _run_workers() with + async_run_remote_workers_only to complete.""" + raise NotImplementedError + + +class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + if self.parallel_worker_tasks is None: + # Start model execution loop running in the parallel workers + self.parallel_worker_tasks = asyncio.create_task( + self._start_worker_execution_loop()) + + # Only the driver worker returns the sampling results. + return await self._driver_execute_model_async(execute_model_req) + + async def stop_remote_worker_execution_loop_async(self) -> None: + if self.parallel_worker_tasks is None: + return + + await self._driver_execute_model_async() + parallel_worker_tasks = self.parallel_worker_tasks + self.parallel_worker_tasks = None + # Ensure that workers exit model loop cleanly + # (this will raise otherwise) + await parallel_worker_tasks + + @abstractmethod + async def _driver_execute_model_async( + self, + execute_model_req: Optional[ExecuteModelRequest] = None, + ) -> List[SamplerOutput]: + """Execute the model asynchronously in the driver worker. + + Passing None will cause the driver to stop the model execution + loop running in each of the remote workers. + """ + raise NotImplementedError + + @abstractmethod + async def _start_worker_execution_loop(self): + """Run execution loop on all workers. It guarantees all workers run + the loop or None of them is running the loop. Loop can be stopped by + `stop_remote_worker_execution_loop`. + The API is idempotent (guarantee only 1 loop run at any moment).""" + raise NotImplementedError diff --git a/vllm/executor/executor_base.py b/vllm/executor/executor_base.py new file mode 100644 index 00000000..c96cb0f2 --- /dev/null +++ b/vllm/executor/executor_base.py @@ -0,0 +1,150 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, Set, Tuple + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig) +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import ExecuteModelRequest + + +class ExecutorBase(ABC): + """Base class for all executors. + + An executor is responsible for executing the model on a specific device + type (e.g., CPU, GPU, Neuron, etc.). Or it can be a distributed executor + that can execute the model on multiple devices. + """ + + uses_ray: bool # whether the executor uses Ray for orchestration. + + def __init__( + self, + model_config: ModelConfig, + cache_config: CacheConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + speculative_config: Optional[SpeculativeConfig], + prompt_adapter_config: Optional[PromptAdapterConfig], + observability_config: Optional[ObservabilityConfig], + ) -> None: + self.model_config = model_config + self.cache_config = cache_config + self.lora_config = lora_config + self.load_config = load_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.speculative_config = speculative_config + self.prompt_adapter_config = prompt_adapter_config + self.observability_config = observability_config + self._init_executor() + + @abstractmethod + def _init_executor(self) -> None: + pass + + @abstractmethod + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available blocks for the GPU KV cache and + swappable CPU KV cache. + + Normally, this should simply delegate to the underlying Worker. Some + ExecutorBase may require modification of the result, e.g. to ensure the + selected cache sizes are compatible with all workers. + + Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks + are blocks that are "active" on the device and can be appended to. + num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be + appended to. + """ + raise NotImplementedError + + @abstractmethod + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache with the given size in blocks. + """ + raise NotImplementedError + + @abstractmethod + def execute_model( + self, execute_model_req: ExecuteModelRequest + ) -> Optional[List[SamplerOutput]]: + """Executes at least one model step on the given sequences.""" + raise NotImplementedError + + def stop_remote_worker_execution_loop(self) -> None: + """Releases parallel workers from model loop.""" + return + + @abstractmethod + def add_lora(self, lora_request: LoRARequest) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_lora(self, lora_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def pin_lora(self, lora_id: int) -> bool: + raise NotImplementedError # type: ignore + + @abstractmethod + def list_loras(self) -> Set[int]: + raise NotImplementedError + + @abstractmethod + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError # type: ignore + + @abstractmethod + def list_prompt_adapters(self) -> Set[int]: + raise NotImplementedError + + @abstractmethod + def check_health(self) -> None: + """Checks if the executor is healthy. If not, it should raise an + exception.""" + raise NotImplementedError + + def shutdown(self) -> None: + """Shutdown the executor.""" + return + + def __del__(self): + self.shutdown() + + +class ExecutorAsyncBase(ExecutorBase): + + @abstractmethod + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + """Executes one model step on the given sequences.""" + raise NotImplementedError + + async def stop_remote_worker_execution_loop_async(self) -> None: + """Releases parallel workers from model loop.""" + return + + async def check_health_async(self) -> None: + """Checks if the executor is healthy. If not, it should raise an + exception.""" + self.check_health() diff --git a/vllm/executor/gpu_executor.py b/vllm/executor/gpu_executor.py new file mode 100644 index 00000000..ed30d318 --- /dev/null +++ b/vllm/executor/gpu_executor.py @@ -0,0 +1,191 @@ +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union + +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import ExecuteModelRequest, PoolerOutput +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port, + make_async) +from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase + +logger = init_logger(__name__) + + +def create_worker(worker_module_name: str, worker_class_name: str, + worker_class_fn: Optional[Callable[[], Type[WorkerBase]]], + **kwargs): + wrapper = WorkerWrapperBase( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + worker_class_fn=worker_class_fn, + ) + wrapper.init_worker(**kwargs) + return wrapper.worker + + +class GPUExecutor(ExecutorBase): + + uses_ray: bool = False + + def _init_executor(self) -> None: + """Initialize the worker and load the model. + """ + assert self.parallel_config.world_size == 1, ( + "GPUExecutor only supports single GPU.") + + self.driver_worker = self._create_worker() + self.driver_worker.init_device() + self.driver_worker.load_model() + + def _get_worker_kwargs( + self, + local_rank: int = 0, + rank: int = 0, + distributed_init_method: Optional[str] = None) -> Dict[str, Any]: + """Return worker init args for a given rank.""" + if distributed_init_method is None: + distributed_init_method = get_distributed_init_method( + get_ip(), get_open_port()) + return dict( + model_config=self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + device_config=self.device_config, + cache_config=self.cache_config, + load_config=self.load_config, + local_rank=local_rank, + rank=rank, + distributed_init_method=distributed_init_method, + lora_config=self.lora_config, + speculative_config=self.speculative_config, + prompt_adapter_config=self.prompt_adapter_config, + is_driver_worker=(not self.parallel_config) + or (rank % self.parallel_config.tensor_parallel_size == 0), + observability_config=self.observability_config, + ) + + def _get_worker_module_and_class( + self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]: + worker_class_fn = None + if self.scheduler_config.is_multi_step: + worker_module_name = "vllm.worker.multi_step_worker" + worker_class_name = "MultiStepWorker" + elif self.speculative_config: + worker_module_name = "vllm.spec_decode.spec_decode_worker" + worker_class_name = "create_spec_worker" + else: + worker_module_name = "vllm.worker.worker" + worker_class_name = "Worker" + return (worker_module_name, worker_class_name, worker_class_fn) + + def _get_create_worker_kwargs( + self, + local_rank: int = 0, + rank: int = 0, + distributed_init_method: Optional[str] = None) -> Dict: + worker_kwargs = self._get_worker_kwargs(local_rank, rank, + distributed_init_method) + + (worker_module_name, worker_class_name, + worker_class_fn) = self._get_worker_module_and_class() + worker_kwargs.update( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + worker_class_fn=worker_class_fn, + ) + + return worker_kwargs + + def _create_worker(self, + local_rank: int = 0, + rank: int = 0, + distributed_init_method: Optional[str] = None): + return create_worker(**self._get_create_worker_kwargs( + local_rank=local_rank, + rank=rank, + distributed_init_method=distributed_init_method)) + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks by invoking the + underlying worker. + """ + return self.driver_worker.determine_num_available_blocks() + + def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None: + """Initialize the KV cache by invoking the underlying worker. + """ + # NOTE: This is logged in the executor because there can be >1 worker + # with other executors. We could log in the engine level, but work + # remains to abstract away the device for non-GPU configurations. + logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, + num_cpu_blocks) + max_concurrency = (num_gpu_blocks * self.cache_config.block_size / + self.model_config.max_model_len) + logger.info("Maximum concurrency for %s tokens per request: %.2fx", + self.model_config.max_model_len, max_concurrency) + + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + def execute_model( + self, execute_model_req: ExecuteModelRequest + ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]: + output = self.driver_worker.execute_model(execute_model_req) + return output + + def add_lora(self, lora_request: LoRARequest) -> bool: + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." + return self.driver_worker.add_lora(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + assert lora_id > 0, "lora_id must be greater than 0." + return self.driver_worker.remove_lora(lora_id) + + def pin_lora(self, lora_id: int) -> bool: + assert lora_id > 0, "lora_id must be greater than 0." + return self.driver_worker.pin_lora(lora_id) + + def list_loras(self) -> Set[int]: + return self.driver_worker.list_loras() + + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + assert prompt_adapter_request.prompt_adapter_id > 0, \ + "prompt_adapter_id must be greater than 0." + return self.driver_worker.add_prompt_adapter(prompt_adapter_request) + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + assert prompt_adapter_id > 0, \ + "prompt_adapter_id must be greater than 0." + return self.driver_worker.remove_prompt_adapter(prompt_adapter_id) + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + assert prompt_adapter_id > 0, \ + "prompt_adapter_id must be greater than 0." + return self.driver_worker.pin_prompt_adapter(prompt_adapter_id) + + def list_prompt_adapters(self) -> Set[int]: + return self.driver_worker.list_prompt_adapters() + + def check_health(self) -> None: + # GPUExecutor will always be healthy as long as + # it's running. + return + + def start_profile(self) -> None: + self.driver_worker.start_profile() + + def stop_profile(self) -> None: + self.driver_worker.stop_profile() + + +class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[Union[SamplerOutput, PoolerOutput]]: + output = await make_async(self.driver_worker.execute_model + )(execute_model_req=execute_model_req) + return output diff --git a/vllm/executor/msgspec_utils.py b/vllm/executor/msgspec_utils.py new file mode 100644 index 00000000..c467115f --- /dev/null +++ b/vllm/executor/msgspec_utils.py @@ -0,0 +1,27 @@ +from array import array +from typing import Any, Type + +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE + + +def encode_hook(obj: Any) -> Any: + """Custom msgspec enc hook that supports array types. + + See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder + """ + if isinstance(obj, array): + assert obj.typecode == VLLM_TOKEN_ID_ARRAY_TYPE, ( + f"vLLM array type should use '{VLLM_TOKEN_ID_ARRAY_TYPE}' type. " + f"Given array has a type code of {obj.typecode}.") + return obj.tobytes() + + +def decode_hook(type: Type, obj: Any) -> Any: + """Custom msgspec dec hook that supports array types. + + See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder + """ + if type is array: + deserialized = array(VLLM_TOKEN_ID_ARRAY_TYPE) + deserialized.frombytes(obj) + return deserialized diff --git a/vllm/executor/multiproc_gpu_executor.py b/vllm/executor/multiproc_gpu_executor.py new file mode 100644 index 00000000..2dbde778 --- /dev/null +++ b/vllm/executor/multiproc_gpu_executor.py @@ -0,0 +1,258 @@ +import asyncio +import os +from functools import partial +from typing import Any, List, Optional + +import torch + +from vllm.executor.distributed_gpu_executor import ( # yapf: disable + DistributedGPUExecutor, DistributedGPUExecutorAsync) +from vllm.executor.gpu_executor import create_worker +from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper, + ResultHandler, WorkerMonitor) +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.triton_utils import maybe_set_triton_cache_manager +from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless, + cuda_is_initialized, get_distributed_init_method, + get_open_port, get_vllm_instance_id, make_async, + update_environment_variables) + +logger = init_logger(__name__) + + +class MultiprocessingGPUExecutor(DistributedGPUExecutor): + """Python multiprocessing-based multi-GPU executor""" + + uses_ray: bool = False + + def _init_executor(self) -> None: + self._check_executor_parameters() + + # Create the parallel GPU workers. + world_size = self.parallel_config.world_size + tensor_parallel_size = self.parallel_config.tensor_parallel_size + + # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers + os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id() + + # Disable torch async compiling which won't work with daemonic processes + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" + + # Configure thread parallelism if OMP_NUM_THREADS isn't set + # + # Helps to avoid CPU contention. The default of spawning a thread per + # core combined with multiprocessing for each GPU can have a negative + # impact on performance. The contention is amplified when running in a + # container where CPU limits can cause throttling. + default_omp_num_threads = 1 + if "OMP_NUM_THREADS" not in os.environ and ( + current_parallelism := + torch.get_num_threads()) > default_omp_num_threads: + logger.warning( + "Reducing Torch parallelism from %d threads to %d to avoid " + "unnecessary CPU contention. Set OMP_NUM_THREADS in the " + "external environment to tune this value as needed.", + current_parallelism, default_omp_num_threads) + os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads) + torch.set_num_threads(default_omp_num_threads) + + # workaround for https://github.com/vllm-project/vllm/issues/6103 + if world_size > 1: + maybe_set_triton_cache_manager() + + # Multiprocessing-based executor does not support multi-node setting. + # Since it only works for single node, we can use the loopback address + # 127.0.0.1 for communication. + distributed_init_method = get_distributed_init_method( + "127.0.0.1", get_open_port()) + + self.workers: List[ProcessWorkerWrapper] = [] + # This is the list of workers that are rank 0 of each TP group EXCEPT + # global rank 0. These are the workers that will broadcast to the + # rest of the workers. + self.tp_driver_workers: List[ProcessWorkerWrapper] = [] + # This is the list of workers that are not drivers and not the first + # worker in a TP group. These are the workers that will be + # broadcasted to. + self.non_driver_workers: List[ProcessWorkerWrapper] = [] + + if world_size == 1: + self.worker_monitor = None + else: + result_handler = ResultHandler() + for rank in range(1, world_size): + worker = ProcessWorkerWrapper( + result_handler, + partial( + create_worker, + **self._get_create_worker_kwargs( + rank=rank, + local_rank=rank, + distributed_init_method=distributed_init_method, + ))) + self.workers.append(worker) + if rank % tensor_parallel_size == 0: + self.tp_driver_workers.append(worker) + else: + self.non_driver_workers.append(worker) + + self.worker_monitor = WorkerMonitor(self.workers, result_handler) + result_handler.start() + self.worker_monitor.start() + + # Set up signal handlers to shutdown the executor cleanly + # sometimes gc does not work well + + self.driver_worker = self._create_worker( + distributed_init_method=distributed_init_method) + self._run_workers("init_device") + self._run_workers("load_model", + max_concurrent_workers=self.parallel_config. + max_parallel_loading_workers) + + def _check_executor_parameters(self): + world_size = self.parallel_config.world_size + tensor_parallel_size = self.parallel_config.tensor_parallel_size + + # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers + if "CUDA_VISIBLE_DEVICES" not in os.environ: + update_environment_variables({ + "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size)))) + }) + + if (cuda_is_initialized() + and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"): + logger.warning("CUDA was previously initialized. We must use " + "the `spawn` multiprocessing start method. Setting " + "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.") + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + cuda_device_count = cuda_device_count_stateless() + # Use confusing message for more common TP-only case. + assert tensor_parallel_size <= cuda_device_count, ( + f"please set tensor_parallel_size ({tensor_parallel_size}) " + f"to less than max local gpu count ({cuda_device_count})") + + assert world_size <= cuda_device_count, ( + f"please ensure that world_size ({world_size}) " + f"is less than than max local gpu count ({cuda_device_count})") + + def shutdown(self): + if (worker_monitor := getattr(self, "worker_monitor", + None)) is not None: + worker_monitor.close() + + def _driver_execute_model( + self, execute_model_req: Optional[ExecuteModelRequest] + ) -> Optional[List[SamplerOutput]]: + """Run execute_model in the driver worker. + + Passing None will cause the driver to stop the model execution + loop running in each of the remote workers. + """ + return self.driver_worker.execute_model(execute_model_req) + + def _run_workers( + self, + method: str, + *args, + async_run_tensor_parallel_workers_only: bool = False, + max_concurrent_workers: Optional[int] = None, + **kwargs, + ) -> Any: + """Runs the given method on all workers. + + Args: + async_run_tensor_parallel_workers_only: If True the method will be + run only in the remote TP workers, not the driver worker. + It will also be run asynchronously and return a list of futures + rather than blocking on the results. + """ + + if max_concurrent_workers: + raise NotImplementedError( + "max_concurrent_workers is not supported yet.") + + if async_run_tensor_parallel_workers_only: + # Run only non-driver workers and just return futures. + return [ + worker.execute_method(method, *args, **kwargs) + for worker in self.non_driver_workers + ] + + # Start all remote workers first. + worker_outputs = [ + worker.execute_method(method, *args, **kwargs) + for worker in self.workers + ] + + driver_worker_method = getattr(self.driver_worker, method) + driver_worker_output = driver_worker_method(*args, **kwargs) + + # Get the results of the workers. + return [driver_worker_output + ] + [output.get() for output in worker_outputs] + + def check_health(self) -> None: + """Raises an error if engine is unhealthy.""" + if self.worker_monitor is not None and not self.worker_monitor.is_alive( + ): + raise RuntimeError("Worker processes are not running") + + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None: + """Wait for futures returned from _run_workers() with + async_run_remote_workers_only to complete.""" + for result in parallel_worker_tasks: + result.get() + + +class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor, + DistributedGPUExecutorAsync): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.driver_exec_model = make_async(self.driver_worker.execute_model) + self.pp_locks: Optional[List[asyncio.Lock]] = None + + async def _driver_execute_model_async( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + if not self.tp_driver_workers: + return await self.driver_exec_model(execute_model_req) + + if self.pp_locks is None: + # This locks each pipeline parallel stage so multiple virtual + # engines can't execute on the same stage at the same time + # We create the locks here to avoid creating them in the constructor + # which uses a different asyncio loop. + self.pp_locks = [ + asyncio.Lock() + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + + tasks = [ + asyncio.create_task( + _run_task_with_lock(self.driver_exec_model, self.pp_locks[0], + execute_model_req)) + ] + for pp_rank, driver_worker in enumerate(self.tp_driver_workers, + start=1): + tasks.append( + asyncio.create_task( + _run_task_with_lock(driver_worker.execute_method_async, + self.pp_locks[pp_rank], + "execute_model", execute_model_req))) + results = await asyncio.gather(*tasks) + + # Only the last PP stage has the final results. + return results[-1] + + async def _start_worker_execution_loop(self): + coros = [ + worker.execute_method_async("start_worker_execution_loop") + for worker in self.non_driver_workers + ] + return await asyncio.gather(*coros) diff --git a/vllm/executor/multiproc_worker_utils.py b/vllm/executor/multiproc_worker_utils.py new file mode 100644 index 00000000..e14ecc13 --- /dev/null +++ b/vllm/executor/multiproc_worker_utils.py @@ -0,0 +1,274 @@ +import asyncio +import multiprocessing +import os +import sys +import threading +import traceback +import uuid +from dataclasses import dataclass +from multiprocessing import Queue +from multiprocessing.connection import wait +from multiprocessing.process import BaseProcess +from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO, + TypeVar, Union) + +import vllm.envs as envs +from vllm.logger import init_logger + +logger = init_logger(__name__) + +T = TypeVar('T') + +_TERMINATE = "TERMINATE" # sentinel + +# ANSI color codes +CYAN = '\033[1;36m' +RESET = '\033[0;0m' + +JOIN_TIMEOUT_S = 2 + + +@dataclass +class Result(Generic[T]): + """Result of task dispatched to worker""" + + task_id: uuid.UUID + value: Optional[T] = None + exception: Optional[BaseException] = None + + +class ResultFuture(threading.Event, Generic[T]): + """Synchronous future for non-async case""" + + def __init__(self): + super().__init__() + self.result: Optional[Result[T]] = None + + def set_result(self, result: Result[T]): + self.result = result + self.set() + + def get(self) -> T: + self.wait() + assert self.result is not None + if self.result.exception is not None: + raise self.result.exception + return self.result.value # type: ignore[return-value] + + +def _set_future_result(future: Union[ResultFuture, asyncio.Future], + result: Result): + if isinstance(future, ResultFuture): + future.set_result(result) + return + loop = future.get_loop() + if not loop.is_closed(): + if result.exception is not None: + loop.call_soon_threadsafe(future.set_exception, result.exception) + else: + loop.call_soon_threadsafe(future.set_result, result.value) + + +class ResultHandler(threading.Thread): + """Handle results from all workers (in background thread)""" + + def __init__(self) -> None: + super().__init__(daemon=True) + self.result_queue = get_mp_context().Queue() + self.tasks: Dict[uuid.UUID, Union[ResultFuture, asyncio.Future]] = {} + + def run(self): + for result in iter(self.result_queue.get, _TERMINATE): + future = self.tasks.pop(result.task_id) + _set_future_result(future, result) + # Ensure that all waiters will receive an exception + for task_id, future in self.tasks.items(): + _set_future_result( + future, + Result(task_id=task_id, + exception=ChildProcessError("worker died"))) + + def close(self): + self.result_queue.put(_TERMINATE) + + +class WorkerMonitor(threading.Thread): + """Monitor worker status (in background thread)""" + + def __init__(self, workers: List['ProcessWorkerWrapper'], + result_handler: ResultHandler): + super().__init__(daemon=True) + self.workers = workers + self.result_handler = result_handler + self._close = False + + def run(self) -> None: + # Blocks until any worker exits + dead_sentinels = wait([w.process.sentinel for w in self.workers]) + if not self._close: + self._close = True + + # Kill / cleanup all workers + for worker in self.workers: + process = worker.process + if process.sentinel in dead_sentinels: + process.join(JOIN_TIMEOUT_S) + if process.exitcode is not None and process.exitcode != 0: + logger.error("Worker %s pid %s died, exit code: %s", + process.name, process.pid, process.exitcode) + # Cleanup any remaining workers + if logger: + logger.info("Killing local vLLM worker processes") + for worker in self.workers: + worker.kill_worker() + # Must be done after worker task queues are all closed + self.result_handler.close() + + for worker in self.workers: + worker.process.join(JOIN_TIMEOUT_S) + + def close(self): + if self._close: + return + self._close = True + logger.info("Terminating local vLLM worker processes") + for worker in self.workers: + worker.terminate_worker() + # Must be done after worker task queues are all closed + self.result_handler.close() + + +class ProcessWorkerWrapper: + """Local process wrapper for vllm.worker.Worker, + for handling single-node multi-GPU tensor parallel.""" + + def __init__(self, result_handler: ResultHandler, + worker_factory: Callable[[], Any]) -> None: + self.mp = get_mp_context() + self._task_queue = self.mp.Queue() + self.result_queue = result_handler.result_queue + self.tasks = result_handler.tasks + self.process: BaseProcess = self.mp.Process( # type: ignore[attr-defined] + target=_run_worker_process, + name="VllmWorkerProcess", + kwargs=dict( + worker_factory=worker_factory, + task_queue=self._task_queue, + result_queue=self.result_queue, + ), + daemon=True) + + self.process.start() + + def _enqueue_task(self, future: Union[ResultFuture, asyncio.Future], + method: str, args, kwargs): + task_id = uuid.uuid4() + self.tasks[task_id] = future + try: + self._task_queue.put((task_id, method, args, kwargs)) + except SystemExit: + raise + except BaseException as e: + del self.tasks[task_id] + raise ChildProcessError("worker died") from e + + def execute_method(self, method: str, *args, **kwargs): + future: ResultFuture = ResultFuture() + self._enqueue_task(future, method, args, kwargs) + return future + + async def execute_method_async(self, method: str, *args, **kwargs): + future = asyncio.get_running_loop().create_future() + self._enqueue_task(future, method, args, kwargs) + return await future + + def terminate_worker(self): + try: + self._task_queue.put(_TERMINATE) + except ValueError: + self.process.kill() + self._task_queue.close() + + def kill_worker(self): + self._task_queue.close() + self.process.kill() + + +def _run_worker_process( + worker_factory: Callable[[], Any], + task_queue: Queue, + result_queue: Queue, +) -> None: + """Worker process event loop""" + + # Add process-specific prefix to stdout and stderr + process_name = get_mp_context().current_process().name + pid = os.getpid() + _add_prefix(sys.stdout, process_name, pid) + _add_prefix(sys.stderr, process_name, pid) + + # Initialize worker + worker = worker_factory() + del worker_factory + + # Accept tasks from the engine in task_queue + # and return task output in result_queue + logger.info("Worker ready; awaiting tasks") + try: + for items in iter(task_queue.get, _TERMINATE): + output = None + exception = None + task_id, method, args, kwargs = items + try: + executor = getattr(worker, method) + output = executor(*args, **kwargs) + except SystemExit: + raise + except KeyboardInterrupt: + break + except BaseException as e: + tb = traceback.format_exc() + logger.error( + "Exception in worker %s while processing method %s: %s, %s", + process_name, method, e, tb) + exception = e + result_queue.put( + Result(task_id=task_id, value=output, exception=exception)) + except KeyboardInterrupt: + pass + except Exception: + logger.exception("Worker failed") + + logger.info("Worker exiting") + + +def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None: + """Prepend each output line with process-specific prefix""" + + prefix = f"{CYAN}({worker_name} pid={pid}){RESET} " + file_write = file.write + + def write_with_prefix(s: str): + if not s: + return + if file.start_new_line: # type: ignore[attr-defined] + file_write(prefix) + idx = 0 + while (next_idx := s.find('\n', idx)) != -1: + next_idx += 1 + file_write(s[idx:next_idx]) + if next_idx == len(s): + file.start_new_line = True # type: ignore[attr-defined] + return + file_write(prefix) + idx = next_idx + file_write(s[idx:]) + file.start_new_line = False # type: ignore[attr-defined] + + file.start_new_line = True # type: ignore[attr-defined] + file.write = write_with_prefix # type: ignore[method-assign] + + +def get_mp_context(): + mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD + return multiprocessing.get_context(mp_method) diff --git a/vllm/executor/multiproc_xpu_executor.py b/vllm/executor/multiproc_xpu_executor.py new file mode 100644 index 00000000..a66afbf9 --- /dev/null +++ b/vllm/executor/multiproc_xpu_executor.py @@ -0,0 +1,26 @@ +import vllm.envs as envs +from vllm.executor.multiproc_gpu_executor import ( + MultiprocessingGPUExecutor, MultiprocessingGPUExecutorAsync) +from vllm.executor.xpu_executor import XPUExecutor +from vllm.logger import init_logger +from vllm.utils import make_async + +logger = init_logger(__name__) + + +class MultiprocessingXPUExecutor(MultiprocessingGPUExecutor, XPUExecutor): + """Python multiprocessing-based multi-XPU executor""" + + def _check_executor_parameters(self): + mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD + if mp_method != "spawn": + raise RuntimeError( + "XPU multiprocess executor only support spawn as mp method") + + +class MultiprocessingXPUExecutorAsync(MultiprocessingXPUExecutor, + MultiprocessingGPUExecutorAsync): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.driver_exec_model = make_async(self.driver_worker.execute_model) diff --git a/vllm/executor/neuron_executor.py b/vllm/executor/neuron_executor.py new file mode 100644 index 00000000..f2fcfa58 --- /dev/null +++ b/vllm/executor/neuron_executor.py @@ -0,0 +1,115 @@ +from typing import List, Set, Tuple + +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port, + make_async) + +logger = init_logger(__name__) + + +class NeuronExecutor(ExecutorBase): + + uses_ray: bool = False + + def _init_executor(self) -> None: + assert (self.lora_config is + None), "LoRA is not supported for Neuron backend." + assert (not self.speculative_config + ), "Speculative decoding not yet supported for Neuron backend." + + # Instantiate the worker and load the model to the device. + self._init_worker() + + def _init_worker(self): + from vllm.worker.neuron_worker import NeuronWorker + distributed_init_method = get_distributed_init_method( + get_ip(), get_open_port()) + self.driver_worker = NeuronWorker( + model_config=self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + device_config=self.device_config, + cache_config=self.cache_config, + local_rank=0, + rank=0, + distributed_init_method=distributed_init_method) + self.driver_worker.init_device() + self.driver_worker.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks by invoking the + underlying worker. + """ + return self.driver_worker.determine_num_available_blocks() + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache by invoking the underlying worker. + """ + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + assert (not execute_model_req.blocks_to_swap_in + and not execute_model_req.blocks_to_swap_out + and not execute_model_req.blocks_to_copy), ( + "Cache operations are not supported for Neuron backend.") + assert execute_model_req.num_lookahead_slots == 0, ( + "lookahead not supported for Neuron backend.") + + output = self.driver_worker.execute_model(execute_model_req) + return output + + def add_lora(self, lora_request: LoRARequest) -> bool: + return self.driver_worker.add_lora(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + return self.driver_worker.remove_lora(lora_id) + + def pin_lora(self, lora_id: int) -> bool: + return self.driver_worker.pin_lora(lora_id) + + def list_loras(self) -> Set[int]: + return self.driver_worker.list_loras() + + def add_prompt_adapter(self, prompt_adapter_request) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the Neuron backend.") + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the Neuron backend.") + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the Neuron backend.") + + def list_prompt_adapters(self) -> Set[int]: + raise NotImplementedError( + "Soft prompt is currently not supported by the Neuron backend.") + + def check_health(self) -> None: + # NeuronExecutor will always be healthy as long as + # it's running. + return + + +class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[SamplerOutput]: + output = await make_async(self.driver_worker.execute_model + )(execute_model_req=execute_model_req, ) + return output + + async def check_health_async(self) -> None: + # NeuronExecutor will always be healthy as long as + # it's running. + return diff --git a/vllm/executor/openvino_executor.py b/vllm/executor/openvino_executor.py new file mode 100644 index 00000000..4a39839a --- /dev/null +++ b/vllm/executor/openvino_executor.py @@ -0,0 +1,213 @@ +from typing import List, Set, Tuple + +import openvino as ov +import openvino.properties.hint as hints +import torch + +import vllm.envs as envs +from vllm.config import CacheConfig, ModelConfig +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip, + get_open_port, make_async) + +logger = init_logger(__name__) + + +def is_openvino_cpu() -> bool: + return "CPU" in envs.VLLM_OPENVINO_DEVICE + + +def is_openvino_gpu() -> bool: + return "GPU" in envs.VLLM_OPENVINO_DEVICE + + +class OpenVINOExecutor(ExecutorBase): + + uses_ray: bool = False + + def _init_executor(self) -> None: + assert self.device_config.device_type == "openvino" + assert self.lora_config is None, "OpenVINO backend doesn't support LoRA" + assert is_openvino_cpu() or is_openvino_gpu(), \ + "OpenVINO backend supports only CPU and GPU devices" + + self.ov_core = ov.Core() + self.model_config = _verify_and_get_model_config(self.model_config) + self.cache_config = _verify_and_get_cache_config( + self.ov_core, self.cache_config) + + # Instantiate the worker and load the model to CPU. + self._init_worker() + + def _init_worker(self): + from vllm.worker.openvino_worker import OpenVINOWorker + + assert ( + self.parallel_config.world_size == 1 + ), "OpenVINOExecutor only supports single CPU socket currently." + + distributed_init_method = get_distributed_init_method( + get_ip(), get_open_port()) + self.driver_worker = OpenVINOWorker( + ov_core=self.ov_core, + model_config=self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + device_config=self.device_config, + cache_config=self.cache_config, + load_config=self.load_config, + local_rank=0, + rank=0, + distributed_init_method=distributed_init_method, + lora_config=self.lora_config, + kv_cache_dtype=self.cache_config.cache_dtype, + is_driver_worker=True, + ) + self.driver_worker.init_device() + self.driver_worker.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks by invoking the + underlying worker. + """ + return self.driver_worker.determine_num_available_blocks() + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache by invoking the underlying worker.""" + # NOTE: We log here to avoid multiple logs when number of workers is + # greater than one. We could log in the engine, but not all executors + # have GPUs. + # NOTE: In case of a CPU device, `cpu block` for OpenVINO backend + # is located on CPU memory but is referred as `gpu block`. + # Because we want to reuse the existing block management procedure. + device_blocks = num_gpu_blocks + swap_blocks = num_cpu_blocks + logger.info("OpenVINO %s: # device blocks: %d; # swap blocks: %d", + envs.VLLM_OPENVINO_DEVICE, device_blocks, swap_blocks) + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + output = self.driver_worker.execute_model(execute_model_req) + return output + + def add_lora(self, lora_request: LoRARequest) -> bool: + return self.driver_worker.add_lora(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + return self.driver_worker.remove_lora(lora_id) + + def pin_lora(self, lora_id: int) -> bool: + return self.driver_worker.pin_lora(lora_id) + + def list_loras(self) -> Set[int]: + return self.driver_worker.list_loras() + + def add_prompt_adapter(self, prompt_adapter_request) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the OPENVINO backend.") + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the OPENVINO backend.") + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the OPENVINO backend.") + + def list_prompt_adapters(self) -> Set[int]: + raise NotImplementedError( + "Soft prompt is currently not supported by the OPENVINO backend.") + + def check_health(self) -> None: + # OpenVINOExecutor will always be healthy as long as + # it's running. + return + + +class OpenVINOExecutorAsync(OpenVINOExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + output = await make_async(self.driver_worker.execute_model + )(execute_model_req=execute_model_req, ) + return output + + async def check_health_async(self) -> None: + # OpenVINOExecutor will always be healthy as long as + # it's running. + return + + +def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig: + if config.dtype != torch.float32: + logger.warning( + f"Only float32 dtype is supported on OpenVINO, casting from {config.dtype}." # noqa: G004, E501 + ) + config.dtype = torch.float32 + if not config.enforce_eager: + logger.warning( + "CUDA graph is not supported on OpenVINO backend, fallback to the " + "eager mode.") + config.enforce_eager = True + return config + + +def _verify_and_get_cache_config(ov_core: ov.Core, + config: CacheConfig) -> CacheConfig: + if envs.VLLM_OPENVINO_CPU_KV_CACHE_PRECISION == "u8": + if not is_openvino_cpu(): + logger.info("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION is" + "ignored for GPU, f16 data type will be used.") + config.cache_dtype = ov.Type.f16 + else: + logger.info("KV cache type is overridden to u8 via " + "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION env var.") + config.cache_dtype = ov.Type.u8 + else: + if is_openvino_cpu(): + ov_device = envs.VLLM_OPENVINO_DEVICE + inference_precision = ov_core.get_property( + ov_device, hints.inference_precision) + if inference_precision == ov.Type.bf16: + config.cache_dtype = ov.Type.bf16 + else: + config.cache_dtype = ov.Type.f16 + else: + config.cache_dtype = ov.Type.f16 + + if is_openvino_cpu(): + if config.block_size != 32: + logger.info( + f"OpenVINO CPU optimal block size is 32, overriding currently set {config.block_size}" # noqa: G004, E501 + ) + config.block_size = 32 + else: + if config.block_size != 16: + logger.info( + f"OpenVINO GPU optimal block size is 16, overriding currently set {config.block_size}" # noqa: G004, E501 + ) + config.block_size = 16 + + kv_cache_space = envs.VLLM_OPENVINO_KVCACHE_SPACE + if kv_cache_space >= 0: + if kv_cache_space == 0 and is_openvino_cpu(): + config.openvino_kvcache_space_bytes = 4 * GiB_bytes # type: ignore + logger.warning( + "Environment variable VLLM_OPENVINO_KVCACHE_SPACE (GB) " + "for OpenVINO backend is not set, using 4 by default.") + else: + config.openvino_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore + else: + raise RuntimeError( + "Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE" + f" {kv_cache_space}, expect a positive integer value.") + + return config diff --git a/vllm/executor/ray_gpu_executor.py b/vllm/executor/ray_gpu_executor.py new file mode 100644 index 00000000..be2948a2 --- /dev/null +++ b/vllm/executor/ray_gpu_executor.py @@ -0,0 +1,586 @@ +import asyncio +import os +from collections import defaultdict +from itertools import islice, repeat +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import msgspec + +import vllm.envs as envs +from vllm.executor.distributed_gpu_executor import ( # yapf: disable + DistributedGPUExecutor, DistributedGPUExecutorAsync) +from vllm.executor.msgspec_utils import encode_hook +from vllm.executor.ray_utils import RayWorkerWrapper, ray +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (_run_task_with_lock, get_distributed_init_method, + get_ip, get_open_port, get_vllm_instance_id, + make_async) + +if ray is not None: + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +if TYPE_CHECKING: + from ray.util.placement_group import PlacementGroup + +logger = init_logger(__name__) + + +class RayGPUExecutor(DistributedGPUExecutor): + + uses_ray: bool = True + + def _init_executor(self) -> None: + self.forward_dag: Optional["ray.dag.CompiledDAG"] = None + # If the env var is set, it uses the Ray's compiled DAG API + # which optimizes the control plane overhead. + # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it. + # Currently, this requires USE_RAY_SPMD_WORKER=True. + self.use_ray_compiled_dag = envs.VLLM_USE_RAY_COMPILED_DAG + # If the env var is set, then we do not distinguish between the + # "driver worker" vs other workers. Also, the rank 0 worker will + # be executed in a remote Ray worker. Currently this requires + # USE_RAY_COMPILED_DAG=True. + self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER + if self.use_ray_compiled_dag: + assert self.use_ray_spmd_worker, ( + "VLLM_USE_RAY_COMPILED_DAG=1 requires " + "VLLM_USE_RAY_SPMD_WORKER=1") + if self.use_ray_spmd_worker: + # TODO: Support SPMD worker for non-DAG Ray executor. + assert self.use_ray_compiled_dag, ( + "VLLM_USE_RAY_SPMD_WORKER=1 requires " + "VLLM_USE_RAY_COMPILED_DAG=1") + + assert self.uses_ray + placement_group = self.parallel_config.placement_group + + # Disable Ray usage stats collection. + ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0") + if ray_usage != "1": + os.environ["RAY_USAGE_STATS_ENABLED"] = "0" + + # Create the parallel GPU workers. + self._init_workers_ray(placement_group) + + self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook) + self.output_decoder = msgspec.msgpack.Decoder( + Optional[List[SamplerOutput]]) + + def shutdown(self) -> None: + if hasattr(self, "forward_dag") and self.forward_dag is not None: + self.forward_dag.teardown() + import ray + for worker in self.workers: + ray.kill(worker) + self.forward_dag = None + + def _configure_ray_workers_use_nsight(self, + ray_remote_kwargs) -> Dict[str, Any]: + # If nsight profiling is enabled, we need to set the profiling + # configuration for the ray workers as runtime env. + runtime_env = ray_remote_kwargs.setdefault("runtime_env", {}) + runtime_env.update({ + "nsight": { + "t": "cuda,cudnn,cublas", + "o": "'worker_process_%p'", + "cuda-graph-trace": "node", + } + }) + + return ray_remote_kwargs + + def _get_worker_wrapper_args(self) -> Dict[str, Any]: + (worker_module_name, worker_class_name, + worker_class_fn) = self._get_worker_module_and_class() + + return dict( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + worker_class_fn=worker_class_fn, + trust_remote_code=self.model_config.trust_remote_code, + ) + + # child class could overwrite this to return actual env vars. + def _get_env_vars_to_be_updated(self): + return self._env_vars_for_all_workers + + def _init_workers_ray(self, placement_group: "PlacementGroup", + **ray_remote_kwargs): + if (self.parallel_config.tensor_parallel_size == 1 + and self.parallel_config.pipeline_parallel_size == 1): + # For single GPU case, we use a ray worker with constrained memory. + num_gpus = self.cache_config.gpu_memory_utilization + else: + # Otherwise, the ray workers are allocated with a full GPU. + num_gpus = 1 + + # The driver dummy worker does not actually use any resources. + # It holds the resource for the driver worker. + self.driver_dummy_worker: Optional[RayWorkerWrapper] = None + # The remaining workers are the actual ray actors. + self.workers: List[RayWorkerWrapper] = [] + + # Used in ray compiled DAG: indexed first by PP rank, + # and then TP rank. In other words, the inner list is + # the TP group of workers for a PP rank. + self.pp_tp_workers: List[List[RayWorkerWrapper]] = [] + + if self.parallel_config.ray_workers_use_nsight: + ray_remote_kwargs = self._configure_ray_workers_use_nsight( + ray_remote_kwargs) + + logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker) + + # Create the workers. + driver_ip = get_ip() + worker_wrapper_kwargs = self._get_worker_wrapper_args() + # vllm_multi_node_nccl_id = os.environ.get("VLLM_MULTI_NODE_NCCL_COMM_ID",None) + nccl_socket_name = os.environ.get("NCCL_SOCKET_IFNAME",None) + runtime_env={} + if nccl_socket_name is not None: + runtime_env["env_vars"] = {"NCCL_SOCKET_IFNAME" :f"{nccl_socket_name}"} + for bundle_id, bundle in enumerate(placement_group.bundle_specs): + if not bundle.get("GPU", 0): + continue + scheduling_strategy = PlacementGroupSchedulingStrategy( + placement_group=placement_group, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=bundle_id, + ) + + worker = ray.remote( + num_cpus=0, + num_gpus=num_gpus, + scheduling_strategy=scheduling_strategy, + runtime_env=runtime_env, + **ray_remote_kwargs, + )(RayWorkerWrapper).remote(**worker_wrapper_kwargs) + + if self.use_ray_spmd_worker: + self.workers.append(worker) + else: + worker_ip = ray.get(worker.get_node_ip.remote()) + if worker_ip == driver_ip and self.driver_dummy_worker is None: + # If the worker is on the same node as the driver, we use it + # as the resource holder for the driver process. + self.driver_dummy_worker = worker + self.driver_worker = RayWorkerWrapper( + **worker_wrapper_kwargs) + else: + # Else, added to the list of workers. + self.workers.append(worker) + + logger.debug("workers: %s", self.workers) + logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker) + if not self.use_ray_spmd_worker and self.driver_dummy_worker is None: + raise ValueError( + "Ray does not allocate any GPUs on the driver node. Consider " + "adjusting the Ray placement group or running the driver on a " + "GPU node.") + + worker_ips = [ + ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined] + for worker in self.workers + ] + ip_counts: Dict[str, int] = {} + for ip in worker_ips: + ip_counts[ip] = ip_counts.get(ip, 0) + 1 + + def sort_by_driver_then_worker_ip(worker): + """ + Sort the workers based on 3 properties: + 1. If the worker is on the same node as the driver (vllm engine), + it should be placed first. + 2. Then, if the worker is on a node with fewer workers, it should + be placed first. + 3. Finally, if the work is on a node with smaller IP address, it + should be placed first. + """ + ip = ray.get(worker.get_node_ip.remote()) + return (ip != driver_ip, ip_counts[ip], ip) + + # After sorting, the workers on the same node will be + # close to each other, and the workers on the driver + # node will be placed first. + self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip) + + # Get the set of GPU IDs used on each node. + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids", + use_dummy_driver=True) + + node_workers = defaultdict(list) # node id -> list of worker ranks + node_gpus = defaultdict(list) # node id -> list of gpu ids + + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids): + node_workers[node_id].append(i) + # `gpu_ids` can be a list of strings or integers. + # convert them to integers for consistency. + # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs), + # string sorting is not sufficient. + # see https://github.com/vllm-project/vllm/issues/5590 + gpu_ids = [int(x) for x in gpu_ids] + node_gpus[node_id].extend(gpu_ids) + for node_id, gpu_ids in node_gpus.items(): + node_gpus[node_id] = sorted(gpu_ids) + + all_ips = set(worker_ips + [driver_ip]) + n_ips = len(all_ips) + n_nodes = len(node_workers) + + if n_nodes != n_ips: + raise RuntimeError( + f"Every node should have a unique IP address. Got {n_nodes}" + f" nodes with node ids {list(node_workers.keys())} and " + f"{n_ips} unique IP addresses {all_ips}. Please check your" + " network configuration. If you set `VLLM_HOST_IP` or " + "`HOST_IP` environment variable, make sure it is unique for" + " each node.") + + VLLM_INSTANCE_ID = get_vllm_instance_id() + + # Set environment variables for the driver and workers. + all_args_to_update_environment_variables = [({ + "CUDA_VISIBLE_DEVICES": + ",".join(map(str, node_gpus[node_id])), + "VLLM_INSTANCE_ID": + VLLM_INSTANCE_ID, + "VLLM_TRACE_FUNCTION": + str(envs.VLLM_TRACE_FUNCTION), + **({ + "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND + } if envs.VLLM_ATTENTION_BACKEND is not None else {}) + }, ) for (node_id, _) in worker_node_and_gpu_ids] + + self._env_vars_for_all_workers = ( + all_args_to_update_environment_variables) + + self._run_workers("update_environment_variables", + all_args=self._get_env_vars_to_be_updated()) + + if len(node_gpus) == 1: + # in single node case, we don't need to get the IP address. + # the loopback address is sufficient + # NOTE: a node may have several IP addresses, one for each + # network interface. `get_ip()` might return any of them, + # while they might not work for communication inside the node + # if the network setup is complicated. Using the loopback address + # solves this issue, as it always works for communication inside + # the node. + driver_ip = "127.0.0.1" + distributed_init_method = get_distributed_init_method( + driver_ip, get_open_port()) + + # Initialize the actual workers inside worker wrapper. + init_worker_all_kwargs = [ + self._get_worker_kwargs( + local_rank=node_workers[node_id].index(rank), + rank=rank, + distributed_init_method=distributed_init_method, + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids) + ] + self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs) + + self._run_workers("init_device") + self._run_workers("load_model", + max_concurrent_workers=self.parallel_config. + max_parallel_loading_workers) + + if self.use_ray_spmd_worker: + for pp_rank in range(self.parallel_config.pipeline_parallel_size): + self.pp_tp_workers.append([]) + for tp_rank in range( + self.parallel_config.tensor_parallel_size): + # PP=2, TP=4 + # pp_tp_workers = [[0, 1, 2, 3], [4, 5, 6, 7]] + rank = (pp_rank * self.parallel_config.tensor_parallel_size + ) + tp_rank + assert len(self.pp_tp_workers[pp_rank]) == tp_rank + assert pp_rank < len(self.pp_tp_workers) + self.pp_tp_workers[pp_rank].append(self.workers[rank]) + + # This is the list of workers that are rank 0 of each TP group EXCEPT + # global rank 0. These are the workers that will broadcast to the + # rest of the workers. + self.tp_driver_workers: List[RayWorkerWrapper] = [] + # This is the list of workers that are not drivers and not the first + # worker in a TP group. These are the workers that will be + # broadcasted to. + self.non_driver_workers: List[RayWorkerWrapper] = [] + + # Enforce rank order for correct rank to return final output. + for index, worker in enumerate(self.workers): + # The driver worker is rank 0 and not in self.workers. + rank = index + 1 + if rank % self.parallel_config.tensor_parallel_size == 0: + self.tp_driver_workers.append(worker) + else: + self.non_driver_workers.append(worker) + + def _driver_execute_model( + self, execute_model_req: Optional[ExecuteModelRequest] + ) -> Optional[List[SamplerOutput]]: + """Run execute_model in the driver worker. + + Passing None will cause the driver to stop the model execution + loop running in each of the remote workers. + """ + assert not self.use_ray_spmd_worker, ( + "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1") + return self.driver_worker.execute_method("execute_model", + execute_model_req) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + if not self.use_ray_spmd_worker: + return super().execute_model(execute_model_req) + + if self.forward_dag is None: + self.forward_dag = self._compiled_ray_dag(enable_asyncio=False) + + serialized_data = self.input_encoder.encode(execute_model_req) + outputs = ray.get(self.forward_dag.execute(serialized_data)) + output = self.output_decoder.decode(outputs[0]) + return output + + def _run_workers( + self, + method: str, + *args, + async_run_tensor_parallel_workers_only: bool = False, + all_args: Optional[List[Tuple[Any, ...]]] = None, + all_kwargs: Optional[List[Dict[str, Any]]] = None, + use_dummy_driver: bool = False, + max_concurrent_workers: Optional[int] = None, + **kwargs, + ) -> Any: + """Runs the given method on all workers. Can be used in the following + ways: + + Args: + - async_run_tensor_parallel_workers_only: If True the method will be + run only in the remote TP workers, not the driver worker. + It will also be run asynchronously and return a list of futures + rather than blocking on the results. + - args/kwargs: All workers share the same args/kwargs + - all_args/all_kwargs: args/kwargs for each worker are specified + individually + """ + if self.use_ray_spmd_worker: + assert not async_run_tensor_parallel_workers_only, ( + "async_run_tensor_parallel_workers_only is not supported for " + "spmd mode.") + + if max_concurrent_workers: + raise NotImplementedError( + "max_concurrent_workers is not supported yet.") + + count = len(self.workers) if not \ + async_run_tensor_parallel_workers_only \ + else len(self.non_driver_workers) + # If using SPMD worker, all workers are the same, so we should execute + # the args on all workers. Otherwise, we skip the first worker's args + # because those args will go to the driver worker. + first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1 + all_worker_args = repeat(args, count) if all_args is None \ + else islice(all_args, first_worker_args_index, None) + all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \ + else islice(all_kwargs, first_worker_args_index, None) + + # Start the ray workers first. + ray_workers = self.workers + if async_run_tensor_parallel_workers_only: + ray_workers = self.non_driver_workers + ray_worker_outputs = [ + worker.execute_method.remote(method, *worker_args, **worker_kwargs) + for (worker, worker_args, worker_kwargs + ) in zip(ray_workers, all_worker_args, all_worker_kwargs) + ] + + if async_run_tensor_parallel_workers_only: + # Just return futures + return ray_worker_outputs + + driver_worker_output = [] + # In SPMD mode, the driver worker is the same as any other worker, + # so we only explicitly execute on the driver worker if using a + # non-SPMD worker class. + if not self.use_ray_spmd_worker: + driver_args = args if all_args is None else all_args[0] + driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0] + + # Start the driver worker after all the ray workers. + if not use_dummy_driver: + driver_worker_output = [ + self.driver_worker.execute_method(method, *driver_args, + **driver_kwargs) + ] + else: + assert self.driver_dummy_worker is not None + driver_worker_output = [ + ray.get( + self.driver_dummy_worker.execute_method.remote( + method, *driver_args, **driver_kwargs)) + ] + + # Get the results of the ray workers. + if self.workers: + ray_worker_outputs = ray.get(ray_worker_outputs) + + return driver_worker_output + ray_worker_outputs + + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None: + """Wait for futures returned from _run_workers() with + async_run_remote_workers_only to complete.""" + ray.get(parallel_worker_tasks) + + def _check_ray_adag_installation(self): + import pkg_resources + from packaging import version + + required_version = version.parse("2.35") + current_version = version.parse( + pkg_resources.get_distribution("ray").version) + # TODO: update the constraint once we adapt to the backward + # incompatible API change from ray 2.36 + if current_version != required_version: + raise ValueError(f"Ray version {required_version} is " + f"required, but found {current_version}") + + import importlib.util + adag_spec = importlib.util.find_spec( + "ray.experimental.compiled_dag_ref") + if adag_spec is None: + raise ValueError("Ray accelerated DAG is not installed. " + "Run `pip install ray[adag]` to install it.") + + cupy_spec = importlib.util.find_spec("cupy") + if cupy_spec is None and envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: + raise ValueError( + "cupy is not installed but required since " + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL is set." + "Run `pip install ray[adag]` and check cupy installation.") + + def _compiled_ray_dag(self, enable_asyncio: bool): + assert self.parallel_config.use_ray + self._check_ray_adag_installation() + from ray.dag import InputNode, MultiOutputNode + from ray.experimental.channel.torch_tensor_type import TorchTensorType + + logger.info("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL = %s", + envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL) + with InputNode() as input_data: + # Example DAG: PP=2, TP=4 + # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501 + # -> 1 -> (ExecuteModelReq, IntermediateOutput) -> 5 -> SamplerOutput # noqa: E501 + # -> 2 -> (ExecuteModelReq, IntermediateOutput) -> 6 -> SamplerOutput # noqa: E501 + # -> 3 -> (ExecuteModelReq, IntermediateOutput) -> 7 -> SamplerOutput # noqa: E501 + + # All workers in the first TP group will take in the + # ExecuteModelRequest as input. + outputs = [input_data for _ in self.pp_tp_workers[0]] + for pp_rank, tp_group in enumerate(self.pp_tp_workers): + # Each PP worker takes in the output of the previous PP worker, + # and the TP group executes in SPMD fashion. + outputs = [ + worker.execute_model_spmd. + bind( # type: ignore[attr-defined] + outputs[i]) for i, worker in enumerate(tp_group) + ] + + last_pp_rank = len(self.pp_tp_workers) - 1 + if pp_rank < last_pp_rank: + # Specify how intermediate tensors should be passed + # between pp stages, no need to specify for the last + # pp stage. + transport = "nccl" \ + if envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL \ + else "auto" + outputs = [ + output.with_type_hint( + TorchTensorType(transport=transport)) + for output in outputs + ] + + forward_dag = MultiOutputNode(outputs) + + return forward_dag.experimental_compile(enable_asyncio=enable_asyncio) + + def __del__(self): + self.shutdown() + + +class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pp_locks: Optional[List[asyncio.Lock]] = None + self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER + if not self.use_ray_compiled_dag: + self.driver_exec_method = make_async( + self.driver_worker.execute_method) + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + if not self.use_ray_spmd_worker: + return await super().execute_model_async(execute_model_req) + + if self.forward_dag is None: + self.forward_dag = self._compiled_ray_dag(enable_asyncio=True) + + serialized_data = self.input_encoder.encode(execute_model_req) + dag_future = await self.forward_dag.execute_async(serialized_data) + outputs = await dag_future + return self.output_decoder.decode(outputs[0]) + + async def _driver_execute_model_async( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + assert not self.use_ray_spmd_worker, ( + "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1") + if not self.tp_driver_workers: + return await self.driver_exec_method("execute_model", + execute_model_req) + if self.pp_locks is None: + # This locks each pipeline parallel stage so multiple virtual + # engines can't execute on the same stage at the same time + # We create the locks here to avoid creating them in the constructor + # which uses a different asyncio loop. + self.pp_locks = [ + asyncio.Lock() + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + + tasks = [ + asyncio.create_task( + _run_task_with_lock(self.driver_exec_method, self.pp_locks[0], + "execute_model", execute_model_req)) + ] + for pp_rank, driver_worker in enumerate(self.tp_driver_workers, + start=1): + tasks.append( + asyncio.create_task( + _run_task_with_lock(driver_worker.execute_method.remote, + self.pp_locks[pp_rank], + "execute_model", execute_model_req))) + + results = await asyncio.gather(*tasks) + + # Only the last PP stage has the final results. + return results[-1] + + async def _start_worker_execution_loop(self): + assert not self.use_ray_spmd_worker, ( + "worker loop is disabled for VLLM_USE_RAY_SPMD_WORKER=1") + coros = [ + worker.execute_method.remote("start_worker_execution_loop") + for worker in self.non_driver_workers + ] + return await asyncio.gather(*coros) + + def __del__(self): + self.shutdown() diff --git a/vllm/executor/ray_tpu_executor.py b/vllm/executor/ray_tpu_executor.py new file mode 100644 index 00000000..d02fecb4 --- /dev/null +++ b/vllm/executor/ray_tpu_executor.py @@ -0,0 +1,363 @@ +import asyncio +import os +from collections import defaultdict +from itertools import islice, repeat +from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple, + Union) + +import vllm.envs as envs +from vllm.executor.executor_base import ExecutorAsyncBase +from vllm.executor.ray_utils import RayWorkerWrapper, ray +from vllm.executor.tpu_executor import TPUExecutor +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port, + get_vllm_instance_id, make_async) + +if ray is not None: + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + +if TYPE_CHECKING: + from ray.util.placement_group import PlacementGroup + +logger = init_logger(__name__) + + +class RayTPUExecutor(TPUExecutor): + + uses_ray: bool = True + + def __init__(self, *args, **kwargs): + # This is non-None when the execute model loop is running + # in the parallel workers. It's a coroutine in the AsyncLLMEngine case. + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None + # Updated by implementations that require additional args to be passed + # to the _run_workers execute_model call + self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {} + + super().__init__(*args, **kwargs) + + def _init_executor(self) -> None: + assert self.parallel_config.distributed_executor_backend == "ray" + placement_group = self.parallel_config.placement_group + + # Disable Ray usage stats collection. + ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0") + if ray_usage != "1": + os.environ["RAY_USAGE_STATS_ENABLED"] = "0" + + # Create the parallel TPU workers. + self._init_workers_ray(placement_group) + + def _init_workers_ray(self, placement_group: "PlacementGroup", + **ray_remote_kwargs): + # The driver dummy worker does not actually use any resources. + # It holds the resource for the driver worker. + self.driver_dummy_worker: Optional[RayWorkerWrapper] = None + # The remaining workers are the actual ray actors. + self.workers: List[RayWorkerWrapper] = [] + + # Create the workers. + driver_ip = get_ip() + for bundle_id, bundle in enumerate(placement_group.bundle_specs): + if not bundle.get("TPU", 0): + continue + scheduling_strategy = PlacementGroupSchedulingStrategy( + placement_group=placement_group, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=bundle_id, + ) + + assert self.speculative_config is None + if self.scheduler_config.is_multi_step: + worker_module_name = "vllm.worker.multi_step_tpu_worker" + worker_class_name = "MultiStepTPUWorker" + else: + worker_module_name = "vllm.worker.tpu_worker" + worker_class_name = "TPUWorker" + + # GKE does not fetch environment information from metadata server + # and instead sets these from within the Ray process. Therefore we + # need to override the Ray environment variables manually. + override_env = {} + if "TPU_CHIPS_PER_HOST_BOUNDS" in os.environ: + override_env.update({ + "TPU_CHIPS_PER_HOST_BOUNDS": + os.environ["TPU_CHIPS_PER_HOST_BOUNDS"] + }) + if "TPU_HOST_BOUNDS" in os.environ: + override_env.update( + {"TPU_HOST_BOUNDS": os.environ["TPU_HOST_BOUNDS"]}) + + worker = ray.remote( + num_cpus=0, + resources={"TPU": 1}, + scheduling_strategy=scheduling_strategy, + **ray_remote_kwargs, + )(RayWorkerWrapper).remote( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + trust_remote_code=self.model_config.trust_remote_code, + ) + if override_env: + worker.override_env_vars.remote(override_env) + + worker_ip = ray.get(worker.get_node_ip.remote()) + if worker_ip == driver_ip and self.driver_dummy_worker is None: + # If the worker is on the same node as the driver, we use it + # as the resource holder for the driver process. + self.driver_dummy_worker = worker + self.driver_worker = RayWorkerWrapper( + worker_module_name=worker_module_name, + worker_class_name=worker_class_name, + trust_remote_code=self.model_config.trust_remote_code, + ) + else: + # Else, added to the list of workers. + self.workers.append(worker) + + logger.debug("workers: %s", self.workers) + logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker) + if self.driver_dummy_worker is None: + raise ValueError( + "Ray does not allocate any TPUs on the driver node. Consider " + "adjusting the Ray placement group or running the driver on a " + "TPU node.") + + worker_ips = [ + ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined] + for worker in self.workers + ] + ip_counts: Dict[str, int] = {} + for ip in worker_ips: + ip_counts[ip] = ip_counts.get(ip, 0) + 1 + + def sort_by_driver_then_worker_ip(worker): + """ + Sort the workers based on 3 properties: + 1. If the worker is on the same node as the driver (vllm engine), + it should be placed first. + 2. Then, if the worker is on a node with fewer workers, it should + be placed first. + 3. Finally, if the work is on a node with smaller IP address, it + should be placed first. + """ + ip = ray.get(worker.get_node_ip.remote()) + return (ip != driver_ip, ip_counts[ip], ip) + + # After sorting, the workers on the same node will be + # close to each other, and the workers on the driver + # node will be placed first. + self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip) + + # Get the set of TPU IDs used on each node. + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids", + use_dummy_driver=True) + + node_workers = defaultdict(list) + for i, (node_id, _) in enumerate(worker_node_and_gpu_ids): + node_workers[node_id].append(i) + + VLLM_INSTANCE_ID = get_vllm_instance_id() + + # Set environment variables for the driver and workers. + all_args_to_update_environment_variables = [({ + "VLLM_INSTANCE_ID": + VLLM_INSTANCE_ID, + "VLLM_TRACE_FUNCTION": + str(envs.VLLM_TRACE_FUNCTION), + }, ) for _ in worker_node_and_gpu_ids] + self._run_workers("update_environment_variables", + all_args=all_args_to_update_environment_variables) + + if len(node_workers) == 1: + # in single node case, we don't need to get the IP address. + # the loopback address is sufficient + # NOTE: a node may have several IP addresses, one for each + # network interface. `get_ip()` might return any of them, + # while they might not work for communication inside the node + # if the network setup is complicated. Using the loopback address + # solves this issue, as it always works for communication inside + # the node. + driver_ip = "127.0.0.1" + distributed_init_method = get_distributed_init_method( + driver_ip, get_open_port()) + + # Initialize the actual workers inside worker wrapper. + init_worker_all_kwargs = [ + self._get_worker_kwargs( + local_rank=node_workers[node_id].index(rank), + rank=rank, + distributed_init_method=distributed_init_method, + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids) + ] + self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs) + + self._run_workers("init_device") + self._run_workers("load_model", + max_concurrent_workers=self.parallel_config. + max_parallel_loading_workers) + + def _driver_execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + """Run execute_model in the driver worker. + + Passing None will cause the driver to stop the model execution + loop running in each of the remote workers. + """ + return self.driver_worker.execute_method("execute_model", + execute_model_req) + + def _run_workers( + self, + method: str, + *args, + async_run_remote_workers_only: bool = False, + all_args: Optional[List[Tuple[Any, ...]]] = None, + all_kwargs: Optional[List[Dict[str, Any]]] = None, + use_dummy_driver: bool = False, + max_concurrent_workers: Optional[int] = None, + use_ray_compiled_dag: bool = False, + **kwargs, + ) -> Any: + """Runs the given method on all workers. Can be used in the following + ways: + + - async_run_remote_workers_only: If True the method will be run only + in the remote workers, not the driver worker. It will also be + run asynchronously and return a list of futures rather than blocking + on the results. + - args/kwargs: All workers share the same args/kwargs + - all_args/all_kwargs: args/kwargs for each worker are specified + individually + """ + + if max_concurrent_workers: + raise NotImplementedError( + "max_concurrent_workers is not supported yet.") + + count = len(self.workers) + all_worker_args = repeat(args, count) if all_args is None \ + else islice(all_args, 1, None) + all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \ + else islice(all_kwargs, 1, None) + + # Start the ray workers first. + ray_worker_outputs = [ + worker.execute_method.remote(method, *worker_args, **worker_kwargs) + for (worker, worker_args, worker_kwargs + ) in zip(self.workers, all_worker_args, all_worker_kwargs) + ] + + if async_run_remote_workers_only: + # Just return futures + return ray_worker_outputs + + driver_args = args if all_args is None else all_args[0] + driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0] + + # Start the driver worker after all the ray workers. + if not use_dummy_driver: + driver_worker_output = self.driver_worker.execute_method( + method, *driver_args, **driver_kwargs) + else: + assert self.driver_dummy_worker is not None + driver_worker_output = ray.get( + self.driver_dummy_worker.execute_method.remote( + method, *driver_args, **driver_kwargs)) + # Get the results of the ray workers. + if self.workers: + ray_worker_outputs = ray.get(ray_worker_outputs) + + return [driver_worker_output] + ray_worker_outputs + + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None: + """Wait for futures returned from _run_workers() with + async_run_remote_workers_only to complete.""" + ray.get(parallel_worker_tasks) + + def determine_num_available_blocks(self) -> Tuple[int, int]: + num_blocks = self._run_workers("determine_num_available_blocks", ) + num_tpu_blocks = min(b[0] for b in num_blocks) + num_cpu_blocks = min(b[1] for b in num_blocks) + return num_tpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, + num_cpu_blocks) + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + self._run_workers("initialize_cache", + num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks) + + def execute_model( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[SamplerOutput]: + if self.parallel_worker_tasks is None: + self.parallel_worker_tasks = self._run_workers( + "start_worker_execution_loop", + async_run_remote_workers_only=True, + **self.extra_execute_model_run_workers_kwargs) + + # Only the driver worker returns the sampling results. + return self._driver_execute_model(execute_model_req) + + def stop_remote_worker_execution_loop(self) -> None: + if self.parallel_worker_tasks is None: + return + + self._driver_execute_model() + parallel_worker_tasks = self.parallel_worker_tasks + self.parallel_worker_tasks = None + # Ensure that workers exit model loop cleanly + # (this will raise otherwise) + self._wait_for_tasks_completion(parallel_worker_tasks) + + +class RayTPUExecutorAsync(RayTPUExecutor, ExecutorAsyncBase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.driver_exec_method = make_async(self.driver_worker.execute_method) + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: + if self.parallel_worker_tasks is None: + # Start model execution loop running in the parallel workers + self.parallel_worker_tasks = asyncio.create_task( + self._start_worker_execution_loop()) + + # Only the driver worker returns the sampling results. + return await self._driver_execute_model_async(execute_model_req) + + async def stop_remote_worker_execution_loop_async(self) -> None: + if self.parallel_worker_tasks is None: + return + + await self._driver_execute_model_async() + parallel_worker_tasks = self.parallel_worker_tasks + self.parallel_worker_tasks = None + # Ensure that workers exit model loop cleanly + # (this will raise otherwise) + await parallel_worker_tasks + + async def _driver_execute_model_async( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + return await self.driver_exec_method("execute_model", + execute_model_req) + + async def _start_worker_execution_loop(self): + coros = [ + worker.execute_method.remote("start_worker_execution_loop") + for worker in self.workers + ] + return await asyncio.gather(*coros) diff --git a/vllm/executor/ray_utils.py b/vllm/executor/ray_utils.py new file mode 100644 index 00000000..37a0afe0 --- /dev/null +++ b/vllm/executor/ray_utils.py @@ -0,0 +1,336 @@ +import os +import time +from collections import defaultdict +from typing import Dict, List, Optional, Tuple, Union + +import msgspec + +from vllm.config import ParallelConfig +from vllm.executor.msgspec_utils import decode_hook, encode_hook +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.sequence import ExecuteModelRequest, IntermediateTensors +from vllm.utils import get_ip, is_hip, is_xpu +from vllm.worker.worker_base import WorkerWrapperBase + +logger = init_logger(__name__) +PG_WAIT_TIMEOUT = 1800 + +try: + import ray + from ray.util import placement_group_table + from ray.util.placement_group import PlacementGroup + try: + from ray._private.state import available_resources_per_node + except ImportError: + # Ray 2.9.x doesn't expose `available_resources_per_node` + from ray._private.state import state as _state + available_resources_per_node = _state._available_resources_per_node + + class RayWorkerWrapper(WorkerWrapperBase): + """Ray wrapper for vllm.worker.Worker, allowing Worker to be + lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES.""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Since the compiled DAG runs a main execution + # in a different thread that calls cuda.set_device. + # The flag indicates is set_device is called on + # that thread. + self.compiled_dag_cuda_device_set = False + + self.input_decoder = msgspec.msgpack.Decoder(ExecuteModelRequest, + dec_hook=decode_hook) + self.output_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook) + + def get_node_ip(self) -> str: + return get_ip() + + def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]: + node_id = ray.get_runtime_context().get_node_id() + gpu_ids = ray.get_gpu_ids() + return node_id, gpu_ids + + def execute_model_spmd( + self, req_or_tuple: Union[bytes, + Tuple[bytes, + Optional[IntermediateTensors]]] + ) -> bytes: + """Execute model in SPMD fashion: used only when SPMD worker and + compiled DAG are both enabled. + + Args: + req_or_tuple: A request or a tuple containing the + request and intermediate tensors. Intermediate tensors are + None unless if it is provided because it is > 0 pipeline + stage. The request is serialized by msgspec. + """ + if isinstance(req_or_tuple, bytes): + serialized_req, intermediate_tensors = req_or_tuple, None + else: + serialized_req, intermediate_tensors = req_or_tuple + + execute_model_req = self.input_decoder.decode(serialized_req) + + # TODO(swang): This is needed right now because Ray aDAG executes + # on a background thread, so we need to reset torch's current + # device. + import torch + if not self.compiled_dag_cuda_device_set: + torch.cuda.set_device(self.worker.device) + self.compiled_dag_cuda_device_set = True + + output = self.worker._execute_model_spmd(execute_model_req, + intermediate_tensors) + # Pipeline model request and output to the next pipeline stage. + if isinstance(output, IntermediateTensors): + output = serialized_req, output + else: + output = self.output_encoder.encode(output) + + return output + + def override_env_vars(self, vars: Dict[str, str]): + os.environ.update(vars) + + ray_import_err = None + +except ImportError as e: + ray = None # type: ignore + ray_import_err = e + RayWorkerWrapper = None # type: ignore + + +def ray_is_available() -> bool: + """Returns True if Ray is available.""" + return ray is not None + + +def assert_ray_available(): + """Raise an exception if Ray is not available.""" + if ray is None: + raise ValueError("Failed to import Ray, please install Ray with " + "`pip install ray`.") from ray_import_err + + +def _verify_bundles(placement_group: "PlacementGroup", + parallel_config: ParallelConfig, device_str: str): + """Verify a given placement group has bundles located in the right place. + + There are 2 rules. + - Warn if all tensor parallel workers cannot fit in a single node. + - Fail if driver node is not included in a placement group. + """ + assert ray.is_initialized(), ( + "Ray is not initialized although distributed-executor-backend is ray.") + pg_data = placement_group_table(placement_group) + # bundle_idx -> node_id + bundle_to_node_ids = pg_data["bundles_to_node_id"] + # bundle_idx -> bundle (e.g., {"GPU": 1}) + bundles = pg_data["bundles"] + # node_id -> List of bundle (e.g., {"GPU": 1}) + node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list) + + for bundle_idx, node_id in bundle_to_node_ids.items(): + node_id_to_bundle[node_id].append(bundles[bundle_idx]) + driver_node_id = ray.get_runtime_context().get_node_id() + + if driver_node_id not in node_id_to_bundle: + raise RuntimeError( + f"driver node id {driver_node_id} is not included in a placement " + f"group {placement_group.id}. Node id -> bundles " + f"{node_id_to_bundle}. " + "You don't have enough GPUs available in a current node. Check " + "`ray status` to see if you have available GPUs in a node " + f"{driver_node_id} before starting an vLLM engine.") + + for node_id, bundles in node_id_to_bundle.items(): + if len(bundles) < parallel_config.tensor_parallel_size: + logger.warning( + "tensor_parallel_size=%d " + "is bigger than a reserved number of %ss (%d " + "%ss) in a node %s. Tensor parallel workers can be " + "spread out to 2+ nodes which can degrade the performance " + "unless you have fast interconnect across nodes, like " + "Infiniband. To resolve this issue, make sure you have more " + "than %d GPUs available at each node.", + parallel_config.tensor_parallel_size, device_str, len(bundles), + device_str, node_id, parallel_config.tensor_parallel_size) + + +def _wait_until_pg_ready(current_placement_group: "PlacementGroup"): + """Wait until a placement group is ready. + + It prints the informative log messages if the placement group is + not created within time. + + """ + # Wait until PG is ready - this will block until all + # requested resources are available, and will timeout + # if they cannot be provisioned. + placement_group_specs = current_placement_group.bundle_specs + + s = time.time() + pg_ready_ref = current_placement_group.ready() + wait_interval = 10 + while time.time() - s < PG_WAIT_TIMEOUT: + ready, _ = ray.wait([pg_ready_ref], timeout=wait_interval) + if len(ready) > 0: + break + + # Exponential backoff for warning print. + wait_interval *= 2 + logger.info( + "Waiting for creating a placement group of specs for " + "%d seconds. specs=%s. Check " + "`ray status` to see if you have enough resources.", + int(time.time() - s), placement_group_specs) + + try: + ray.get(pg_ready_ref, timeout=0) + except ray.exceptions.GetTimeoutError: + raise ValueError( + "Cannot provide a placement group of " + f"{placement_group_specs=} within {PG_WAIT_TIMEOUT} seconds. See " + "`ray status` to make sure the cluster has enough resources." + ) from None + + +def _wait_until_pg_removed(current_placement_group: "PlacementGroup"): + ray.util.remove_placement_group(current_placement_group) + s = time.time() + wait_interval = 10 + while time.time() - s < PG_WAIT_TIMEOUT: + pg = ray.util.get_current_placement_group() + if pg is None: + break + + # Exponential backoff for warning print. + wait_interval *= 2 + logger.info( + "Waiting for removing a placement group of specs for " + "%d seconds.", int(time.time() - s)) + time.sleep(wait_interval) + + +def initialize_ray_cluster( + parallel_config: ParallelConfig, + ray_address: Optional[str] = None, +): + """Initialize the distributed cluster with Ray. + + it will connect to the Ray cluster and create a placement group + for the workers, which includes the specification of the resources + for each distributed worker. + + Args: + parallel_config: The configurations for parallel execution. + ray_address: The address of the Ray cluster. If None, uses + the default Ray cluster address. + """ + assert_ray_available() + + # Connect to a ray cluster. + if is_hip() or is_xpu(): + ray.init(address=ray_address, + ignore_reinit_error=True, + num_gpus=parallel_config.world_size) + else: + import torch + device_count = torch.cuda.device_count() + if device_count >= parallel_config.world_size: + ray.init(address=ray_address, + ignore_reinit_error=True, + num_gpus=parallel_config.world_size) + else: + # For multi-node case + ray.init(address=ray_address, + ignore_reinit_error=True) + + if parallel_config.placement_group: + # Placement group is already set. + return + + device_str = "GPU" if not current_platform.is_tpu() else "TPU" + # Create placement group for worker processes + current_placement_group = ray.util.get_current_placement_group() + if current_placement_group: + # We are in a placement group + bundles = current_placement_group.bundle_specs + # Verify that we can use the placement group. + device_bundles = 0 + for bundle in bundles: + bundle_devices = bundle.get(device_str, 0) + if bundle_devices > 1: + raise ValueError( + "Placement group bundle cannot have more than 1 " + f"{device_str}.") + if bundle_devices: + device_bundles += 1 + if parallel_config.world_size > device_bundles: + raise ValueError( + f"The number of required {device_str}s exceeds the total " + f"number of available {device_str}s in the placement group." + f"Required number of devices: {parallel_config.world_size}. " + f"Total number of devices: {device_bundles}.") + else: + num_devices_in_cluster = ray.cluster_resources().get(device_str, 0) + if parallel_config.world_size > num_devices_in_cluster: + raise ValueError( + f"The number of required {device_str}s exceeds the total " + f"number of available {device_str}s in the placement group.") + # Create a new placement group + placement_group_specs: List[Dict[str, float]] = ([{ + device_str: 1.0 + } for _ in range(parallel_config.world_size)]) + + # vLLM engine is also a worker to execute model with an accelerator, + # so it requires to have the device in a current node. Check if + # the current node has at least one device. + current_ip = get_ip() + current_node_id = ray.get_runtime_context().get_node_id() + current_node_resource = available_resources_per_node()[current_node_id] + if current_node_resource.get(device_str, 0) < 1: + raise ValueError( + f"Current node has no {device_str} available. " + f"{current_node_resource=}. vLLM engine cannot start without " + f"{device_str}. Make sure you have at least 1 {device_str} " + f"available in a node {current_node_id=} {current_ip=}.") + # This way, at least bundle is required to be created in a current + # node. + placement_group_specs[0][f"node:{current_ip}"] = 0.001 + + # By default, Ray packs resources as much as possible. + current_placement_group = ray.util.placement_group( + placement_group_specs, strategy="PACK") + _wait_until_pg_ready(current_placement_group) + + assert current_placement_group is not None + _verify_bundles(current_placement_group, parallel_config, device_str) + # Set the placement group in the parallel config + parallel_config.placement_group = current_placement_group + + +def get_num_tpu_nodes() -> int: + from ray._private.accelerators import TPUAcceleratorManager + cluster_resources = ray.cluster_resources() + total_tpus = int(cluster_resources["TPU"]) + tpus_per_node = TPUAcceleratorManager.get_current_node_num_accelerators() + assert total_tpus % tpus_per_node == 0 + return total_tpus // tpus_per_node + + +def get_num_nodes_in_placement_group() -> int: + pg_table = ray.util.placement_group_table() + current_pg = ray.util.get_current_placement_group() + num_nodes = 0 + + if current_pg: + nodes_in_pg = set() + for pg_key, pg in pg_table.items(): + if pg_key == current_pg.id.hex(): + for _, node in pg["bundles_to_node_id"].items(): + nodes_in_pg.add(node) + num_nodes = len(nodes_in_pg) + + return num_nodes diff --git a/vllm/executor/ray_xpu_executor.py b/vllm/executor/ray_xpu_executor.py new file mode 100644 index 00000000..2b1cdc09 --- /dev/null +++ b/vllm/executor/ray_xpu_executor.py @@ -0,0 +1,37 @@ +import asyncio +from typing import List, Optional + +import vllm.envs as envs +from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync +from vllm.executor.xpu_executor import XPUExecutor +from vllm.logger import init_logger +from vllm.utils import get_vllm_instance_id, make_async + +logger = init_logger(__name__) + + +class RayXPUExecutor(RayGPUExecutor, XPUExecutor): + + def _get_env_vars_to_be_updated(self): + # Get the set of GPU IDs used on each node. + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids", + use_dummy_driver=True) + + VLLM_INSTANCE_ID = get_vllm_instance_id() + + # Set environment variables for the driver and workers. + all_args_to_update_environment_variables = [({ + "VLLM_INSTANCE_ID": + VLLM_INSTANCE_ID, + "VLLM_TRACE_FUNCTION": + str(envs.VLLM_TRACE_FUNCTION), + }, ) for (_, _) in worker_node_and_gpu_ids] + return all_args_to_update_environment_variables + + +class RayXPUExecutorAsync(RayXPUExecutor, RayGPUExecutorAsync): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.driver_exec_method = make_async(self.driver_worker.execute_method) + self.pp_locks: Optional[List[asyncio.Lock]] = None diff --git a/vllm/executor/tpu_executor.py b/vllm/executor/tpu_executor.py new file mode 100644 index 00000000..972649de --- /dev/null +++ b/vllm/executor/tpu_executor.py @@ -0,0 +1,147 @@ +from typing import Any, Dict, List, Optional, Set, Tuple + +import torch + +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port, + make_async) + +logger = init_logger(__name__) + + +class TPUExecutor(ExecutorBase): + + uses_ray: bool = False + + def _init_executor(self) -> None: + assert not self.scheduler_config.chunked_prefill_enabled, ( + "Chunked prefill is not yet supported for TPU backend") + assert not self.speculative_config, ( + "Speculative decoding is not yet supported for TPU backend") + if self.model_config.dtype in (torch.float16, torch.float32): + logger.warning( + "The TPU backend currently does not support %s. " + "Using bfloat16 instead.", self.model_config.dtype) + self.model_config.dtype = torch.bfloat16 + + # Instantiate the worker and load the model to the device. + self.driver_worker = self._create_worker() + self.driver_worker.init_device() + self.driver_worker.load_model() + + def _get_worker_kwargs( + self, + local_rank: int = 0, + rank: int = 0, + distributed_init_method: Optional[str] = None, + ) -> Dict[str, Any]: + """Return worker init args for a given rank.""" + if distributed_init_method is None: + distributed_init_method = get_distributed_init_method( + get_ip(), get_open_port()) + return dict( + model_config=self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + device_config=self.device_config, + cache_config=self.cache_config, + load_config=self.load_config, + local_rank=local_rank, + rank=rank, + distributed_init_method=distributed_init_method, + is_driver_worker=rank == 0, + ) + + def _create_worker( + self, + local_rank: int = 0, + rank: int = 0, + distributed_init_method: Optional[str] = None, + ): + if self.scheduler_config.is_multi_step: + from vllm.worker.multi_step_tpu_worker import MultiStepTPUWorker + worker = MultiStepTPUWorker(**self._get_worker_kwargs( + local_rank, rank, distributed_init_method)) + return worker + else: + from vllm.worker.tpu_worker import TPUWorker + + worker = TPUWorker(**self._get_worker_kwargs( + local_rank, rank, distributed_init_method)) + return worker + + def initialize_cache( + self, + num_gpu_blocks: int, + num_cpu_blocks: int, + ) -> None: + """Initialize the KV cache by invoking the underlying worker.""" + # NOTE: This is logged in the executor because there can be >1 worker + # with other executors. We could log in the engine level, but work + # remains to abstract away the device for non-GPU configurations. + logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, + num_cpu_blocks) + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks by invoking the + underlying worker.""" + return self.driver_worker.determine_num_available_blocks() + + def execute_model( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[SamplerOutput]: + output = self.driver_worker.execute_model(execute_model_req) + return output + + def add_lora(self, lora_request: LoRARequest) -> bool: + raise NotImplementedError( + "LoRA is currently not supported by the TPU backend.") + + def remove_lora(self, lora_id: int) -> bool: + raise NotImplementedError( + "LoRA is currently not supported by the TPU backend.") + + def pin_lora(self, lora_id: int) -> bool: + raise NotImplementedError( + "LoRA is currently not supported by the TPU backend.") + + def list_loras(self) -> Set[int]: + raise NotImplementedError( + "LoRA is currently not supported by the TPU backend.") + + def add_prompt_adapter(self, prompt_adapter_request) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the TPU backend.") + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the TPU backend.") + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + raise NotImplementedError( + "Soft prompt is currently not supported by the TPU backend.") + + def list_prompt_adapters(self) -> Set[int]: + raise NotImplementedError( + "Soft prompt is currently not supported by the TPU backend.") + + def check_health(self) -> None: + # TPUExecutor will always be healthy as long as it's running. + return + + +class TPUExecutorAsync(TPUExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + sexecute_model_req: ExecuteModelRequest, + ) -> SamplerOutput: + output = await make_async(self.driver_worker.execute_model + )(sexecute_model_req) + return output diff --git a/vllm/executor/xpu_executor.py b/vllm/executor/xpu_executor.py new file mode 100644 index 00000000..bada5606 --- /dev/null +++ b/vllm/executor/xpu_executor.py @@ -0,0 +1,96 @@ +from typing import Callable, List, Optional, Tuple, Type, Union + +import torch + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig) +from vllm.executor.executor_base import ExecutorAsyncBase +from vllm.executor.gpu_executor import GPUExecutor +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest, PoolerOutput +from vllm.utils import make_async +from vllm.worker.worker_base import WorkerBase + +logger = init_logger(__name__) + + +class XPUExecutor(GPUExecutor): + + uses_ray: bool = False + + def __init__( + self, + model_config: ModelConfig, + cache_config: CacheConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + prompt_adapter_config: Optional[PromptAdapterConfig], + speculative_config: Optional[SpeculativeConfig], + observability_config: Optional[ObservabilityConfig], + ) -> None: + assert device_config.device_type == "xpu" + assert (not speculative_config + ), "Speculative decoding not yet supported for XPU backend" + + model_config = _verify_and_get_model_config(model_config) + + self.model_config = model_config + self.cache_config = cache_config + self.load_config = load_config + self.lora_config = lora_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.prompt_adapter_config = prompt_adapter_config + self.speculative_config = None + self.observability_config = observability_config + + # Instantiate the worker and load the model to GPU. + self._init_executor() + + def _get_worker_module_and_class( + self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]: + worker_class_fn = None + if self.speculative_config is not None: + raise NotImplementedError( + "XPU does not support speculative decoding") + else: + worker_module_name = "vllm.worker.xpu_worker" + worker_class_name = "XPUWorker" + return (worker_module_name, worker_class_name, worker_class_fn) + + def execute_model( + self, execute_model_req: ExecuteModelRequest + ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]: + output = self.driver_worker.execute_model(execute_model_req) + return output + + +class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase): + + async def execute_model_async( + self, + execute_model_req: ExecuteModelRequest, + ) -> List[SamplerOutput]: + output = await make_async(self.driver_worker.execute_model + )(execute_model_req=execute_model_req) + return output + + +def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig: + if config.dtype == torch.bfloat16: + logger.warning( + "bfloat16 is not fully supported on XPU, casting to float16.") + config.dtype = torch.float16 + if not config.enforce_eager: + logger.warning( + "CUDA graph is not supported on XPU, fallback to the eager " + "mode.") + config.enforce_eager = True + return config diff --git a/vllm/forward_context.py b/vllm/forward_context.py new file mode 100644 index 00000000..77774750 --- /dev/null +++ b/vllm/forward_context.py @@ -0,0 +1,22 @@ +from contextlib import contextmanager +from typing import Any + +_forward_context: Any = None + + +def get_forward_context() -> Any: + """Get the current forward context.""" + return _forward_context + + +@contextmanager +def set_forward_context(context: Any): + """A context manager that stores the current forward context, + can be attention metadata, etc.""" + global _forward_context + prev_context = _forward_context + _forward_context = context + try: + yield + finally: + _forward_context = prev_context diff --git a/vllm/inputs/__init__.py b/vllm/inputs/__init__.py new file mode 100644 index 00000000..a8c8672c --- /dev/null +++ b/vllm/inputs/__init__.py @@ -0,0 +1,44 @@ +from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt, + LLMInputs, PromptType, SingletonPrompt, TextPrompt, + TokensPrompt, build_explicit_enc_dec_prompt, + to_enc_dec_tuple_list, zip_enc_dec_prompts) +from .registry import InputContext, InputRegistry + +INPUT_REGISTRY = InputRegistry() +""" +The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine` +to dispatch data processing according to the target model. + +See also: + :ref:`input_processing_pipeline` +""" + +__all__ = [ + "TextPrompt", + "TokensPrompt", + "PromptType", + "SingletonPrompt", + "ExplicitEncoderDecoderPrompt", + "LLMInputs", + "EncoderDecoderLLMInputs", + "build_explicit_enc_dec_prompt", + "to_enc_dec_tuple_list", + "zip_enc_dec_prompts", + "INPUT_REGISTRY", + "InputContext", + "InputRegistry", +] + + +def __getattr__(name: str): + if name == "PromptInput": + import warnings + + msg = ("PromptInput has been renamed to PromptType. " + "The original name will be removed in an upcoming version.") + + warnings.warn(DeprecationWarning(msg), stacklevel=2) + + return PromptType + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/inputs/__pycache__/__init__.cpython-310.pyc b/vllm/inputs/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ecbfd4b57076e924c54b03c5fbb65e7b4813e7e GIT binary patch literal 1075 zcmZ8fO>fgM7`C0XO}l0v9TVb93dbC_vJ(f;1llGvX-v~rgviB8Ytx$9PUN`VI&~Mu zm0vJ%;l_c#zz^}2)BXSlPS|Nj2bS{0@AvEXeX=lY0DCsS{}}x80r=^J+sCuQ3;Pva z3j`3zfq)4VC_%zYJmDw4s3a9pO{$`n)I^X3qMp=6BWXbAkNARUCQT70Ap|tM1j%9@ z^a1g{!eoi~boo6Z6;l1=Co80O4C3G#+Cp)|+})MEj8L^h%isR~>phtdbzvUvp5>hN znEuz{P-P<5rapIUOjVt6PR-IWlYLHgCTD()4n5Ped$W_BP+82RN8KUg1k+g+Ol1!f z+Qa#@z?#nfs`N1D6mwQ+vwF_*zxt_b5hK$qo8Hc(w&hG%N=LMBBh)w!Rn5Lt1r90> zst(-5m0bl;$U)tK+mTwZU>5fd4&QX}(eA6g z%fiI&?*yqef(YQ?0ut{6P2dNp(F9#W;*-i}1cCOBf%ZvNSFRALO~4iU;$Om#Fb)oq zh3SyWcU;?y;GEvlM1!=5x|GUDQP&NLbQVqRZVMYxXF#J&u|AV2kDNzz$~dW&vXx z;#$S{vCUSv6$UVXK5BVBx`WgMt2l^T#wV%GU$S6MiaAl{9MX{}3MpZqO z$xu0j0^%HW$T2W-2#|{ox#b)n{~$R9=u3d)5cn@7TJpW>o{ggI+%iNLyQ*Hjdi9p? zz3OaYqN(9`@ehC6bzDvRCv`@DZFH{Vk$=%OjcH7ewM6&y#PE#7^vuNatfc1E65F$r zx>rve&q*3yLnm4zZYC|Sm9)KfGT}|=8rQ`e7&qg|WXhY;wRbgUvD!n8)p+~J^rq3b zSsnel>d){7pXJkhX2)FCyt#(PCz!+M_#A7n<|E6SN53`FU%>h{o51=B-sTpc=L77Pze$YG9SYWTQ zQ~WerK?2f@+VXvb15?i>X zbzc9HY~5)T6I(y|aKnG+2OIDF@aE>bh52^6UsyMyFfZ!w@|24xES#G;7s2hA7uI`` z%nRp(ZXRW65Er$rUKhi)k5lwJX3^Tr@{jnvUL-gx8e98a&PrDF`lCQ}^kVk?UYtko zXDo4)m|r#IPxIQB3)M zkfvE4kP@ZuixzeibY@|qd8~OMjEfsw{VVkvEaHew7r&2qS8&PGT)IK(@{)F)mBcQ5lgF-d zlS<_)YE_895Rr`1J8D(6Q-66yiYev>Zl70askNqJu(itv)I>H;Bv)Jr)#YIx?eTbj z#Z7{J_co^pOoBY;Oz{Z9FcXYq$TRGia!&vNCA-|s zdNHFj1-W9GXWiA9-~fHyeVy}cD|0|JwK>G=Nkn9z9H?{^vPa*fs~KrcInq$^YP-@_ z@USb}m1G2#K-_Jmf-B^&SGFt9=ufFVcOo7$S=u|2O4TkTjP-J~Z; zfof6zXfL=$l*&AS;^dm5D<;RKDJI87sBooyBCecA*p`su(Ax!S{#S5$8N^$7Bw=t< z1N_dljU_|Sc!+20S3~bQ9{CBH7lC3FcnKt^)}Z94(jLHRGv%0G>lS8#Fb4(ysI*wI17ac%3v>~f|E@~sX&~g zW|#_;@l+}pq}UVl!E0FdGdz;g3z@8K8~QYI!n9@ROO!Q<;sxWG7`%>0esk9RCgWB0 z%HqZt$`1|p`~Q!@28gOGM+U2cYPFB_%Jh}ZyruDw8%gJ{KuyY!Lz?4d)@G-9Vk~~W zVF&fnNP2%AoBf9D(l3Ap<|I?nY+)k3cML&AfN-mUOPfN8`6>-k)lt}j1B%j+4Bh|H zJ4Nj~cqCO5hX&Jnny@47fst#6+M$jS{gH{7Gd3Ta$9Q&h1q9;TSUKX0#HcuTyBEdG z=T#o|p-mrpt}yeDv>{cf4o+cIfA&+cLQ7jjTP2X9HC!Z5l%38XZ=m@cg>@MU1nz+-S!|GQ*X_ z1uT{DwW2^t6$msz_ZqS9-jCwApKMf0RBpe=F;%&DKWkTo!(Vm#25=fUsDD!(@F z8MjK`daNt&Kn-iGy*YG543m2gBc^CxB}-5_c`hiR zSatt87CYh+sQ-pX(zgp+b4>eLDL4+_q!PC*4%A(%FmaZSDOPERHEM?HxJX?UjdLd) z_|KS?MC?phdJ}(+Wk0d)CR*_w3>Q=5F)PQkLB=%jDCi&+Glbbg7_kZ@bKKrUN=T!W zH?;Qy{ScSpWvGtn_%2|S_6C>@!HYl<1EfKZ%AgSxIZ1E-0G+BnuN)xIf;?K~IJ?H=lQT>Kn-W=O$9 z78bn~vp4997pgA@KQ5<==1{m|^6mcW4Iwh2QYn1=d)QdKg{CkQdFQx&TD}iJrhR?? zUm#M^13wZ<`rMy~GbPKYOAY-b_xKJfmgSu4ydNbuB zR7ceRDphhh+d|KyhGJ! zD!Yp}sN122lB0@D)%YH5vY}m*j5RahVg9DG@drC<`ZMadSU}Wn;4rlo4Wnncsf~Y Q{>L~E`scGo^VzKPU%qBpKL7v# literal 0 HcmV?d00001 diff --git a/vllm/inputs/__pycache__/parse.cpython-310.pyc b/vllm/inputs/__pycache__/parse.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4664dc3c781cd500d8eb00971ffeea9dfd0e588d GIT binary patch literal 3434 zcma)8OK;pp5+<9E;n38~NU~(fFHCG_gUKdVyh%3ML@*3Qc3{9O7?zR)fdxTNw`C%H zjGH}5WOy%YUvt>~7Xp&MkRQTpPC5IQgO#tE9Eq|dWCvhPS9Nvw*VR>DcQH5T8~EM& z`Io0jU>LuW(*Ai+?%*r`XBq}KxS1N68JZajS!RV+W`}m>gihv$u1WnY?POl)>9&=2 zGe7jvwz-p@%Yrb-=EAwG7xuD#*v|&xAe#^8vxRWMG`=>t%R8?O-VydoD_n%`@hcvi)hVlGZO`wE-|YVnwWq zi%%>)*D}9=xh|ovk3LV#Y3mAKh4nJO2=AT?89wbW0IIX-b@oXycv8cM+dlSJolCi3sZ}OzbE3eoUGA&|0GAoyQ@0V!#Ng1V> zBPyQ!hO;(Co$A8fd|YrLH-!Fs@Zg{K^T|||)#bZCOwweWsN)U~Wsyx()je|bYZEO} z^|z9ICl#v58$W+r{Ggh4uw8s7^0L;(w5RsZf^r96c@07sp~;PqePiHMY;LX_!r?d> zw5@gI8lvJ<{=-<7g3}6BXI$h;JCkSP{FqHX{$w09eU_#D* zWIN_oZMR^5r0f^QOOspQF?mTj9E|iIQ@PEk_qfB|SD0f1J{U#z4n}>fJj~WPoQ>>< zvvIIQGvh{kMzZVCkHC`cS-ZSTtB(B5d;3hEw?1x3v44JTl8U$#L&R}6;bIu8AsN)L z$h8s0dn^PLWJ=2|e6Lp^b^!lh-oH(R0wu`JAK8D|Glqg=_vE|iAJw_L&nkBFqskVk z$leNg<*yUTj6oE)*DyjSk!u8|d+_}I)6e_ndRmO*w7iYRZ0PoxI3<T!m2H(QP1p=g}HA$XAI9{KMUF&e6MMuHo0(96DuQou2H~Kz^N|*TDam#wVEGYx!S{oP z=Zmigw#yY1MH^DMstQKG0;MNOc7fUIdgSqNn?r7T$unm}~8G)CXW)*K! zAM=PE?Iax36;t1Fys_ws)GUI?XxN`)z^UM_;RPa^$5S*L%j@@5Gsk0EkwGjl`33$e zw+`S5c;utu5refst;{AqaWs_UAZg;gR)*=@_BxR5#%aP&z{0a7sv+!PoQCFEavkl> z(N*O3-UOl6&xiWaKshKK5>&;q0};;GUf4TcVkJGJB%uy{dKBH9sw6En?ctbXJfY6f zKzbI*ITAe*eGs@c%Td!Y#x2b~4 zM-W3JU;(?q2F}1?4l{cuJ8;e5fZEH4?W<@lF#pZq+RTp~1khQZX&)c{KDAiy%^ycx B*F69L literal 0 HcmV?d00001 diff --git a/vllm/inputs/__pycache__/preprocess.cpython-310.pyc b/vllm/inputs/__pycache__/preprocess.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0ab20168dc22f23a43ebceaf72984774ef410d2 GIT binary patch literal 13412 zcmbVT%X1V-daqYkw_1%5;wfOD5D(EBkbvPaqj~HyARfD*VFzaHsU5p1byZ6R)m6P& z)qpx$5xa(nJ@F4Xd>DK(;qciPhcAw>BYbu^9Ch`@bN0oa0p#D8S&!~kH^W%Fon&Qw z`TFwrJ+mg*wyiAT@9f|G2IXsW*#eu~{-pGSR5sShH-Fd0q3yn-#Od>jiIHbHbe9b=}+EoHQqSz3A;| zPMK4@Uh;M}cbU6*eaze4++*%Ry=;$rdz<^rea&fey1C!n-#lO*;O!OfVDmlmz2+hF zkSsY0`xf(W^A0x+)4=nDbL6RP&NxL!caBa=PQf{bQnx3lbTsFvqc)UjXPtC#)t zSi9=hqgcD;hEZJlq7}J;Z+UUyUc2Qvap5Z;&s90DSYha}sOCG59ERud+kx$PR|3D` zuF|9HyHU*xR#(N_Tfvu?zjS`rcESj6cimw@vlU&otybi)ynX6(D|GG!4;|m_IP5wL z+AUB|wV3NiwZ|6oU4OL}_;u%d9JW-|;=N0L9rLoQ4*%!Yt=l*KRyzvgvAkLqligcy zIq~+p7{PO*z!!~GC7yA^n)9UPxpgeq5yRDNF88u}llhHMvX{aMA- zZPh6>6uW4bo@r)rOtU|>$3R;mTKqdkXduKrE12aGZ!7jTydBTz6??+oj?pS;*~Uju z>`8kES|-M(>`&~S7?-zWj$O|7j7qn6+j}t5q`g}{0nBlZl|vF|MuN9|*vn6{5Qd+ig> zetXuzikyQaKiJQE7~`Z}#Vm)gj>CKgYSryI`+c+;1Ffg*(`Y?n&ttTF&oiK3u+QMl z(J>8c`4qkSSKCWlw$Iw8bRc{6dnYuH2PsN>@Rt+R|O=rMwI|E`;rt!{Sn{#`Q*x zO`{W|o>ev0LdR?5EidJGtX8XgP(ZcXOX-JC7ouisq2;ztM;2RkqSKd}?X~9F=Hms= zU0G}>ljcePs80L5 zf?v1~MOS(N!_t=6SR~u>Q~58j+XvWXRax%*;+vpt)Ggl+BBO=H8 z1|}{hwjrLPJ@6%GOpGflK`6LUbM2Rk;Mgn{Jjk|5>fB3wmbOL?Mp-S(nlhnuW`=XE zx6?=_v6C2yfH1iIrT^#lS(P?H2g}|^VOE@;jk43!J^&29!(OV7iD&`EsGao^_d%bI zU&D_>e+8AU^qurV3d;icIC_O2y<#hEDclWkC=X!kAQ}fip*|&WfFjb8H=34Q4S=h+46#Q2Z&lXEZ;~> z`$9sU4a61OyzwPhV_^+|1;rV_ma6kEC|Rsy{RW z-}nSl`xzJCpV-ET*XhPGK2`O2ANgVfE*M`g+YW?r6JU<5HBvKFguq%+#MnIELOqf$ zgahK#YyFEK6odUF=)?E9gdw=aDiuo*`8^7iB}J39%@XP*RZ}{rhAV;L18xx-5*#ix zWZi_&aM;UjIV}H5g2j+m3G9mXHRLM;&)t?&cN=aUr?AesJ{+5iS%+1}cbFA9h6Mp> zSa4G_K@m2Gk3wcA>q5c=)ys@fOy&UUKBlAgWV*y&&6l%IOX9pzxk_cE9l4~cdj&M> zlFSeBwcSR;VU8aekFE9aW5a@~cOIu>SS*(?%eEc+l;Oe-Kq&mk@$rn&El51cNu%x8 zDOAZOdF*-~jEMmwo6rWdc)MMP#^BQe%^CG@7KNw4vlhA&@PYZQBLB!@t`n{I@Z<7I zNc*t_6Dnm)NhxTrlSnSen$)#T|^%7n+zb54}uu%S!xb6Sg{R=BvZ0Et^`ugn8l3u{8I8<6XT zh*^FJW;7iZF06RL%0jlf^JhM|WSv{NuyXd?r4KKCaM8JN@uKC_&z^J6Ui|3nnKS1u zUAS=mf@9@9LRRk|Fkk-yY3vXYQNFUo3<#SupjzgU1L3^MAd#J24XeembMphnoUG;} zUS_N9CL}8C!|4{@E<#QLghZDB4x~@=uq_{N80zl`twq*>t0E7Htc+#LjAFK`%Fg2- zb^%4KdX8V!L|AZ`h={x^C*w&W zcDY0&GsWY_nk7 zE5mxnx>#3SPC`=XiaP0p^sj&l{jT7bMC25fY-nBW0rrdR1N6jmmHewt?kilTg>qZQ z<#m?umLwqwc~)?VFs~fsq*eK#x@~?Rv5nBG6T}-H9#Nr|n2nbdpB&jm_UygMS z)S6)qhk)V~}XrfZ;G>7K(j$5Y18C|tqc zq%nR)#ekOIkINGdtyIi%bp0F1!w*rE_Esp4hFapP>9tnS5{z=F03pu!cX#LxlUV5UfGrhJq{HfbeD_emxOX67f%dmWaruYc$=_03@M~$D~z|L zt|x!w0Plm*I95oTq%DVkilQrjt8Hi-g$*6rTtU6GF}6`gy^OTK%yvb%flku?a(Aq& z(`7(#h`)tBd9G}XArKksmb&Gx3TvW)LVaF@zSfpIxA|S-X|8kJXuV$tb6uS*ejvIq z5TWo4IWAFJ0i!5--N7|0cVw$Yah}|($fVfZm+~Q%bn4AuK@oXMl@TS^@_e*J# zeuLnfuC26P&(302T>aoCg|3Cq5bDV1s_1abHPSiSrdCv-hx9iJe-?U(>#6_kygvkC zqsHzFQ;V1*^DHPB5Xy(0;9dl-h7f)tGY&l9Vubt)4KAv!A!yx4>cuZ|IxV3obm>Ax zccmA~hVD}?9U*JhUnNfieR1CKO;Gn`S12#|EVy_foaryf`U7|%=Qwwx7x|^!;{7F&61+NaDUX{hABkC9-}iVD_mFu(juOA< zabS=w7vQ7ykvt=QX*>svKu=$iBqDKV(Ak&2V`AGB$aZ)tn-|) zG=gcPrq8+>oag=A>6sy;TMm$^<{3I94}NO8@Kn~%v2glfd*!s)uW>Tne~%9A@UY1j3K8m^K}Wn>&R`c9uDwOCa@)Bwv>vB+pU={9Y=XN5p}|b=nts z-}z#DV#ADnxMMALusA4yS+6<7r2N|&=+(Eo!0#-g+1QNqH!hmFNBR}&%P+NM{Bh;} z{U-k47wkqm`+^GY7krHx3=hu_@&uJF&KNK$pBW1tneW-X}rvbz^6*dw^;n;un^?WX4_TK_@RCgtMIx7fP7OS7$7u(g{sCz` zl(0A2njYI>}Y~M@-3_sY7_+ z%%E%@&*f#&8$Ka;j!z1zs#V2$G6hC%PBP7Do;?sb`P`|v^LF>fNOflRN#RgO=IxrB zYrDi+W-IGc+o_8)hfAgWlbase1ULhk-gi^?LzbS6%tJp`1786XuEX!6{!$-Q-?L~H zfZ@;?K*+&N2>XsyEc)LG9|S&aZ;Ab`O z8YBH5VnHZ7KW#gIsfI7iPIQ(vm?Tqk&P7=Iof`^ok;0i|T9HhIj zL{i9)=|%KBw+BCbbfxF(w!{(3VpV|zN~aGI#xyscVGL&E9P#wGRB(gBW6c#jj>wY_ zyMid^cShq1SiK+JOC_P0^f#pfl2TdFr@!hP+{*BLfT4`#z`KS~(_A^*-pg76zl_0q z!C-v(#6Jqm8LU0mQo3rPb8Hka^OXRGWPX>KjgmPiQf~bjHRRl?(_@2*0cheLDLqSp znpxt`K~TatCj6-k@Tc->*p0&AKz*OP0NwcFGqGs&9UTeh1AN@VXewwM{*@;`=i_J? zSS1I*6YPH|tK1;k| zWm*9V3!9*V>abR6{_)rCx)|C zD%PkVHpknN4`=I+=hbRG3^$16C>8rq(B{L>aC9X}p}#k}s=QXTiB}V4rKD}b-QL7^ zvTu~}ygpT)I-tyIuW_mOMgg_1m0u~d$m8<9{Ha1CiFe8?y-@y9ncUK{LwnWRrV!QQ zkhT*kRasMBajGL}i$XKs?0Y&zPht;vw+kN1kLi0I{q=(KqoS98P)>||1G}}gPnr4{ zquxv&~gt zfdKtC796ihi}(tbQL@t9!D+~am``Ft4q)>DZx>29y^GB#2(;!br|A9Mpe8;ENH5ZG zsZU=LbNMjKoSGBxkMm-UQg{NP9E}{I9P|Z#t043eIf>?W-k0YSd9suL7Zc3IEAdGy zmmH5CeJTPf2$nr~ouNm9^gJ+l5Cj2`6k8%GawHP8tvFI(l_CYygO&)(BxOZy*^S2Z9iYkX znc?jofS3$iDMM9K7VKpA6r4pPC zpR%9g`ZK|~(6X)YY5Qq@uLjSA=k4>TyCg0gsrE&&=*@}+vEa>ps@u;B!<+XOK2_gT zyc6EyM~b(2q@mU&)H*4uV(#wc6~$ZnRI_WSamrgpjb%|e((UK)zT&OoeN||9e_oj4 zk~p)Yi8BptMX|3G71TKGok5K=N1A=rQ*SG^vwxy}-mIyK`CcUCb-x}XU-aDAtp{!t z)r!f?oe$o9-Pw5S^^Lc0Y`&T3uQd-6bHfb+cRLV?K~0jvO}E|ln>&g5vv%yanr@Jo z@5)x(s<(n9cc;^)ns*M`0-bBkCepcgrHG~LH-)#+YBu~GlohDPFIC!>6SQ`A zgska_wIgB&L$>yX=iJ+O<&M*81_w@SPe|!|B1#t8(r?BNJ~c59e65S*v=mV**b`0| zI&ImiizsTzw0ZO=AW6YM#dqGhd866x#1V~pv*n2(n~#<(-0TFgk1{v7jt$|Jj+EUJ zJAM?)gU{8Za$DT*2#iVf5_O}dCAsZZr|BgHPc&R~a|hk$8?x0!E!y(Y-w@s?6^gBT zimkl|nA^Ii0TBAjN>LHHhUyt2FA4xaP81r3m-h-E)KG z9B(f{@YLBqq0NR;w}GlPiPfuIKqF&VvK1 zzANhYFr4MbkwxoVD-9~%a{Wj|>w|ZiC87LvDO+;gx-rtbGfS@2G!Ne!>b*=z;V8@-4s5ebQz>pnMa;nSOK)@BmZahh%@949F` z4rbd4C|_}$`yDsPz7!nCYt>Oa>o{%`m}A+997k4ZsufC3Q}P@o1tf_MMJGv5$t#r5 zxsN2YBzX=AwTh^HNij4tr>U#j>H>dBR2Ug~2A}G9#(pz+dlgUg3X;D1^Hh;3Kaqpd zGf25X{S~0>OfN5r9@HQ!yi%s{N}0l&yWR=I1Jc6~|F|U&61^TjND7g-?*yWm6k!bN z5Q8{MPE4^2@)=A*u2OOeN!>uw|IzvhTW{lu=mVQj&HWIC188kX;l{%9XciXPQRq!l zp^SZ!NSLYCP^XOOip^Fm#x=z{wO`+21i+l825hUZC}45})(>fS!x2aNB#_>LPG7G> zsjqKMS@4$SN?{EQDG9(HEUIU@@nB8V3mIe|dZ(M3i`WkZtqiKDBgm5Ql2XjWcC;dq zeH!b8%!%7*Ks9moW+&Vhl4YxPPrx9x@z&QsLn+&2s6k(`B>fm0FaMGf5)=6nB||*2 zbU_H}U<;0O9UobcjQQ@$Brw+cwKi*Id6l{$^CPcOLSRmG9F-&;B7Z~)!@o?qA|-!C z$#7uKu^{{fKKu=yXc>vAX!QF=)3vWlhW0;NPW!5wuYM`1))S)8M_WXaHxz|F4*&-+ zK30eac)5=%cK&0^c4X1Xr>Nh85s;P(E01Fz5P&^l zfl1**2$BRn$W8Zm1$aRmix8v^_6^7F)(M1Y-VQo2u{cFHw!9V)K&DGP03k}xEz$3Z z0~V$+YflR#uCL>Xeu|`9h?OJdA?*7j^?l_w78Gj-7gXgCtiC%xu={#Xd1yRRQrmy1 zgB$2au>Zf+JkD*l;jNzFelY3AQ+Uw6+wae%<*U;;IB5}`jpHz_XHDM#HW zGugm(LEs+&C&__iaQ+0jzr_;~=Q0&TeJUrZ>o61b1j0iqA9r;A=q-=e&Z* zd!{JB+sLzy)}Wg~N4-*}pG~jqRX|9~-i)Y#o>skCyv=xX@MdQ3&M{?$$Ahx@tOof# zh7Escq{c~a31ugWr@SlPGW;AGb(Ops_(hW=mb|mxQy|7Cd4A`_5^5|p@}A{AjXq9! z&xmErPPv&@|(*rqg3S?k7(uJK zBcj-9x1z|0=WCHeHRy}=VBKwwS^EEfJ5S4?pCno$SCG)D&k)&O1L0wpl*x=e58pF% zVluOtA*dCm+EdzZyPCp=I1?e)UP|9tqY{4HfXhcT44?REC)>{qR@64b)WHGzjmVf* zs#VQUO|4s-0KoBWlR|bVW445B2w{N!enHIAQ)BWG6*7_^tB`TsGd`*u!jHts=iKK< zX<90_pd0SAvnNZJ`N6EoM;O)h4gw83u#+&L2Np!S;Rh(V!XO!tr$(n)XSW)r$Zzgp z+sS{n>P!v?sNr4xb4Vg%6=)=jHXOV&u+)TsXt&h~JOUtK*l81$;t&MUt9>%lq+(e1 zt$nTurykawH~?%WTt169-H>&GDYK%e2T^Lr31j^q<%tiS!1D#kUr+f&y{qiOdx%w* zKDd?vS<)|u%Dy+b4~sXe@6T<25n&<3<zU!Vc?bSxhAeHxnSz%MAr zL!X$^+3)a`or~q7QSLtdorCkz#^i#mLti(3LC!R}z{IfdS9=6mkZ+-|zgapshTKE} zwukvOZPlb;pCTqQvf{o_{{eNsASPje%4m>C_X9j}XnC8GJCwXf2?K1%JPCRKO6A1+y6fXd)E{+%2+)k4 zfVKR9da)^)05T%26bl7|NDW1{Fg6nZjQS~{3jp+l(*KrrV<;KLwEU1}kYSkD|0^mW z3`=6zli)o!0>jaX^+Eh^)Mg7w_v-lGd`G)CIQEmqJqpM$XmwJT z{U7-NMLZjLBHSSi=}QjuC5N!5`k{81KP>bNxd_)s>*YaZ%w`UGJn&OB1CE5EvkF{+2NakV!a z&mL*AjTW;*BkbZpuZ7Mc2%#U+P}G1d@`5rN3>&6C<_ygVyE}i`j6xX7NDOI|vG{rLRHDh1Vk+ z7*1|e;vlh?d5@;-r<|ucFHu7AA@-Z?xpc2HYVCVbq9e>GNy3xrYY{HhhzF*zgB(Q@ zB6)$n837b#Z6UR}7InultvcmAN_bXej1$#?zfAGOzQ!6xzHD>A(F=e8ll2n9YUy;M zM7@`&dr6T>cx<~uS&`iiOlr#Ibd;2~~g5i@4C$R(p*X4hA#~DBed;Pi)7)o#C#A-tT|Yuxyg_ z$z>eZBqpaus&{in4wz{)2QrLEFah3*Qk^%-{7LH>U`_cL zrFX%39wI>WNbTj1bg->_oKLw^Lj!9n5NqO~(H{r+dkvQ^xWkw(i&UE0DPkSG(p0{p z3CYYp{U1?}vCTY2Y)D&5K?&I}#C~=t&o*X{awqzqBp)K#H^iJ7#1?&G{Wvg5GN3XB zAe%$R`2g(6Os4BnJg;DEj#O646*3z+M1SaN_v>n&j6wJG_>qsbJr;ZsdFnB_wQy>| zgNZ4s-NxXYGMYH$pe{#XahVqDtMp7w)zZ6>K_}TAd_6tNA)7*st;jmH)e54d&~{d@rUqz!=Yi!xQ8dYg@kS-N;%COy)vujN?%n^E-qD##YL@X zeAWM^4X)ElU*=!Z5P#6r(&K7waXD`c-p+9UwKe$wy(eY-f207E7mO^DwB<>e$PUM% zL?p=wCwcsD2MCcQtDN5XDd8)1dp_+G z#0z1oh)18LW@WabX}C&KsKzdFNhkIZb(^O#h)A+@CI(HFH#}>X-nR1D+p`n)m~XGy z6O}iq2MX(^@VHGmx~NY<@pqJaff5D)-37@*O6Xjs9av(%4D|(6yh1u9`T-JCselt+ jQ7bzAt!RtL8Cpf1!<(s9HR68@=jZgfrMX#6UC{p*9YeVx literal 0 HcmV?d00001 diff --git a/vllm/inputs/data.py b/vllm/inputs/data.py new file mode 100644 index 00000000..724cdd2e --- /dev/null +++ b/vllm/inputs/data.py @@ -0,0 +1,242 @@ +from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, + Optional, Tuple, Union) + +from typing_extensions import NotRequired, TypedDict, TypeVar + +if TYPE_CHECKING: + from vllm.multimodal import MultiModalDataDict + + +class TextPrompt(TypedDict): + """Schema for a text prompt.""" + + prompt: str + """The input text to be tokenized before passing to the model.""" + + multi_modal_data: NotRequired["MultiModalDataDict"] + """ + Optional multi-modal data to pass to the model, + if the model supports it. + """ + + mm_processor_kwargs: NotRequired[Dict[str, Any]] + """ + Optional multi-modal processor kwargs to be forwarded to the + multimodal input mapper & processor. Note that if multiple modalities + have registered mappers etc for the model being considered, we attempt + to pass the mm_processor_kwargs to each of them. + """ + + +class TokensPrompt(TypedDict): + """Schema for a tokenized prompt.""" + + prompt_token_ids: List[int] + """A list of token IDs to pass to the model.""" + + multi_modal_data: NotRequired["MultiModalDataDict"] + """ + Optional multi-modal data to pass to the model, + if the model supports it. + """ + + mm_processor_kwargs: NotRequired[Dict[str, Any]] + """ + Optional multi-modal processor kwargs to be forwarded to the + multimodal input mapper & processor. Note that if multiple modalities + have registered mappers etc for the model being considered, we attempt + to pass the mm_processor_kwargs to each of them. + """ + + +SingletonPrompt = Union[str, TextPrompt, TokensPrompt] +""" +Set of possible schemas for a single LLM input: + +- A text prompt (:class:`str` or :class:`TextPrompt`) +- A tokenized prompt (:class:`TokensPrompt`) + +Note that "singleton" is as opposed to a data structure +which encapsulates multiple prompts, i.e. of the sort +which may be utilized for encoder/decoder models when +the user desires to express both the encoder & decoder +prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt` + +A prompt of type :class:`SingletonPrompt` may be employed +as (1) input to a decoder-only model, (2) input to +the encoder of an encoder/decoder model, in the scenario +where the decoder-prompt is not specified explicitly, or +(3) as a member of a larger data structure encapsulating +more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt` +""" + +_T1_co = TypeVar("_T1_co", + bound=SingletonPrompt, + default=SingletonPrompt, + covariant=True) +_T2_co = TypeVar("_T2_co", + bound=SingletonPrompt, + default=SingletonPrompt, + covariant=True) + + +# TODO: Make fields ReadOnly once mypy supports it +class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]): + """ + Represents an encoder/decoder model input prompt, + comprising an explicit encoder prompt and a decoder prompt. + + The encoder and decoder prompts, respectively, may be formatted + according to any of the :class:`SingletonPrompt` schemas, + and are not required to have the same schema. + + Only the encoder prompt may have multi-modal data. mm_processor_kwargs + should be at the top-level, and should not be set in the encoder/decoder + prompts, since they are agnostic to the encoder/decoder. + + Note that an :class:`ExplicitEncoderDecoderPrompt` may not + be used as an input to a decoder-only model, + and that the :code:`encoder_prompt` and :code:`decoder_prompt` + fields of this data structure themselves must be + :class:`SingletonPrompt` instances. + """ + + encoder_prompt: _T1_co + + decoder_prompt: Optional[_T2_co] + + mm_processor_kwargs: NotRequired[Dict[str, Any]] + + +PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt] +""" +Set of possible schemas for an LLM input, including +both decoder-only and encoder/decoder input types: + +- A text prompt (:class:`str` or :class:`TextPrompt`) +- A tokenized prompt (:class:`TokensPrompt`) +- A single data structure containing both an encoder and a decoder prompt + (:class:`ExplicitEncoderDecoderPrompt`) +""" + + +class LLMInputs(TypedDict): + """ + The inputs in :class:`~vllm.LLMEngine` before they are + passed to the model executor. + + This specifies the data required for decoder-only models. + """ + prompt_token_ids: List[int] + """The token IDs of the prompt.""" + + prompt: NotRequired[Optional[str]] + """ + The original prompt text corresponding to the token IDs, if available. + """ + + multi_modal_data: NotRequired[Optional["MultiModalDataDict"]] + """ + Optional multi-modal data to pass to the model, + if the model supports it. + """ + + mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]] + """ + Optional multi-modal processor kwargs to be forwarded to the + multimodal input mapper & processor. Note that if multiple modalities + have registered mappers etc for the model being considered, we attempt + to pass the mm_processor_kwargs to each of them. + """ + + +class EncoderDecoderLLMInputs(LLMInputs): + """ + The inputs in :class:`~vllm.LLMEngine` before they are + passed to the model executor. + + This specifies the required data for encoder-decoder models. + """ + encoder_prompt_token_ids: List[int] + """The token IDs of the encoder prompt.""" + + encoder_prompt: NotRequired[Optional[str]] + """ + The original encoder prompt text corresponding to the token IDs, if + available. + """ + + encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]] + """ + Optional multi-modal data to pass to the encoder model, + if the model supports it. + """ + + +_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt) +_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt) + + +def build_explicit_enc_dec_prompt( + encoder_prompt: _T1, + decoder_prompt: Optional[_T2], + mm_processor_kwargs: Optional[Dict[str, Any]] = None, +) -> ExplicitEncoderDecoderPrompt[_T1, _T2]: + if mm_processor_kwargs is None: + mm_processor_kwargs = {} + return ExplicitEncoderDecoderPrompt( + encoder_prompt=encoder_prompt, + decoder_prompt=decoder_prompt, + mm_processor_kwargs=mm_processor_kwargs) + + +def zip_enc_dec_prompts( + enc_prompts: Iterable[_T1], + dec_prompts: Iterable[Optional[_T2]], + mm_processor_kwargs: Optional[Union[Iterable[Dict[str, Any]], + Dict[str, Any]]] = None, +) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]: + """ + Zip encoder and decoder prompts together into a list of + :class:`ExplicitEncoderDecoderPrompt` instances. mm_processor_kwargs + may also be provided; if a dict is passed, the same dictionary will be + used for every encoder/decoder prompt. If an iterable is provided, it will + be zipped with the encoder/decoder prompts. + """ + if mm_processor_kwargs is None: + mm_processor_kwargs = {} + if isinstance(mm_processor_kwargs, Dict): + return [ + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt, + mm_processor_kwargs) + for (encoder_prompt, + decoder_prompt) in zip(enc_prompts, dec_prompts) + ] + return [ + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt, + mm_proc_kwargs) + for (encoder_prompt, decoder_prompt, mm_proc_kwargs + ) in zip(enc_prompts, dec_prompts, mm_processor_kwargs) + ] + + +def to_enc_dec_tuple_list( + enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]], +) -> List[Tuple[_T1, Optional[_T2]]]: + return [(enc_dec_prompt["encoder_prompt"], + enc_dec_prompt["decoder_prompt"]) + for enc_dec_prompt in enc_dec_prompts] + + +def __getattr__(name: str): + if name == "PromptInput": + import warnings + + msg = ("PromptInput has been renamed to PromptType. " + "The original name will be removed in an upcoming version.") + + warnings.warn(DeprecationWarning(msg), stacklevel=2) + + return PromptType + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/inputs/parse.py b/vllm/inputs/parse.py new file mode 100644 index 00000000..e5fa1e41 --- /dev/null +++ b/vllm/inputs/parse.py @@ -0,0 +1,106 @@ +from typing import List, Literal, Sequence, TypedDict, Union, overload + +from typing_extensions import TypeIs + +from vllm.utils import is_list_of + +from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt, + LLMInputs, PromptType, SingletonPrompt, TextPrompt, + TokensPrompt) + + +class ParsedText(TypedDict): + content: str + is_tokens: Literal[False] + + +class ParsedTokens(TypedDict): + content: List[int] + is_tokens: Literal[True] + + +@overload +def parse_and_batch_prompt( + prompt: Union[str, List[str]]) -> Sequence[ParsedText]: + ... + + +@overload +def parse_and_batch_prompt( + prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]: + ... + + +def parse_and_batch_prompt( + prompt: Union[str, List[str], List[int], List[List[int]]], +) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]: + if isinstance(prompt, str): + # case 1: a string + return [ParsedText(content=prompt, is_tokens=False)] + + if isinstance(prompt, list): + if len(prompt) == 0: + raise ValueError("please provide at least one prompt") + + if is_list_of(prompt, str): + # case 2: array of strings + return [ + ParsedText(content=elem, is_tokens=False) for elem in prompt + ] + if is_list_of(prompt, int): + # case 3: array of tokens + return [ParsedTokens(content=prompt, is_tokens=True)] + if is_list_of(prompt, list): + if len(prompt[0]) == 0: + raise ValueError("please provide at least one prompt") + + if is_list_of(prompt[0], int): + # case 4: array of token arrays + return [ + ParsedTokens(content=elem, is_tokens=True) + for elem in prompt + ] + + raise TypeError("prompt must be a string, array of strings, " + "array of tokens, or array of token arrays") + + +class ParsedStrPrompt(TypedDict): + type: Literal["str"] + content: str + + +class ParsedTextPrompt(TypedDict): + type: Literal["text"] + content: TextPrompt + + +class ParsedTokensPrompt(TypedDict): + type: Literal["tokens"] + content: TokensPrompt + + +def parse_singleton_prompt( + prompt: SingletonPrompt, +) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]: + if isinstance(prompt, str): + return ParsedStrPrompt(type="str", content=prompt) + elif isinstance(prompt, dict): + if "prompt_token_ids" in prompt: + return ParsedTokensPrompt(type="tokens", + content=prompt) # type: ignore + elif "prompt" in prompt: + return ParsedTextPrompt(type="text", content=prompt) + + raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt") + + +def is_explicit_encoder_decoder_prompt( + prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]: + return isinstance(prompt, dict) and "encoder_prompt" in prompt + + +def is_valid_encoder_decoder_llm_inputs( + inputs: Union[LLMInputs, EncoderDecoderLLMInputs], +) -> TypeIs[EncoderDecoderLLMInputs]: + return "encoder_prompt_token_ids" in inputs diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py new file mode 100644 index 00000000..64387fd2 --- /dev/null +++ b/vllm/inputs/preprocess.py @@ -0,0 +1,580 @@ +import asyncio +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from typing_extensions import assert_never + +from vllm.config import ModelConfig +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup +from vllm.utils import print_warning_once + +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType, + SingletonPrompt) +from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt + +if TYPE_CHECKING: + from vllm.multimodal import MultiModalDataDict + +logger = init_logger(__name__) + +PromptComponents = Tuple[Optional[str], List[int], + Optional["MultiModalDataDict"], Optional[Dict[str, + Any]]] +DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]], + Optional["MultiModalDataDict"], + Optional[Dict[str, Any]]] + + +class InputPreprocessor: + + def __init__( + self, + model_config: ModelConfig, + tokenizer: Optional[BaseTokenizerGroup], + ) -> None: + super().__init__() + + self.model_config = model_config + self.tokenizer = tokenizer + + def get_tokenizer_group(self) -> BaseTokenizerGroup: + if self.tokenizer is None: + raise ValueError("You cannot pass text prompts when " + "`skip_tokenizer_init` is True") + + return self.tokenizer + + def get_bos_token_id(self, + lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + if self.tokenizer is None: + logger.warning("Using None for BOS token id because tokenizer " + "is not initialized") + return None + + return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id + + def get_eos_token_id(self, + lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + if self.tokenizer is None: + logger.warning("Using None for EOS token id because tokenizer " + "is not initialized") + return None + + return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id + + def get_decoder_start_token_id(self) -> Optional[int]: + ''' + Obtain the decoder start token id employed by an encoder/decoder + model. Returns None for non-encoder/decoder models or if the + model config is unavailable. + ''' + + if not self.is_encoder_decoder_model(): + print_warning_once("Using None for decoder start token id because " + "this is not an encoder/decoder model.") + return None + + if (self.model_config is None or self.model_config.hf_config is None): + print_warning_once("Using None for decoder start token id because " + "model config is not available.") + return None + + dec_start_token_id = getattr(self.model_config.hf_config, + 'decoder_start_token_id', None) + if dec_start_token_id is None: + print_warning_once("Falling back on for decoder start token " + "id because decoder start token id is not " + "available.") + dec_start_token_id = self.get_bos_token_id() + + return dec_start_token_id + + def _get_default_enc_dec_decoder_prompt(self) -> List[int]: + ''' + Specifically for encoder/decoder models: + generate a default decoder prompt for when + the user specifies only the encoder prompt. + + Encoder/decoder models utilize the decoder + prompt in different ways; as new models are + added, it is intended that this function + will be extended to produce differing + default decoder prompts, depending on the + model variety. + + Absent a special case, the default behavior + of this method is to mirror the behavior of + the HuggingFace (HF) GenerationMixin for a None + decoder prompt, which is to employ a logit processor + setting to force the first decoded token to be . + Here, this behavior is approximated by having the + "default" decoder prompt be . + + However, it is possible that in the future + other models may have different or more + complex logic for the default decoder prompt. + This motivates having a special helper method + for default decoder prompts. + + Returns: + + * prompt_token_ids + ''' + + bos_token_id = self.get_bos_token_id() + assert bos_token_id is not None + return [bos_token_id] + + def _prepare_decoder_input_ids_for_generation( + self, + decoder_input_ids: Optional[List[int]], + force_bos: bool = True, + ) -> List[int]: + """ + Prepares `decoder_input_ids` for generation with encoder-decoder models. + + Based on + + https://github.com/huggingface/transformers/blob/ + 4037a2b5b1278736e566aec12e169100275545ea/ + src/transformers/generation/utils.py + + specifically GenerationMixin._prepare_decoder_input_ids_for_generation() + + Arguments: + + * decoder_input_ids: input token ids to preprocess + + Returns: + + * Processed token list + """ + + decoder_start_token_id = self.get_decoder_start_token_id() + assert decoder_start_token_id is not None + + if decoder_input_ids is None: + # no decoder prompt input -> + # use decoder_start_token_id as decoder_input_ids + decoder_input_ids = self._get_default_enc_dec_decoder_prompt() + + if force_bos and (len(decoder_input_ids) == 0 + or decoder_input_ids[0] != decoder_start_token_id): + decoder_input_ids = [decoder_start_token_id] + decoder_input_ids + + return decoder_input_ids + + def _apply_prompt_adapter( + self, + prompt_token_ids: List[int], + prompt_adapter_request: Optional[PromptAdapterRequest], + ) -> List[int]: + if prompt_adapter_request: + prompt_token_ids = ( + [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens + + prompt_token_ids) + + return prompt_token_ids + + def _tokenize_prompt( + self, + prompt: str, + request_id: str, + lora_request: Optional[LoRARequest], + ) -> List[int]: + """ + Apply the model's tokenizer to a text prompt, returning the + corresponding token IDs. + """ + tokenizer = self.get_tokenizer_group() + + return tokenizer.encode(request_id=request_id, + prompt=prompt, + lora_request=lora_request) + + async def _tokenize_prompt_async( + self, + prompt: str, + request_id: str, + lora_request: Optional[LoRARequest], + ) -> List[int]: + """Async version of :meth:`_tokenize_prompt`.""" + tokenizer = self.get_tokenizer_group() + + return await tokenizer.encode_async(request_id=request_id, + prompt=prompt, + lora_request=lora_request) + + def _extract_prompt_components( + self, + prompt: SingletonPrompt, + request_id: str, + lora_request: Optional[LoRARequest] = None, + ) -> PromptComponents: + ''' + Extract the components of any single encoder or decoder input prompt. + + Arguments: + + * request_id + * prompt: single encoder or decoder input prompt + * lora_request: this is only valid for decoder prompts + + Returns: + + * prompt + * prompt_token_ids + * multi_modal_data + * mm_processor_kwargs (request-level input processor/mapper overrides) + ''' + + parsed = parse_singleton_prompt(prompt) + + if parsed["type"] == "str": + prompt_text = parsed["content"] + prompt_token_ids = self._tokenize_prompt( + prompt_text, + request_id=request_id, + lora_request=lora_request, + ) + multi_modal_data = None + mm_processor_kwargs = None + elif parsed["type"] == "tokens": + prompt_text = None + prompt_token_ids = parsed["content"]["prompt_token_ids"] + multi_modal_data = parsed["content"].get("multi_modal_data") + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs") + elif parsed["type"] == "text": + prompt_text = parsed["content"]["prompt"] + prompt_token_ids = self._tokenize_prompt( + prompt_text, + request_id=request_id, + lora_request=lora_request, + ) + multi_modal_data = parsed["content"].get("multi_modal_data") + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs") + else: + assert_never(parsed) + + return (prompt_text, prompt_token_ids, multi_modal_data, + mm_processor_kwargs) + + async def _extract_prompt_components_async( + self, + prompt: SingletonPrompt, + request_id: str, + lora_request: Optional[LoRARequest] = None, + ) -> PromptComponents: + """Async version of :meth:`_extract_prompt_components`.""" + parsed = parse_singleton_prompt(prompt) + + if parsed["type"] == "str": + prompt_text = parsed["content"] + prompt_token_ids = await self._tokenize_prompt_async( + prompt_text, + request_id=request_id, + lora_request=lora_request, + ) + multi_modal_data = None + mm_processor_kwargs = None + elif parsed["type"] == "tokens": + prompt_text = None + prompt_token_ids = parsed["content"]["prompt_token_ids"] + multi_modal_data = parsed["content"].get("multi_modal_data") + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs") + elif parsed["type"] == "text": + prompt_text = parsed["content"]["prompt"] + prompt_token_ids = await self._tokenize_prompt_async( + prompt_text, + request_id=request_id, + lora_request=lora_request, + ) + multi_modal_data = parsed["content"].get("multi_modal_data") + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs") + else: + assert_never(parsed) + + return (prompt_text, prompt_token_ids, multi_modal_data, + mm_processor_kwargs) + + def _build_enc_dec_llm_inputs( + self, + encoder_comps: PromptComponents, + decoder_comps: DecoderPromptComponents, + mm_processor_kwargs: Dict[str, Any], + ) -> EncoderDecoderLLMInputs: + encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps + decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if decoder_mm_data is not None: + raise ValueError( + "Multi-modality decoder inputs of encoder-decoder models are " + "not supported yet") + + # For Multi-Modal models (e.g., mllama), the text input can be + # <|image|><|begin_of_text|>hello world. And we should not add + # another <|begin_of_text|> to the beginning. + decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation( + decoder_prompt_ids, + force_bos=(encoder_mm_data is None and decoder_mm_data is None))) + + return EncoderDecoderLLMInputs( + prompt_token_ids=decoder_prompt_ids, + prompt=decoder_prompt, + multi_modal_data=decoder_mm_data, + mm_processor_kwargs=mm_processor_kwargs, + encoder_prompt_token_ids=encoder_prompt_ids, + encoder_prompt=encoder_prompt, + encoder_multi_modal_data=encoder_mm_data, + ) + + def _process_encoder_decoder_prompt( + self, + prompt: PromptType, + request_id: str, + ) -> EncoderDecoderLLMInputs: + ''' + For encoder/decoder models only: + Process an input prompt into an + :class:`EncoderDecoderLLMInputs` instance. + + There are two types of input prompts: + singleton prompts which carry only the + encoder prompt, and explicit encoder/decoder + prompts which carry both the encoder and the + decoder prompts as member variables. + + This function handles the following scenarios: + * Singleton encoder prompt: extract encoder prompt + token ids & infer default decoder prompt token ids + * Explicit encoder/decoder prompt: extract encoder + and decoder prompt token ids + + Note that for Explicit encoder/decoder prompts, + each sub-prompt (encoder or decoder prompt) can + have any possible singleton type; thus this + method relies on helper functions to obtain + token ids for the sub-prompts. + + Arguments: + + * prompt: an input prompt + * request_id + + Returns: + + * :class:`EncoderDecoderLLMInputs` instance + ''' + + encoder_comps: PromptComponents + decoder_comps: DecoderPromptComponents + + if is_explicit_encoder_decoder_prompt(prompt): + encoder_comps = self._extract_prompt_components( + prompt["encoder_prompt"], + request_id=request_id, + ) + + if (decoder_input := prompt["decoder_prompt"]) is None: + decoder_comps = None, None, None, None + else: + decoder_comps = self._extract_prompt_components( + decoder_input, + request_id=request_id, + ) + # Handle this carefully in case it was directly initialized by user + mm_processor_kwargs = prompt.get("mm_processor_kwargs", {}) + else: + encoder_comps = self._extract_prompt_components( + prompt, + request_id=request_id, + ) + # If there are no decoder components, we assume the + # mm_processor_kwargs are in the encoder prompt + mm_processor_kwargs = encoder_comps[-1] if encoder_comps[ + -1] is not None else {} + decoder_comps = None, None, None, None + + return self._build_enc_dec_llm_inputs( + encoder_comps, + decoder_comps, + mm_processor_kwargs, + ) + + async def _process_encoder_decoder_prompt_async( + self, + prompt: PromptType, + request_id: str, + ) -> EncoderDecoderLLMInputs: + """Async version of :meth:`_process_encoder_decoder_prompt`.""" + encoder_comps: PromptComponents + decoder_comps: DecoderPromptComponents + + if is_explicit_encoder_decoder_prompt(prompt): + encoder_task = self._extract_prompt_components_async( + prompt["encoder_prompt"], + request_id=request_id, + ) + + if (decoder_input := prompt["decoder_prompt"]) is None: + encoder_comps = await encoder_task + decoder_comps = None, None, None, None + else: + decoder_task = self._extract_prompt_components_async( + decoder_input, + request_id=request_id, + ) + + encoder_comps, decoder_comps = await asyncio.gather( + encoder_task, decoder_task) + mm_processor_kwargs = prompt["mm_processor_kwargs"] + else: + encoder_comps = await self._extract_prompt_components_async( + prompt, + request_id=request_id, + ) + # If there are no decoder components, we assume the + # mm_processor_kwargs are in the encoder prompt + mm_processor_kwargs = encoder_comps[-1] if encoder_comps[ + -1] is not None else {} + decoder_comps = None, None, None, None + + return self._build_enc_dec_llm_inputs( + encoder_comps, + decoder_comps, + mm_processor_kwargs, + ) + + def _build_decoder_only_llm_inputs( + self, + prompt_comps: PromptComponents, + prompt_adapter_request: Optional[PromptAdapterRequest], + ) -> LLMInputs: + (prompt, prompt_token_ids, multi_modal_data, + mm_processor_kwargs) = prompt_comps + + prompt_token_ids = self._apply_prompt_adapter( + prompt_token_ids, prompt_adapter_request=prompt_adapter_request) + + return LLMInputs(prompt_token_ids=prompt_token_ids, + prompt=prompt, + multi_modal_data=multi_modal_data, + mm_processor_kwargs=mm_processor_kwargs) + + def _process_decoder_only_prompt( + self, + prompt: SingletonPrompt, + request_id: str, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + ) -> LLMInputs: + ''' + For decoder-only models: + Process an input prompt into an :class:`LLMInputs` instance. + + Arguments: + + * prompt: input prompt + * request_id + * lora_request + * prompt_adapter_request + + Returns: + + * :class:`LLMInputs` instance + ''' + + prompt_comps = self._extract_prompt_components( + prompt, + request_id=request_id, + lora_request=lora_request, + ) + + return self._build_decoder_only_llm_inputs( + prompt_comps, + prompt_adapter_request=prompt_adapter_request, + ) + + async def _process_decoder_only_prompt_async( + self, + prompt: SingletonPrompt, + request_id: str, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + ) -> LLMInputs: + """Async version of :meth:`_process_decoder_only_prompt`.""" + prompt_comps = await self._extract_prompt_components_async( + prompt, + request_id=request_id, + lora_request=lora_request, + ) + + return self._build_decoder_only_llm_inputs( + prompt_comps, + prompt_adapter_request=prompt_adapter_request, + ) + + def preprocess( + self, + prompt: PromptType, + request_id: str, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]: + """Preprocess the input prompt.""" + if self.is_encoder_decoder_model(): + # Encoder-decoder model requires special mapping of + # input prompts to encoder & decoder + return self._process_encoder_decoder_prompt( + prompt, + request_id=request_id, + ) + + if is_explicit_encoder_decoder_prompt(prompt): + raise ValueError("Cannot pass encoder-decoder prompt " + "to decoder-only models") + + # Decoder-only operation + return self._process_decoder_only_prompt( + prompt, + request_id=request_id, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + + async def preprocess_async( + self, + prompt: PromptType, + request_id: str, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]: + """Async version of :meth:`preprocess`.""" + if self.is_encoder_decoder_model(): + # Encoder-decoder model requires special mapping of + # input prompts to encoder & decoder + return await self._process_encoder_decoder_prompt_async( + prompt, + request_id=request_id, + ) + + if is_explicit_encoder_decoder_prompt(prompt): + raise ValueError("Cannot pass encoder-decoder prompt " + "to decoder-only models") + + # Decoder-only operation + return await self._process_decoder_only_prompt_async( + prompt, + request_id=request_id, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + + def is_encoder_decoder_model(self): + return self.model_config.is_encoder_decoder_model diff --git a/vllm/inputs/registry.py b/vllm/inputs/registry.py new file mode 100644 index 00000000..5bd3e1c8 --- /dev/null +++ b/vllm/inputs/registry.py @@ -0,0 +1,314 @@ +import functools +from collections import UserDict +from dataclasses import dataclass +from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional, + Protocol, Tuple, Type) + +from torch import nn +from transformers import PretrainedConfig +from typing_extensions import TypeVar + +from vllm.logger import init_logger +from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once, + resolve_mm_processor_kwargs) + +from .data import LLMInputs + +if TYPE_CHECKING: + from vllm.config import ModelConfig + from vllm.multimodal import MultiModalDataDict, MultiModalRegistry + from vllm.sequence import SequenceData + +logger = init_logger(__name__) + +C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig) + + +@dataclass(frozen=True) +class InputContext: + """ + Contains information about the model which may be used to + modify the inputs. + """ + + model_config: "ModelConfig" + """The configuration of the model.""" + + def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C: + """ + Get the HuggingFace configuration + (:class:`transformers.PretrainedConfig`) of the model, + additionally checking its type. + + Raises: + TypeError: If the model is not of the specified type. + """ + + hf_config = self.model_config.hf_config + if not isinstance(hf_config, hf_config_type): + raise TypeError("Invalid type of HuggingFace config. " + f"Expected type: {hf_config_type}, but " + f"found type: {type(hf_config)}") + + return hf_config + + def get_hf_image_processor_config(self) -> Dict[str, Any]: + """ + Get the HuggingFace image processor configuration of the model. + """ + + return self.model_config.hf_image_processor_config + + +N = TypeVar("N", bound=Type[nn.Module]) + + +class DummyDataFactory(Protocol): + + def __call__( + self, + ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], + **mm_processor_kwargs: Any, + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]: + """ + Create dummy data to be inputted into the model. + + Note: + :data:`InputProcessor` is not applied to the dummy data. + + The :code:`mm_processor_kwargs` are overrides provided at + initialization time to values in the config whose values + may affect the number of tokens per instance. + """ + ... + + +class _MultiModalCounts(UserDict): + """ + Wraps `mm_counts` for a more informative error message + when attempting to access a plugin that does not exist. + """ + + def __getitem__(self, key: str) -> int: + try: + return super().__getitem__(key) + except KeyError as exc: + msg = (f"There is no multi-modal plugin with the key: {key}. " + f"Available keys: {set(self.keys())}") + raise KeyError(msg) from exc + + +InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs] +"""Preprocess the inputs to the model.""" + + +class InputRegistry: + """ + A registry to dispatch data processing + according to the target model. + """ + + def __init__(self) -> None: + self._dummy_factories_by_model_type: Dict[Type[nn.Module], + DummyDataFactory] = {} + self._dummy_encoder_factories_by_model_type: Dict[ + Type[nn.Module], DummyDataFactory] = {} + self._input_processors_by_model_type: Dict[Type[nn.Module], + InputProcessor] = {} + + def _default_dummy_data_factory( + self, + ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]: + """ + The default dummy data factory represents the longest possible text + that can be inputted to the model. + + Note: + :data:`InputProcessor` is not applied to the dummy data. + """ + # Avoid circular import + from vllm.sequence import SequenceData + + dummy_seq_data = SequenceData.from_token_counts((0, seq_len)) + dummy_multi_modal_data = None + + return dummy_seq_data, dummy_multi_modal_data + + def register_dummy_data(self, factory: DummyDataFactory): + """ + Register a dummy data factory to a model class. + + During memory profiling, the provided function is invoked to create + dummy data to be inputted into the model. The resulting memory usage + should be an upper bound of what the model would use at inference time. + """ + + def wrapper(model_cls: N) -> N: + if model_cls in self._dummy_factories_by_model_type: + logger.warning( + "Model class %s already has dummy data " + "registered to %s. It is overwritten by the new one.", + model_cls, self) + + self._dummy_factories_by_model_type[model_cls] = factory + + return model_cls + + return wrapper + + def _get_dummy_data_factory(self, model_cls: Type[nn.Module]): + return self._dummy_factories_by_model_type \ + .get(model_cls, self._default_dummy_data_factory) + + def register_dummy_encoder_data(self, factory: DummyDataFactory): + """ + Register a dummy encoder data factory to a model class + + This is similar to :meth:`~register_dummy_data`, but for encoder input. + """ + + def wrapper(model_cls: N) -> N: + if model_cls in self._dummy_encoder_factories_by_model_type: + logger.warning( + "Model class %s already has dummy encoder data " + "registered to %s. It is overwritten by the new one.", + model_cls, self) + + self._dummy_encoder_factories_by_model_type[model_cls] = factory + + return model_cls + + return wrapper + + def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]): + return self._dummy_encoder_factories_by_model_type \ + .get(model_cls, self._default_dummy_data_factory) + + def dummy_data_for_profiling( + self, + model_config: "ModelConfig", + seq_len: int, + mm_registry: "MultiModalRegistry", + is_encoder_data: bool = False, + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]: + """ + Create dummy data for profiling the memory usage of a model. + + The model is identified by ``model_config``. + + See also: + :ref:`enabling_multimodal_inputs` + + Note: + This should be called after + :meth:`~MultiModalRegistry.init_mm_limits_per_prompt`. + """ + # Avoid circular import + from vllm.model_executor.model_loader import get_model_architecture + + model_cls, _ = get_model_architecture(model_config) + if is_encoder_data: + dummy_factory = self._get_dummy_encoder_data_factory(model_cls) + else: + dummy_factory = self._get_dummy_data_factory(model_cls) + mm_counts = mm_registry.get_mm_limits_per_prompt(model_config) + mm_processor_kwargs = get_allowed_kwarg_only_overrides( + dummy_factory, overrides=model_config.mm_processor_kwargs) + + seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len, + _MultiModalCounts(mm_counts), + **mm_processor_kwargs) + + # Having more tokens is over-conservative but otherwise fine + num_tokens = seq_data.prompt_token_ids + if len(num_tokens) < seq_len: + if is_encoder_data: + print_warning_once( + f"Expected at least {seq_len} dummy encoder tokens for " + f"profiling, but found {len(num_tokens)} tokens instead.") + else: + raise AssertionError( + f"Expected at least {seq_len} dummy tokens for profiling, " + f"but found {len(num_tokens)} tokens instead.") + if mm_data is not None: + for k, v in mm_data.items(): + num_items = len(v) if isinstance(v, list) else 1 + num_expected = mm_counts[k] + assert num_items >= num_expected, ( + f"Expected at least {num_expected} dummy '{k}' instances " + f"for profiling, but found {num_items} instances instead.") + + return seq_data, mm_data + + def _default_input_processor(self, ctx: InputContext, + inputs: LLMInputs) -> LLMInputs: + """The default input processor is a no-op.""" + return inputs + + def register_input_processor(self, processor: InputProcessor): + """ + Register an input processor to a model class. + + The provided function is invoked on each input to the model. This + happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`. + + See also: + :ref:`input_processing_pipeline` + """ + + def wrapper(model_cls: N) -> N: + if model_cls in self._input_processors_by_model_type: + logger.warning( + "Model class %s already has input processor " + "registered to %s. It is overwritten by the new one.", + model_cls, self) + + self._input_processors_by_model_type[model_cls] = processor + + return model_cls + + return wrapper + + def _get_model_input_processor(self, model_cls: Type[nn.Module]): + return self._input_processors_by_model_type \ + .get(model_cls, self._default_input_processor) + + def process_input(self, model_config: "ModelConfig", + inputs: LLMInputs) -> LLMInputs: + """ + Apply an input processor to an instance of model inputs. + + The model is identified by ``model_config``. + + See also: + :ref:`input_processing_pipeline` + """ + # Avoid circular import + from vllm.model_executor.model_loader import get_model_architecture + + model_cls, _ = get_model_architecture(model_config) + processor = self._get_model_input_processor(model_cls) + + # Handle multimodal processor kwargs with priority: + # Inference kwargs -> Init kwargs -> {} + # If it's empty, it'll fall back to the default kwarg values + mm_processor_kwargs = resolve_mm_processor_kwargs( + model_config.mm_processor_kwargs, + inputs.get("mm_processor_kwargs"), + processor, + ) + + return processor(InputContext(model_config), inputs, + **mm_processor_kwargs) + + def create_input_processor(self, model_config: "ModelConfig"): + """ + Create an input processor (see :meth:`_process_input`) for a + specific model. + """ + return functools.partial(self.process_input, model_config) diff --git a/vllm/logger.py b/vllm/logger.py new file mode 100644 index 00000000..77dddbfb --- /dev/null +++ b/vllm/logger.py @@ -0,0 +1,155 @@ +"""Logging configuration for vLLM.""" +import datetime +import json +import logging +import os +import sys +from functools import partial +from logging import Logger +from logging.config import dictConfig +from os import path +from typing import Dict, Optional + +import vllm.envs as envs + +VLLM_CONFIGURE_LOGGING = envs.VLLM_CONFIGURE_LOGGING +VLLM_LOGGING_CONFIG_PATH = envs.VLLM_LOGGING_CONFIG_PATH +VLLM_LOGGING_LEVEL = envs.VLLM_LOGGING_LEVEL + +_FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s" +_DATE_FORMAT = "%m-%d %H:%M:%S" + +DEFAULT_LOGGING_CONFIG = { + "formatters": { + "vllm": { + "class": "vllm.logging.NewLineFormatter", + "datefmt": _DATE_FORMAT, + "format": _FORMAT, + }, + }, + "handlers": { + "vllm": { + "class": "logging.StreamHandler", + "formatter": "vllm", + "level": VLLM_LOGGING_LEVEL, + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "vllm": { + "handlers": ["vllm"], + "level": "DEBUG", + "propagate": False, + }, + }, + "version": 1, + "disable_existing_loggers": False +} + + +def _configure_vllm_root_logger() -> None: + logging_config: Optional[Dict] = None + + if not VLLM_CONFIGURE_LOGGING and VLLM_LOGGING_CONFIG_PATH: + raise RuntimeError( + "VLLM_CONFIGURE_LOGGING evaluated to false, but " + "VLLM_LOGGING_CONFIG_PATH was given. VLLM_LOGGING_CONFIG_PATH " + "implies VLLM_CONFIGURE_LOGGING. Please enable " + "VLLM_CONFIGURE_LOGGING or unset VLLM_LOGGING_CONFIG_PATH.") + + if VLLM_CONFIGURE_LOGGING: + logging_config = DEFAULT_LOGGING_CONFIG + + if VLLM_LOGGING_CONFIG_PATH: + if not path.exists(VLLM_LOGGING_CONFIG_PATH): + raise RuntimeError( + "Could not load logging config. File does not exist: %s", + VLLM_LOGGING_CONFIG_PATH) + with open(VLLM_LOGGING_CONFIG_PATH, encoding="utf-8", + mode="r") as file: + custom_config = json.loads(file.read()) + + if not isinstance(custom_config, dict): + raise ValueError("Invalid logging config. Expected Dict, got %s.", + type(custom_config).__name__) + logging_config = custom_config + + if logging_config: + dictConfig(logging_config) + + +def init_logger(name: str) -> Logger: + """The main purpose of this function is to ensure that loggers are + retrieved in such a way that we can be sure the root vllm logger has + already been configured.""" + + return logging.getLogger(name) + + +# The root logger is initialized when the module is imported. +# This is thread-safe as the module is only imported once, +# guaranteed by the Python GIL. +_configure_vllm_root_logger() + +logger = init_logger(__name__) + + +def _trace_calls(log_path, root_dir, frame, event, arg=None): + if event in ['call', 'return']: + # Extract the filename, line number, function name, and the code object + filename = frame.f_code.co_filename + lineno = frame.f_lineno + func_name = frame.f_code.co_name + if not filename.startswith(root_dir): + # only log the functions in the vllm root_dir + return + # Log every function call or return + try: + last_frame = frame.f_back + if last_frame is not None: + last_filename = last_frame.f_code.co_filename + last_lineno = last_frame.f_lineno + last_func_name = last_frame.f_code.co_name + else: + # initial frame + last_filename = "" + last_lineno = 0 + last_func_name = "" + with open(log_path, 'a') as f: + if event == 'call': + f.write(f"{datetime.datetime.now()} Call to" + f" {func_name} in {filename}:{lineno}" + f" from {last_func_name} in {last_filename}:" + f"{last_lineno}\n") + else: + f.write(f"{datetime.datetime.now()} Return from" + f" {func_name} in {filename}:{lineno}" + f" to {last_func_name} in {last_filename}:" + f"{last_lineno}\n") + except NameError: + # modules are deleted during shutdown + pass + return partial(_trace_calls, log_path, root_dir) + + +def enable_trace_function_call(log_file_path: str, + root_dir: Optional[str] = None): + """ + Enable tracing of every function call in code under `root_dir`. + This is useful for debugging hangs or crashes. + `log_file_path` is the path to the log file. + `root_dir` is the root directory of the code to trace. If None, it is the + vllm root directory. + + Note that this call is thread-level, any threads calling this function + will have the trace enabled. Other threads will not be affected. + """ + logger.warning( + "VLLM_TRACE_FUNCTION is enabled. It will record every" + " function executed by Python. This will slow down the code. It " + "is suggested to be used for debugging hang or crashes only.") + logger.info("Trace frame log is saved to %s", log_file_path) + if root_dir is None: + # by default, this is the vllm root directory + root_dir = os.path.dirname(os.path.dirname(__file__)) + sys.settrace(partial(_trace_calls, log_file_path, root_dir)) diff --git a/vllm/logging/__init__.py b/vllm/logging/__init__.py new file mode 100644 index 00000000..b9aec380 --- /dev/null +++ b/vllm/logging/__init__.py @@ -0,0 +1,5 @@ +from vllm.logging.formatter import NewLineFormatter + +__all__ = [ + "NewLineFormatter", +] diff --git a/vllm/logging/__pycache__/__init__.cpython-310.pyc b/vllm/logging/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84691e35306720775fa6a500b67c3328df08aa43 GIT binary patch literal 240 zcmd1j<>g`k0>icYGNOU>V-N=!FabFZKwK;aBvKes7;_kM8KW2(8B&78QZC`)M*qiIwH#AG;rR$}k$lqd* zk5A0WiH~2&P{aaM4<>%u=$GUc=oe%b=$0fFrKgtYn&*~g*r)9=@(R%WaQ`R tgG|+jnyMckpP83g5+AQuP-EKfYPQQMe{h)+=N9Ry{a4td3697R3 zRb)(F6X6NACW1+FML)B#C+J%e`adCuj3$VO!oax%HBSK~8B;;tw+{2Mvyh(HUs?b?%t<vX-u3*GaTL0XA`8d7qpw8FF{QYzygsQuObz#yozmn3S?mvItrOa^Cx5SJxSwW%8GHGI(DzI~uDlx`y$!u)V8ayztC;Xl$T6ui@hXjxBaQ|uuN!E1DI{!aY@izeuDPAp zwRS59{-sJ`A6k|wp2*PGNWe9g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Heeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_eolT-qJDgQW?p7Ve7s&kARd7B1`#-My<0(b?8)b$s9hsMLhhIRrimUocpjv7mED>my7U-6) zQn~OA%iv!#NLeW^)BJRhu`-<2{cJE{O>o-qbHSuF8RV@z=S_bqC|Ctfr~G}vv^5>f zSTjMOTkCf-;n z>+$5p56``3pZ@-9r{8>I@l33rt%XsXe!CUbnhnQ~jdvSJESA-{a87!WbZQOHJ>6_n zYs)Cho^75#)ys*v6nTxXDea)?dcNIqq~rTYK-$YrwBpI#T1d}rS3HdQ;IbFl<42NC z<5Jm(51nrM?Vxe4_kgpQk|W=*MJqI$_~ExaxlD81rOn}_%{yvjHAMWr?ET>`&a?+yz{Tve*7tGeP!dR^_R*aEHNXG^$TuhNuG)Rd*wm0D$`9u^P;@Q5l z=E*0@+_2J{82`}hNcj)r4i7+FQ@Tn`>8ig`KUbR*UCeY&sVkVJSd1s=X*OoVn{LhU zRP5eVY@SM4#JVGw!`QsEO1~RwJTWwp4du(%7NVfF(5khLMQGqf$DR+`E5Xyj>VjWe zT4=3BE6v8jC%zvn(0~hDoW{)FGGIDU9? z9{uEg^eGEj#E-m>1Qo~`60;ET!JP-9`jA#EXX?776&qPk$9o?@x4VhLBZ zG*@_f-EcLMRo&HH1IO5OO)gD&Df*pLdi5rQfNrI|jF)w@-b6hI7?=bEO<$G^CQ-ZS8LTOmbO%t&7ibXH)>y6 zg7*2fO0BV6;{2%+*0njGD%yY-s zWaV;V6X?>GjS6XI|1yxE`!w!b&p|}WWo2D-mAYET%FZe4T32+nqOz_-V*%7Z5?vjC z+I>h>1?u^vZHUF2H0*{Z$sEM_Pjn4D0c~942FzlV>l&BUq^>^M z)ljR{^RDTpK0_^@pmp`nX@|56%0_ze<{ey;5K zJlPE8LsXQi`b{T#=C}j_z}~mojatRA2`XE#NLhMPxFTzfOEQfH78iJ*ww#9BQ1PYd2+WnNX45X{k-eYKxXgQenD78lfnrg~tL7bD z6rm3vHh)d@eE438+`- zko7N0r*MOakZ(Yo#eFLeVP*vuDETwL>O8xbCE7lT znK*6R4JYtyJI>m@B|SCg`Bw^_klUciH6^CHt6`2;eOOakvK>>W)w(M&cF zv1l}6^DSO=Nh>O8NhKX$oa|=P=NF16d#_sYz>|8f%fLdf%63T}M2eoqdXrXkb=FL$ z$n@!I_KujPMZj^sEj019(8YgDLwqI7gI{vL*kBScf+B=X8QipGp}?7pl&+}5iV&XG zMP0kD({C4u0zA>X%5?*HqSVbEo+w?NOZ0w87b)P0zHUS*;D*L+2~jYri(j-ORMAkI zL0y(n#f&Dp+GX`s<)gxS3N3TjC%a}hML0zLD3u`1$1g?su8G;1u2@fR_BzbHrl?&q z{(6Q+sTW`Yl5s?xYrE-ghH=RRE+PHAvY!6f!kmq+0fd6ZfYnSpA7CNaChbljhXA_- zkQnMQmNRZPmkzTxkBwjmmE)eUmQHEat1YiYB|5>tCT4tz9wj!C8(xIz48Xo&Itg05 zM#U@D5}p`FUW?5$S$DQnr7GAY8>6Kjsf6nDFrPSbjca#dz{68sYDe@K-~?=6BciTD zZ@ia7$IazDRBeF%l1!w08sefn4q4t8YrfZDz?R>px@SocR?6o|5V-T{li#7#cS$gw z8!*z#o_vuCPLLoFm*0bk3k2;6lG|)q%h>^RON*MQ4fztaf0;6=1hm6ggMqgue6I>j zuU12NJx8b!AIgEr&)PwyWi#s9s16AmvJ*vhS{wv+(@kieGxBN{_@35ukb#0~5~?%4 z)2TJT%%-zAveH543ht3>lTiK~#ajubXLu$|u|_lp2SxYv(Gn0%!!1Nz4>OnC*rkF6^Id2LiQ+M-dTi}-adPI{v&V*=AGtr#Ek=YlQ_k2`3zhXTY zp_{vR-by;b9eId^d_GQ;@_(nFO!gKfc@!helVH8ivqFhX;t{z@TM2VWW6>!s^W@0^ z(GWTS3aNy0%uhvEfR%!YQF)J@6 zFg%9RvYrqUVyxD_go`A88KZzz=ojrkTiJyD5}nxvJOC9fKv{DltLk?Wsei4To#%Gn zs7<&ye`H0 zlO(oGfeh-OafdXQQ3vFJV1sqLsBLpUj@)&s_wGRbm1fI)-%PkZo6WB}$L~6)ZKq1U zjj@JSWT5nyQ1J(P&=Cmc*Qp_XT@;DEzJo{K0nNQF#KGH=`s}ReOQ?l?Jd%LG-R9c) z#ddYyrRNx>Fa7^aqsD8L_=n+Nkm(yan8xY8S?i@?4%ti=$)It-ih^J1NO3{uUsW!v zKUS|QS5?He5mXp1hY=k%or270m*q7sA8SShnPs$sL@UUlRy-3nwt|c_T0x=}WWUw2 z)-$~6#PCWNFUP@(ZGl~~UlcYBi8w5S0DP2PSFdphhG_s8+0Vqs6lYg?00bQ$qQS^= z!zmW?PsM5tb!_4Ij1oe`*%RcOSDHcV?;sIgY>#8Su*bPRH+>6B`cia(LiHTD91KBkH4>3u8c%o#`A(C5kh4vW z51byL#E8?QAUF?4hh3mSU>>o<;h=n3TV!Ztlm$GH%g*@72!qHX<3&WJj4%WmP=W^H z!n=pxmma}hh7{t;!I{ac6j$vW8@rW58ZgYZmKZoj{6XT-bn~kdRY+dFA9qKL9R_P# z)Ln3i*o?X(g(w-~?t6;5BksmgcPt@NFpxKrsJo6x7}ZTCv3icc=eq{{O_O^}!I|QE zM8S+324Y@_=yf$jy5N<Wn`$S0|4 zGu9$cQFe1*$06>Mw2Md4Y#I@jv2k)Ut>)5Mimp)n^L7?SqjTi0`+V9_40*Kc`x2xh{kiY&_qm__-1BP;(;B#J8EBk zFgEew2HycC@sq?W<);#pVAs@2^Jkd(MuXi@D~JCZXoLtI2Kdy-ld~r1Q}2B8`GGxD z>me?2L4afpQAf{3Z4mI+8p15Q*-B!2q-56NZH@c*M;~hv5or1u_D(g9;jpGdx{oWvp~?!y5!EpiC;Oj;0 literal 0 HcmV?d00001 diff --git a/vllm/lora/__pycache__/layers.cpython-310.pyc b/vllm/lora/__pycache__/layers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d1ee4c20c556161cb1ff8d12782414848d2ea52 GIT binary patch literal 34024 zcmd6Q3y>VgdEUBt%&gO3S^QIbaX_ zIQHxT#QI>#0%cON=-7(mII`sciYy08l!xNjN;yg?k)1e>N_jbsYGNmHtjJ-N%W{HB zLP8{v@B4dZcXn_0?jYM0-_`VX_w;M}@%Q)t{daeLXegV&-yL82r>DRBtBJ%P&=LPj z!SMhd|64{P;U-+8l5qHIR*b6YmRyF0MsWni} zopolbTb-@dZO*ppc4xc1&s26)cRD-8K2X_J%{zIqXDhp_dz?Mhz0Th1K4)L`7U!1g zerLbD8?4+~J>VP=d#+Nb-sap^z1_K8+(VUv)kDr9BjH)F}cE zj=EznTFw!7+?{wa;oRYw?@YLp?v@u5?iO$2V)CNl+zFp4cN#v^jCnEX9EHz}I}4v# z?hjXKH10XPdymEw}YLhyhgF%)%?0DR_m@; zDK3{(sZxPOm1<{#ef{5`tE-Az^vg{z*yb-+%B^%tm3r+|5ns|My5(w+@0~m{DV{1d zPJ3#-5Xy5`7QA44?=*CLe3b2$&z4;ewxK6iYUPE}2NXWoQ-xG8dAwd(sn(thKkIQ+ zPe}z^pYYTv)>DsnWAA&2}cUe98AvtLIT+HX3?G z)fPY*T7R@B@f^qFKLewYSOwL%XsUtL#FDXOKCOLB_>99R+44!kXB&L1mX8IWeeg+% zkL{*b6F-;uta;J+m{G8s_dF%J`*}28-u2Fw7rgvpUFFZAExmlBo?ot%;KjG>YhnBQ zbAf%%E1x>u2!`}8y4u1S&kfSghyKPx&6)St{FUY9xO9yvQ7k<@O!+I!Jt$gK7&Nb43f+B{!s%~YNBr>~?CC|Z_GE8&$hH02L#?9vKMbucaGqYG4 zbN5y--YWk6b4WoFESTyJB%A}^ z6^55^3^(DJ??(kYNzd|9u6Zw}byi?7$iWlR?MH)b*)J|DZ?RmdoWKN;DHdy`s#h!q z*&+$|N`>}ZvH1K-siNPgw;+eWny=R@!AP-Ks-YfA>_vXDsJ}}-P7HpWx(mjmc;pMR ziBVI0)_lPrf;_-rTS0Q7FK`VHQq}%o`=ixH)&|#gsDy7%{MDx+Cr=!VlAha--D9o8}ne z;1{LN2(+-A5qHuXMT{-(6k?2d}G$tO9K@9JFDPLa9 z%Us(K0svB>C&v78IK7qq>f8xw-798b`CesFaV*E$Tl#U%XQG;>m7ZXZ%-*t^JJ%F> zE*>;2|9sz)D-Obf*}OAW>iN!6pF>@&S8VkT7Wg3=ogFlYmwNtb26zz024$WP2CAhO zG<{LGA$*Y2Ku43K1y*(RM|;!;(U)v}tOnYP*}Liax)rLWhC=Q6QAJz2#E&4dWS}^R zy30PXpgh0_UFCYd^FN4iy5iH#Th>;5j~GF!49fn(hSh)jQ=acN^z0!$C(JQlYUCva z%&iM&I6)kVzQWXB@$MdF&nl)SMW4rOU;2vK+`hKR?SR3U7oi+K1egnCX`(;cyQ_Nl zd(V36oGLe>n%14$ZE6;?*pIubJrfmt5ifPYGtI4Q3mydvhL;6D7CCVO zK(482emjS#Pws8iSWOhch&`RSVgSRdC*gE$0AA$rR zuk(0W_BDQ!Aaj7G6DR78$Ex5tsvaPQDFK!Q(#* zBa<-Atw11K&Fpp4ZrQTd8>VSqHMQ3nQ{O#TMF|Y=S>SD1h_C;4ld=ug3}yD9Q?{mIp&Td%#b@FZ%mq))ya-yj`c>| zDQ|RXOmdpW{MpLQo4|}ohMY6zuhoYvv$!ULi7bAd%q1fVExYjHQl=-CRn_uN5BPqY zwLp%*t0BC-l$PJjjE-+Fkqb;L!M%u@C}zdgO39Vjmj;RRjYQ3MnGz&NOs|n!8bS@4 zOT!XpWKrjCE{%q1jJf73@GFjoAtfJ%pH9G>KzMU$vMcNs9oE%pjYv4FN4P1(+j41| z=F*I7@C(HT{$nz-TeFGF{H9s9Y$0_*ab5xZNx6;lXx)8qy-KfFNo@;TrH!3EwUufS z9G2%6V-ppEQvlKsZcFo-7UeQdp26$tRNuhCcr2N4wsnStyFuS_i5Hw{o!?n9_(dOF zXoIi37lL7?9C`z1I9uWY!OMlgMq^nBDsxig2a4)1;lqMd(^GXn*dhqidjXQ5c#=b{ zxD?8$GGa*0bd)=sZ-b!-F=-)CR+5z342H#CV)2D}*KYY7aa(80-Z_k`)1_rE80!pN zywg?hWX3`>+k}&Z_>*}2MHo3d1CH9_kh*T)$fRr#(TqvceiNh-zZh5KPx^M+-^?==f#)Re&)`PkVhBWq-@H}BkFsP+I~ir8ok%^|&!vAPEq@XYA_ zs|J`j+Q31FHdAZ_3a0L#g``?R1jQ*(DH@zQx@(5B)7SC%HVl;87&e-P4NBg|U>f>A z#B_;`IzR#wOyiNFLxzjNbO{CL@Ye;$;5U~{4!Mt;p&bO~a2@~2S?jnXK!Y9FBMKLTlv0|uCBBqU)iB$-mTKUi(&&O)TN-U)%-lrJv zQii6E7upa@NCJJq!;vH)h9((lo#t%f0?Fe5)A4`!-GD{fwF*I(F2QW#QgSuLS|Blr zo`S6#O)a6uPF^vbT&-R!&O>r=27H)01K&bcLkVfr7vK@37V69Ai_*Y>aZB_Sl7_`+Nfy&arx)nn|3oASX&JVRpHI zhG)fILKnGgT(BCcOK9>{3o`>6=dy|B0E6Cs&ot1q#-)K(%m$N*%Zz2O+85HR_EOeO zo-x(&Rr}H)Tsg**U!*^oa4n=}qak&uXI4`Pl?p@gw;hUIz{H$}(69}=VE36AdiP5>?$f*ngZXeZ3c!OEp(IQ6Zyy%a8Eon4TdT^)rCG zU@81F>PIEbDB~$gyg~*)HfcG7;$L5B;1isYn1i!hcTc;BB)@~lKL%qI2xpkR3XgSt z*s_frdYpxxWf_yOWkHTXbkWN|nBUB#fn5#s$2Y;>?B9T7g%zQh2=Wgdq>5;WQ#bqC zRm%4u*)}d34Pzxja*!!nGS?ILsSnWD-!cTR5wwF?X^tzGX&%|2M15F~sczOoG9(yO zAEMEh!{7|9o~0qY1}j#5m`0Jtb2RvwZ4M(C3SsAhhjxq$>=qeLKB@%U$ME=%!pJ11 zz%z8Q7gqL-;kTKoZuVkoyScL$YcY9EG0g=$9e=EJ{`ud8u_0@*1CDs3-WXQ*#+N39-Pr0)#@G!C=(f4r$!<_YCk)3l#`?}+rh9ScrdSP1 zP1mv-Bz}OTgtsQ<$3zjNkM@folSRPmkOe8E6)Rgw4|2vJ4Hd7>Ak&^uk^Q2HmsiXfI_w7w53{{3gcex1w{Q722MF2LEkH;>x&{yusu2`|O-v4&DE`3p%{qd(SqR9RP#z=5t0xPOVThfv78 zOy~pL3rK-M0-GKj#^n@%2G)1L2K=E;>`&ftB`ruI$YJfK2Kk_tJEb5dteh8Q;baH} z*s0sS;z}rWW&zcwVFb4B?%kE~vv_wrq70MfFz0WuNX?yn`-^1U!RN4V@RNlO+zX5T zT%RBuL=CHe1`!{iGe6ywPe*mAnb><)gtf?p5wrO#k80$@^~`?7$jTBWmPPXqVkPic z%S9heBah{^tPcy%by+C(8+kw|03t4xLVEdJo0dK|*G?}KgLWo)D6gxpBcDTgwtUO$ zM8VQ6HRPWuFBkc)h?bG+SFGod9tF0}SikrTUIY;-XPoX9x`+VpW_O*0#U9Mc%4N<1 zW*1w=Z1%0Bxy>Ud^AaQ3CtwV*ekt4)m^9c0lPHCqT`Psnj1tXSM)n5K z?4-;GYoxJL*ChP~B;E1Xf=t)p-G)pK*d0%ZUMP_T^8tx$YQTarQ7DbY8fL~DkhMz- z5?M%JA(73xInQ1+U94e3t~v-gEW~nHS%ukBh#GdsJA5bHNrW9?%rLxuzXjM0pms%K96>lDUo|1PAjTp6(~wT^j2>OtE=Z?_kZh1{ENdJ`K;z8D zph>Y)=qTXk00ONK_MT^lW+x-qg9Zxz0rzg{Rclc_-#p~wgsUN6!;or#`o~hzAA)Js zY#>^}wx|LVaF8PB2o4hRQQ({n;oxBFZDJ`Wu{z%mA?y%XambcXu_FWnP$AJwIKiP7 zv5abr$NE&4X&j^>fN`9*O`=A{&kC|(6JW-;(i27ovf5iL75leUo7xDWA*ejhc44_} zNFZ}&&X9$jb)aSD+Bn$Jb=&Qx0jBy1RtJHk`pY!>A;_OWVbxC}MDx~l3($oiJFo%g z2qdGqbAklwG|5QN#Q1)A#mKT$)C2JQ3?k{ZKZrEf%{{JwAbJ5E`P)dVg-zZH2S>2T zpA6s-b2T}cxB%|N#R`eoiAy2a`?v|dB?asXG^_QnD4AVCd@SH{T&d5|9!lsQf<4%( ziAzsC$*h+2i#FsyDB5>l2+|_FcVFnPfQOOZXITNv2_id;;2ec0H&GSyYh}7|2!EB$ z+6m!h2?+}^K&d9gY8tR5VIxwd3|Gr}Q(6)#03Z5DEr4%h+@@myA7VO*LCLPL0f<3+ zJVvq|BLeuWViByJ6o7waKm+{C^nQQ_>*nV1{x707{XC*((^-?$urKOgpGI~de*tiP zfxr4IEa^|v_!%01okkZ)AcFbFD4O~?Sekp*?dNN8cSqkQdO<@0S8ss3Ej0_$3@zvj zRt^0hya!!sq*XRa`Iyu?D2ZBSca1j&k5>x<~Qk3g2Ia|1by5A%9+ zZrx({!S>s_uzip&7Tx*+M3hnvmxb9Z$bW!=WxR`2luh`jhx?b1$`ZfzhhWJhEMv+x zb3Fk6jwx#@X=b`*qnXJV=%48e`k6~#(0>&h95x5~^EV6ny&-RSX~YF5P4!B`^iH1^ zGcZbKpz~eN(3(jY3z-C_JuW$rNq{THB#eVe;CnI&@&@VQcQIlTz#4d4-0j}f(zGxK zJG@*MbHM!+-OR!6U{8Nt{Adg{U+>jbL*=E$X==6k`GtCQ8G-vsRu0lL4J>=&R=sc&Fy_pc)>uYwJt-+w&{S+IHxE?H#%cA=hz|YEf0MRP*!a zy!`3XSr2)U74Z+|y%M#yA+{-N`P`w1l-$QQ|C(&1x5MLudmRaK`j^G@u*IEeDO%MZ786jkNq`UN)8n z{M%t>!GjF)7xC zBS>k4C0QCJra_D`;YBhhgaZ-WA|SE(Fx$LWrwlb;mG}Hg8Mv7}k-E0%E~59^(ZFjq z3Q#YNQV!uDeiYOFhsG`#4Un^!fz4i)`oZ@mQA*1-Ua_A6@_rryS#V3qHnVgBVyJ$s z5Y%GKxg&v5g#7UH$VhWkkI*Jsi!uQcqXz@`p7Lr~epL6T0rCAA_?aKI5@yDFokTH` zlv~%bH=2)aqSN$XV%oa*t-Siz$ek#F)W~F5BY^QQi-y8h)C}qb7#42e&sM!8$wlgV zIm0g`Ckf^GSmg+_EevFX0rt`O3Jg@IEGP7+7{$&!+N(;awIcthlES!7;2$u>9Wb0+ zjE(fwE%a|QvSb59M8Hi~U2H8h8}&XfygTK!CBVYQYA&TG|LSCFMcwr@tob!&DeM>d z)c?g9lHX|C79eqXRUjvdz1Hux8#sZ-!DAy%@qF z*V34QLSoDX+|%KT-i513_?U!l=ymzs?QFIT!VY64dl)(r%|n~`q|I`_>K9oGs;KG; z@5lM>K`Aunn`|E5ME=r<;Zpuu{x^mE_#DUM*I-Zx17wg`fqaN8nzpN9^5x_JG;d7p z3Ss;Zf_i$wC5JJEJBGAgbH`B{oxZ0`%{8m9Axp5#pTm#(D(zuse25-bXnc{z-=@(8 zuZjF_Ws{7I=JOn3Zn9Y7-%NZt%}@es&FyQ)@m_r?wJA%X-GqTLCPb;AuJEA`ug~;7mo{khaD+~-snO(By_aPORAWvm>7ML6P3EZZR*^=Z?K=4D$i;0fOkl^oJ`_19l^j3>mK?w;|Mh_oF^6dg?ki@ znzjo6mSA?x4rSqvKFX(K@#*0e#T|I%+9LKm*A_gD?!)!yX!G_#;rgPjS6~7~ zFGa6*3D1WuH>HJz6>R$TP?(nDIX)8Jb;T{>+@nRV5FiA#(I|u8Bu4Z@Hu8O%yU15d z=QRiEmKUW+4Q>ox<}Pe(?Rk$>!7~vja^T!)?k}Dv3reZ8qiG9K7nF0`+=Y?egd z7VvWt{)8WN9$3IJue-lsDo%3eLGC;8vZz84GnBuk6Hv>IVq)1e7PIB*ikWTGCXtey!IGMBEerJ1O-H))*---5+bHWPdz>@`7(Y!zHnLS!f%Sbd#AehJ2v5ROdIV_i7%d6qVdV8ba)tEfQhM)UUm z6}JY76xL|@3J8&m1%!xjMZksN3c)0VKN?~50vCgPE9ivqv#sUZA8Y&u!Y%Scj=&;V zBR7>Z$Gb6vIbl-eT0WD?{@5&NPF+`wy@03VFG3CKAHut^JJ_XRTHl%}J%YJ<6Vr5D z?s{~ya#u{Rm{{{7bX_d@r=hYXGtL1`t09Oz6G>$CG@pk+Hkyjp5yl=}PZ%p>q?a`I zFTgaa{t;S5OAmg9Hjf5@rlusf(#GER6?6*qtFS~ILErXk&k>I>vW&P#VOy|ZM1Lt* zI96n~c$geA=ZQq5?-5gfNm1zqN5Z6KF6S(s5*?4oM`m%vl^I;#=46(kw#cx580HA% zwWIvSTdgxPwlt2pL8}l?;0?D4ZlMIP@dLc!#>C=OE2dUF=GMZssX{YF%CX6Jk}@|T z@~iosS@D^{innY#gt@h2d+95v(ZV)e-OY5O-c z{w>-F0p;%wBWF&}`?N6e=WJH_)TCPx=34<#X zQj)!1Uo&SUiHc``(UQtgT6x<>AN^-7s+>kK)IUQpng=(^I)*<=y`_{k(h)NAHCSBZ z+2jRi3Ja{%GD<`31~Cx3$QmK5#8A_qRBUcC21>D`k`*i>y89w~7>^(~DPUG?9r3tl z0s86^?}L$j6Y-!!L=EOs$?(*s)Cr)`6R~}PT4-YF!7M{q1~Z3nhfMU}!$bWC8oy1W zYt#`@Y-i)&4NDeM_|XoOm`kqj&JrcwOk4F3nszV{uW?F9T!K_N+8Vl!8j%PxrDorz z1(>qkGA=w##l(!X$3={oNLuPeB7&5ZKCE{Tf{6f=s8<0*>}khRqCK9HNISyqsYz}I zm6R?bCD^f-kSW22)~Cu-I+hac@s!fBGKo-EQu-oN$S_=(f)@9-1-g+to|5)!f1!wJ zCAEJiskOzi@+MZM$laM*hd37E;pi)`Nowt{lQ$iy;a%5P>M*UcMZ#TfdtG85CFRoU zTDp_>*4g6JqZg%CX}ds!b5g`jy_+8Q(s+mlITN-1V4zmWVGafYAMhTc*hTSi#@H#4 z7mU<(3XRuw?AVtt(>X3546C;fZdxE^}K;G&~?ssh4P&c`o4T_=}i4{|$JzMZ+FQbB|MC_T1D~O??7P zbi#{1MQw}b%Zsg2+)wRlW93kEw8QYfX-0 z#b!;dShQ0Vi?~N(mp2pJ*g4y=>tL(wI@l(=4rbS!(HZFzttZz+D{8qWhp(1v#t)I9 z^bB~PvaN7KK!a10-UfgZ3PQyj5Iwd{duOLJ)838LrSdO0{>K^m0F8Y#`muk{a~ypN zA)0&oe%+=RKcX=*f5-9o3Pv~Mr;l|2nHE#8^S^-V77+{?Ac*UzAi-Z8FoIp8ui4ogNGnwQc^ZkZ zK>AJYwdSGz&2!VN$!7KWKf}&JcCERuf9~t?2<#M`!lYABQW?sQF@^P0rq^opT8`XS zrq>Fpp(Z_BFlrf@|Nj;|HqZ@t06~6%pYwNMY3}d;IqUNs9Lz*7{Ge|A4Ds8~7h(w0 z`!JkeV76a?Ws}+VfY2ORX*|rrCrIdo(AY%F{B$fh|e){_#i)12W!gR=9sV_gHOx9fBc4iK9dw8FMiVfL`&7OWxO z#dq9NW%{QPMHKb1ur0c7g5X5=yQ>rYEJA*jACrY;9P;;Y%j~ZH#a^qsPeE^MC|!a> z7gu4E)CIFP#znFVN$7No;6T**u!#zT2|Vqk)Me4q!C5Kn9Q+WVLv>Jo!A+s-9K_zU z6g2#8X!xguH$o@yAC(X|0R_kF+>i537%w}GQ+j!VikrNUSxv8IsP!)=s>s{6U%@Kf zhM^MM46x>NH?tEU+<~{LJ-PYE7DedGrN;PQxN(<9-iGIzJ}3M9yoM~P%E~O3W8?dM zd#2O52RMKDa6a6H_h+&l7{|d3bD^oVZtDxTbOOs#vLrdCC;Zq~=#520!{Sggd8__D zL#9J%;QIRoO|)35gR?^uBsE+F^Zu620}D%@1J=eHfMV`nl`w~Cia%wVToBIKrtiH?*(_g^T@kflvKmQqYr8aXDZK}p0m6qtU;23I5 zcD6SFT@^?>ApgM;(Th+H$I0tF;4bua6dnV*~Xxg#G+U4Pv zw2#70T^HKNU?&ShJN8)U(`R6xfSpG`(|>Xx;~vG$5-IG5*n%4+2E3`T6dAcyBIQiG zc{yTpx4Q=n(+pkiUX***-6!TPhy#~s@%5ek-W4FrU?FQp?hkGY?WH@_-ZQ6@rcULgob&EkXrc2*P ztqYx%t=HiRxw7rLE5<>s4kL#U0(PG&FHnsInkX(bY@lc&iO5C?EVlEohH!&~q+8NA zR*1?96zQN0Bi?6uYT`--*C&LkFP)pw=3BQT97FhyTNt$BPHTcVT}Gl%jm8d(V|ne6 z?G{lXM2n~uT24JYmggamc^)y`0X_yLBAfvU z4Ia6nBfqHX)z+c@^3kk29@Lc=SF2>4FqOC9K_v)EQH<5DgE`(<&?mzKg*nW-`4iZk zqBWtSnYUI)dQuqb9i79;k~|aGMYxzUP%s#)zt}O7(|vLaPiiH>ivDPCG1=q?)-c&z zg2Zk&VY9(#;|x+;REn_UUVaC7avRSljhOAhE@oT(9+GXBu1nKivzh+|K_ae%^L2L| zzVEdFI{;KC7dhZqJu2?tpOazMXm)qJXyCN?@ins{XT;-pu2NejSP%P0Z$9W;Qc8!2mLAny_TORdanG03fqjF@W^XVfDdKKZn zA?z%Qpne~g=AM29Xs2=$d8d4+0OFWlmT7;>5Pt0s;M~#5c1OQ#C4rmBG2_=48^wjv za?eb^gG_g~GVR;yQ5rXqZ*~QDyy!iwJ#+r=$oZC5&V6gSoyz9(RyEdauR7+7LMzk# z{pz^$4BO3T96&I(P+4*D;ma%3`IwImjPF^R@1it^TBSMCuQdJAyoqw{@lV%RaK#bG zczN-B@s#pB_k37Dz-wNo(zAqrh!Wn>D&etyCEP^nonovGy$z(_=7zrshhDi}GAuys zGYN%VY+w^d8<6^P51o++1#%SFbqn-Tp+HZI?5su>M`sFejj*|GIE0hGSWfK}e@WzZfOip#!bIM0?rGECS+sq**Um%sBZ z_Z+$NF1?O*8ufI|Is~oVLej~~*~w>lT~K?b0J+WrD$;Oacs>> z2Nv(s!W~=sqMvU3z){opROPpzoA{3-v8l8yC=iKv2r%^t$vnD~hk^-8$4SZ%ZT@d- zC`Y*m{xff6A?(!uxi_<^=I#Bu^i2ykeUfwCb6Xk~a%eL8tw_{)#31x3s6heUPfnNW zf4~4!*wS`Ue?(uYw)fE1HAC?0T~>n)3#C%n_0^YfcF9KD3zcgW6kF20&mq}P=^n-2 z=*fR>N9C>R^ON5z`nBtxpsZ_7VA4;am^f{cwb5U)CzW`DLAolDd}NuGNC5#()H9+} z^z7^H6ZN>4$4pw+_0hxj4J@bjpBqClUoaHGN!(53`9Zp(^|yoJ)?vAF&RPfUW8Jsv zpR6G28b|$8?guF#L})^3Mkq|X2TK&#+yWMd!Zr9OlpGTe>Z3#5O5afZBPsH0tARU! z@SaKpSY(&&aoQyd{@PdGQyUBi>$}NUQdscNdhBSAB^*4AG_idSu|Xw4dvih)!Jgim zz@fxLBGSXA3LaQ|X%s&CoNE3e4k#^c*obl2yflWFBvOK& zb?XaoHxbH2GfYcsVX5}oxFfJ)C;?0D*+o&syxWTw~3u&kN zC=4oJL!u<(P}BiYQ`jz#8%b28$_8m1c}6FV zq0Eg8PEI(QBCbNh_6cVYWh4cq+)h(>q8n!fk@ODBioop{bs_b~zz$4bzFh9J!mi8Tki zzY~TQ&s~IAZZ&ZVWchQ(Q@rQRU+hu0o5ZBAQnL|fW2;k`DQ z2O$iEG`agu8yq~%k}crr_@l6sfBrbydab|@$Dmju)C*5hgl+)uf};SAdIA*n(VoHS zinl-`VE~7mWZj|G&2T(jao8P!xNHd9+jujazRPBqF5G7`f>R(S#oU58aK#Qo8N*>H zd_ODZRxy*_IDh$YCPC_Q;4}n-Ha6eka1`bNf79LJ?i8`x6wXhCHUoryELq;$+Af{9 z4dOWw4&n>8BmNG=+w1OwAaJJ$>vn-c?+^B^cfY9S`?2c|n^qI8o11({#GicRknqIF zvvp0E7TRe-o#$JMfuh}ZAis?=2rXvxryj)Blya+Vc&J4QchX^}p4^e<=FfPl=2b!= z9ZwGBWuWVoQs?^wGgvQ}2)^{rH4M{8nkeOxrTuoHu(zOHBnZc%5{bl2C>5#OItM{c z;&r!BswHev^LX1=KjXJa+VBX}Y#v(ggQC{iSUj{=95WKOPjRudfQzsEo{Z@0D8{!) z^nM?f=3SdAOINxb69nlkpW}Ge7AdRba$C7xrk5Du8e;u!t2Bo+r%v_7tG8 zTt4|R;z5;ZkSoRbuNkK_3IyhB@M>FPB%D+y#}-4h*f1LD)f6T;TuZ^Nti*~9>i1y9 zTukm#Op4xr64;J*D7nb*&_$@k6oDke_2t5_Gplbu#$nCHuDS#x#|xZslP7kZLIWS- z1L=%gQDUsz=>ZouXkg>|VBE15B%yGqHwm!*a055)G9<*J*gSTx1J0<10NfeEVq?v# zzkus`&I1&2N{U3Bo%N^KH=GHP7oP3Ui>^%@|r+KFsXNXEe-kth^P zZh$S>*PCzMR0D6CV~qOURgSIpSQK#4kL~+E>}_8__~s^iTkD#`E;ZFP++S20#qkh3 z@}nQ`^g2uO7)!D_iV$g;1_M!B`$6h?2rAA8$ur*h4sM7Q!Vw_7e+m}#OeiLcy1J~X z;-ny=d6Tz{r}f@NA23b&-jfLL;F>00*v&2*f>-d@0(c=eN+{l&p}@uTgmNZuj%-x0 zY_8?8@$EO72RGIDA;P-}8Z*<`SBd2H2lb?XIBPvIg7Rtlci*PU)Hj2tQB&)msvu`1 z0W*Vu@NSaj!O)J=`#>JvkNBwdmzWGkyxK;iA6A`Uv40&Qns;xi*lU36no1?ymM%i2 z7Iu+KxEn7*@!?`aR$m*Yyk*t1LJ1>wvB8-b7CbgLtYV425Ah25o@iRC!^L|gc$xG` zk>qW9)ms2E#R3`|b69B65W7J)RK`cdwqb(G;#zTjUH=W4uOY@akZ4=OH*H!WO74Y- zr`9fen**VUP40Od<~@%?Gr-!t5OMC>)SzOXh5OX2_z4EN1SuC0dcI=)2R7Ct0nt+~P_NhYHkdr3&PsExY;{VmdioAy@M; zNUI(I26+-^8}CP-3(;Q@8$YYKPI^wCv!~dxoIwep@75J&#u*hK7gL`q&#z!k(^@d1 z!a2+mj?fsTL5YAf)g69rrBTKWqvL$36CcuxKO5g-H%*ev?ts5gmEG<;s9_!B0bux;C#0qC!c$N0V zu7u-zj)mPv;{c7LH14ADb{ZubpP+Gx#xK+O4H~~g<6AVyTuE<=&F(UF-ijAN=3dks zZVaJi&pS+-@%cH(uKqG23Uaa}icqq+{_#tmvPMV($iAXreGk+kxX zOV2K4i=}|35|F&OZC?7?7JVrE(7({9{)$E4`s8On)R)BWcV>CBBqm;A4(D=a=FFLM z`9|@~jA!7x@Xvp(-#ls<|E0#{XQ6QwC8>i5L$Jtbu{vw$;q2<ZD=Z zZbTK9xi{tQu534D4UOYr5_H0L(2}iKZ3T_ZMi@0#BH5(rlixfVS5cDxfJmdxgi+^$ z)y*O=a`CD#r6nwB3me6e&PG8LGe3R%*qf8_A&EXvxf?(aeit z)d(6hwv$+8g{0Yt(2+Sxrd`!;y10emM|FFH2x4}*s=yil~)94q5wfn4ADX0Qg zRJ4aG5+VCmB_d@aGawZ=vl1Dtz69%iLmkG9CzqDeR%fXbcFv~_wItqZL? zOHsJ8)Y(ec1*Ry+45Ub zt>Kb%-{8Xhu$_`o;Idb$4`|B@*E-WGywB7ey0l7iLPyHnc(D4n&`L<*9=F&W&S|j& zPB-Mw}fONMHDdq}B z_g4ELKixAZ%J{HCecBy0}nc6+qdCR-3c8^8oo3b5a+fDg~iBMy$afBDA?+Sj;+Aiu|t8c#0 z>-7uj97Zf*BFGH6?KX1BHq?Z$g&fi^_Pu^d*!LXVm{ZHlW~W;*6}f_X9VGL{`RaiV zM%te2sUfOjXJ(QrWz%Fkja75KZug{$lRCv>Q7L1OD9MA@NLl%Eqg~avY5(sK*%JUM zoK5)dVpGXRDg)7NwYCE7_=B{u494V#AoHHbaC!K1zVfVLjTfD#d0me@vW_{YS^n$j2w3ab#VM7MbfPale1RhI8r|Gyhc2wX4FWMmPC9P z`c82pu*szET+>FW-XhkqPIZlfqCNwuK3M^A4T}MHFzXTL{24d-SKQ)XT2Ptg@F#A8 zKXzRHh!?&8c?;Z~W89o57B9Xyk5o5(E((d=i&Dg!<$?apXCRF*TcAZ7;R#;PS|#i1Mt ziA6Cd4g)tGl5;>-gq!BYkpTi4_8b+*bkDrzIRc&qaa{Kt{S4WyeheOVGF$k$j(U47 zt9Ooi{4QY|0z7$a%u!F`zaiJE{_R_v0q%UGQmu_se^$!j3_+V@!dWO|Yin#0pcsBs zCs;uim3#n_8atGBSs&mMs0BfSQ1QJ2Gy3M|gk;bLZaJ$}WeRK8{TvA9nz6$$Y7%~- zQn3dK?rY$S7Kj4e3xQn6eRPspCzL1F5XwI(Y(pe$WSla-;J1MiwTV`kjuJrGy!v*I zj>efqXL4^j0{kY7cx96oq*&@0oV(AR=e>&uD}IDls>>0gf0SH7Ww=VkLbC3{%W}WF zjP-H|R>+aW7_8}`ajc@#-e*}c2($;nx3#F+wdkzmRS9JY2W~a_=R9_bF7ueh%g?Om zy))k+bY|1_C%OBOGKU{YHcBNuG*TpG!FLh%>GpQmHmBMrrERls3KMrcf4kH-|H}Sm zFatHd!M80?a~miYc7aqO?BQ(TOwGUvF$m z>_vN2N6YQAI_(0`OF~fRQ?j}uSsl!sDX8XGqy|pLts}WM!24WLLdh)B`9##zwpUzU zlWlpoqps}Gs)U00?@k)M>XaMoKqC_-H=JDCCn&9S6XhY9i*}FFNYGNBZ5Wi$2X3O1 z4$FPxD%kf3soq83Jrs?%UNw44FDKl{AEpYY-lyFL+iN$tN*&|OqAj1c{{!`319`c% zG;F;}a}HSRkKeeKzFn;}Z=^I&unCpDGFp4uE`I{YD3r`1lpqk1e^W3skyZQ_EXnI2^RCNWLkjQGdWIUy2TM4 zPTa;bjoLO6>yYm<63H^S^Y*eqJn(l@;eRoPet6U@e`$D zbiZzsq3Z~mDr^q_Nv5vT5aC<(JtE@=!X;|@Alb|ysWxR4A+e6*!jFhW z2O1JDKuNBE(xI17{wAQMCgdGmd*&|ZN$d*QNCL4wjsA|=BjLCKOOnWAMol=l1C5(jiCw2|bJ8}cv^{M+Jx!askE5Qx z9NSGzTSC9@zcc#+DEs6TID7ZrJNNnD|NigU*63(1fxmmd_>~Ly{X`=14Ho+U#!xto zkN-zTBH<=nqn>c|x8WF4HXT#GlTK2;EhmL)A%m$u<kw)Ih zH%6V&M!_jG#+)&EZq$p;xb!bMC45`;Qe(oIkY}m-WaEHypfTl4HKv_usk7@djag^5 zanL!~IOH5^%sF$7!_MKxEzT|SJYB!FQFh9b&(v>g-0s{i`E31;#u4X8q z)aB}THQwU9r7`c!OL?UJ*2dk=-ICAOk2Q`v#~TaILgR#UqH&LNPvc(aUU@cJe_P`| z=f1}M&izs@)ZgBCz=D>zF2=pGcfy^#mT=yM`UCD1>ZiP^t)z1roQ*e*CN3W%v z$6e#;#QdGF@#Gih%`od$gX(g<>ic18rRLS$c{@y=XBhAhgca z{2z}J#%&4n>RwU*_=;?V`tpzdf=&ZPd$6Kx_rS4^HIh5mh08SiT=|0 zYO}iPDNHm{b=^u;_Y6y}S6vQjm#cx-ts7bOf^K27jupfuKP)J((YoAgJK^`*RvOjy z^;&Zk>&u;MJ$2@6tGQBRX{6Sy1(kYhl?zCQbo0Dp6jBhTd8U|!nWt5NzAawq8`VqtzHQHaO%nG@GnAF{L>rj>n#=dvgHXsDpeXA z^`M4U)jAehIM0$C;QPF*K5L$Q@_fzr0SMVEYPci-y3)W|sH>IgFOBaczB69dwXPdZ&Kp5pdc~9$sZ$Bp2L7hQ zsdKI7DvmYqUI@5}DBHe2%9K}Hs!S-9BmDBq!G&s2j<$+N&DHX0)IM9;2x~lx!wHhiemOK<(15xsr1;ZWh!oCoR)%V#RWE?#MOE8S(ON+Ot+nH}8&K zvp|hU-2!kfjo#^5wi|PcXjkyYvg28^_wU}(owbCSCuHX0+Bh-Oox}(SFhX&ygwhmp zTvzlIHPfiE@ExylCTd}`H|l3mp73nRbG0+>LHE$L6jnFsjkzZN$0cQM?lWo7dmGeV zdVu2Vv6IpF;9BCDf6$DRZrQ!+Aq9=uhdt-N4EHJ4L3*Z7ZlqPpPN{m68TPNI_NSV)bI;Tjh$ zzw};68J)yh;?s%Gf`7hjbWp08og~tvo5X6Z>Up3^t{nZHQLFyRZi$rxt2|kL=7Lwo zB?B#0<(ezBB0v>eMLQel%}t%uPnlg9cqn^4$!?&ynmYDKU_5?rS-f|N z_CmFM8H^CCCTQ^OUubRA-SU!GUR54gwIW$f6_K<+|GTgWEl!Ju z@4_-QfQBHFS_ZH3p2lM#*p_7)mXR^+*V|(~Fi4wa9$EayJV~s;;C5sP12q=lO1b7W zDQ%lOiN_LK#uh|EeT^0eh9DUE@%foT@)2}<-rmH$TuXc)VI(d=9{?vX7QqCHV8oC) zF1MDeOBKJ?_8?}M3-^Q;F{h6s$SG$}t$I!Gg?05vg=1%sm@iog(qAyIET@O;g+G)M}RJI3p|^G*?$}2$H1x$w;JZyv{J3FU#N!Z2q{C`^)A%EDWjllY70t!&lwXS74IbySGZkd~YDc{Z_@>ZCfho4U>s z@^~`7ZSN#D$AdJ+1|uPJ?Tl~PozYhkrxKl_s&~eEeYP_@Sivh8r;i@i=|!%VB+^+vB|NxWETG2ewN)yue@WkS?Nk zC+lXfTfo>{`<}Bx*@3Iw8&RfOf(nYGRR5GFlo6Q=lgsNHdr1dRoVVSdLn5RjOVaU5 z;&MWn9ivA;)MIEMsE&K^IP#%!5!bhO0XT~K5hO3+BZ8y((m^n%?)nzA+C*%RY42WA zfx(~VK+hoAL+84SCA*NtrsKuF*+MPyd-dZSRtQaZrJq7oA6+Z%s<*Qdsq(xboNs<2 zO!|RRM^P1~YJu1A)ve51po~Fi2CXpHQ?1l29~nVt>pLBNFNMh}l=T$Y5bkmcy36tf zMY-O|t!y;)Vb#MVE~t7h2gtObzc)yor*LX@zmFyVD4wgQm~^3}o@Ma^OoXF9$J_^* zRGE-PJ0lw4LK+=g7o05ALcLCBO1`_a!jBHf=}{Cvq8rNPco|Y0BAI%Ea|l!gz65PJ zzUQD{$_2)Z>D9stXPSQ6{^5hYf2?0%UD%s*21Hb6dtIMnRpe?Nn!M$Ji`B2c^b>ps|%S63xuL z3ID+>h5^cABz_DZ>2G&Nb`14%pz`CZiOo`wUdweKV5;SUZd&JG|7;bJhu{qV5>X_gGYn{?|zB9RlIW9c~8vo&7)E(&j)`=% zQ$kw!Y+BTWc;xKnG-fGurgx0acxU=l;?j$I*8P|I*WG-7wDLSw{>NN(r|6Dh9W%jL zXC^3anf@0$GpoktOlJlZwS@7f_gU?4$!akxR{JHa_Upm89;X8y<$wLn)*h6=fAXD_ zTO`kFrh`%0<+xUWTzhjiTHz>G_@{trX?voB{N`+PEP7JdF+OQ1TSh8iq>?*vJ-IoH z6-;6$2l0IX--kNI?MZjawQ<68824~z4!!2V6YY%za^=q4W->SsOl?nh<~qqki56}WGr*5PJojZP1-UWLpqww?=BU+XQjarVZ8a#Y^;BAqNRQhS365Zx?2 z3F#d^VJv1F+C`E=|Q>fHCKZR3*~2h&&7m$ zW{b2%(K_n11o}a2%Ifxg&x&Y($8;ksN$a(D`Flo-j9^jEa-=454$mFjhtMH_#;Zfs zKMCm|9u0pv>LEzVVL^WGR$S=rK@|UhBRDnEmiAS{RXsgH8`D? zQ%{y36M40Kr9keOzFmd1>ew+AoIY zdN6M|*%bvWz0y(_Lu28!UHr8R&t0g&_CQEb{cImT)`zfSD(+%*tyyjt%1avoaCX^) zF5;Hk4=j|Qtb4G8moIy&wsKXYy*3;Wwi*KZP7vbDErrfOSQs=zLkY~o?heMkET+u% zOxbODe)BGkomEU7LE{;SVm^42laI_Kl~%KUH8hl>5E2$5l&t8BFhBBVlv3sohGxqT zEmDNgS_4%Ht=77Sl&3|_wCaX-&0hiV!jZnyadcVVsu3%MUun^3YW`YaUrv7(^jWH~1c^>t63XXOI-YvH0`fjEx3 zy2x5uMCWf&Ys@#8G?|dizqzR36y=I)l!WdUT!?D)%5+Uh+5n+s8MSIDrqpLcBFy8T z5?#)K`GHa%KGv>jLQdSxry)P?&VAE-DL-x{*#?rKQIH(u!xZF9wCwLuOchL6(O$VnJXGo zmYFis{UEWv9k&Ktzn_=Z=8a6+{BAL0{$D$5{;8ca|4(7W{7x=!e%l<)f6IKtwEIV} zzrztA%)idLY}>qn8NQo|dKb_;hu&ts_-$G3G8Q`Umyb&RA3}5McSE=)oB&BL1vf;B zZU~t6?UkHp_mFI_d<*R3hIW8#b$af=u$>*e8ypv%~T zdjd5Wg)U>bCE#WlLutyLMhOy}lxEynl*aq|(tTjTy*}qnxLNlwM!Utm6*bz-pMa77 zcJ~grpr_m;-n4tvy%R0&a^Hd$GugxL<1*4bR`OPF);kCX#ohQm>*1-~pV(sgiK-qGX39ua>b zOo5ug)A^~O(r!lUFM~Y|Co>cB>w+K z0RlFCB=mxaK^ww&5{9CJHi$trOS%RHb=R1LL9znR>MGkMrIZ%m3QSITl7+$rFUp!2 z(5};;GE!rYd@#C|fWl-@b%Dy0SSz}zEmM6U7~d|jv^D`%Dp8r}q}C?YB5EeK4?u-r z>$t^~w3rH}*?!At{rPB|^cH8Hk$Gk%Jt%8A)Jbg5(I^Fz6naz+!?cyC+#=qnZ1p#B z|Mn^m(P5RVO*i)QipQnT%u1=PD|qhJAmp%4idZRGE#OCk4avXp8S+GZe&f&Zq!wxWXP7TA&WGzOXJK8xSqF8hmZQpC_1yf`36Vy)k|zX zg`FUXB(|%^yH>aJ;YcrPqk$C;v<%ZYEC~%6MPLb{W}5z$HM3o%RfCdtCoCiioaC_>UlCPuMVwO^PUHQy7EDlX=5V zQShQjWWyAfPwHIG$|Wtx1gM2rMS-h5+ZBYOo8I;Jsdr-Z0V^)11o6;DuWu36Kt({= zo=#|K$&}?H77b_zbF~1#SMNi+KG}tmrTP?-Dn|Mi>=lSm*k=(jirsX6^|v?>&Dzj( zh?`E9^AgPa&-t@x<6%zqDl&a`#CY3sK!VCQozo|q;GW|h8;Yaux5aQYK4KZNhKa?Q zWSi|<_CS1~X_%Ik)Z76tSQhip?kM)guOcJ9R@(W~3^#+%(R7p75&;PRT5`(}FO3B+ zjoC3>Ya^lN0)%pC$3AR-$xQ@za3dW9z8Ly$4A`ksfxVq}Z7`fnC%FS_qvSANCn>G6 z+c~i!mNA08neL>vM>=3go0(2VeXx_+&I@ZvZ;yiEW!4J#jNwz9dpIp*VT}Wd?GdV%ji6etv`ag4IUsxRr94>09SSr3fT17GmeU*%Rl!pcd;v_VUTb*CE)=+nKy7(I9?y`_y+ zMOvc%I+H)ggbIqPA#pN{`Xf%NW!8xcq2INjHDB-$FK#%W)fA1ubjDU{3gnUdt#BJ~ z#Bd+#6+Da>h_F5FjQSY6#J*seRLEYJ_Jzga2*w6a<{os*rAYcp_*idHBP4mFM#+=P zneVD8U10gjzH;{ z>I8*ounEwZH5}R%iQngdaLNFwy^fH5WB z`MdUlc;8V508Bnd{1MzCfr2Csu#y}KAtw-lIjBwQT7pP(H!GZrB??^Z)kt*n5yK-v zr|L@>O<&$$Lk`SPaEbdK-fkX|z1{o&Ic}2tEC<{v812J*x7F8r&t|wG*$i(4gBE%A z^X84CPl^!S9V{l^TDMaW)}fm%w4X(@CP`gv3)kVXn%H+c$E+Dv`f?ohJp53U5jb%< z8qA{;i>=^24JeNdL=$AKaH|){*EH zYA(_Sm-X%K5$kL#vkPq6z&?i&}aXVeuLeHQN3 zE3*EBKyEm%X;s)hq^TD~!#T7)EaIN9konBwNDF$Q z@mdk~kCiWx)D7oF4n8f)eXG%pgSKcAN3Jj2sA<< zWo@W zaG2J9$XJz)W>3wgA}xBN`W1FfLxLrxQZI6}&oL1vMVYxdCUV7phPjV2nP$Qpr0zn} zRn!qTubCkEpEF98z5DWkI?U$VOs0^8W3liTtt6c3ii2=J^lQJ)2sv3q5QGka*9tp1 z$I&R8g(*4}e5W+f6K_NKfs>;fL+{p^9)@ZRhQUjzK94bfotKb8C$0Jgco9s{Gu$W9_H5a=z$Qs=|t0(@)+%(pmnsDvL$9yjyxIfYk_b* z1lROhT-?)wV~ijmV*+c+A~?}Ri^;VEuyi8S<$8KE-<%8>$iZ1BF(+4;UYkM=K`_16 zJN9P2li!{u%W9tO+bM%GGVFzUMiI7B2xdE@RJfnN8}J+3Eb7sP%@@JyF-r$wExnnd zQg=2fV}1QIb_&}EI|z&cC-_2e2(%P|CR`7w_u&qNXuv)Nb~(~M9vN)go@yQM6Owx< zBk~cbPGe(TlfOlwe_^4eR{KJUC3Y?x?M6uO%^V4(Q$Gd-f?#!^`zVetW0Tk3fO8xK zlRhYIg!>)l{xK6Fn1c60#D14`LiTo;`z#VCw*;5CelJV?Q`QRM`%UCRTl=lm3akGa z6NcwGv(b%pHyVxT-Jr<9?G!{Bs#f3^CQ%a~T2zT}>8O$#t){zwo4J>m97IAF_VPuE zx0@gDQ^&+j+PztNi@mG+HS|OzUUPjzUS)D}t&JectKa9)6vM-O6uUx^)Q38~vBZ`7 z>I-ZUFc~C@;rO0Seuo6%r_i*JrXous2$w6=T=R9)qMA!CYeu65_~%l!TyS&>$VCD{A3@ctt(K#)o>ae+-FOcE&f20P>_O)E4g< zAmVk$TpwrT>&4i@!`Sd-dvf8eA6oeEKorI15uo$vK-D5%R5TT$JB6`_HcU;(L6>?Q zGpc{dL;@t`oVXMTm3;_{^Ghg6V7+C%E?{~C?@zqme#agF?JG3wV;;h+9%rEWS}ih> zS!I5inwRTAPEEMrG_1_`+jstO(+!+Sd>+Eg9>7>Q2i!79o{(N4dBVL11)l0WUgm%d ziXiSa(DL;e$)zq+k(W1v+P~fPg#7;!=DGt83Uv!Y*My|CpbaRBdbGjAK(?`bp$v-w zb%e`^4r#$=*t+7WV_F96jt5~-OKDIwaant=6D=Cznk62ex3%;n#7eC4e+AV}77mLh zf^raxG9qJBr19$WR!>9DYo#x>W%h+! z$K3u+<;t)aO|`NCy@n#|z#*qF0Y7r_C}ZJxp@V7Lj2tnQ3r4QS%0;qcA;Ld-(NHsD zt1+Ogz_{g_KM91`F+Qd*F09G8xE8N1`a^}$GS9rd*psn^r|h}SM7s%m{4$bNAl>u% zP25|Re+61F7$)>%#5o#2ZC>IHHuP|WUwj=&G2s*WH~7+5gSR5gonjo&a;vfa$TRWf zuoIJz5!>(FdpWuS)PNZ4_g%;CXut+RgwJt43hQAKK~w4l7Rkw=h`x*YKV(9NpasZ3 zU`~$x>&(f`{U^x1FD4p1@Pa`@Bq=&VsdwA+KlDP?-^EY^2c1O)4_a>XOUU3@MCFEL z@(?5w82urwSgrw?FNwP1CNw}CI}P~oS3~B54hS7Ct!~9jASt}ekn-=ilnSgMMT8Iu zX9SX-)yd2v>gCcr?48I%7MZ8!+JUYL8&;-cq9zp?_#S@-mSExb#5`?M=F-9x-tmL! z`QNgZ+*kc`CJ!TtO~>4y0MaS;v2KLTEK&5o5JxtK`0#%MRVY zzmEqW~_Mfy zCgd<3t#JU6xIhjhi1b!ugwu+Fl155=k~#U*T4jlOF$N3f{CgIMR=_47#tPJbK&CGQE=K6s z)}?gQHan!zF%PPRPK9) zM@#9_DE><&tCTHSQx<*{fio}TGw{bP^G`f`y{JGB?-6>Ims~X%vByZW?z@S3$ChX_ z`j%5+p7byF8OYfT{Q8NgiYv`M#vQ_qnC26TK>`t4G~MGRp2$Qmcg<9v!BaW<*qS4U z|6f=u5PchS0@l;Wy%OVH$!s6!UspHO0;G?WwlB6+FpT_MeS^tqCjW_v3@u6T?EW(z z{1hjl=OL4T&SxClRjl0uS6*HI`CmrTgDZY+2yk6XA>I;iPU3?fS3taFdLZ7C(Uw{K z=t3@v{EL3>!OeTQD2fn2^nj?pQFjczC9Y6Tr|6C&TC*TECf|`ko3UOSchWt88od33 z_|7=)ay%;OxXY59>;z8sU^o+>>M)0EKa6-n;Wv#R7*$BqFy3LoP?1MUYxbW2P==y! z_Jj$h(MC|@A!K?ef|v&jX7Cg^qxgA1o?z?(o1ig1=8-ICdTb-xLnJvZO+(~9e~Bv* zY9iD_*#m9dylKn*%5npgDuu~p{9aH=+xYrUS11C3KQILt(Ov=G-lFn;v*J|GiiKblb zv#7z>(C>;;{}D4j^Egc9zkKY8rJL@5C<^E1qD&cc#G(v;2}LocLoDLN zjM&1lD5g-HaZ>9na;X0eUH8N5@W8;vW1-?J+zAOZwS76ig`2K_-vI^=5rfGg3d&QC z07By++TDn!4J@Fg88=1;394SD&^NwI|{TgC40OeG<}?#Paq8;Dkv8mk#gxe_)AUetfB)f4;_Af;+$MYx=-xgQBXifkET&Mh%LSOBK8}S+g!YgK zvnc2ctp6+y zT#(q;0ByjwZ&+#d9XyHcJ*=PTY0{abA|R!!%* z#hMfoL9sM*JR(ISm^0q@n&E=}xrR+XTxCLhbq?`2+ZJLi8#coReid#(WJh0(uoPOM@e zh?aXhTg}UO`?g|ZT>Gw(`wq4*ze#(&AQCtyC(U%lQB-K+A_CL{EZ)iFK_(QlMMu;k zvfSlwvW}!c5ULL~1xYDDg_%=&b&o)&P53U73~Xcizi5D3-!@v?Ygv|T+44w`wCgo(hw5Q9XE@?b z)lF$;(o8lhIlI|Jo9re?5bOpR?VBG01VIoWFL}r>$U}etjUWh+CqFyLuH>~Q->K?u zc9WEBBa3i`#p=48sycOUbxw6>ax$;rbL?;aY3*4{QT~e_;-3XPyoRsyx~eElsVGck zT2pOl6|I%3q*{7KZ>1~gmQgWUnMy{b`cyMp$>LXU=34nmURCToe;v>1=0vMdDTp>k zbFx*e6h%JMoN7&1rbRy6oN3KgW<@^NoNLWj=0!f=+|^pBEC80qF96m=b9ZY`Wsj&U zG#6WYD|Kbx`*`i;H^Z2B|nnMI|pvuQSSORqeZOR@cImd&ww zTes8pfkkCeVY}=DYysch`0laOY!PL{&-EwSZcG@TMr`GKDl6kJYWWH}NnHBuL*Fzq^m9<`H$$SI3 zQ}!0>sLfO6JG|knwXIfcY95@arm;`w%V?F zu5thvT4~z8ey-uTe)fFVZFJgJ(@$USby1xDb{o&jCDqTr zb@Ed6^2M{4&(qVy%4+L+)!yh@ZHC90*kjdcHfna%##!Ty_Emaxl3iktgD#?Q(1n)y zuGQ^gm4L8tuG3yU)oHu-hD$X;mS9(FJ;&{|s-3Rm&$ykdcDvf_aI5OtZKuP1gWGP8 zw`(Bv=#xV(gYPMPoj*n5D(k8z*xsIn`CnPTCc%Da3~U$ggo=A z&D(afY`)iU*F>pVw=FE*c1{RL$I9l}x>@VA9lO?Z8`o_C$97x?z(t1@%dM@MJ;#6*X4Gd26tR*RiH%zTEc2-RJf-Si6}1R&I|wrtnh2T;y_Bzi$t z^E*W?ukKX~_{*!hI<4!fp=MQmhd)Kj&9o$s^C^0o4i|54RNYRs!Tg+921diouXkz| zR!TFUh8zHo$4X|JkPwpdu?-|A_%uM)3?$)a4>I(e!q-tW;(HIAe^cGoz_o^Fh~G@` z3w77ed1=1iO|9!Lc=IOIB=QDJ-A?s$KTzsWhb(<73mBOp z@2g&VJ-;3Gx})_cy-6=eEw3FV>EUHoQ~e@ly&3d!MX%y^!OM}9!u%(_qF9B-vRKEI zD9OiGHtnXii{2E=Eh?TquLQZ2r|(wgl~t00Gmwn=J&HHu&5#tsY^PX$QR&Zm)BQPb zZhH#YoDbT}d-Jjl#+haYa5EpnqwdYil6hq}`T&2s@LjM-amCBpzf#2~N}tc^yi*D}HAD>f&AN#n%^8}JJncbMHYMG0i~ zRv>t3O2fQ$!aPmf7Hca7u~{8fa{Awb;rmy9J8doV=VJcCuN+HIQ)%@$~5p@3I} z?j4yndIHH#3?B{Kk#GX-83zJ-jVyr?P!Z971te6v7K5t&YM37xDTbQ_-s8A1Z;)+6 z!@wdK@Ibg-mx6UCHU~?Wn*Q)H6IKkjyG^SGY9vr2lZdz}3ET!li7}mrZ1`~uLyRs| zR77DTL&Vn^!6jKk8Xud~;NX7{Bu$p=yQqyb0!pCUT-g*hjo|eq6UJSq&Ef#adZfy- zkTuo_l)(VWMGqMHYq8V_{P=<|b}pO-N7fx1;ua?e%+obtSjnXdnJ^@?!o6MYz|e#Q z8I>EXV>|7qUC1p+&g2q;iIX7|2v$e~X-GLjiU2j%N(ZKJDCNzwu8>Mrqdki6LvaBE zR3~M?NI}34OCuN=U_2zJC9y7PzXX#m$)zm&dacK-lI9!Cz787k)6CuM+KCelY#Aku z?;Ke+2a&A8viWGTui~7c!19=c<)L`cfv_;tpJ>-K=srytww$~Hux`^B)!x7;TKTB2 zZhWHhL#PmTA%7A%KmT$Q=p{q%)fS$eED}S}wD}!lXX|m$9I4Q1p!>=%H({(bomY_w zdku#BMB_&Ycp1s3Gm zt!^~9pXOG3)#gXhx-uIOTXkhKz%KJsux)(fq2yUgmXK6-C1x9LYGo$c=%}a=DtzMhO3Z2v=E+|ncv;zsZTM=V zG9eyheWegk2Tl|?wP%8-%B~~|!c)L(G=WHP`Gq(U`%^@N%z1Vrq|rWy@O*iXh)}+X8t11-iqB-}Sb)F0sv$M*=To|7+}HJKwU8?AXsWie zN23;n-~f@;wC`ytl&ks|dYX-pXJ+kUF(jpEWc}>5sr)ZQc+GkTV zCR+SDYoLEhLL-QPa*v)1y z1EOp$R%WutMA>|->>ztwl z|6x`}{oV)GA7MvP|Hzp7V{$Lp@!*%>71(p^d9>g6mD|6-UPSvx$F!I0dx?Dm_4{+b z#ois$My}xm`zH4AzysQhpXbZ$74$K48PLiy%dl5}t|s7rOW^pm5YucJ%1Nvmdrl5f z(9A4*ot?UssXUf|@kWw*>M687J*EsRWoJ;Xvp4JTgb9d6_7*#fl^^_)xh3aOU~jW` zZW(Z@k_YU(KPlax?{N~;eCt1i7EP3qg~V*9vD$_?2Sc@GwaFz!W`*!MG20}E1hZT# z@{~!Zg(-lA4ztU!0$Nb`k%x-_3wZEpQ8`&qP!Dd>3rEJ8fFu30WVbAd!O=AwpLzla z{9CX&hbIw?FJSl!WP0M{Ruwj+&ENomFVpcSs9csgLSxqSm_VXDxin6ZNctBf!j-8C z_hpKyw^N={h5Hij&19KArYucm46ewiw~pS%n3fr)tRf}==YUepihAiV4M)t4Da(&3 zn_x6TL8Ox`jeHRf5T!aL(rKoE5HidB1+0s|h~!hUaRzq8nbJ&P64+4xoBUv~T5k(HQH8dT{ zD|_F7l1u)QVnoRo20U<7Q^-8HtpP=9KtXOTB>1fy4!11LA@bQh&?b`ZH*&LwAIg)} zC}{NrGM+F>;rxbkiJaee47kzNb(QZyjRvXJbKvz)~q?_y_3na3gOR`BTBr;i&%!2oCui^LfIs zjy)*A_nFtW9!+9#s9!CPqe=9qXMv-CMFuWPb%3Fp#)n2f)6e#E{d|7{&PC!_#mlV2 z1X)x>SR&4v)P>>UL=O0&!{L>fPat5EQCYFRQdnf;}_FtqqZG1a`8SLSA>G6F^NXW%x-s5;gz(eqySn2_) zrp~E%tHfl$78s8396Uks#hxe+#kP)f0O82`0GlY3Joz#%Qz#AQ0d#@88QKZ(g^<_b zy4`Nri1FUA;L<@bt%frqED_icZJV+d-;@>y0Se`guquMJ9OgE{U$BOPU?4^45SWn> zv7~DR(ZE5_h*Vd?ZAX4e#Kg$z;jY>3I1P&Y#aziW14QWtfvNHR14Wn|`nj@j_=acV zfm)1DffQ%x6_aKaJk5~T@W{i56QPslK_58xTDH}8%yvh5lby9ruZgf9?7?og*|1r1 z4$qfk&bkxk>%v1%esI&O!(20KO@x(g1h7}l2E(2p5Z{!51D%ls_d*#KWEgP6GK57b zz41Vi#HGXq)*yCNm~=@XFP5?E6u_oPodhDD1BZSxOgrehTrwo*3APz9ok^@jHbzO6 ztR@VHf%Pz*WB}KChWLr@jufYgI4(ngeX+Ga$veYc5?li^7qK;x0H2W*2M>U25W5lF zL8c|~3wR>6_48;4AHWbq^>G>iDRaa-yKrvOrhq*}G*}`i0+v9oc``;ZUxz@1c3Osq zjBGrinUE)GRVl`d2khstRH~xQtH9!CIx=~qsBW1_{KdKlbo4+B@)f`Urmhc&J&7vF z5rpc+$tNZ$5;$|Nv!~ z+>k9zPM={!kkWkiF229i3cNu@?*d&;usOF zitLMURkOi<1m}Qpiy|Z90`Z90-86Q0NV5zrEC4mKKI%?9;+RN=d4y>nYm!sQ4GE%r<$b!L5_T31V#=;1vjDi`#V+t?`k0Gc0Y_PB|C6ossDi)KBkWT1>Z-1DT&f(^Z z#V>)kAdfC~peQ$o5XP+m;eAS*9wCg(*$`oq1A}x7p58`FD`KPrNog|Z84$0pZt~;k z;b(4C-6}Q_4s1-&uF=6c{7^n5!Osrifk=?U{JW&M@~ok*!f&vy{X+HhpQsm=AH#~L z+E3KwQc9dpc@f>Xh2)d8)Gam@(9pb2$topJM9GuL`9*oVD?I3~V^yC~^ufr|c`5k& zLf5rlYw5!0TlGK8#1J);KS|q9YIweE1oD2qH zh@&yvqe+rpTgYrZGIm0;ou7l?0lmZ68kt&%YraOqiAe-lEKtEiu-F|lzoNW)7h>^a zOprIxZWv$5W+Rgd@x>{a6JmxA7!otr#*91}fRro#OB%RC0}tUP(aO(mR^>6vFKo)o zej+@YIP}tiH$;*{QJaSoq?i7Q>KSpuOdyqx&=RS^seXtwU8?UaCc4;OM9 zt~3~Mf}&HGg-855W~~Ryv+cr!g4T!z2uxvl4>553!e+ga6jWCRPUXRggzT~Sf#f{k z@*52&n1ej>$-Ni`pO8)w>}XOC9z%%aPc(VJCh?3@1Nq5>Joi&{eYW6;%e-<3!9ouf zlX;SNvgJP$fssNmW(_zQ_p5qpVMkZD_9O{y(BblLlECN+_J9QA1ST$L>;FFlwldK4 z!vuDPNa#07U;&Q@^!J-4umE~MeTh52bZ&SEfsr^335@c~pTsDPZ=oclFgZqy!p5l^ zL1D7xzY~R#+x~Y3vVFccnVK6e9^lDk7rDxX`Vq6 zjr`GZf*aRM5FuR?CBp-)OZrwR;?PTtR-25bmXMRa64ftDwbaL4`R`IPPR>sc zdP=4GKM{te2tzNbWo`KAKHLSDK8%%K9rv*P>xc}FRHk6yk` zw`y@`9?oYSH&WqU+|IzqNGc_LN5``vdY!@DdFp)z=!N1z3vn0!whnjm1j;ZMDubsI zckG#QTLcrf^K|dEkFXZDA!9v;BSVDgOEv4a&X@42vNgR4>WUIA|1!=F}L zqw6)e&e=XJuF?%eAHmrefpeCu;s8owW+C22dXUbpT2DLX4ZE?r<|aD@Xd2&XWqc=r z74uNN*|A)#EI7282+Cs3$@z!&s(C1UBL*AdVnaK@3o__`UR+=kE?W2&g|C**lfqew z!(d{*gp*pQ*=x1Q8rd8kOxl^k@%WGdFOg#s-D#A~5S-|mgeUnqHud6*WYzH;)9`x$}T=Z?!brAL+D zqvZERC3UM>PHAHJuy~W&+@gd`5wvu- zGXf<1jKY;fQY|o!=@vJ|7a-E`>Ra7x(Q9gSJ=^K62ol!>{ZIqEiwKhN`8O@(Iyc&8l3C?h^qdW5Kf>3^nj3AT|vKT`;p$-kTVN!=?hjggnW}qWUgCcGR&6s9s9J#18j}js;3d!Q_ zHF*%j8zqu}NK0n7z8UTN1H8nh)kKJj+|T5Sr$a;3V_10bO|0@VCv&D$i0x}qeDFU& zy%Zh9KKyqmd5e|Cf^QWTSX$sFb#pE1{QRoxH zf@2}QfPEfzS`YP?=W+EiLZHi~Cw$z*z&o6Fwd&`q)mDe~nv^e8tJiu~GpNZ_tE^K) zaS93ruYS3le@K(mDM?dug%S!m%WEx$RK=WPoqjfq8FD(y$82YTrn6LbhLX3baau@Y zew7N^RG_bPI!(^$*`eeHB^#6^MYZIzA5+DjP%^-x$Pp^}yGZ^HU*|_4>8yeqn}&uL zy|l7^SJO1y+tj|$gcDDCMvP4UuJ*`->*k-LxcVNga9^97RkdvJ&4hU!Fm=F`j=}ur z+U!V4ZlvT(+8Al=b3=G-Ft>$x??g=@UZbTk_5uC#kup=i2M6X(SZ-kL1VuCQ_q2V` zJ$JR7@ymD%xe_@8Bct8T=CogGx%@AS&y($rzx&z_UDFY{Uu$TQ&uYKYviV z;aOBx7DYL!#xgtv<=D%tpW+b8?Xmj}og7Mj@Zy zSSZ0LlX&ScUJS;yvGQqZty610gRD5Z^1q^_ANo+eAVP~x!P4u8KR5*sIIE6`nsy?t*ykv_tD?} zKK^#o^S-6Q`q#zaQ#A8Ai1egicq#n{g_&4_3Hi!k^U&HNKY zd8seGH25RzNkbK-9Tlst3?6x^hyU;q^rstcu7kPcY|~lbr+u}h;v)|{J@CZoHpV?S z-nhiKv*Js@w<-H?ymVLX$t}74#(V9j`{?h;UDv;X{+`@-{R8xG$OG5EscxPf%A4mS zd8ls5Tk7^GlDFm2nn}fBIzjx^_q>GRTTZZ1t(joN z!oJiTLb9P^6Qd-{EBF`QE@lJvTL`MFBCnL&Zj_$p z)#(G4TZZjgWp!DqN@AC-SyfwxZS_l%qYX`=v#x14u})528LNb3jjpG;R7_y&SZP*> z8E8FXVrZ-u8HJt1G+IqnWtVbCx-Ojo zf%4b>UvOK1uI>{}mwi5cK344hJJ#K2!djgpl8Rg+6=IVK3pa>LvX^;LkZZLn>*>NZ zf{iA2q7|D06Nu-4eNiZ!#$jV2olbopxO1Qwd>26ZTfa?=71@gdkuNl*m-&%1g_v0; zDhdHKsj{)N<2c>p9FUo0mXC5(NW*#Cdnt4UbeJ}xUc3MK(pXsW)nZBaAlRK23L~S6 z(OXa7#N5Ard1A}vq{*9yR_Ie@AAVd;#^p!l%abA>o-{L{tvZ<&MR`J2o#;hA8Z@(Z zi}R*7Kw6bs&Od}Ovk#K^kstU$82ItOq>kn#uVT0V-Sk&Gq?ya z(lpjLh>+O}a)@4=EN?y#e}eJ%{?|Jf$|r$dKYBA zyRPI~yq=%ul6V)4>&_BU&_BLjJS|0=|nHNihpI^QI zGq^9~T5r(-wn5%TdYAfpL2$BQl(z+{H#^ru!scb&Vk^w_*;aT!4J)?Zl$6Z?)cG;a{Va98&bi*}WE+sqvt@gjy zkE$|A@)RXJF8;R+B=5Dy1|zKx7jm<@D3q)Ec$G}DF%$S#*0|atbGIUNuFQj#LFYk3 zbcZv-NUj-$7kl0po1l$I=lio+k8@(+{7)1N@@&$bAFLe;L0B*x-)+H6`cQ8YP8E7m zP`}SPL3O#(9YQ0l) eo#k!s(IOO|P@!@4x99a?a6d>wSBL%NKmP@_Vp=@_ literal 0 HcmV?d00001 diff --git a/vllm/lora/__pycache__/utils.cpython-310.pyc b/vllm/lora/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76b259e1d6f96312169a11b507d43a003c25ba96 GIT binary patch literal 5748 zcma)A&2tpT6`z@%*)OeDNCJdD5Q7k)#bRY_V{A;>0SqAtV1i_vOgR}w+r8Qm^Re!k zm5^%jA+mGxF{f0j#LYL_j^6@;9yz9 zb>Z*-Y=u*r_Ah#P@v7qC7r2vu>zekd#x$k}nxp=8$H21@GbYEy;8-~BjE57>L^$b8hEvW|c-%QI*Gj>3c)~dm+KwHbbWVmd&P;g9ITg-2 zv*Bsyba=)&6P|U>hI7uG#4iW6@SJl_jt7D_!g*(2*Z3Gezok2MKF4i-il64Q{477i zXZSdu;uCz5ALl1qRcC?cd6AzO*TyyeCaYZ0N;*5s2Kn&n1y<#6u_1nh9pWY%<`y3v z*VqWHG%IYB9sbI4-tJ+aV@LRWAAb*b2AD_L7%<;q$0W^hevwVck)l*&lWgiM(|MO2 z$I1vV@QeIX(-_y9CYxp_u<{`To&n_;Jb^*)=G`!8bOX?6xP zmzln()#jd&AKb0wGxN5eq*?KkcIwBG7i76bo@UlkrycOjT-s>!%=$FKQq9apZbYfr zxErT8<4(kG2oZ~{daHS>vwUl5>7IHXzIF3~7x>I0a%yVqK5xf~pT-CF*7VGXBEUoU z1W$$MN1R=cqo%)tx$^D!{?-1|fFJp(8^kLsT-2=W)b%*%gwZ`uctOB}+t}%e&-}Et z*zyG9jP_==J6x;~#h*t#qfmeQn+LBUDZI@7d5rt<`fF(1zgjD2!`Hlo-}W}RWLfe$ z^;2BR)mbIrVgrheR_@2CCpK<`%bYPhlGwLnFgJEPUh&i9o`@SfN#Z@?Gp|>Ke8q{^ zXh2dQf2HVh=@{%A_zf?G;`^46RL5Vb4Y@S^AZ~ceyQ(@M4b-G(LkHx+RCO2s_MKbY zW7)6^`@=1S=QaW_NqCZ30Vy^sh2CSikYodDT{>1a+V&ccICI08bpoEa0XcsKPdsoN z3MCI`?1-pgQH1oa0{-c~jyw4fAk{wB{xI>x>RPLMSL+(xT-Rj!H>Q~CT3u~Rf0FO! zw~elj74vCM*SdK_)3rtI3!@8IGw#+*r;@}S(crG+Sj>W|I8ERTKrJUu;?Egwc#$ih zUvC#N8*K;EBTm3FXK8BsnAm5Dym-%6qR~m6FmLhjKdB)fL26rYQqMNAf;HI(L?7(qC zuiZuji8;`z<&_4{WBh^|$rhqUfV8H-8yIDidwQU}gurv#yOH-xAS4xlVxgkvb@Rt+ z&eR3TnOV}+vH|6PZsdhLD@T02=a7vYkajO}z`mBQ;GwH+qe5>(OWOw1pX$tDxv!D= zpJMz3TI1T?(WOX)!2_RD?#L^Ss~%6m!?R~;sW>-ZxMJKAj`Ks!J{-AP+lel zKB&s!UH|TL^(nR{MS$sCRj=q!-Umd?By}~=s`M`dBU00K+{r%xl!~AZJ%yUKprS{X zSnBEwN@3`H_*-dgB4izXydQj-l$3QuQ8Q6Kahn@k}%- z*mt8h9wgfReTDwM9vipwLLVV{sPj=%&A4ilk&W}dinf54*Rt2^%jRvLQ0u} z%~;^YQZ=q;<3X~ptmG5#zzLp_TL|(Kr@SZVGn!ETd67m70MCAeb31uS;cD;ynsZ1# z@8y-NFU?%roFpDPQPOF*V}Z&i72Pvc%@mhFJj(?<5|kygGJ2v8m+er^5ERRqg~&*g zgRwAySy+Y)=t=`B1(TWoh+WADKzU%+sOZP^Y2Da4W=!iq&!DvwKk*HgU`7S;Bz9&%e^1AgwSH|2ay_slH_-Lkvk5DHxSa%%#S5j^(@B zQ{#!*HN{-l+%{R^Dbh77BJwOG9t*JtFQ;9#l82*!CT)A~#?2+<9;vd#Zlhu&{n`(E zxuM76L&{0gi(?T|5D31~2|Qs#Ac4*XePuFlOL|GCfsB>d53v=Vd~WUwO2Vt%N9u1) z{?%q5*6pPh%+>5f4QfltbRpW!V8iYtFkKoea*U^3gi!rHmrFX$rvG@Csh61{YQz}Q z+%A%2?%^v9@KE{bzAQ$LN!e}@cS6NzvzTf~^GNKv_8F|Q2s5SHsW;jKRWLxHxWRa#0?VtoDyjyMf8dMT`-H5(HwgTi zz#YQ0LN5hkUb)Rd&c}X&#}2k8hXIPEVxerAdJ!RR>W5$wR6nYRSxIIk*tE6Jc}VyH z+%kwaF@U2`$%6BUA~l;Ri8<;P#Hd^`X$1v{u+XYv{)8M;Kx2NhlonR8j~U;fh<*lp zN#`r2<#ZseG|i1G$_3NG?JCQWgLbt%98$WY>74~T();i8Bo5YiA|rXlU*nO=)5Jdo zoQJ++87Q5UV*wt8bda`sHSSkUxveiSg_w^bk@`&^@rsD|>|33c70MnrsiTqczlS1J zDe+VEG(ljqcs-)rD2XamFJj0m)aUo!L>wfM*G^h-+GB+(*(X$a*8PN|*rj$#owpKc zk2lGU5}}FslbazPlWM@3VvQru%ia5>enDOoek6FrYOk{I5ubwx)-EG`_2@!#lo|;^ z2|bTo5wZ zHca!fP08B(c~&X#x&CEMcM53!V%KMzXJvKl+lov}yLx8cvvO@HGvXvONxIDP6TC4i zBVM6oYGAW6(k`@*#4fTte~j#rWCb8e{GpcenkF5mTAgKQc-Q9cu9JuoVn|R#3ChWW zzMy635a`oGR^pEvTz)A*8GnEkXH;6=?USL`91Qw#YMnjiwP9g*S3@pg{sknG8Gy=& zsh4vVy@K1wnK()ncg#}pzsSiuWfKRg@5D0DRMz}J{0`&HlKHuppK#itqYPyOkt0C) zS5R*(sLTsGNC+3eIdVV;b$PZ^!b;gmVyOVf!Z({nOHgreD)>SZB~8?1xTzSVsR244 z*443D(6_|ij8c{uqM2!lL;hN)biUC)gvnEHA`Ssb3=^O_Aw~&M9&`@vZS0{Qp*1>h ziZKFo`V->>CJ0Otm?ChTz%+pKhNK}Q-sO*ZqXT*Bidy~Pp-r5i9dEy8#~OXHfZ5bX zC%4}p3L87KqAQzX*A)-(C{OR%kn4I;gzq)fbtJBvD49(MTO} zmg6{(?VL>d%IJGXPdSq3&*Bw`g)`cR)I21A1}JMq-A3IwrdLoq=$_KwELO@D%o#a# zK7SQF{2$|J`FpJF7>4m9Fn`FK#`mU)xufQHFU}j7&s)ZKM!x*KQiYT5z-f0TN+V;` KGLWktwf_elm1AlE literal 0 HcmV?d00001 diff --git a/vllm/lora/__pycache__/worker_manager.cpython-310.pyc b/vllm/lora/__pycache__/worker_manager.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68fe57247571c5ade105c1e5cfa3f9f1b3bd0ae7 GIT binary patch literal 7501 zcmb7J+ix3JdY?Hn9Fil7q9jX}9CHy2$J z^YP4gzH|G|cV{v`?;7~L`49iR_p@t;@$d99`myoy4xaSi0Jy=;$cRnfG^x!Z7F)g* z+rAwa{6g&bPF(bhamg>mWxpJ|zN_i1XwIJly&YBJd4E2x`qg;BUx;gdP4^d~#dyhI z(rqVNj?ek$bh{X>#H;?QZkM9-@tVIDulwuqhQASC@Goe5Il35M@-OMO8(oe!{Y}#l ztHL`r{VN|De2!P17`!5ujxGNx-sgE0?^SUX@7K8Xu~A?66;5WdOtbr3RSq*0dBi)tEG_h zAmm{e)WM@fJ`hr^gxzj*IK-wy^tlKVPhmGw0c1049f-+3>~!Vboivl^{!CB((O|=- za_=N}-@To5c3byBHrML3vLH(C(`jr~rp}KNE+SPsZ4Sn&wL5q3-42_3Vyb7kDMgqG z%p=2K$Qw*fgm=ZmeUWBZYyCcLcyQuyCrzS#jOm8iUX$26`Z+gvNk1_SLm0lv4WHeE zX!(|~{ep0Wy>5siXKxvz#I0S%ZBgb0JPw|saCu40wJW?V=0#OhciFnZU9ljl_?^S( z5`HV9Cd%u^?$S?;T~lBByuT=x_yVszF(4Yt;v8QTE0}wUFF&z?S>@+2>-qK?@awxa zU*W4yY=48F7Z-pj0kg)}fw`zL8`uerclZT<5!g%o(g0)g%X|}<%iI&2VjX+Ag1yl1 zBF^IQtNa>zJ^sp|m#{X!&R+%g3V%(k>v?T{gV%w%Ix_FK`0K!2<8SbrPYnMR&~557 zGTz`@=(+wA!#4a^A=0(nBwl!V4qF6F=0` zA4dS%6v1!nJ|N%tD!1G0SuZ^x~(d4_&Dh7$3ZF{rfPNgwi9L$WFBP6 z1JOy9dyq84ogi)XgqqV*3`K{AD2I!KR#T`&5$_1jTb=s>&cBcCs`WEe7rH#!Rm~NF z7)85K=V{q)lD5bo1gGQDrZ%htS*Nx&c+&5p(J%Fl!00=GX1@r?xCv9Fu;9-j2>>AZGTXoQwswH9qvnRVioBDsHLDun#}{gAIFE9 zPHs@|1p*fdTq1B4Ko!#cF05542(aMa=5$(<#ahZmlE+nrEXL)X_ascjxX&MC2 zjjw-yD~r2Z-B$Nz7D`x(o4*t9@5OJ%kG7)L&Q|v@+eGn5p%b4xf|YfqkAY{ z#muzFbQQp@G5f^RNyEwgrGeT9+Hl<0l!w+c!oq?AA zl1`i%B=>E4!2LGlz745w7r6OT=ZI--aQduW)Y4ur%#uY_%k?t1cBGm0l3c;uvxwz7 zI#lh{VGZ1xyan`gE58Y9`<2m2C)lB9{{v8cNgj>a%(PFx&eu-KF?-jiIWxnXs(h7! zAsyos*^N0o`jQY$8NJ)!3phnwX|p2>&^og38%MAf?-`#jA33lo_5n1yUwCT6b~x(> zS&N4YcrWywyq!6Hvt2wkk6?M)rFIz>sDPFW>>TMryV9Nqtm@XndDxnE4bLKHKP?}X zJ~KKiwvjCz{=hVj%BImPXUp8`7k_~e@0ZrG*)RWyarFFU`H}eumactH&$IHtmYr;s z7YGAd*gmYYf>!aDg=6zaCh+GqZUL0Xpezvv{jhzYTo|Qt%tLwgHTj#^ zx)w#X3S-<&By7d0b=U4`wHr1c06B11u$0{{_O2XECpug?c*n0ikC{|iFGQi3yhcn} z*_BC*`I;dd*q&%U=q9aBmRtYaKjJk23(4U6t{$!Vs9lA>GIQT&oK)raA2)>#k5%RF zekW_i;(aL-scd2r;{Q>eDfiPb+Rq96+9WsXZmx@D^)yH1?^5gc^fAQau4qoN#>sgi zzXKX&(_WP+CNx*gBy#10;<@L!+-m4x~| zHJ=$!>N45rOqXUXxK*^J58#g3WJr(k$#Wg?3bOiHHi#DCr0-ss7@?q@8&kEevXI zBnhh9BNs`B_T3Tk38z=3foI(ZAlH7n3f6#S zOsX`n`Kc-nRCr7nXlDgHv^&iK?I-D4wWn-b$95hA%I|>!Jg(vgk>Gfyncvc~&L{^7 z+F-Hzz}0<>Ve%6K6E~3$8c#!i47!v=l$c*SndReVB(JJAyn*47r|#iY?8$VGMg(W^ zz;R*pypE(BlYdBnY^FayTqBK=ELBx0;^ZuWmVXQyT&Rw-KBGb7r#y7_1hC>&|lFBnhWSqhNdVXr!4s)Un!y18QlHK-&i! z={6`IAFMe%(4P|B#DQqU-{MJKfPq0X?QdpD<%q6MT?~mKiJ;sjphYcL3;6>=kolZk znL7TC@OJ@b>F4;>Oa-RIG%fWc(Ya1O8{|8q8&J!`d7@~@kW|hHUFDx)o}c57^Z zWHw`}m+y-t4;KkdY_nt7ihNg=oCiL?E-jX`wn80$Ph^@mGhD=&0j|GBxmYKRMGz=A z7`S}2D?#vZKa2)F#US8GGYI4|%}n8~ECaw81YxI>WFcjzNXByWZIQ#TQGX#zWOGkP zOOkw#vQSx)WOQVaz^er4HsnnJRqDzF>0ySl*iMo}a*L3x(^1YxI&+pJD|y(xN-ZtD zcc?{{QBu$^KP5oArWG=mA-Wzd?{Lj|-q;kl(qKfcZV<`q1Sn^t$UgmZ0N1e1T7k{M zKYzs-d%-OB4Px>CVFmWzw!^-5i|oIg68lf5%>Ki1t6#Bo*JQ<-$t*`xyWcD=tX^N5 zcP;Yj@Zu+{g{8uhQ?fQJy;2j4jD3hr@8C(l0QkS_SKJbauG=muSaYcI6u5&rO$9Y9 zRIX4FvN_e2%G?EZ0kx|t=xV@J_&hL+yo4yXs@8|4xhX>wLM4nTEJdcjW?2VWZi*U= zhn2kBR4Zuw`jsA?^S-RwGpGjDb)hE`VMt=y+|5K=E#)DHvv#ALHhU`^r|Z$WRLUW2h~D` z8KYL88U49p>!p5+22Pk;lF8?U|80QY&6i$VV^$8Br!Pb2xIK1^v7nGlD>pfNg3$0i z)H074>=0~FZi|pPUCu0S!BhgWqi$=Mz= zd8GKXI0Xng0BOPGeO@|3fFzlYGR>c{hYaSASbDC9{m*&S9WPX+5n?^b;( z0VAnG^>8rc!|&hA%Z@sD@NV>2y-|0R6^TwBUF4aN?v?LgM4OC*(bSAdjKO@ zgehOoGl*l8n>w{e1OldL_lw`}H@dEZZ0cxKP~~aYSQ3Yei`urBWo6hdmUl`PXLBKhl3x4J*2GfhgxR zWnPQd!N>rU|6wH;iKeD$D2s3CGGeOZmpsVPEGC=lFP>2i45%b-`U_|He8W@^(buIQ xAN04F_VAivvv{paZ{^j5w<$qR$yrp48>ls}F;dcsxn@=v)t+mX@mJ?6{|nNKV%Gow literal 0 HcmV?d00001 diff --git a/vllm/lora/fully_sharded_layers.py b/vllm/lora/fully_sharded_layers.py new file mode 100644 index 00000000..a7887a04 --- /dev/null +++ b/vllm/lora/fully_sharded_layers.py @@ -0,0 +1,343 @@ +# pylint: disable=unused-argument +from typing import TYPE_CHECKING, List, Optional, Union + +import torch +import torch.nn as nn +from transformers import PretrainedConfig + +from vllm.config import LoRAConfig +from vllm.distributed.communication_op import ( + tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce) +from vllm.distributed.parallel_state import get_tensor_model_parallel_rank +from vllm.lora.layers import (ColumnParallelLinearWithLoRA, + MergedColumnParallelLinearWithLoRA, + MergedQKVParallelLinearWithLora, + QKVParallelLinearWithLora, + RowParallelLinearWithLoRA) + +if TYPE_CHECKING: + pass + + +def _fully_sharded_can_replace(can_replace): + """ + decorator which adds the condition of fully sharded loras + intended to wrap can_replace_layer() + """ + + def dec(*args, **kwargs): + return (can_replace(*args, **kwargs) + and kwargs["lora_config"].fully_sharded_loras) + + return dec + + +# these layers are based on the tensor parallelism strategy given in +# Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023, +# https://arxiv.org/abs/2311.03285. + + +class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA): + """ + Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also. + + Based on S-LoRA, slicing happens along the rank dim. + """ + + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: + tp_rank = get_tensor_model_parallel_rank() + shard_size = self.lora_a_stacked.shape[2] + start_idx = tp_rank * shard_size + lora_a = lora_a[:, start_idx:start_idx + shard_size] + return lora_a + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + + x = x.view(-1, x.shape[-1]) + output, out_orig_shape = output.view(-1, + output.shape[-1]), output.shape + buffer = torch.zeros( + (x.shape[0], self.lora_a_stacked.shape[2]), + dtype=torch.float32, + device=x.device, + ) + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0) + buffer = tensor_model_parallel_all_gather(buffer) + self.punica_wrapper.add_expand(output, + buffer, + self.lora_b_stacked, + add_input=True) + # now have column partitioned output + output = output.view(*out_orig_shape) + return output + + @classmethod + @_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + # specifying kwargs so they can be easily accessed in decorator + return super().can_replace_layer( + source_layer=source_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + decorate=False, + ) + + +def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora): + """ + MergedColumnParallelLinearWithShardedLoRA and + MergedQKVParallelLinearWithShardedLora share the same + LoRa weight application method. + + The main difference is the step by shard_size for lora_b which can + vary for MergedQKVParallelLinearWithShardedLora but is constant for + MergedColumnParallelLinearWithShardedLoRA. + """ + # expecting 2 for column parallel and 3 for qkv + n = len(layer.lora_a_stacked) + output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias) + + x = x.view(-1, x.shape[-1]) + output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape + buffers = torch.zeros( + (n, x.shape[0], layer.lora_a_stacked[0].shape[2]), + dtype=torch.float32, + device=x.device, + ) + for idx in range(n): + layer.punica_wrapper.add_shrink(buffers[idx], x, + layer.lora_a_stacked[idx], 1.0) + + buffers = tensor_model_parallel_all_gather(buffers) + left_offset = 0 + for idx in range(n): + shard_size = layer.lora_b_stacked[idx].shape[2] + layer.punica_wrapper.add_expand_slice( + output, + buffers[idx], + layer.lora_b_stacked[idx], + left_offset, + shard_size, + add_input=True, + ) + left_offset += shard_size + + output = output.view(*out_orig_shape) + # now have column partitioned and packed output + return output + + +class MergedColumnParallelLinearWithShardedLoRA( + MergedColumnParallelLinearWithLoRA): + """ + Differs from MergedColumnParallelLinearWithLoRA by slicing the + LoRA A's also. + + Based on S-LoRA, slicing happens along the rank dim. + """ + + def slice_lora_a( + self, lora_a: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + if lora_a[0] is None or lora_a[1] is None: + return lora_a + output_shard_size = self.lora_a_stacked[0].shape[2] + output_start_idx = self.tp_rank * output_shard_size + lora_a = [ + lora_a[0][:, + output_start_idx:output_start_idx + output_shard_size], + lora_a[1][:, + output_start_idx:output_start_idx + output_shard_size], + ] + return lora_a + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + return _mcp_apply(x, bias, self) + + @classmethod + @_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + # specifying kwargs so they can be easily accessed in decorator + return super().can_replace_layer( + source_layer=source_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + decorate=False, + ) + + +class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora): + """ + Differs from QKVParallelLinearWithLora by slicing the + LoRA A's also. + + Based on S-LoRA, slicing happens along the rank dim. + """ + + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: + tp_rank = get_tensor_model_parallel_rank() + shard_size = self.lora_a_stacked.shape[2] + start_idx = tp_rank * shard_size + lora_a = lora_a[:, start_idx:start_idx + shard_size] + return lora_a + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + + x = x.view(-1, x.shape[-1]) + output, out_orig_shape = output.view(-1, + output.shape[-1]), output.shape + buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]), + dtype=torch.float32, + device=x.device) + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0) + buffer = tensor_model_parallel_all_gather(buffer) + self.punica_wrapper.add_expand(output, + buffer, + self.lora_b_stacked, + add_input=True) + # now have column partitioned output + output = output.view(*out_orig_shape) + return output + + @classmethod + @_fully_sharded_can_replace + def can_replace_layer(cls, source_layer: nn.Module, + lora_config: LoRAConfig, packed_modules_list: List, + model_config: Optional[PretrainedConfig]) -> bool: + # specifying kwargs so they can be easily accessed in decorator + return super().can_replace_layer( + source_layer=source_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + decorate=False, + ) + + +class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora): + """ + Differs from MergedQKVParallelLinearWithLora by slicing the + LoRA A's also. + + Based on S-LoRA, slicing happens along the rank dim. + """ + + def slice_lora_a( + self, lora_a: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + if lora_a[0] is None or lora_a[1] is None or lora_a[2] is None: + return lora_a + shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)] + start_idx = [self.tp_rank * shard_size[i] for i in range(3)] + lora_a = [ + lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]], + lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]], + lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]], + ] + return lora_a + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + return _mcp_apply(x, bias, self) + + @classmethod + @_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + # specifying kwargs so they can be easily accessed in decorator + return super().can_replace_layer( + source_layer=source_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + decorate=False, + ) + + +class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA): + """ + Differs from RowParallelLinearWithLoRA by slicing the + LoRA B's also. + + Based on S-LoRA, slicing happens along the output dim. + This yields a combined partial sum from the row parallel base + layer and column partitioned output from the LoRA. + """ + + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: + shard_size = self.lora_b_stacked.shape[2] + start_idx = self.tp_rank * shard_size + end_idx = (self.tp_rank + 1) * shard_size + lora_b = lora_b[:, start_idx:end_idx] + return lora_b + + def apply(self, x: torch.Tensor) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x) + + x = x.view(-1, x.shape[-1]) + output, out_orig_shape = output.view(-1, + output.shape[-1]), output.shape + buffer = torch.zeros( + (x.shape[0], self.lora_a_stacked.shape[2]), + dtype=torch.float32, + device=x.device, + ) + + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0) + buffer = tensor_model_parallel_all_reduce(buffer) + + # following S-LoRA, allows the fusing of all_gather and all_reduce + # by adding the column partitioned lora output to a slice of output + # tensor, which is a partial sum due to row parallel. All that + # remains is a standard all_reduce. User should be aware though that + # the output is not the same as a normal row_parallel, it should be + # reduced before being used + shard_size = self.lora_b_stacked.shape[2] + start_idx = self.tp_rank * shard_size + self.punica_wrapper.add_expand_slice(output, buffer, + self.lora_b_stacked, start_idx, + shard_size) + output = output.view(*out_orig_shape) + return output + + @classmethod + @_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + # specifying kwargs so they can be easily accessed in decorator + return super().can_replace_layer( + source_layer=source_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + decorate=False, + ) diff --git a/vllm/lora/layers.py b/vllm/lora/layers.py new file mode 100644 index 00000000..6254c675 --- /dev/null +++ b/vllm/lora/layers.py @@ -0,0 +1,1312 @@ +# pylint: disable=unused-argument +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig + +from vllm.adapter_commons.layers import AdapterMapping +from vllm.config import LoRAConfig +from vllm.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, + tensor_model_parallel_gather) +from vllm.distributed.utils import divide +from vllm.lora.punica import PunicaWrapper +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.rotary_embedding import ( + LinearScalingRotaryEmbedding, RotaryEmbedding) +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) + +if TYPE_CHECKING: + pass + + +def _get_lora_device(base_layer: nn.Module) -> torch.device: + # code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34 + """Returns the device for where to place the LoRA tensors.""" + # unquantizedLinear + if hasattr(base_layer, "weight"): + return base_layer.weight.device + # Compressed Tensor + elif hasattr(base_layer, "weight_packed"): + return base_layer.weight_packed.device + # GPTQ/AWQ + elif hasattr(base_layer, "qweight"): + return base_layer.qweight.device + # marlin + elif hasattr(base_layer, "B"): + return base_layer.B.device + else: + raise ValueError(f"Unsupported base layer: {base_layer}") + + +def _not_fully_sharded_can_replace(can_replace): + """ + decorator which adds the condition of not using fully sharded loras + intended to wrap can_replace_layer() + """ + + def dec(*args, **kwargs): + decorate = kwargs.pop("decorate") if "decorate" in kwargs else True + condition = (not kwargs["lora_config"].fully_sharded_loras + if decorate else True) + return can_replace(*args, **kwargs) and condition + + return dec + + +@dataclass +class LoRAMapping(AdapterMapping): + is_prefill: bool = False + + +class BaseLayerWithLoRA(nn.Module): + + def slice_lora_a( + self, lora_a: Union[torch.Tensor, List[Union[torch.Tensor, None]]] + ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]: + """Slice lora a if splitting for tensor parallelism.""" + ... + + def slice_lora_b( + self, lora_b: Union[torch.Tensor, List[Union[torch.Tensor, None]]] + ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]: + """Slice lora b if splitting with tensor parallelism.""" + ... + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + """Initializes lora matrices.""" + ... + + def reset_lora(self, index: int): + """Resets the lora weights at index back to 0.""" + ... + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + """Overwrites lora tensors at index.""" + ... + + def set_mapping( + self, + punica_wrapper: PunicaWrapper, + ): + self.punica_wrapper: PunicaWrapper = punica_wrapper + + @classmethod + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + """Returns True if the layer can be replaced by this LoRA layer.""" + raise NotImplementedError + + +class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): + + def __init__(self, base_layer: VocabParallelEmbedding) -> None: + super().__init__() + self.base_layer = base_layer + self.embeddings_slice: Optional[Tuple[int, int]] + self.embeddings_weights: Optional[torch.Tensor] + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None) -> None: + + if self.base_layer.num_added_embeddings_per_partition > 0: + # We can start adding lora weights + self.embeddings_weights = self.base_layer.weight.data[ + self.base_layer.num_org_embeddings_per_partition:self. + base_layer.num_org_embeddings_per_partition + + self.base_layer.num_added_embeddings_per_partition] + self.embeddings_slice = ( + self.base_layer.shard_indices.added_vocab_start_index - + self.base_layer.org_vocab_size, + self.base_layer.shard_indices.added_vocab_end_index - + self.base_layer.org_vocab_size) + self.base_layer.weight.data[ + self.base_layer.num_org_embeddings_per_partition:].fill_(0) + else: + self.embeddings_slice = None + self.embeddings_weights = None + + self.embeddings_tensors = torch.zeros( + ( + max_loras, + lora_config.lora_extra_vocab_size, + self.base_layer.embedding_dim, + ), + dtype=self.base_layer.weight.dtype, + device=self.base_layer.weight.device, + ) + self.lora_a_stacked = torch.zeros( + ( + max_loras, + self.base_layer.org_vocab_size + + lora_config.lora_extra_vocab_size, + lora_config.max_lora_rank, + ), + dtype=lora_config.lora_dtype, + device=self.base_layer.weight.device, + ) + self.lora_b_stacked = torch.zeros( + ( + max_loras, + 1, + self.base_layer.embedding_dim, + lora_config.max_lora_rank, + ), + dtype=lora_config.lora_dtype, + device=self.base_layer.weight.device, + ) + self.lora_a_stacked_2d = self.lora_a_stacked.view( + self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1], + self.lora_a_stacked.shape[2], + ) + + def reset_lora(self, index: int): + self.lora_a_stacked[index] = 0 + self.lora_b_stacked[index] = 0 + self.embeddings_tensors[index] = 0 + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + self.lora_a_stacked[index, :lora_a.shape[0], :lora_a.shape[1]].copy_( + lora_a, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + if embeddings_tensor is not None: + self.embeddings_tensors[ + index, :embeddings_tensor.shape[0], :embeddings_tensor. + shape[1], ].copy_(embeddings_tensor, non_blocking=True) + if self.embeddings_slice is not None: + # TODO(yard1): Optimize this copy, we don't need to copy + # everything, just the modified part + embeddings = self.embeddings_tensors.view( + self.embeddings_tensors.shape[0] * + self.embeddings_tensors.shape[1], + self.embeddings_tensors.shape[2], + )[self.embeddings_slice[0]:self.embeddings_slice[1]] + assert self.embeddings_weights is not None + self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + added_tokens_mask = x > self.base_layer.org_vocab_size - 1 + embeddings_indices = self.punica_wrapper.embeddings_indices + indices = embeddings_indices[1].view_as(x) + full_lora_a_embeddings = F.embedding( + x + indices, + self.lora_a_stacked_2d, + ) + indices = embeddings_indices[0].view_as(x) + full_output = self.base_layer.forward( + x.add_(indices * added_tokens_mask)) + + full_output_org = full_output + if full_output.ndim == 3: + full_output = full_output.view( + full_output.shape[0] * full_output.shape[1], -1) + if full_lora_a_embeddings.ndim == 3: + full_lora_a_embeddings = full_lora_a_embeddings.view( + full_lora_a_embeddings.shape[0] * + full_lora_a_embeddings.shape[1], + -1, + ) + + # Embedding layer only need expand op + self.punica_wrapper.add_expand(full_output, + full_lora_a_embeddings, + self.lora_b_stacked, + add_input=True) + return full_output.view_as(full_output_org) + + @classmethod + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return type(source_layer) is VocabParallelEmbedding + + +class ReplicatedLinearWithLoRA(BaseLayerWithLoRA): + + def __init__(self, base_layer: ReplicatedLinear) -> None: + super().__init__() + self.base_layer = base_layer + self.input_size = self.base_layer.input_size + self.output_size = self.base_layer.output_size + self.device = _get_lora_device(self.base_layer) + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + self.lora_config = lora_config + lora_a_output_size = lora_config.max_lora_rank + self.lora_a_stacked = torch.zeros( + max_loras, + 1, + lora_a_output_size, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ) + self.lora_b_stacked = torch.zeros( + max_loras, + 1, + self.output_size, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ) + + def reset_lora(self, index: int): + self.lora_a_stacked[index] = 0 + self.lora_b_stacked[index] = 0 + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + + self.lora_a_stacked[index, + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_( + lora_a.T, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked, + self.lora_b_stacked, 1.0) + return output + + def forward(self, input_): + """Forward of ReplicatedLinearWithLoRA + + Args: + input_: Tensor whose last dimension is `input_size`. + + Returns: + - output + - bias + """ + bias = (self.base_layer.bias + if not self.base_layer.skip_bias_add else None) + + # Matrix multiply. + output = self.apply(input_, bias) + + output_bias = (self.base_layer.bias + if self.base_layer.skip_bias_add else None) + return output, output_bias + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return type(source_layer) is ReplicatedLinear + + +class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): + """ + LoRA on top of ColumnParallelLinear layer. + + LoRA B is sliced for tensor parallelism. + """ + + def __init__(self, base_layer: ColumnParallelLinear) -> None: + super().__init__() + self.base_layer = base_layer + self.tp_size = get_tensor_model_parallel_world_size() + self.input_size = self.base_layer.input_size + self.output_size = self.base_layer.output_size_per_partition + self.device = _get_lora_device(self.base_layer) + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + self.lora_config = lora_config + self.tp_size = get_tensor_model_parallel_world_size() + lora_a_output_size_per_partition = ( + lora_config.max_lora_rank if not lora_config.fully_sharded_loras + else divide(lora_config.max_lora_rank, self.tp_size)) + self.lora_a_stacked = torch.zeros( + max_loras, + 1, + lora_a_output_size_per_partition, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ) + self.lora_b_stacked = torch.zeros( + max_loras, + 1, + self.output_size, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ) + self.output_dim = self.lora_b_stacked.shape[2] + + def reset_lora(self, index: int): + self.lora_a_stacked[index] = 0 + self.lora_b_stacked[index] = 0 + + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: + return lora_a + + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: + tensor_model_parallel_rank = get_tensor_model_parallel_rank() + shard_size = self.output_dim + start_idx = tensor_model_parallel_rank * shard_size + end_idx = (tensor_model_parallel_rank + 1) * shard_size + lora_b = lora_b[:, start_idx:end_idx] + return lora_b + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + + if self.tp_size > 1: + lora_a = self.slice_lora_a(lora_a) + lora_b = self.slice_lora_b(lora_b) + + self.lora_a_stacked[index, + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_( + lora_a.T, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked, + self.lora_b_stacked, 1.0) + return output + + def forward(self, input_): + """Forward of ColumnParallelLinear + + Args: + input_: Tensor whose last dimension is `input_size`. + + Returns: + - output + - bias + """ + bias = (self.base_layer.bias + if not self.base_layer.skip_bias_add else None) + + # Matrix multiply. + output_parallel = self.apply(input_, bias) + if self.base_layer.gather_output: + # All-gather across the partitions. + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = (self.base_layer.bias + if self.base_layer.skip_bias_add else None) + return output, output_bias + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return type(source_layer) is ColumnParallelLinear or ( + type(source_layer) is MergedColumnParallelLinear + and len(packed_modules_list) == 1) + + +class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): + """ColumnParallelLinear layer that is composed of 2 sublayers (slices) + packed together (eg. gate_proj + up_proj -> gate_up_proj). + + This means we have 2 LoRAs, each applied to one half of the layer. + + Both slices must have the same size. + """ + + def __init__(self, base_layer: MergedColumnParallelLinear) -> None: + super().__init__(base_layer) + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + self.lora_config = lora_config + n_slices = 2 + if not (len(self.base_layer.output_sizes) == n_slices + and self.base_layer.output_sizes[0] + == self.base_layer.output_sizes[1]): + raise ValueError( + "LoRAColumnParallelLinear2Slice requires 2 slices with " + "the same size.") + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + + lora_a_output_size_per_partition = ( + lora_config.max_lora_rank if not lora_config.fully_sharded_loras + else divide(lora_config.max_lora_rank, self.tp_size)) + + self.lora_a_stacked = tuple( + torch.zeros( + max_loras, + 1, + lora_a_output_size_per_partition, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ) for _ in range(n_slices)) + self.lora_b_stacked = tuple( + torch.zeros( + max_loras, + 1, + self.output_size // 2, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ) for _ in range(n_slices)) + + self.output_dim = self.lora_b_stacked[0].shape[2] + + def reset_lora(self, index: int): + self.lora_a_stacked[0][index] = 0 + self.lora_a_stacked[1][index] = 0 + self.lora_b_stacked[0][index] = 0 + self.lora_b_stacked[1][index] = 0 + + def slice_lora_a( + self, lora_a: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + return lora_a + + def slice_lora_b( + self, lora_b: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + if lora_b[0] is None or lora_b[1] is None: + return lora_b + shard_size = self.output_dim + start_idx = self.tp_rank * shard_size + end_idx = (self.tp_rank + 1) * shard_size + lora_b = [ + lora_b[0][:, start_idx:end_idx], + lora_b[1][:, start_idx:end_idx], + ] + return lora_b + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + + if self.tp_size > 1: + lora_a = self.slice_lora_a(lora_a) + lora_b = self.slice_lora_b(lora_b) + + if lora_a[0] is not None: + self.lora_a_stacked[0][ + index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_( + lora_a[0].T, non_blocking=True) + self.lora_b_stacked[0][ + index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_( + lora_b[0].T, non_blocking=True) + if lora_a[1] is not None: + self.lora_a_stacked[1][ + index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_( + lora_a[1].T, non_blocking=True) + self.lora_b_stacked[1][ + index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_( + lora_b[1].T, non_blocking=True) + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + self.punica_wrapper.add_lora_packed_nslice( + output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0, + (self.output_dim, self.output_dim)) + return output + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return (type(source_layer) is MergedColumnParallelLinear + and len(packed_modules_list) == 2) + + +class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA): + """ + ColumnParallelLinear layer that is specifically designed for + qkv_proj. Certain models, such as chtglm3 and baichuan-7b, + only contains a single LoRA within their qkv_proj layer. + + During inference with Tensor Parallel, the weights of lora_b + must be accurately partitioned according to the respective ranks. + + Q slice may have different shape than K and V slices (which both have + the same shape). + """ + + def __init__(self, base_layer: QKVParallelLinear) -> None: + super().__init__(base_layer) + self.tp_size = get_tensor_model_parallel_world_size() + self.q_proj_total_size = (self.base_layer.total_num_heads * + self.base_layer.head_size) + self.q_proj_shard_size = (self.base_layer.num_heads * + self.base_layer.head_size) + self.kv_proj_shard_size = (self.base_layer.num_kv_heads * + self.base_layer.head_size) + self.kv_proj_total_size = (self.base_layer.total_num_kv_heads * + self.base_layer.head_size) + + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: + tp_rank = get_tensor_model_parallel_rank() + self.q_shard_id = tp_rank + self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas + lora_b_q = lora_b[:, self.q_proj_shard_size * + self.q_shard_id:self.q_proj_shard_size * + (self.q_shard_id + 1)] + k_offset = self.q_proj_total_size + lora_b_k = lora_b[:, k_offset + + self.kv_proj_shard_size * self.kv_shard_id:k_offset + + self.kv_proj_shard_size * (self.kv_shard_id + 1)] + v_offset = k_offset + self.kv_proj_total_size + lora_b_v = lora_b[:, v_offset + + self.kv_proj_shard_size * self.kv_shard_id:v_offset + + self.kv_proj_shard_size * (self.kv_shard_id + 1)] + lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1) + return lora_b + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + if self.tp_size > 1: + lora_a = self.slice_lora_a(lora_a) + lora_b = self.slice_lora_b(lora_b) + + self.lora_a_stacked[index, + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_( + lora_a.T, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer(cls, source_layer: nn.Module, + lora_config: LoRAConfig, packed_modules_list: List, + model_config: Optional[PretrainedConfig]) -> bool: + return type(source_layer) is QKVParallelLinear and len( + packed_modules_list) == 1 + + +class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA): + """ColumnParallelLinear layer that is composed of 3 sublayers (slices) + packed together in qkv proj fashion + (q_proj + k_proj + v_proj -> qkv_proj). + + This means we have 3 LoRAs, each applied to one slice of the layer. + + Q slice may have different shape than K and V slices (which both have + the same shape). + """ + + def __init__(self, base_layer: QKVParallelLinear) -> None: + super().__init__(base_layer) + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + self.lora_config = lora_config + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + self.q_proj_shard_size = (self.base_layer.num_heads * + self.base_layer.head_size) + self.kv_proj_shard_size = (self.base_layer.num_kv_heads * + self.base_layer.head_size) + self.q_shard_id = self.tp_rank + self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas + + lora_a_output_size_per_partition = ( + lora_config.max_lora_rank if not lora_config.fully_sharded_loras + else divide(lora_config.max_lora_rank, self.tp_size)) + # q, k, v + self.lora_a_stacked = ( + torch.zeros( + max_loras, + 1, + lora_a_output_size_per_partition, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ), + torch.zeros( + max_loras, + 1, + lora_a_output_size_per_partition, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ), + torch.zeros( + max_loras, + 1, + lora_a_output_size_per_partition, + self.input_size, + dtype=lora_config.lora_dtype, + device=self.device, + ), + ) + self.lora_b_stacked = ( + torch.zeros( + max_loras, + 1, + self.q_proj_shard_size, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ), + torch.zeros( + max_loras, + 1, + self.kv_proj_shard_size, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ), + torch.zeros( + max_loras, + 1, + self.kv_proj_shard_size, + lora_config.max_lora_rank, + dtype=lora_config.lora_dtype, + device=self.device, + ), + ) + + self.output_slices = ( + self.q_proj_shard_size, + self.kv_proj_shard_size, + self.kv_proj_shard_size, + ) + self.packed_indices: Optional[torch.Tensor] = None + self.standard_indices: Optional[torch.Tensor] = None + # lazily initialized. + self.indices: torch.Tensor + self.indices_len: List[int] + + def reset_lora(self, index: int): + self.lora_a_stacked[0][index] = 0 + self.lora_b_stacked[0][index] = 0 + self.lora_a_stacked[1][index] = 0 + self.lora_b_stacked[1][index] = 0 + self.lora_a_stacked[2][index] = 0 + self.lora_b_stacked[2][index] = 0 + + def slice_lora_a( + self, lora_a: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + return lora_a + + def slice_lora_b( + self, lora_b: List[Union[torch.Tensor, None]] + ) -> List[Union[torch.Tensor, None]]: + lora_b_q, lora_b_k, lora_b_v = None, None, None + if lora_b[0] is not None: + lora_b_q = lora_b[0][:, self.q_proj_shard_size * + self.q_shard_id:self.q_proj_shard_size * + (self.q_shard_id + 1), ] + if lora_b[1] is not None: + lora_b_k = lora_b[1][:, self.kv_proj_shard_size * + self.kv_shard_id:self.kv_proj_shard_size * + (self.kv_shard_id + 1), ] + if lora_b[2] is not None: + lora_b_v = lora_b[2][:, self.kv_proj_shard_size * + self.kv_shard_id:self.kv_proj_shard_size * + (self.kv_shard_id + 1), ] + lora_b = [lora_b_q, lora_b_k, lora_b_v] + return lora_b + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + + if self.tp_size > 1: + lora_a = self.slice_lora_a(lora_a) + lora_b = self.slice_lora_b(lora_b) + + if lora_b[0] is not None: + lora_b_q = lora_b[0] + self.lora_b_stacked[0][ + index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_( + lora_b_q.T, non_blocking=True) + if lora_b[1] is not None: + lora_b_k = lora_b[1] + self.lora_b_stacked[1][ + index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_( + lora_b_k.T, non_blocking=True) + if lora_b[2] is not None: + lora_b_v = lora_b[2] + self.lora_b_stacked[2][ + index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_( + lora_b_v.T, non_blocking=True) + + if lora_a[0] is not None: + self.lora_a_stacked[0][ + index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_( + lora_a[0].T, non_blocking=True) + if lora_a[1] is not None: + self.lora_a_stacked[1][ + index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_( + lora_a[1].T, non_blocking=True) + if lora_a[2] is not None: + self.lora_a_stacked[2][ + index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_( + lora_a[2].T, non_blocking=True) + + def apply(self, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + self.punica_wrapper.add_lora_packed_nslice(output, x, + self.lora_a_stacked, + self.lora_b_stacked, 1.0, + self.output_slices) + return output + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return (type(source_layer) is QKVParallelLinear + and len(packed_modules_list) == 3) + + +class RowParallelLinearWithLoRA(BaseLayerWithLoRA): + + def __init__(self, base_layer: RowParallelLinear) -> None: + super().__init__() + self.base_layer = base_layer + self.input_size = self.base_layer.input_size_per_partition + self.output_size = self.base_layer.output_size + self.device = _get_lora_device(self.base_layer) + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + self.lora_config = lora_config + self.tp_rank = get_tensor_model_parallel_rank() + self.lora_a_stacked = torch.zeros( + ( + max_loras, + 1, + lora_config.max_lora_rank, + self.input_size, + ), + dtype=lora_config.lora_dtype, + device=self.device, + ) + tp_size = get_tensor_model_parallel_world_size() + lora_b_output_size_per_partition = ( + self.output_size if not lora_config.fully_sharded_loras else + divide(self.output_size, tp_size)) + + self.lora_b_stacked = torch.zeros( + ( + max_loras, + 1, + lora_b_output_size_per_partition, + lora_config.max_lora_rank, + ), + dtype=lora_config.lora_dtype, + device=self.device, + ) + + def reset_lora(self, index: int): + self.lora_a_stacked[index] = 0 + self.lora_b_stacked[index] = 0 + + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: + tensor_model_parallel_rank = get_tensor_model_parallel_rank() + shard_size = self.input_size + start_idx = tensor_model_parallel_rank * shard_size + end_idx = (tensor_model_parallel_rank + 1) * shard_size + lora_a = lora_a[start_idx:end_idx, :] + return lora_a + + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: + return lora_b + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + + if self.base_layer.tp_size > 1: + lora_a = self.slice_lora_a(lora_a) + lora_b = self.slice_lora_b(lora_b) + + self.lora_a_stacked[index, + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_( + lora_a.T, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + + def apply(self, x: torch.Tensor) -> torch.Tensor: + output = self.base_layer.quant_method.apply(self.base_layer, x) + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked, + self.lora_b_stacked, 1.0) + return output + + def forward(self, input_): + """Forward of RowParallelLinear + + Args: + input_: tensor whose last dimension is `input_size`. If + `input_is_parallel` is set, then the last dimension + is `input_size // tp_size`. + + Returns: + - output + - bias + """ + # Set up backprop all-reduce. + if self.base_layer.input_is_parallel: + input_parallel = input_ + else: + # TODO: simplify code below + tp_rank = get_tensor_model_parallel_rank() + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.base_layer.tp_size) + input_parallel = splitted_input[tp_rank].contiguous() + + # Matrix multiply. + output_parallel = self.apply(input_parallel) + if self.base_layer.reduce_results and self.base_layer.tp_size > 1: + output_ = tensor_model_parallel_all_reduce(output_parallel) + else: + output_ = output_parallel + + if not self.base_layer.skip_bias_add: + output = (output_ + self.base_layer.bias + if self.base_layer.bias is not None else output_) + output_bias = None + else: + output = output_ + output_bias = self.base_layer.bias + return output, output_bias + + @property + def weight(self): + return (self.base_layer.weight if hasattr(self.base_layer, "weight") + else self.base_layer.qweight) + + @classmethod + @_not_fully_sharded_can_replace + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + return type(source_layer) is RowParallelLinear + + +class LogitsProcessorWithLoRA(BaseLayerWithLoRA): + """ + LoRA wrapper for LogitsProcessor, with extra logic to handle the + application of the LoRA adapter and added LoRA vocabulary. + + Args: + base_layer: LogitsProcessor layer + hidden_size: hidden size of the model + dtype: data type of the model + device: device of the model + sharded_to_full_mapping: index mapping from sharded vocab to full vocab + received from base_layer.get_sharded_to_full_mapping(). If None, + no reindexing will be done. + """ + + def __init__(self, base_layer: LogitsProcessor, hidden_size: int, + dtype: torch.dtype, device: torch.device, + sharded_to_full_mapping: Optional[List[int]]) -> None: + super().__init__() + self.base_layer = base_layer + self.hidden_size = hidden_size + self.dtype = dtype + self.device = device + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + self.sharded_to_full_mapping = sharded_to_full_mapping + + @property + def logits_as_input(self): + return self.base_layer.logits_as_input + + @property + def vocab_size(self): + return self.base_layer.vocab_size + + @property + def scale(self): + return self.base_layer.scale + + @property + def soft_cap(self): + return self.base_layer.soft_cap + + @property + def use_gather(self): + return self.base_layer.use_gather + + @property + def org_vocab_size(self): + return self.base_layer.org_vocab_size + + @property + def include_gpu_probs_tensor(self): + return self.base_layer.include_gpu_probs_tensor + + @property + def should_modify_greedy_probs_inplace(self): + return self.base_layer.should_modify_greedy_probs_inplace + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + # TODO: Verify if this condition can be further relaxed + if 32000 < self.base_layer.vocab_size > 257024: + raise ValueError("When using LoRA, vocab size must be " + "32000 >= vocab_size <= 257024") + self.lora_a_stacked = torch.zeros( + ( + max_loras, + 1, + lora_config.max_lora_rank, + self.hidden_size, + ), + dtype=lora_config.lora_dtype, + device=self.device, + ) + self.lora_b_stacked = torch.zeros( + ( + max_loras, + 1, + # Pad for kernel compatibility + math.ceil(self.base_layer.vocab_size / + lora_config.lora_vocab_padding_size) * + lora_config.lora_vocab_padding_size, + lora_config.max_lora_rank, + ), + dtype=lora_config.lora_dtype, + device=self.device, + ) + self.embeddings_tensors = torch.full( + (max_loras, lora_config.lora_extra_vocab_size, self.hidden_size), + fill_value=float("-inf"), + dtype=self.dtype, + device=self.device, + ) + if self.sharded_to_full_mapping is not None: + self.sharded_to_full_mapping_gpu = torch.tensor( + self.sharded_to_full_mapping, + device=self.device, + dtype=torch.long) + else: + self.sharded_to_full_mapping_gpu = None + + def reset_lora(self, index: int): + self.lora_a_stacked[index] = 0 + self.lora_b_stacked[index] = 0 + self.embeddings_tensors[index] = float("-inf") + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + self.reset_lora(index) + self.lora_a_stacked[index, + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_( + lora_a.T, non_blocking=True) + self.lora_b_stacked[index, + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_( + lora_b.T, non_blocking=True) + if embeddings_tensor is not None: + self.embeddings_tensors[ + index, :embeddings_tensor.shape[0], :embeddings_tensor. + shape[1], ] = embeddings_tensor + + def _get_logits( + self, + hidden_states: torch.Tensor, + lm_head: VocabParallelEmbedding, + embedding_bias: Optional[torch.Tensor] = None, + ) -> Optional[torch.Tensor]: + # Get the logits for the next tokens. + logits = lm_head.linear_method.apply(lm_head, hidden_states) + if embedding_bias is not None: + logits += embedding_bias + logits = tensor_model_parallel_gather(logits) + if logits is None: + return None + + if self.sharded_to_full_mapping_gpu is not None: + # Reindex full logits tensor to ensure 1:1 mapping between + # index and token_id + # Example for: + # org_vocab_size = 4 + # added_vocab_size = 2 + # pad_to_size = 8 + # tp_size = 2 + + # indices: [0, 1, 2, 3, 4, 5, 6, 7] + # token_id: [0, 1, 4, -1, 2, 3, 5, -1] + + # Therefore, the mapping is expected to be: + # [0, 1, 4, 6, 2, 3, 5, 7] so that when we reindex, + # we get: + # indices: [0, 1, 2, 3, 4, 5, 6, 7] + # token_id: [0, 1, 2, 3, 4, 5, -1, -1] + logits = logits[:, self.sharded_to_full_mapping_gpu] + + lora_logits = torch.empty( + self.embeddings_tensors.shape[0] + 1, + self.embeddings_tensors.shape[1], + hidden_states.shape[0], + dtype=self.embeddings_tensors.dtype, + device=self.embeddings_tensors.device, + ) + torch.matmul(self.embeddings_tensors, + hidden_states.T, + out=lora_logits[:-1]) + lora_logits[-1] = float("-inf") + lora_logits = lora_logits.mT + indices_padded = self.punica_wrapper.sampler_indices_padded + lora_logits = (lora_logits.reshape( + lora_logits.shape[0] * lora_logits.shape[1], + lora_logits.shape[2], + ).index_select(0, indices_padded).nan_to_num_(nan=float("-inf"), + posinf=float("inf"), + neginf=float("-inf"))) + logits[:, + self.base_layer.org_vocab_size:self.base_layer.org_vocab_size + + lora_logits.shape[1], ] = lora_logits + + # LogitsProcessorWithLoRA always using bgmv + self.punica_wrapper.add_lora_logits(logits, hidden_states, + self.lora_a_stacked, + self.lora_b_stacked, 1.0) + + # Remove paddings in vocab (if any). + logits = logits[:, :self.base_layer.vocab_size] + return logits + + def forward(self, *args, **kwargs): + return type(self.base_layer).forward(self, *args, **kwargs) + + @classmethod + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + # Special handling for the LogitsProcessor. + return False + + +class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA): + """Implements RoPE-scaled embeddings with linear scaling for + multiple LoRA adapters with a specialized kernel. + + Replace LinearScalingRotaryEmbedding with MultiLinearScalingRotaryEmbedding + which can handle multi lora adapters in a specialied kernel. + """ + + def __init__(self, base_layer: RotaryEmbedding) -> None: + super().__init__() + self.base_layer = base_layer + + @property + def scaling_factors(self): + return self.base_layer.scaling_factors + + @property + def rotary_dim(self): + return self.base_layer.rotary_dim + + def create_lora_weights( + self, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, + ) -> None: + scaling_factors = (list(lora_config.long_lora_scaling_factors) + if lora_config.long_lora_scaling_factors else []) + base_scaling_factor = (self.base_layer.scaling_factor if isinstance( + self.base_layer, LinearScalingRotaryEmbedding) else 1.0) + scaling_factors = sorted( + list(set([base_scaling_factor] + scaling_factors))) + self.base_layer = LinearScalingRotaryEmbedding( + self.base_layer.head_size, + self.base_layer.rotary_dim, + self.base_layer.max_position_embeddings, + self.base_layer.base, + self.base_layer.is_neox_style, + scaling_factors, + self.base_layer.dtype, + ) + + def reset_lora(self, index: int): + ... + + def set_lora( + self, + index: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor], + ): + ... + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + return self.base_layer( + positions, + query, + key, + offsets=self.punica_wrapper.long_lora_indices, + ) + + @property + def scaling_factor_to_offset(self) -> Dict[float, int]: + return self.base_layer.scaling_factor_to_offset + + @classmethod + def can_replace_layer( + cls, + source_layer: nn.Module, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig], + ) -> bool: + """Returns True if the layer can be replaced by this LoRA layer.""" + return (type(source_layer) is LinearScalingRotaryEmbedding + or type(source_layer) is RotaryEmbedding) + + def extra_repr(self) -> str: + return self.base_layer.extra_repr() diff --git a/vllm/lora/lora.py b/vllm/lora/lora.py new file mode 100644 index 00000000..14081b5b --- /dev/null +++ b/vllm/lora/lora.py @@ -0,0 +1,169 @@ +from typing import List, Optional +from typing import Sequence as GenericSequence + +import torch +import torch.types + +from vllm.utils import is_pin_memory_available + + +class LoRALayerWeights: + """LoRA weights for a layer composed of two low rank matrixes.""" + + def __init__( + self, + module_name: str, + rank: int, + lora_alpha: int, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + embeddings_tensor: Optional[torch.Tensor] = None, + scaling: Optional[float] = None, + ) -> None: + self.module_name = module_name + self.rank = rank + self.lora_alpha = lora_alpha + self.lora_a = lora_a + self.lora_b = lora_b + self.embeddings_tensor = embeddings_tensor + + if scaling is None: + self.scaling = self.lora_alpha / self.rank + else: + self.scaling = scaling + + def optimize(self) -> "LoRALayerWeights": + """Optimize the LoRA by merging the scaling into lora_b.""" + if self.scaling == 1: + return self + self.lora_b *= self.scaling + self.scaling = 1 + return self + + @property + def input_dim(self) -> int: + return self.lora_a.shape[0] + + @property + def output_dim(self) -> int: + return self.lora_b.shape[1] + + @property + def is_packed(self) -> bool: + return False + + @property + def extra_vocab_size(self) -> int: + return self.embeddings_tensor.shape[ + 0] if self.embeddings_tensor is not None else 0 + + @classmethod + def create_dummy_lora_weights( + cls, + module_name: str, + input_dim: int, + output_dim: int, + rank: int, + dtype: torch.dtype, + device: torch.types.Device, + embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights": + pin_memory = str(device) == "cpu" and is_pin_memory_available() + lora_a = torch.zeros([input_dim, rank], + dtype=dtype, + device=device, + pin_memory=pin_memory) + lora_b = torch.zeros([rank, output_dim], + dtype=dtype, + device=device, + pin_memory=pin_memory) + embeddings_tensor = torch.rand( + 10, + embeddings_tensor_dim, + dtype=dtype, + device=device, + pin_memory=pin_memory) if embeddings_tensor_dim else None + return cls( + module_name, + rank=rank, + lora_alpha=1, + lora_a=lora_a, + lora_b=lora_b, + embeddings_tensor=embeddings_tensor, + ) + + +class PackedLoRALayerWeights(LoRALayerWeights): + """LoRA used for packed layers (eg. qkv_proj).""" + + def __init__( + self, + module_name: str, + rank: int, + lora_alphas: List[Optional[int]], + lora_a: List[Optional[torch.Tensor]], + lora_b: List[Optional[torch.Tensor]], + scaling: Optional[List[float]] = None, + ) -> None: + super().__init__( + module_name=module_name, + rank=rank, + lora_alpha=0, + lora_a=lora_a, + lora_b=lora_b, + scaling=scaling, # type: ignore + embeddings_tensor=None, + ) + self.lora_alphas = lora_alphas + if scaling is None: + self.scaling = [ # type: ignore + lora_alpha / self.rank # type: ignore # noqa + for lora_alpha in self.lora_alphas + ] + + @classmethod + def pack( + cls, loras: GenericSequence[Optional["LoRALayerWeights"]] + ) -> "PackedLoRALayerWeights": + """Pack a list of LoRAs into a single LoRA. + + If LoRA is None, it signifies that the submodule does not have a LoRA. + """ + first_lora = next(lora for lora in loras if lora is not None) + for lora in loras: + if lora is None: + continue + lora.optimize() + rank = first_lora.rank + module_name = first_lora.module_name + obj = cls( + module_name, + rank, + [lora.lora_alpha if lora is not None else None for lora in loras], + [lora.lora_a if lora is not None else None for lora in loras], + [lora.lora_b if lora is not None else None for lora in loras], + scaling=[ + 1 if lora is not None else None # type: ignore + for lora in loras + ]) + return obj + + def optimize(self) -> "PackedLoRALayerWeights": + """Optimize the LoRA by merging the scaling into lora_b.""" + for i in range(len(self.lora_b)): + if self.scaling[i] == 1 or self.lora_b[i] is None: # type: ignore + continue + self.lora_b[i] *= self.scaling[i] # type: ignore + self.scaling[i] = 1 # type: ignore + return self + + @property + def input_dim(self) -> int: + raise NotImplementedError() + + @property + def output_dim(self) -> int: + raise NotImplementedError() + + @property + def is_packed(self) -> bool: + return True diff --git a/vllm/lora/models.py b/vllm/lora/models.py new file mode 100644 index 00000000..8805d054 --- /dev/null +++ b/vllm/lora/models.py @@ -0,0 +1,747 @@ +import copy +import json +import math +import os +import regex as re +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Type + +import safetensors.torch +import torch +from torch import nn + +from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel, + AdapterModelManager) +from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter, + get_adapter, list_adapters, + remove_adapter, set_adapter_mapping) +from vllm.config import LoRAConfig +from vllm.logger import init_logger +from vllm.lora.layers import (BaseLayerWithLoRA, + LinearScalingRotaryEmbeddingWithLora, + LoRAMapping) +from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights +from vllm.lora.punica import PunicaWrapper +from vllm.lora.utils import (from_layer, from_layer_logits_processor, + is_regex_target_modules, + parse_fine_tuned_lora_name, replace_submodule) +from vllm.model_executor.models import SupportsLoRA, supports_multimodal +from vllm.model_executor.models.module_mapping import MultiModelKeys +from vllm.model_executor.models.utils import PPMissingLayer +from vllm.utils import is_pin_memory_available + +logger = init_logger(__name__) + +_GLOBAL_LORA_ID = 0 + + +@dataclass +class LongContextLoRAContext: + """Context for lora adapters that support long context.""" + # The scaling factors to support long context lora fine tuned models. + scaling_factors: List[float] + # dimension to apply rotary embedding. + rot_dim: int + # offsets to the sin_cos_cache for each lora_id loaded. + # This value is dynamically modified. + offsets_by_lora_id: Dict[int, int] = field(default_factory=dict) + + +def get_lora_id(): + global _GLOBAL_LORA_ID + _GLOBAL_LORA_ID += 1 + return _GLOBAL_LORA_ID + + +class LoRAModel(AdapterModel): + """A LoRA fine-tuned model.""" + + def __init__( + self, + lora_model_id: int, + rank: int, + loras: Dict[str, LoRALayerWeights], + scaling_factor: Optional[float] = None, + ) -> None: + """ + Args: + lora_model_id: The integer id for the lora model. + rank: lora rank. + loras: module name -> weights for lora-replaced layers. + scaling_factor: Scaling factor to support long context lora model. + None if the lora is not tuned for long context support. + """ + self.id = lora_model_id + # Scaling factor for long context lora model. None if it is not + # fine tuned for the long context. + self.scaling_factor = scaling_factor + assert (lora_model_id > + 0), f"a valid lora id should be greater than 0, got {self.id}" + self.rank = rank + self.loras: Dict[str, LoRALayerWeights] = loras + + def clone(self, lora_model_id: int) -> "LoRAModel": + """Return a copy of the object with different ids. + + Will share the underlying tensors.""" + return self.__class__( + lora_model_id, + rank=self.rank, + loras=self.loras.copy(), + ) + + @property + def extra_vocab_size(self) -> int: + return max(lora.extra_vocab_size + for lora in self.loras.values()) if self.loras else 0 + + def get_lora(self, module_name: str) -> Optional[LoRALayerWeights]: + """Get LoRA for a given module by name""" + return self.loras.get(module_name, None) + + # (yard1): TODO see if we can derive target_embedding_padding automatically + @classmethod + def from_lora_tensors( + cls, + lora_model_id: int, + rank: int, + lora_alpha: int, + tensors: Dict[str, torch.Tensor], + device: str = "cuda", + dtype: Optional[torch.dtype] = None, + embeddings: Optional[Dict[str, torch.Tensor]] = None, + target_embedding_padding: Optional[int] = None, + scaling_factor: Optional[float] = None, + embedding_modules: Optional[Dict[str, str]] = None, + embedding_padding_modules: Optional[List[str]] = None, + ) -> "LoRAModel": + """Create a LoRAModel from a dictionary of tensors.""" + pin_memory = str(device) == "cpu" and is_pin_memory_available() + loras: Dict[str, LoRALayerWeights] = {} + for tensor_name, tensor in tensors.items(): + module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name) + if module_name not in loras: + lora_embeddings_tensor = None + if embeddings: + assert embedding_modules is not None + embeddings_module = next( + (k for k in embedding_modules if k in module_name), + None) + if embeddings_module: + lora_embeddings_tensor = embeddings[ + embedding_modules[embeddings_module]].to( + device=device, dtype=dtype) + if pin_memory: + lora_embeddings_tensor = ( + lora_embeddings_tensor.pin_memory()) + loras[module_name] = LoRALayerWeights(module_name, rank, + lora_alpha, None, None, + lora_embeddings_tensor) + if is_lora_a: + loras[module_name].lora_a = tensor.to(device=device, + dtype=dtype).t() + if pin_memory: + loras[module_name].lora_a = loras[ + module_name].lora_a.pin_memory() + else: + loras[module_name].lora_b = tensor.to(device=device, + dtype=dtype).t() + assert embedding_padding_modules is not None + if any(name in module_name + for name in embedding_padding_modules + ) and target_embedding_padding is not None: + lora_b = loras[module_name].lora_b + assert target_embedding_padding >= lora_b.shape[1] + addition = target_embedding_padding - lora_b.shape[1] + loras[module_name].lora_b = torch.nn.functional.pad( + lora_b, (0, addition)) + if pin_memory: + loras[module_name].lora_b = loras[ + module_name].lora_b.pin_memory() + + for lora in loras.values(): + lora.optimize() + return cls(lora_model_id, rank, loras, scaling_factor=scaling_factor) + + @classmethod + def from_local_checkpoint( + cls, + lora_dir: str, + expected_lora_modules: List[str], + *, + max_position_embeddings: Optional[int] = None, + lora_model_id: Optional[int] = None, + device: str = "cuda", + dtype: Optional[torch.dtype] = None, + target_embedding_padding: Optional[int] = None, + embedding_modules: Optional[Dict[str, str]] = None, + embedding_padding_modules: Optional[List[str]] = None, + ) -> "LoRAModel": + """Create a LoRAModel from a local checkpoint. + + Args: + lora_dir: The local path that has lora data. + expected_lora_modules: Name of modules that are expected to be + replaced by lora. + max_position_embeddings: Max position embedding length. Used to + scaling the largest context length. If None, the lora model's + context length is not scaled. + lora_model_id: Lora model id. If not given, automatically set by + a global counter. + device: Device where the lora model is loaded. + dtype: dtype of the lora model weights. + + Returns: + Loaded LoRA Model. + """ + lora_config_path = os.path.join(lora_dir, "adapter_config.json") + lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors") + lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin") + new_embeddings_tensor_path = os.path.join( + lora_dir, "new_embeddings.safetensors") + new_embeddings_bin_file_path = os.path.join(lora_dir, + "new_embeddings.bin") + with open(lora_config_path) as f: + config = json.load(f) + if os.path.isfile(lora_tensor_path): + tensors: Dict[str, torch.Tensor] = {} + # Find unexpected modules. + # Use safetensor key as a source of truth to find expected modules. + # in peft if you have target_modules A, B, C and C does not exist + # in the model it won’t error and model will be trained with A, B + # loraified. C won’t exist in the safetensor but it will exist in + # the target_modules of the adapter_config.json. + unexpected_modules = [] + with safetensors.safe_open(lora_tensor_path, + framework="pt") as f: # type: ignore + for lora_module in f.keys(): # noqa + module_name, _ = parse_fine_tuned_lora_name(lora_module) + part_name = module_name.split(".")[-1] + if part_name not in expected_lora_modules: + unexpected_modules.append(module_name) + if unexpected_modules: + raise ValueError( + f"While loading {lora_dir}, expected" + f" target modules in {expected_lora_modules}" + f" but received {unexpected_modules}." + f" Please verify that the loaded LoRA module is correct" + ) + # Load tensors if there are only expected modules. + for module in f.keys(): # noqa + tensors[module] = f.get_tensor(module) + elif os.path.isfile(lora_bin_file_path): + # When a bin file is provided, we rely on config to find unexpected + # modules. + unexpected_modules = [] + target_modules = config["target_modules"] + if not isinstance(target_modules, list): + target_modules = [target_modules] + for module in target_modules: + # Compatible with more modules, + # such as:layers.11.self_attn.k_proj + part_name = module.split(".")[-1] + if part_name not in expected_lora_modules: + unexpected_modules.append(module) + # loaded lora's target modules must be a subset of + # expected_lora_modules. It is not reliable. See + # https://github.com/vllm-project/vllm/pull/5909. But there's no + # other better mechanism. + if unexpected_modules and not is_regex_target_modules( + config["target_modules"], expected_lora_modules): + raise ValueError( + f"While loading {lora_dir}, expected" + f" target modules in {expected_lora_modules}" + f" but received {unexpected_modules}." + f" Please verify that the loaded LoRA module is correct") + tensors = torch.load(lora_bin_file_path, map_location=device) + else: + raise ValueError(f"{lora_dir} doesn't contain tensors") + + embeddings = None + if os.path.isfile(new_embeddings_tensor_path): + embeddings = safetensors.torch.load_file( + new_embeddings_tensor_path) + elif os.path.isfile(new_embeddings_bin_file_path): + embeddings = torch.load(new_embeddings_bin_file_path, + map_location=device, weights_only=True) + + rank = config["r"] + lora_alpha = config["lora_alpha"] + context_length = config.get("context_length", None) + scaling_factor = None + if context_length: + if max_position_embeddings is None: + max_position_embeddings = context_length + scaling_factor = float( + math.ceil(context_length / max_position_embeddings)) + + return cls.from_lora_tensors( + lora_model_id=get_lora_id() + if lora_model_id is None else lora_model_id, + rank=rank, + lora_alpha=lora_alpha, + tensors=tensors, + device=device, + dtype=dtype, + embeddings=embeddings, + target_embedding_padding=target_embedding_padding, + scaling_factor=scaling_factor, + embedding_modules=embedding_modules, + embedding_padding_modules=embedding_padding_modules, + ) + + +class LoRAModelManager(AdapterModelManager): + """A manager that manages multiple LoRA-fine-tuned models.""" + + def __init__( + self, + model: SupportsLoRA, + max_num_seqs: int, + max_num_batched_tokens: int, + vocab_size: int, + lora_config: LoRAConfig, + ): + """Create a LoRAModelManager and adapter for a given model. + + Args: + model: the model to be adapted. + max_num_seqs: the maximum number of sequences model can run in a + single batch. + max_num_batched_tokens: the maximum number of tokens model can run + in a single batch. + vocab_size: the vocab size of the model. + lora_config: the LoRA configuration. + """ + self.lora_config = lora_config + self.max_num_seqs = max_num_seqs + assert self.capacity >= self.lora_slots + self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8 + self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots + self.vocab_size = vocab_size + self.long_lora_context: Optional[LongContextLoRAContext] = None + self.punica_wrapper = PunicaWrapper(max_num_batched_tokens, + max_batches=self.max_num_seqs, + device="cuda") + # Scaling factor -> offset to the sin_cos_cache to it. + # Used for long context lora. + self.scaling_factor_to_offset: Dict[float, int] = {} + super().__init__(model) + if hasattr(self.model, "supported_lora_modules"): + self.supported_lora_modules = copy.deepcopy( + self.model.supported_lora_modules) + if lora_config.long_lora_scaling_factors: + # We need to replace rotary emb layer to do batch computation + # for long lora. + self.supported_lora_modules.append("rotary_emb") + self.packed_modules_mapping = copy.deepcopy( + self.model.packed_modules_mapping) + # Used to indicate whether the model is a multimodal model + self.supports_mm: bool = ( + supports_multimodal(self.model) + # In case the model only supports LoRA for + # text modules (e.g. ChatGLM) + and hasattr(self.model, "get_mm_mapping")) + self.packed_modules: Dict[str, List[str]] = {} + self.modules: Dict[str, "BaseLayerWithLoRA"] = {} + # Dict instead of a Set for compatibility with LRUCache. + self._last_mapping: Optional[LoRAMapping] = None + self._create_lora_modules() + self.model.lora_manager = self + self.adapter_type = 'LoRa' + + @property + def capacity(self) -> int: + return self.lora_config.max_cpu_loras + + @property + def lora_slots(self) -> int: + return self.lora_config.max_loras + + @property + def adapter_slots(self) -> int: + return self.lora_slots + + def activate_adapter( + self, + lora_id: int, + ) -> bool: + """Move LoRA into a GPU buffer to be used in the forward pass.""" + if lora_id in self._active_adapters: + return False + first_free_slot = next( + ((i, lora_id) for i, lora_id in enumerate(self.lora_index_to_id) + if lora_id is None), None) + if first_free_slot is None: + raise ValueError("No free lora slots") + index, _ = first_free_slot + self._active_adapters[lora_id] = None + lora_model = self._registered_adapters[lora_id] + logger.debug("Activating LoRA. int id: %d, slot index: %d", + lora_model.id, index) + self.lora_index_to_id[index] = lora_model.id + for module_name, module in self.modules.items(): + module_lora = lora_model.get_lora(module_name) + if module_lora: + module_lora.optimize() + module.set_lora(index, module_lora.lora_a, module_lora.lora_b, + module_lora.embeddings_tensor) + else: + module.reset_lora(index) + return True + + def _deactivate_adapter(self, lora_id: int): + try: + index = self.lora_index_to_id.index(lora_id) + self.lora_index_to_id[index] = None + except ValueError: + pass + + def _set_long_lora_context(self, lora: LoRAModel): + if self.long_lora_context is None: + return + + if lora.scaling_factor is None: + return + + if (lora.scaling_factor not in self.scaling_factor_to_offset): + raise ValueError(f"Long LoRA scaling factor {lora.scaling_factor}" + " has not been initialized.") + + offsets = self.scaling_factor_to_offset.get(lora.scaling_factor) + if offsets: + self.long_lora_context.offsets_by_lora_id[lora.id] = offsets + + def _add_adapter(self, lora: LoRAModel): + self._create_merged_loras_inplace(lora) + self._registered_adapters[lora.id] = lora + self._set_long_lora_context(lora) + + def pin_adapter(self, lora_id: int) -> bool: + """Pin a LoRAModel in the manager cache.""" + raise NotImplementedError( + "Pinning is not supported in LoRAModelManager." + "Use LRUCacheLoRAModelManager for pinning") # type: ignore + + def _set_adapter_mapping(self, mapping: LoRAMapping) -> None: + # update lora states + self.punica_wrapper.update_metadata( + mapping, + self.lora_index_to_id, + self.lora_slots + 1, + self.vocab_size, + self.lora_config.lora_extra_vocab_size, + self.long_lora_context, + ) + + def remove_all_adapters(self): + """Remove all LoRAModels from the manager.""" + self._registered_adapters.clear() + self.lora_index_to_id = [None] * self.lora_slots + self._active_adapters.clear() + + def _create_lora_modules(self): + for module_name, module in self.model.named_modules( + remove_duplicate=False): + if isinstance(module, PPMissingLayer): + continue + if not self._match_target_modules(module_name): + continue + # A temporary approach for multimodal models to support LoRA + # TODO: Remove this restriction + if self._filter_unsupported_mm_module(module_name): + logger.warning( + "Regarding multimodal models, vLLM currently only supports " + "adding LoRA to language model, %s will be ignored.", + module_name, + ) + continue + parts = module_name.split(".")[-1] + packed_moduled_lst = self.packed_modules_mapping.get(parts, []) + new_module = replace_submodule( + self.model, module_name, + from_layer(module, self.lora_slots, self.lora_config, + packed_moduled_lst, self.model.config)) + + # LinearScalingRotaryEmbeddingWithLora is used to handle + # long context lora. Register relevant metadata. + if isinstance(new_module, LinearScalingRotaryEmbeddingWithLora): + self.long_lora_context = LongContextLoRAContext( + new_module.scaling_factors, new_module.rotary_dim) + self.scaling_factor_to_offset = \ + new_module.scaling_factor_to_offset + # (yard1): TODO make this more robust + if "lm_head" in module_name: + logits_processor_module = self.model.get_submodule( + "logits_processor") + new_module = replace_submodule( + self.model, "logits_processor", + from_layer_logits_processor(logits_processor_module, + module, self.lora_slots, + self.lora_config, + self.model.config)) + + # In some models, especially multimodal ones, layers with the same + # name may have different types, such as nn.Linear and + # ReplicatedLinear. The nn.Linear layers cannot be replaced with + # LoRA layers, leading to assertion error. The following check + # aims to prevent this error + if self.supports_mm and not isinstance(new_module, + BaseLayerWithLoRA): + continue + self.register_module(module_name, new_module) + self._register_packed_modules(module_name) + # All lora layers share the same punica_wrapper based on reference. + new_module.set_mapping(self.punica_wrapper) + + def register_module(self, module_name: str, module: "BaseLayerWithLoRA"): + assert isinstance(module, BaseLayerWithLoRA) + self.modules[module_name] = module + + def create_dummy_lora( + self, + lora_id: int, + rank: int, + scaling_factor: Optional[float], + embedding_modules: Optional[Dict[str, str]] = None) -> LoRAModel: + """Create zero-initialized LoRAModel for warmup.""" + model = LoRAModel(lora_id, rank, {}, scaling_factor) + for module_name, module in self.model.named_modules(): + if (not self._match_target_modules(module_name) + or not isinstance(module, BaseLayerWithLoRA) + or isinstance(module, LinearScalingRotaryEmbeddingWithLora) + or self._filter_unsupported_mm_module(module_name)): + continue + parts = module_name.split(".") + if module_name not in self.packed_modules: + assert embedding_modules is not None + if parts[-1] in embedding_modules: + input_dim = (module.base_layer.org_vocab_size + + self.lora_config.lora_extra_vocab_size if + hasattr(module.base_layer, "org_vocab_size") + else module.base_layer.weight.shape[1]) + output_dim = module.base_layer.embedding_dim if hasattr( + module.base_layer, + "embedding_dim") else module.base_layer.weight.shape[0] + embeddings_tensor_dim = (module.base_layer.embedding_dim if + hasattr(module.base_layer, + "embedding_dim") else + module.base_layer.weight.shape[1]) + lora = LoRALayerWeights.create_dummy_lora_weights( + module_name, + input_dim, + output_dim, + rank, + module.lora_a_stacked.dtype, + "cpu", + embeddings_tensor_dim=embeddings_tensor_dim) + else: + lora = LoRALayerWeights.create_dummy_lora_weights( + module_name, + module.lora_a_stacked.shape[-1], + module.lora_b_stacked.shape[-2], + rank, + module.lora_a_stacked.dtype, + "cpu", + ) + lora.optimize() + else: + parts = module_name.split(".") + replacements = self.packed_modules_mapping[parts[-1]] + subloras: List[Optional["LoRALayerWeights"]] = [] + for i, r in enumerate(replacements): + lora = LoRALayerWeights.create_dummy_lora_weights( + module_name + "." + r, + module.lora_a_stacked[i].shape[-1], + module.lora_b_stacked[i].shape[-2], + rank, + module.lora_a_stacked[i].dtype, + "cpu", + ) + lora.optimize() + subloras.append(lora) + lora = PackedLoRALayerWeights.pack(subloras) + model.loras[module_name] = lora + return model + + def _match_target_modules(self, module_name: str): + return any( + re.match( + r".*\.{target_module}$".format(target_module=target_module), + module_name) or target_module == module_name + for target_module in self.supported_lora_modules) + + def _filter_unsupported_mm_module(self, module_name: str) -> bool: + """ + Regarding multimodal models, vLLM currently only supports adding LoRA to + language model. LoRA for other modules, such as the vision tower, will + be filtered out. + """ + if self.supports_mm: + prefix = module_name.split(".")[0] + module_mapping: MultiModelKeys = self.model.get_mm_mapping() + return (prefix in module_mapping.connector + or prefix in module_mapping.tower_model) + return False + + def _register_packed_modules(self, module_full_name: str) -> None: + parts = module_full_name.split(".") + module_name = parts[-1] + replacements = self.packed_modules_mapping.get(module_name, []) + # When replacements is less than or equal to 1, it indicates that this + # module is not a packed module. + if len(replacements) <= 1: + return + prefix = ".".join(parts[:-1]) + self.packed_modules[module_full_name] = [ + prefix + "." + r if prefix else r for r in replacements + ] + + def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None: + for module_name, new_module_names in self.packed_modules.items(): + replacement_loras: List[Optional[LoRALayerWeights]] = [] + has_replacement = False + for r in new_module_names: + lora = lora_model.get_lora(r) + replacement_loras.append(lora) + if lora: + has_replacement = True + if not has_replacement: + continue + for i in range(len(replacement_loras)): + if replacement_loras[i]: + continue + replacement_loras[i] = None + lora_model.loras[module_name] = PackedLoRALayerWeights.pack( + replacement_loras) + + def deactivate_adapter(self, adapter_id: int) -> bool: + return deactivate_adapter(adapter_id, self._active_adapters, + self._deactivate_adapter) + + def add_adapter(self, adapter: LoRAModel) -> bool: + logger.debug( + "Adding lora. Model id: %d, " + "int id: %d, " + "scaling factor: %s", adapter.id, adapter.id, + adapter.scaling_factor) + return add_adapter(adapter, self._registered_adapters, self.capacity, + self._add_adapter) + + def set_adapter_mapping(self, mapping: LoRAMapping) -> None: + self._last_mapping = set_adapter_mapping(mapping, self._last_mapping, + self._set_adapter_mapping) + + def remove_adapter(self, adapter_id: int) -> bool: + return remove_adapter(adapter_id, self._registered_adapters, + self.deactivate_adapter) + + def list_adapters(self) -> Dict[int, Any]: + return list_adapters(self._registered_adapters) + + def get_adapter(self, adapter_id: int) -> Optional[Any]: + return get_adapter(adapter_id, self._registered_adapters) + + +class LoRALRUCache(AdapterLRUCache[LoRAModel]): + + def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], + bool]): + super().__init__(capacity, deactivate_lora_fn) + + +class LRUCacheLoRAModelManager(LoRAModelManager): + """A model manager that manages multiple LoRAs with LRU cache.""" + + def __init__( + self, + model: nn.Module, + max_num_seqs: int, + max_num_batched_tokens: int, + vocab_size: int, + lora_config: LoRAConfig, + ): + super().__init__(model, max_num_seqs, max_num_batched_tokens, + vocab_size, lora_config) + self._registered_adapters: LoRALRUCache = LoRALRUCache( + self.capacity, self.deactivate_adapter) + self._active_adapters: LoRALRUCache = LoRALRUCache( + self.lora_slots, self._deactivate_adapter) + + def list_adapters(self) -> Dict[int, LoRAModel]: + """List all registered LoRAModels.""" + return dict(self._registered_adapters.cache) + + def add_adapter(self, lora: LoRAModel) -> bool: + """Add a LoRAModel to the manager.""" + logger.debug( + "Adding lora. Model id: %d, " + "int id: %d, " + "scaling factor: %s", lora.id, lora.id, lora.scaling_factor) + if lora.id not in self._registered_adapters: + self._add_adapter(lora) + was_added = True + else: + # We always touch to update the LRU cache order + self._registered_adapters.touch(lora.id) + was_added = False + return was_added + + def activate_adapter( + self, + lora_id: int, + ) -> bool: + if lora_id not in self._active_adapters and len( + self._active_adapters) >= self.lora_slots: + self._active_adapters.remove_oldest() + result = super().activate_adapter(lora_id) + # We always touch to update the LRU cache order + self._active_adapters.touch(lora_id) + return result + + def remove_oldest_adapter(self) -> bool: + if len(self._registered_adapters) > 0: + self._registered_adapters.remove_oldest() + return True + return False + + def pin_adapter(self, lora_id: int) -> bool: + """Pin a LoRAModel in the manager cache.""" + self._pin_lora_in_cpu_cache(lora_id) + self._pin_lora_in_gpu_cache(lora_id) + return True + + def _pin_lora_in_cpu_cache(self, lora_id: int): + try: + self._registered_adapters.pin(lora_id) + except ValueError as err: + raise ValueError("Pinning failed. " + f"LoRA {lora_id} is not registered.") from err + + def _pin_lora_in_gpu_cache(self, lora_id: int): + if lora_id not in self._active_adapters: + # move lora to gpu if not already active + self.activate_adapter(lora_id) + + self._active_adapters.pin(lora_id) + + +def create_lora_manager( + model: nn.Module, + max_num_seqs: int, + max_num_batched_tokens: int, + vocab_size: int, + lora_config: LoRAConfig, + lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager, + **kwargs) -> LoRAModelManager: + """Create a LoRA adapter for a given model.""" + if not hasattr(model, "supported_lora_modules"): + raise ValueError(f"Model {type(model)} is not supported for LoRA.") + lora_manager = lora_manager_cls( + model=model, + max_num_seqs=max_num_seqs, + max_num_batched_tokens=max_num_batched_tokens, + vocab_size=vocab_size, + lora_config=lora_config, + **kwargs) + return lora_manager diff --git a/vllm/lora/ops/__init__.py b/vllm/lora/ops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/lora/ops/__pycache__/__init__.cpython-310.pyc b/vllm/lora/ops/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85eb3c184325a8abe767c6100a413cad56d974ca GIT binary patch literal 154 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hmeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_eolT-qJDlsv3`7fW?p7Ve7s&k X{Er0JMhL|0ilYeP(-nhEF@cOVc8Y}VyQ@>1xkD^ z&@CM)3~v>w%G9kA)g;v=ElOIFv@B@Lpd-|HPM#sF%+SHep;D^Q8Cqp%%b=r@)+8O1 zbevV_1jS%ANiRHC!K%7nfu7N=8l7TQT5BUZ{jy+qiJ^9c2@3me_v89XsmUZWOk> zcAV)SuKn`v1ABeQJjnEoM{6H%f<#+=yt-~b$jU4C?%5x&Kl;_?MpnAJvaxAzK6$k2 zP6+3N=Nhy+ym#S^{|4hF%9h;Pm#9A>`2xkjPbSI%PStdBS~*1h3+V+)wlSZj>IPUG z6evoohqymA;48pSr#1ebv{KT^RGU<$m2@iAMGGEg!g279(>MA+hir=^(_R+(Ghk2B zLOKfmX47&0IGs6AQ?#XR>B2{f(qei+dY(vUCzZ4$`2JjqVZ@Qwa*$n2iV)*$T6tEu zr~Jn6UrH~f(;&YLBb3t1sfhSWdYP(J{Q^_9g=dv?b{irV5+Rcki0qEie+%@XuW!m3 z%%xY(S;-iFU{{v0yhU|6tGClJ{*|=72%ZetXVT(J+mcI@r5cVp?W%Xll|ffKq+e)Bv(E#mBYF~^WX zMT6SkQ}T5iOzdPYQ*)ha;OvUCka+E0)Qhu1+zT=_P9o0g*Q}Bkc6*84O}JH* z2S;#vl#4R7o}mYpAv?T~dM=Bt(r#eK3HK-icKej|Zw%RrR6p&>KVf+WHtuV4cPQlq zX3JGUG(p5c<3&MAKrF56Q9BN;lH22U(@ETpRUXE~cCBL6YQ=Eyw@?WAL=O*vWB6-|b6r(7&qXpMmXoV8u6EER3BJ(lL??Z9i(D&_8BovabZ zb$db2cVI=oRQNOm{LP(45_B6~uX{Za8IoLoKj?LWrQm79_nM9FUebxe#*Xg?4Ka;I z)Qua>cCcf!-L4bT#qM4fg$P_&%MNH5X8b|m9WBC4#XjAZA!*iIt;7#939YaMOOu zymlvvvkFXBq)NiVIO5sFa05hA425MlKK8&~<1Fd%&=tEnd@8`y;4Q!#r*J4zWlKrW z7KXO~FKHEal(_N><+ljH>$InZK{_L)%7IGJA@o9v579P`uMOndp%VW@^7ugTHwIpG zS^xlEl@?%0h0X0jA3z-?0NNo*OKzVNU8y_ z>Zz9MQwl9^<9;zE{Sqzp%Y!)y*#G>8v*4f8GH4Va`r;w_y^=spf1#x6vqFF5cS;LU zgI1mw`$kep4FFaJfLjz34WL~)s0tg|<|~kYBajcjtI&@JdZGn+tCH#f{-v}`NB>09 zkw2q9Ah>_%0Dm>-AFu@ypjLQ0UY4A|!+ZzLdB|wDvj~+LHoU%^i_LuhPj$~{$s0DJ z`(4M(ljzjwTq?a`llT7Lfd(Kn$hS8v=Os8uBwm0k#%71*7YZPyYM9*rwcH{=;G-h z(iMx%d8i?SMC>GppDf za=GZ%&5VfEw<=zIwi*&hfWXq@&w31Qtxy*`PM2jG)T$&3tx+5crp%@C$;-%62~ROv~@58EHc;%@^rMLRYSoadtrj zLY;(!ituSy{#ZA*1-;q=x@R#_-o<9KRwH#y_?DSR>yOsGtPnW6=Db5d0lh@wZqr SzQJonk*CsYiShi|F>;h7#N{O!p zx~(IH;SGZpSYe|`HA!_z4M~fVnu4}UbcB|kk*CNu8QL4!SBhmiM=K0%l<26WW0Ibd zw93kK9Nr05p_3H1#^@BCepUeM(cLoi7~42SXV@qmYa=@QT(zqoDHN|ML}BAQ9i)gK z8?~#CrSyqXJH7k4`GFfVYDM9ad9%a9dF%edy!DvPTQ;;G^?--g?)0L0>lSb~*)Hfr zpYK`sxEFUqH|X)CvomikFIbnaUA=a_wqThLd!g5Km#hapKk=@tv(OC_>wfgJWowP` zEiY_CpH7l=nG?NVJtlE33}E%v`>8%elMy@Wb`Btax*I zZQWUa{BWfS$9?o%1Wt$dCcN>NFkYf;$RmD^`V*2@DF%KrQTA|JNGGS2ebk>yrzqLP ze3lm0z+z9OC>`C${pkT;0e&Vu#ov)uN;;Wp6UwxbPN%wP!NY7=1@AaLp@TkTQv{p# zTIIxz44Pr|K=`m#zNU^lUlV2huK0PON zeM`>%^cz-k{5R}OIgj&Hmn(WJt@1xh+cV%v&MCF1@f`QxhP4*c+4QXmxK8^xeH&(V zAvBIl`xn60Okdevkk%?vKG_myQ7i7YObebnJnD7ttUP!$Z@H9ui5G>g??b41VZtNY zYeFDegLtu;ZfJQymq%O7@?G9$R>z~1g--14vP-da*k!yG@xTq6On&n`Hf`c=dojll zJ_j%1S*_uW1a7>U6{4iWcpDV*sUbcBOl>5?iEo=QSev_n|_PmCsX5IkQL%2;;eSjE_z|NmpI*o+lIU(g43g1l%drO z-Lp$l;f2&|ve+(e2Tq)Dk1}94k68cekS)vZN1FT-wqsz^d}ZDoO0!bzgL5ZpwPKcV z0&C?3K{P?c72}2=MIg4;^{5kvcCp#xPQy)_9ornvkZs#W?!?(V;zCTrcf2(ml(tMw?vyOZ4zcg5-ND9y1To->vP3WP*?7{vw zN&=bxGbPoZs{Qh>l@_8UT7G8qD@i%60Q4#V>p0b66g5@$D#Av#`JUw82;{@>3h42m zpJ;`gv?^&aHPew4YKcFPwERc(TLg8A4iGu!P6J#?5L@Q$cu8^sGxJlmF2f~gb`~IW zLsZn3a

M$x+|qp>)DVs4q3FJkpMgj>X#vn_T;U2O0p^ASh2*UX~Cd(Rv&4qzwjjnaN4B zcxWR(tz$i2-BjyxBZ~ZaE9&Nn5GHktwOpvsVzCLgT|Z{|8e}HpqBK4@gmiWwa2zTy zm3qN}xYDaK@7)Ga$#JeJ(%|qByvT&(@Wv$|DHJlZ%4bRoD$yEV&2xc3*BY|PgO&B= zFOaPd79vrX&w?^D-t*m{LERg_7mh>%5jaQ=cQ2mA(uMc@s0s7G0ZeXmAf`|wdG)%t;e7A7!<>h zf;(=PWg4W#Bns`(kZnUAj-D_MY0$aG#fC`?wzWKU!uY3!?2I611vw{(09k%s5K9nI zpYlmTWCA_DgiOl|;Ec2(f#;icN%WOFS3S8wk3ku?fXq!3$Snk!hcl##MW!)v74k|2 zmvEJsGV>5LCi`fIgqR>NVU1we`{B=kEcNOK(tqL6Tt{BuaqTNI$roX$-Dz=WmX=Or zJ~;PM&`aC|G7MzZxQ5ni6PfNfG-^5y|Grqr0uXyN-_r#jilUy|W|=YMcp3Crp$FMJ zmOE^#8(spXL=?se1SQXRyyAr|2Db=Gk|2UYN6YanQ{hf#L|g|sQ(K2@&RxDEY7wZY z0{DwzpE@z`tg@WIj2gX!t#A$?B7*oeG6-83spqpB;vU2w10jl`8zQM=qCxK`5htnTU@~Z#Txzw>-cMIl>UPs8R8P4 L(qTz8q3{0yy4d?z literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/__pycache__/bgmv_shrink.cpython-310.pyc b/vllm/lora/ops/__pycache__/bgmv_shrink.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3938bb801de3b546c0b7d9a81989c986e7f54f76 GIT binary patch literal 3329 zcmZuz&2!vH6<7C2ni*Lhe~jagWH(V#1?!=lnK*$BxVA!^4;FA@QgMo91(Zrw_sq!F z$85Lc*sh)n$rj5Y9Jq48uHv!>?wsKN&=smEp9mcH!^`W|*l`kCrPr@tzxUMr`t|QA z9Ur$eXv=^3TmPSvn)VMlINKU9xD7A=D-fc&8bJg{$VIBhE>?Xd)?EW>q(bz_jH_-H zX%uhOh(V35Ix!WkO4>C^jntp(&yZ_TbTD?P)h%+4j8U{@6I;=7MH`Av&~Y+JaMvbN zWcs-QR*ii*Ke07QnzTX2dx*@usJK%fX#{U*x<;q>eu*^s(1ttpv64R3TC@9qwC?zv z5+_Yoth;@hEIAL_OU}o1$#J3oC~3dFT3#$h>7r+9GS#e=b1Bt$^kt(UZ zs0`l`GsGaqrilR3LVLhgH;t1J;*G z{Y3@tf{Nvq^`@+4?RPB)+J`L72Or*l`12*lCnU_mH1VSd@+(X-mXbVx>~Yi~UtMzo zKXJl%z|tM+L_X_LrymkR6OV`c^gXT|4k+uUEcTOtsxd1Q%GLedkTVP^ar6zeb;vP; zWP1bJ1CeSiTY? zxoTR93bbCJHP==XVM4-y!ad-DA3^(yM4+=*G|}RWp_GL4k2D6o8Xms za3W&7+Z_X+CF=!lH47t3Jm0mHohQ%YM7ZhJN8bq&*9eqfGf*m@YitJA z`Ni!{77sduaBww~sgYg18Rz|YIeyxS!p+WLFYBjCXD5o{j$C~w9q`U(FW&KZpM}YG zd$3nDy)y!Do3eyP6X2TP17gqHsDWxoM>d{8Hayd42F5l>On5Ldzg{$SJPUIC4aPA3 zYPKrprB)5-3}&<7=wI+cd-=n{R0q$Ch6<yZ3H+tLKu(zY?L{Z#u6lH&|*VR$QDyrc1^l7hx- z(u<|;flkmN5(s*Hh``5pMsnj&z1l@@3dsdsBMSB2831K6UZ3ac=Q0do(^ zxdZ9rvEs4ff1&sae9JsUzmu!3NtpT+fbUm$XbjD)eqad$z`B}^k@{zvFrQV1wcl!8 zL@YA)yf&<7wx|P4f-S(6Axu#L@YN2+rAO75yD7&As(!SV(KT{Nm55{#^L=mkBGBbVzTrln?D#tAtmk8k0@7^ zJBxt+5IZ$=B)FeyjF8V+RB#JA&PIuaPS+10^rbI_^GyB&97!|-tgnzTK9YAr)q|sB zSptf&Gxt4^b>gH+UY3d`yxayN;E!)X`%LS?|5XdGD`Zr#(?YKP@cxt4&yialtzOC5 zIj|~fA4Gn8IU)BYezVH5loFs1sD{I~+*R)f`~Cqt#xsLCwA z@76X|0G8h`bQvFaJmhDqjSfc?yH);4PT?00>Q3Ju(82`R&(g%5Na!vUob)MWUfT7p zGZh#KPOe#&hL*+VI*wiru>(wJs^wp#P7f;~-;7RkPKFyZE z>Hd_=w3U@JX9|a7$9d*wfSXVgIF$4`&}=n}s^^h3@I3Y{7_$}-ccKhVJA$g7`#oCJ zPB=CO^M#Q^ec~!qZe=UXpdh9R&*<)eDO`6OVbY~g%>wGhDO6;$#DKz3C#BZtR-qO# zpY6$N3jYEz{N1w4(tBeygYR$V8NJUKTr*kH&X7?yf!{o$w`6+q9|O^~nkq3i)+>-# x5DeYXGr literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/__pycache__/sgmv_expand.cpython-310.pyc b/vllm/lora/ops/__pycache__/sgmv_expand.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e54c33948407a5edca1af3847ab7e3dcd212a6e GIT binary patch literal 4663 zcmai1&2!tv6$che@S7ARS(4?Cz~5plQI?(99ZjdTV#}Gg4`tui2#@d zC`+E@rB05X=`Dv|tYc@|YmdG4pTM;RV!JmK-#Z-t&P)9XY`9vm?PtC<UDC9q8A-E(HVY(2@}J}9>T}i1QT1hhU&$88Nm8WhR)Lfx zElWBfX+_dWNvCL$Ov8N)?ipGlvjjC~$Q(KTc?!Ihp9vomTNN@-%cR^=$%z-5Ir)h~ z&?5yabZRe974f5GbMmp2K2@rxp8YcSfgMm{_}*&nR-1Z@#=Y92@t7_eCOkjs0S`~J z-SZcX+rZtT&p;=}Tr}=7Cun=N+hbvSZ_!w*84FjhT)kGU8My~N&uQ4J#{FI=be1-$ zXM3S>&wsdPJfdvJ@mesZ9fsXtb$QulyUtF{XRT$s9xPwIwsNJm^1b)pyOBG9qfj=g zYLw|Xb?Sv|FG_!K@1t9Ht@{IJW5C>v^!1;tZ&-Ju+}iEi){izGe7yN6%HCRgv}tWV zez4v+CY&W#4W4PZZ^0e>5h+TjY{`Ruq4tkSULy$jnNWF&xW*^;QGb>nTwhXVeQY=+sc#8{slhCeh*TOpM}_6l-xPsE_n-J2U0%1Cd@DJ%Cs`4 zY>Q7X{X!EqFz#GDZn&~H$6eyn>}!7UWsa*``K^MOjh~14iu@wKw2y8m{qrzeg~P6a zKMNi!9QFtNxzB#f&yUzG@R75ENXqkuJy*e-IKk1=dY2;(SuhgmqKw(QV=*hqm|d2; zKgy$VyOVr=7Iw|YEwyc^RsBUi^$V3bG7^{hBFR9sm-qr-g4K&SmiT3O6LJko7UX0# z`|M`UfNQf&jiBZ3TnY@j+qFGngy6^bjBUz1+A*BKs3%py_>lR%?uYC5@7JU$Q{kJb zswTE~o!~&cY#h9T3%=463HQ3RiX!aV!FH7L!!~7Al@)|VIYME_%ypUHVzz5JB*G2i z>_oWZ+r-puW_vAamRx(+YWQC0w0eFoF!QdcAKebM{HB?DMwuT(=_Z(j*RDn>X#;+J zmWL^zpmwMWWx@*gB8~WAlnOvat2Hz0c->xTbwg%mIzF={m-b^(q;5p&-Kbo*0{WR1 zgfgL2dd#ZZVWSOfXj`7;^;|OrDl2(-k$T_E4Hipx&7v?F z?#j&g&1OJD7(U>Yka31C*-EIOXj|qX*|;Log$C+f0W}hZ4&kF!ce` zFtf=74fluzcMa)BZ&)HP%xr8A(f^ggPQmuRxw#y=-Q}*+U4r1ZXt;F4?X}$%_vtd! z{pIdn*!I2UoleJH7JFOvyTP)^UMtSsT6Zs+vPJ|}oW*5u9dICplLfV)X5d$XCkC34 zbXuK;_c{2TMw6=kDhCw3npE+tX*3NZa-h}Gyoyx)P}kxo)=H`l5+N(#68M?}FEgsJ zDj}(p9(3R{pkeR80`5e*Ou!&2$-}8zPpQ*the1??D2U7qAo`vShT`+lnecBDpMnfayB7WGPBNT*kfKpq9B+?TvKq*z%AS0ui;zf z%9awUTL^9qZrs#%l%V)y<>xAt&!MJ?KsqO-(1?h-uky?`3eF5A2qY?Kw0#xiOH!^S zBXy8mmy*JAR+?aFpFfjc!8<|V<#D3R5JXbYtW_@!t4tFy zLo3!vA8s-(OVT9s1+=}NqJFNQC)s`h?jp(cOFYj@GtgSLQNPTy{RxuqSCUwWdB1#p z_v}kj=vTtr%RIFB!akB*ju-b4Kmfgo`jcUSPeOxLpot=Aqj>-kn5Q^29vLt3jH494 zt0V74h#pGDiKiG>;6+{%jg?oPAA@3^fnr{=KZ1fGi}D(41*?)1)jd9OV*$R`M!N>s z3sS4P8jFoM(}rWm^YINEp}u4rak>tT#*+38n^^n$M6XrXH>?(9LR*0Y@Dc!|=uk=3 zadzl%X0bg;_8#>b)PULy86Oya(|}Ghln1a=DUco)jh=wFp$}3*cgH+A9*76-YQ}o4 z1+dl`_m<2eTH>egEgILKz=#mu#vFZmWzhiT%9ST^lT4N=x<~TCzHPq`X~So*%#nSJ z%@FVPySI8Y_qSAxw=F;fT;lUL!vrg1H0b)!|GbunB()oD00_{DYQ~xu2UaX?X~l4< z?FEMC8)6AY(Dr*BV${bE(C&6S4kbs+m9A)DEWm+4qRLl8WWq26<3(;TP^=j=!y|la zMhyLl1By2=6dl#gQTI}6K%=cwHlA2DHtS8h$r)e9!Jcux`_)4HyaB_{JT(6TqJ{7$19WIxr?a ziLq6|X%0-}Rh~pjzjB;=YSa0LN5bOkyJDUga_X&!T zgkVuh)*Lg2Xe-yvOkIMtmFuhmqarL0)hs%}2*!&rJ&4C?2I611rc47ofG7|AQuF=D99y276cJpj?D`4-9QVs0M|)S5|h(l z5*0?lMFS*(2W0*{28;kG;cybqLI0lwv`|n5h5%H63TP6?dVx5wIw7fki17?UDn8V) zz!0N9flrQpqZ<4bo@F;+%4ZevRjsa$U>!i>1-BR4AwUDbPC-?@Q=N{|mPP!AWw8~o zV($Yn%W*tw9l+T=yG7wYb;7Zd&{qV$s>Oq>9ze8!iGww@EhhxL>3cy4aErx9QF6Q{ zWugdMt`9g^kDE-S!3P{+k)v|GW*YDtv)P`&u7Jf|%kP@`k<~|~wGgJP_d>eP7?c?S z2Sv8vPj`D}ALu zM0xx*jCh3+I>Z|KAI9jvIEDU$b@UBNqkp3e`WMQgf1(`v8s!WBK#h#>rxp$iS`O^~ E4?BzeZ2$lO literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/__pycache__/sgmv_expand_slice.cpython-310.pyc b/vllm/lora/ops/__pycache__/sgmv_expand_slice.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..51f1d6a3469ce1be2d6cb99bafdaf8538e278a2c GIT binary patch literal 5145 zcmai2&2!tv6$cjJr%3A4lKj2?l$gkrWv6!QcG@`pn6^&TOsq5w(hLS9mLenqU>2Y( zd6t(tJ!Yo&UaVsez4qA2rT+x3Jt=L|cABPb+S>i?f}}_*O^Ct6!`t`X?!JAy?+1qE za#6!??bp9;{q>}#{a(I|{!ZcLRou~EKwQnzT-_yY-O|;QSVTPy-?TEiY%|(fKWF81 zjpd$bjBMpylV!FFZbs3pqB%wLiWVeomfWH{@uB%df1+DOrazt7*9s-~j9X^`d> zI;rTCqSK1bC_2l^?i}u8xaV;%;9g`E_qa=%C)^Y6$qzH|I{8@on%bInPq9gNa#MFt ze`HuQ?`ST0sF@m@-Fu{K@?w+L%)3hZNUNTG{LA8NPQ+Xq2DgfLS}a(huh&-SyKIG8 zc)rsGkEhk@hAZ?gxI64I+N929`ha^;D{%ZSk6U{y^mdIdU%PtkdbLK2Z*~K(;oPEc zblb6aWrGDy5YyMgw{FvijPH2CCUjbH+=*_jt~z|z+o^?obJeLwtJkitU9GKs^Bb?+ zEFNGI!H`h!ke`T>GmiNg` zt#=8%%?QjI`~`TN5ttwNiy!<(TpF>fijlQKB-MIj&M(26T)~mm5|fa>EQok%EZSui zx0lD_R#98?F#f($sXDMoX4Q7YUX2jy98r^_1&YUO=XfxzA2xGi`|5Zgzu43~i_F#e47`#mvF^*Q6~+9!%1<;k4V-i>WV9FZ7Z+i#?x3HRYEh zd|FkvOlQ}N4&=-F!E^j@R%P6HufwV&F?}c6PBJ0-=2e}Sq{U=H;amz8!K$ak(%RCHi`zKS(C)cEmogquAlG z-3}X8UcR?AU=FbbTv>CcFO#68#OUI2bzHwaU_D zI7cg=CM?tkTrlLe69cPc8N41?nKWFPx{BI_(`a~~f>UgrvyVw1&JZu;Gpzve$$MJ-(I z?8U7xSlwy2{Z%=))vyz-j_h1p1+v!JOOD$|#P&Ah0c%g9n`jrLbiSmQ^gON#o+fBs z(K&q%?~Az3kQqJuya-C3&*RMaBX?MF;*|0pRV8ZuvFBwxt+B9xO#KO6hPWOlHy zGA9SZoJt4=k8&UNEjHs;QjEiGrcxy0$ECYsns1T!m?cG=YIc7pSgtuJ=;!V!JMYFXvKEwg$BSX>t3{$b_quwl<~de|FX7Bm>e0gnw5$toyIjoLXKgM%ww#Fbgo}L)uF+coG)M3zl^u~(m5I>42-j= z6QYKuK!X9GP!-%7y&a@+l_R^dMt$Z45nvOQL!ePhVV?T<`Z=9W+hguwq?9Wb(PgX$ z@>IQ2nFzxPjUR~vMLA})3|H{~CdAN>T$esFQ@sou4$s(8=SoUZoa>B_XH|{9-Os8W z#=RyGS(-INhq9+HHScduUzUH&SuS`Ou)(MRO&!;Xox|Xaf{Ve}#`fQm+yD~34*Nc= z4+Db%ihR*^u)P9gnnF%2%3aoUfGaWJkluIN5il+E)A2Vwz|wTZhhvcQ zN7f8X)HGFP9D$;|fXc+>KOG0p07DNXY62EDT+cs{FlzmP%C;$x`x^)N;Kk{u;9`*NNiIktYdOCt$t6iHOR^-%vLv!J^Qt5-Ng_Yoxvacj0|?Dcpiu>$ zQgRk1$;?ss<3S#AkP}rCC!$W0*v0NK|C%Q;KGvndUqp zy7???N}Mx#Nep#(jkZAl6LXP&3*A0mkURR;ts}S$-xDRj8#^(e3@~3*)$dp5lALY3 zVZ*lhb=dNoAl76W=~^3K3teZE;s237$1AdZTH?Z5s$_QoEh8?M+%mSj7&tkUpFF!A zo-R_w3z`fcM)-E~L!kL=TICW0M@VAI>{X+-a=@J&by8w(d{Ovz*s-RM%ps}Vj?po` z7TG<{aokBTJ3`c=Mkh1w1G@hG;JTndQU(+Q2F*D@fW|n+Ga^znmPyR^?ZNBTqX_%|0ih0`5C8xG literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/__pycache__/sgmv_shrink.cpython-310.pyc b/vllm/lora/ops/__pycache__/sgmv_shrink.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..00b0ff3e88b5e6d1b149efc72a6cb2647a0f61b9 GIT binary patch literal 4635 zcmai1OOxBi5e5d}3w$h>OYN@Su=QHnyQEg~BiVALEXyiavXbnw9J>r#g~q=N=5E;Y9dE>=-od(gqiL>PyL#<Fx+aj9KP@FH3QylIXhwN+V#z=&CTz<{@RE0OY;2u&;=lJTlIAy!W*r45r-`gsSy2wv@o8~V zRO^a3-Pd`GX6}J^(B@L@8F7YY`-HzW)nro5S?H%zRa6g2;$cM@pG##hsyHPU>k7!u zi)B%(qgU)*J0u`8p_PF*E6$12%Zl^?e#S3N<>!S4a`b~Mcv&dotSF#A@e<9@%vXeF zI^?91%1o4@RLTrF?ggQWqBtXMUY^sJq)ggO{gkE6MTlTdRK;S40GFq^M`U~{Do_>X zMT9PDqFz@H$wTt4a{vADWwFG64;oEefH+=}+(qEl{59x%Iq6+jR+VuByg@H9KhH*A zp?O+(q@gY7FDClwoR0Nt;u3f_#1*l2NZwS&m%x{ti-5I4`vTY|ZCD4iFMjbmF|$7q zHMz$S`B%UX?!)X3y()-!WiF}}8KGC_qP8F-wEj%bv*PrM(pSWMU#$BXzrwUfcYK|M#J}R-MRCsJVhJZ&8Z8Nnn5Q$((){~9pBtxW;^ghmog65 zh!Y;WXqq=Dbt5T{?2%6bE$Fh8w_4M=;Tg>kWCZ$IKz)oq&6e-)6EuSK+`m zR7-dE-S7w-M{szRyGKvqftw*iqk|#qf=E6M66>B5_Tx+t^%!rcyae(_IVRD-GGK@< zcRbsrvEHWcUaSuShgw;OJARj070=nXVJ(r{9R;J%DtdT%4F}8)I#%WZ<3VWUI$#oA zzZPetO{~!%iZfvpaAp}!6nJjib|`H$t%Bu-^NJO!EH&(afg&jL& zU)W*f@W>tnZ7Yw>0rNwvByC)ux@{K5xv=dFAZAu!--GTPq77_+$_}nh*s`>nYLccu z(H7dr=Ix1;YY-!@Bgr7iBEict)`l*G2Z$Z^twMXm?Hwm-_kfKY+qeCZXJt^Ee*9Ry zV;Pf)vwf?KCesyK`JmGYSp?lDoImCIu?AaXX?(}GGD)y9J2GZPr`;aGRzP%ltoE&3 z?kk;jIm8`1gNKMtBCbX`d0<8fSRgzU}Dp9jfvRd-3YE|gDs@BOe zX!NHQ=s}($mnQ}Re}tdIEz8QZ3*(C12FL!Ax!qnA#%0Jn%*P0lE8u(?`&QCyONH^$ zWa}{JCc>QDa>zk>JW`8p)QgQM=rfrFAxF#T+;2;G1p<6(@Zwmq?*0PKuTXXs zq3$Yu66SxT{8Ytur0x=UwGP=+!X+t%9&^%jNngBo4^@!|>nl?U0f`D4?N9~zbt%`< zo=|tgx1=O9Be|8@W)78bTk`5|mg-QWeGNa+!S zdSk-93wa%4jf3Jbm4~oWDUcr5%@M-oC;%yC`_na`Cn#>g)U0XVYIXr8yYt@CQLyNK z_S(An#(n4!L2J&@XIIzF*X}2!H=QC@h4e_iXssWzcEDkZGi#Xho~-l#FY|cp@3<3Q zGyx2@#0SrZ2xi7vI1He!0Rn(O{zV{-Oj-zX{HWwjPEp0&!&d~gjM672R@r*tj8jUI+1f6 zo)kxzYZAvmfbJ#0!t6M0INwP}3A%V5u5FHY`4K_`z*-u0y(7d5M^6BJ-90jqts;?K z3&1tXya=B-Bkzh7Ep5JG<#*(Gn{UKA?uAu$!x^L%>yQqfl?&k@!rx24Q$1(M;%ov_ zIo4BI+^lQ@eHt}>9%UwymylqI;V&b(faD^QOGqvw!Ro~8NboP=vv3EIj>J5%-UVPK zkwHSvf+jA_R(J%OVFyU{!ZMU<{25Pb`YM#<8kB7VN_0h*`jT2BiC$N$>M|iv%=K@x zS^Uob5F8x;j&=Ii`U-y?hI+7w`MS0G+tQ4-hW`F81tTXccGX9#?KeFy_uEVz)Sl>j071 z43R<4zXgMg2J8oj_(KXrSMs?$qA8+lItFu*{7tWszv@fmFM6H)SwB(vI{Pum0Tt*>KXDMJxjhJIr49kC;uV^@=pLv{~*QE-^u5B^rV)alr#hE{s*a?^(p`W literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/__pycache__/utils.cpython-310.pyc b/vllm/lora/ops/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..762bb6851e16706e7c1911ee9e09a2af04399136 GIT binary patch literal 1518 zcmZWpOK&4Z5bmBCj~|&hNT3kJrG>cokXU;GX@$^=T@F!JUZg}qvV;{=>dOI`h{uKK>Jt}gfY2MpVz-~U+r)nV*U zTI@DDeS+P-LLr&thD~Yo49~r($C+X;ed((}2C5@Fs*{B>l-)PMYd#IzwI_SB|0bAr z-_sh%J>2c>?#f7c3hiXxCoJp9eK|ypzU1d@bg)7HY%=0a@Jy$!>6TUOR+VZxv&5x~ zW^bXTR7Gs{E0y}Bd)sKvu|LIb0~C^>IX-#I-ugg?K%0Qv8H9J@@5f`8SK~@o$1Z`n za>rlh^&)?iUyqHRjjNSgl*M>yOg=UxB;&HOA5A-){i1#>k?;X|3o1i;YJgJay1n9 zSQ5kkSPEPqH%ub(^KbmMZ-Yl_)Ex^TgcOw<0S6xn`LnM`DqH#EdJU(E?)3 zYD|X`wI%*rk}2p4Ay}PjiUaz&oT(yRQH*H&no*-`M`_t1pfs62Qu z^j_#2*s1x?FzJ@cCF%!Q4grTxaWNV+{j4rhSC+;$A^u+o5~ymMzJWSU6D+O9w+C_P)wXSsV2=RkS@s(PQY;KjOpAVKmtDuo?$^`0ig1Kyx<$ literal 0 HcmV?d00001 diff --git a/vllm/lora/ops/bgmv_expand.py b/vllm/lora/ops/bgmv_expand.py new file mode 100644 index 00000000..6a32387a --- /dev/null +++ b/vllm/lora/ops/bgmv_expand.py @@ -0,0 +1,168 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from .utils import get_lora_op_configs + + +@triton.jit +def _bgmv_expand_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + lora_indices, + xm_stride, + xk_stride, + l0_stride, + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + SPLIT_N: tl.constexpr, + EVEN_K: tl.constexpr, + ADD_INPUTS: tl.constexpr, + CAST_TYPE: tl.constexpr, +): + """ + GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's + performance + """ + pid_sn = tl.program_id(axis=0) + cur_batch = tl.program_id(axis=1) + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + offset_k = tl.arange(0, BLOCK_K) + offset_n = tl.arange(0, BLOCK_N) + if EVEN_K: + tiled_a = tl.load(input_ptr + cur_batch * xm_stride + + offset_k * xk_stride, ) # [BLOCK_K] + else: + tiled_a = tl.load( + input_ptr + cur_batch * xm_stride + offset_k * xk_stride, + mask=offset_k < K, + other=0, + ) # [BLOCK_K] + # N must be divisible by SPLIT_N + split_n_length = tl.cdiv(N, SPLIT_N) + if CAST_TYPE: + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty) + # sliding to next row-block + b_ptr = (lora_ptr + l0_stride * lora_index + + pid_sn * split_n_length * lora_k_stride) + c_ptr = out_ptr + cur_batch * cm_stride + pid_sn * split_n_length + for n in range(0, split_n_length, BLOCK_N): + current_n = n + offset_n + current_n_c = tl.max_contiguous(current_n, BLOCK_N) + b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :] + < K) + c_mask = current_n < split_n_length + tiled_b = tl.load( + b_ptr + current_n_c[:, None] * lora_k_stride + + offset_k[None, :] * lora_n_stride, + mask=b_ptr_mask, + other=0.0, + ) # [BLOCK_N,BLOCK_K] + if ADD_INPUTS: + tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask) + accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out + else: + accumulator = tl.sum(tiled_a * tiled_b, 1) + + tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask) + + +@torch.inference_mode() +def _bgmv_expand( + inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + add_inputs: bool = True, +) -> None: + """ + Args: + inputs (torch.Tensor): input tensor + lora_b_weights (torch.Tensor): lora'a weight + output_tensor (torch.Tensor): output tensor + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch, An index of -1 means no lora should be + applied. + batches (int): batch size + add_inputs (bool, optional): Defaults to False, adds the final lora + results to the output. + """ + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] + assert lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(1) == lora_b_weights.size(-1) + + assert inputs.is_contiguous() + assert output_tensor.is_contiguous() + + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank) + assert lora_b_weights.size(1) == 1 + lora_b_weights = lora_b_weights.squeeze(dim=1) + else: + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank) + assert lora_b_weights.is_contiguous() + + # TODO tuning this config + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size + BLOCK_K = triton.next_power_of_2(K) + EVEN_K = K % BLOCK_K == 0 + ADD_INPUTS = add_inputs + CAST_TYPE = False + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ]: + CAST_TYPE = True + batches = lora_indices_tensor.size(0) + config = get_lora_op_configs("expand", batches, N) + grid = lambda META: ( + META["SPLIT_N"], + batches, + ) + _bgmv_expand_kernel[grid]( + inputs, + lora_b_weights, + output_tensor, + N, + K, + lora_indices_tensor, + inputs.stride(0), + inputs.stride(1), + lora_b_weights.stride(0), + lora_b_weights.stride(1), + lora_b_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + BLOCK_K=BLOCK_K, + EVEN_K=EVEN_K, + ADD_INPUTS=ADD_INPUTS, + CAST_TYPE=CAST_TYPE, + **config, + ) + return + + +try: + bgmv_expand = torch.library.custom_op("lora::bgmv_expand", + _bgmv_expand, + mutates_args=["output_tensor"]) +except AttributeError: + bgmv_expand = _bgmv_expand diff --git a/vllm/lora/ops/bgmv_expand_slice.py b/vllm/lora/ops/bgmv_expand_slice.py new file mode 100644 index 00000000..73628fd2 --- /dev/null +++ b/vllm/lora/ops/bgmv_expand_slice.py @@ -0,0 +1,181 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from .utils import get_lora_op_configs + + +@triton.jit +def _bgmv_expand_slice_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + lora_indices, + xm_stride, + xk_stride, + l0_stride, + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + slice_offset, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + SPLIT_N: tl.constexpr, + EVEN_K: tl.constexpr, + ADD_INPUTS: tl.constexpr, + CAST_TYPE: tl.constexpr, +): + """ + GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's + performance + """ + pid_sn = tl.program_id(axis=0) + cur_batch = tl.program_id(axis=1) + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + offset_k = tl.arange(0, BLOCK_K) + offset_n = tl.arange(0, BLOCK_N) + if EVEN_K: + tiled_a = tl.load(input_ptr + cur_batch * xm_stride + + offset_k * xk_stride, ) # [BLOCK_K] + else: + tiled_a = tl.load( + input_ptr + cur_batch * xm_stride + offset_k * xk_stride, + mask=offset_k < K, + other=0, + ) # [BLOCK_K] + # N must be divisible by SPLIT_N + split_n_length = tl.cdiv(N, SPLIT_N) + if CAST_TYPE: + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty) + # sliding to next row-block + b_ptr = (lora_ptr + l0_stride * lora_index + + pid_sn * split_n_length * lora_k_stride) + c_ptr = (out_ptr + cur_batch * cm_stride + pid_sn * split_n_length + + slice_offset * cn_stride) + + for n in range(0, split_n_length, BLOCK_N): + current_n = n + offset_n + b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :] + < K) + c_mask = current_n < split_n_length + tiled_b = tl.load( + b_ptr + current_n[:, None] * lora_k_stride + + offset_k[None, :] * lora_n_stride, + mask=b_ptr_mask, + other=0.0, + ) # [BLOCK_N,BLOCK_K] + + if ADD_INPUTS: + tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask) + accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out + else: + accumulator = tl.sum(tiled_a * tiled_b, 1) + + tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask) + + +@torch.inference_mode() +def _bgmv_expand_slice( + inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + slice_offset: int, + slice_size: int, + add_inputs: bool = True, +) -> None: + """ + Args: + inputs (torch.Tensor): input tensor + lora_b_weights (torch.Tensor): lora'b weight + output_tensor (torch.Tensor): output tensor + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch, An index of -1 means no lora should be + applied. + slice_offset (int): output_tensor's offset + slice_size (int): current output_tensor's size + batches (int): batch size + add_inputs (bool, optional): Defaults to False. + """ + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] + assert lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(1) == lora_b_weights.size(-1) + + assert slice_size == lora_b_weights.size(-2) + assert inputs.is_contiguous() + assert output_tensor.is_contiguous() + + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank) + assert lora_b_weights.size(1) == 1 + lora_b_weights = lora_b_weights.squeeze(dim=1) + else: + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank) + + assert lora_b_weights.is_contiguous() + + # TODO tuning this config + + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size + BLOCK_K = triton.next_power_of_2(K) + EVEN_K = K % BLOCK_K == 0 + ADD_INPUTS = add_inputs + CAST_TYPE = False + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ]: + CAST_TYPE = True + + batches = lora_indices_tensor.size(0) + + config = get_lora_op_configs("expand", batches, N) + + grid = lambda META: ( + META["SPLIT_N"], + batches, + ) + _bgmv_expand_slice_kernel[grid]( + inputs, + lora_b_weights, + output_tensor, + N, + K, + lora_indices_tensor, + inputs.stride(0), + inputs.stride(1), + lora_b_weights.stride(0), + lora_b_weights.stride(1), + lora_b_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + slice_offset, + BLOCK_K=BLOCK_K, + EVEN_K=EVEN_K, + ADD_INPUTS=ADD_INPUTS, + CAST_TYPE=CAST_TYPE, + **config, + ) + return + + +try: + bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice", + _bgmv_expand_slice, + mutates_args=["output_tensor"]) +except AttributeError: + bgmv_expand_slice = _bgmv_expand_slice diff --git a/vllm/lora/ops/bgmv_shrink.py b/vllm/lora/ops/bgmv_shrink.py new file mode 100644 index 00000000..0846ff36 --- /dev/null +++ b/vllm/lora/ops/bgmv_shrink.py @@ -0,0 +1,150 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from .utils import get_lora_op_configs + + +@triton.jit +def _bgmv_shrink_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + lora_indices, + scaling, + xm_stride, + xk_stride, + l0_stride, + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + SPLIT_K: tl.constexpr, +): + """ + GroupGEMV, additionally, introducing SPLIT-K can improve large hidden_size's + performance + """ + pid_sk = tl.program_id(axis=0) + cur_batch = tl.program_id(axis=1) + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + + offset_n = tl.arange(0, BLOCK_N) + offset_k = tl.arange(0, BLOCK_K) + pid_sk * BLOCK_K + a_ptr = input_ptr + cur_batch * xm_stride + b_ptr = lora_ptr + l0_stride * lora_index + accumulator = tl.zeros((BLOCK_N, ), dtype=tl.float32) + for k in range(0, K, BLOCK_K * SPLIT_K): + current_k = k + offset_k + current_k_c = tl.max_contiguous(current_k, BLOCK_K) + tiled_a = tl.load( + a_ptr + current_k_c, + mask=current_k < K, + other=0.0, + ) # [BLOCK_K] + b_ptr_mask = (offset_n[:, None] < N) & (current_k[None, :] < K) + + tiled_b = tl.load( + b_ptr + offset_n[:, None] * lora_k_stride + + current_k[None, :] * lora_n_stride, + mask=b_ptr_mask, + other=0.0, + ) # [BLOCK_N,BLOCK_K] + + accumulator += tl.sum(tiled_a * tiled_b, 1) + accumulator *= scaling + offset_cn = tl.arange(0, BLOCK_N) + c_ptr = out_ptr + cur_batch * cm_stride + offset_cn * cn_stride + c_mask = offset_cn < N + if SPLIT_K == 1: + tl.store(c_ptr, accumulator, mask=c_mask) + else: + tl.atomic_add(c_ptr, accumulator, mask=c_mask) + + +@torch.inference_mode() +def _bgmv_shrink( + inputs: torch.Tensor, + lora_a_weights: torch.Tensor, + output_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + scaling: float = 1.0, +) -> None: + """ + Args: + inputs (torch.Tensor): input tensor + lora_a_weights (torch.Tensor): lora'a weight + output_tensor (torch.Tensor): output tensor + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch. An index of -1 means no lora should be + applied. + batches (int): batch size + scaling (float): Scaling factor. + """ + assert inputs.dtype == lora_a_weights.dtype + assert inputs.dtype in [torch.float16, torch.bfloat16] + assert lora_a_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(1) == lora_a_weights.size(-1) + assert inputs.is_contiguous() + + if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size) + assert lora_a_weights.size(1) == 1 + lora_a_weights = lora_a_weights.squeeze(dim=1) + else: + assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size) + assert lora_a_weights.is_contiguous() + assert output_tensor.is_contiguous() + # TODO tuning this config + batches = lora_indices_tensor.size(0) + N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank + BLOCK_N = triton.next_power_of_2(N) + # First try to load optimal config from the file + config = get_lora_op_configs("bgmv_shrink", batches, K) + + grid = lambda META: ( + META["SPLIT_K"], + batches, + ) + _bgmv_shrink_kernel[grid]( + inputs, + lora_a_weights, + output_tensor, + N, + K, + lora_indices_tensor, + scaling, + inputs.stride(0), + inputs.stride(1), + lora_a_weights.stride(0), + lora_a_weights.stride(1), + lora_a_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + BLOCK_N=BLOCK_N, + **config, + ) + return + + +try: + bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink", + _bgmv_shrink, + mutates_args=["output_tensor"]) +except AttributeError: + bgmv_shrink = _bgmv_shrink diff --git a/vllm/lora/ops/sgmv_expand.py b/vllm/lora/ops/sgmv_expand.py new file mode 100644 index 00000000..adb3ab5b --- /dev/null +++ b/vllm/lora/ops/sgmv_expand.py @@ -0,0 +1,204 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from vllm.triton_utils import libentry + + +@libentry() +@triton.jit +def _sgmv_expand_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + b_seq_start_loc, + seq_lens, + lora_indices, + xm_stride, + xk_stride, # 1 + l0_stride, # hidden_size*max_rank + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + EVEN_K: tl.constexpr, + ADD_INPUTS: tl.constexpr, + CAST_TYPE: tl.constexpr, +): + """ + The sgmv's expand triton kernel is based on GroupGEMM. + """ + pid = tl.program_id(axis=0) + cur_batch = tl.program_id(axis=1) + cta_n_num = tl.cdiv(N, BLOCK_N) + pid_m = pid // cta_n_num + pid_n = pid % cta_n_num + M = tl.load(seq_lens + cur_batch) + if pid_m * BLOCK_M > M: + return + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + cur_seq_start = tl.load(b_seq_start_loc + cur_batch) + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + offset_k = tl.arange(0, BLOCK_K) + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M) + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N) + + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride + + offset_k[None, :] * xk_stride, ) + b_ptr = (lora_ptr + l0_stride * lora_index + + offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride) + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(tl.cdiv(K, BLOCK_K)): + if EVEN_K: + tiled_a = tl.load(a_ptr) + tiled_b = tl.load(b_ptr) + else: + tiled_a = tl.load(a_ptr, + mask=offset_k[None, :] < K - k * BLOCK_K, + other=0) + tiled_b = tl.load(b_ptr, + mask=offset_k[:, None] < K - k * BLOCK_K, + other=0) + if CAST_TYPE: + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty) + accumulator += tl.dot( + tiled_a, + tiled_b, + ) + a_ptr += BLOCK_K * xk_stride + b_ptr += BLOCK_K * lora_n_stride + tiled_c = accumulator.to(lora_ptr.dtype.element_ty) + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride + + offset_cn[None, :] * cn_stride) + M = tl.load(seq_lens + cur_batch) + c_mask = (offset_cm[:, None] < + (cur_seq_start + M)) & (offset_cn[None, :] < N) + if ADD_INPUTS: + tiled_out = tl.load(c_ptr, mask=c_mask) + tiled_c += tiled_out + tl.store(c_ptr, tiled_c, mask=c_mask) + + +@torch.inference_mode() +def _sgmv_expand( + inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + add_inputs: bool = False, +) -> None: + """ + Args: + inputs (torch.Tensor): input tensor + lora_b_weights (torch.Tensor): lora'a weight + output_tensor (torch.Tensor): output tensor + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative + sequence lengths of the sequences in the batch, used to index + into sequence. E.g., if the sequence length is [4, 6], it is + [0, 4, 10]. + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence + length of the sequences in the batch. + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch. An index of -1 means no lora should be + applied. + batches (int): batch size + max_seq_length (int): The max sequence lengths of the sequences in the + batch. + token_nums (int): The token numbers in the batch. Used to verify if the + token numbers in the inputs matches the one in the metadata. + add_inputs (bool, optional): Defaults to False, adds the final lora + results to the output. + """ + + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] + assert lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(0) == token_nums + assert inputs.size(1) == lora_b_weights.size(-1) + assert b_seq_start_loc.size(0) == batches + assert lora_indices_tensor.size(0) == batches + assert inputs.is_contiguous() + assert output_tensor.is_contiguous() + + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank) + assert lora_b_weights.size(1) == 1 + lora_b_weights = lora_b_weights.squeeze(dim=1) + else: + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank) + + assert lora_b_weights.is_contiguous() + + # TODO tuning this config + + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size + BLOCK_M = 32 + BLOCK_N = 32 + BLOCK_K = 16 + EVEN_K = K % BLOCK_K == 0 + ADD_INPUTS = add_inputs + CAST_TYPE = False + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ]: + CAST_TYPE = True + grid = ( + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N), + batches, + ) + _sgmv_expand_kernel[grid]( + inputs, + lora_b_weights, + output_tensor, + N, + K, + b_seq_start_loc, + seq_len_tensor, + lora_indices_tensor, + inputs.stride(0), + inputs.stride(1), + lora_b_weights.stride(0), + lora_b_weights.stride(1), + lora_b_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + BLOCK_M, + BLOCK_N, + BLOCK_K, + EVEN_K, + ADD_INPUTS, + CAST_TYPE, + ) + return + + +try: + sgmv_expand = torch.library.custom_op("lora::sgmv_expand", + _sgmv_expand, + mutates_args=["output_tensor"]) +except AttributeError: + sgmv_expand = _sgmv_expand diff --git a/vllm/lora/ops/sgmv_expand_slice.py b/vllm/lora/ops/sgmv_expand_slice.py new file mode 100644 index 00000000..efa23452 --- /dev/null +++ b/vllm/lora/ops/sgmv_expand_slice.py @@ -0,0 +1,217 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from vllm.triton_utils import libentry + + +@libentry() +@triton.jit +def _sgmv_expand_slice_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + b_seq_start_loc, + seq_lens, + lora_indices, + xm_stride, + xk_stride, # 1 + l0_stride, # hidden_size*max_rank + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + slice_offset, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + EVEN_K: tl.constexpr, + ADD_INPUTS: tl.constexpr, + CAST_TYPE: tl.constexpr, +): + """ + + Similar to the 'sgmv_expand' operator, but with an added parameter + 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator + might be that in the future, we could implement a fusion operator to + achieve the current functionality instead of having to call it multiple + times. + """ + pid = tl.program_id(axis=0) + cur_batch = tl.program_id(axis=1) + cta_n_num = tl.cdiv(N, BLOCK_N) + pid_m = pid // cta_n_num + pid_n = pid % cta_n_num + M = tl.load(seq_lens + cur_batch) + if pid_m * BLOCK_M > M: + return + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + cur_seq_start = tl.load(b_seq_start_loc + cur_batch) + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + offset_k = tl.arange(0, BLOCK_K) + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M) + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N) + + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride + + offset_k[None, :] * xk_stride, ) + b_ptr = (lora_ptr + l0_stride * lora_index + + offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride) + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(tl.cdiv(K, BLOCK_K)): + if EVEN_K: + tiled_a = tl.load(a_ptr) + tiled_b = tl.load(b_ptr) + else: + tiled_a = tl.load(a_ptr, + mask=offset_k[None, :] < K - k * BLOCK_K, + other=0) + tiled_b = tl.load(b_ptr, + mask=offset_k[:, None] < K - k * BLOCK_K, + other=0) + if CAST_TYPE: + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty) + accumulator += tl.dot( + tiled_a, + tiled_b, + ) + a_ptr += BLOCK_K * xk_stride + b_ptr += BLOCK_K * lora_n_stride + tiled_c = accumulator.to(lora_ptr.dtype.element_ty) + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_offset + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride + + offset_cn[None, :] * cn_stride) + M = tl.load(seq_lens + cur_batch) + c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] < + (slice_offset + N)) + if ADD_INPUTS: + tiled_out = tl.load(c_ptr, mask=c_mask) + tiled_c += tiled_out + tl.store(c_ptr, tiled_c, mask=c_mask) + + +@torch.inference_mode() +def _sgmv_expand_slice( + inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + slice_offset: int, + slice_size: int, + add_inputs: bool = False, +) -> None: + """_summary_ + + Args: + inputs (torch.Tensor): input tensor + lora_b_weights (torch.Tensor): lora'a weight + output_tensor (torch.Tensor): output tensor + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative + sequence lengths of the sequences in the batch, used to index + into sequence. E.g., if the sequence length is [4, 6], it is + [0, 4, 10]. + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence + length of the sequences in the batch + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch. An index of -1 means no lora should be + applied. + batches (int): batch size + max_seq_length (int): The max sequence lengths of the sequences + in the batch + token_nums (int): The token numbers in the batch. Used to verify if the + token numbers in the inputs matches the one in the metadata. + slice_offset (int): output_tensor's offset + slice_size (int): current output_tensor's size + add_inputs (bool, optional): Defaults to False, adds the final lora + results to the output. + """ + + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] + assert lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(0) == token_nums + assert inputs.size(1) == lora_b_weights.size(-1) + assert b_seq_start_loc.size(0) == batches + assert lora_indices_tensor.size(0) == batches + assert slice_size == lora_b_weights.size(-2) + assert inputs.is_contiguous() + assert output_tensor.is_contiguous() + + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank) + assert lora_b_weights.size(1) == 1 + lora_b_weights = lora_b_weights.squeeze(dim=1) + else: + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank) + + assert lora_b_weights.is_contiguous() + + # TODO tuning this config + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size + + BLOCK_M = 32 + BLOCK_N = 32 + BLOCK_K = 16 + EVEN_K = K % BLOCK_K == 0 + ADD_INPUTS = add_inputs + CAST_TYPE = False + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [ + torch.float16, + torch.bfloat16, + ]: + CAST_TYPE = True + grid = ( + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N), + batches, + ) + _sgmv_expand_slice_kernel[grid]( + inputs, + lora_b_weights, + output_tensor, + N, + K, + b_seq_start_loc, + seq_len_tensor, + lora_indices_tensor, + inputs.stride(0), + inputs.stride(1), + lora_b_weights.stride(0), + lora_b_weights.stride(1), + lora_b_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + slice_offset, + BLOCK_M, + BLOCK_N, + BLOCK_K, + EVEN_K, + ADD_INPUTS, + CAST_TYPE, + ) + return + + +try: + sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice", + _sgmv_expand_slice, + mutates_args=["output_tensor"]) +except AttributeError: + sgmv_expand_slice = _sgmv_expand_slice diff --git a/vllm/lora/ops/sgmv_shrink.py b/vllm/lora/ops/sgmv_shrink.py new file mode 100644 index 00000000..c003f3dc --- /dev/null +++ b/vllm/lora/ops/sgmv_shrink.py @@ -0,0 +1,201 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +import torch +import triton +import triton.language as tl + +from vllm.triton_utils import libentry + + +@libentry() +@triton.jit +def _sgmv_shrink_kernel( + input_ptr, + lora_ptr, + out_ptr, + N, + K, + b_seq_start_loc, + seq_lens, + lora_indices, + scaling, + xm_stride, # hidden_size + xk_stride, # 1 + l0_stride, # hidden_size*max_rank + lora_k_stride, + lora_n_stride, + cm_stride, + cn_stride, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + EVEN_K: tl.constexpr, + SPLIT_K: tl.constexpr, +): + """ + The sgmv's shrink triton kernel is based on GroupGEMM+SPLIT-K. + The GEMM of Multi-LoRA can be considered as GroupGEMM. Additionally, + introducing SPLIT-K can improve performance + """ + pid = tl.program_id(axis=0) + pid_sk = tl.program_id(axis=1) + cur_batch = tl.program_id(axis=2) + cta_n_num = tl.cdiv(N, BLOCK_N) + pid_m = pid // cta_n_num + pid_n = pid % cta_n_num + + M = tl.load(seq_lens + cur_batch) + if pid_m * BLOCK_M > M: + return + lora_index = tl.load(lora_indices + cur_batch) + if lora_index == -1: + return + cur_seq_start = tl.load(b_seq_start_loc + cur_batch) + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K) + + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M) + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N) + + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride + + offset_k[None, :] * xk_stride) + b_ptr = (lora_ptr + l0_stride * lora_index + rbn[None, :] * lora_k_stride + + offset_k[:, None] * lora_n_stride) + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K * SPLIT_K)): + if EVEN_K: + tiled_a = tl.load(a_ptr) + tiled_b = tl.load(b_ptr) + else: + k_remaining = K - k * (BLOCK_K * SPLIT_K) + tiled_a = tl.load(a_ptr, + mask=offset_k[None, :] < k_remaining, + other=0.0) + tiled_b = tl.load(b_ptr, + mask=offset_k[:, None] < k_remaining, + other=0.0) + accumulator += tl.dot(tiled_a, tiled_b) + + a_ptr += BLOCK_K * SPLIT_K * xk_stride + b_ptr += BLOCK_K * SPLIT_K * lora_n_stride + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M + + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride + + offset_cn[None, :] * cn_stride) + c_mask = (offset_cm[:, None] < + (cur_seq_start + M)) & (offset_cn[None, :] < N) + accumulator *= scaling + # handles write-back with reduction-splitting + if SPLIT_K == 1: + tl.store(c_ptr, accumulator, mask=c_mask) + else: + tl.atomic_add(c_ptr, accumulator, mask=c_mask) + + +@torch.inference_mode() +def _sgmv_shrink( + inputs: torch.Tensor, + lora_a_weights: torch.Tensor, + output_tensor: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + scaling: float, +) -> None: + """ + + Args: + inputs (torch.Tensor): input tensor + lora_a_weights (torch.Tensor): lora'a weight + output_tensor (torch.Tensor): output tensor + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative + sequence lengths of the sequences in the batch, used to index + into sequence. E.g., if the sequence length is [4, 6], it is + [0, 4]. + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence + length of the sequences in the batch. + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index + corresponding to each batch. An index of -1 means no lora should be + applied. + batches (int): batch size + max_seq_length (int): The max sequence lengths of the sequences in the + batch. + token_nums (int): The token numbers in the batch. Used to verify if the + token numbers in the inputs matches the one in the metadata. + scaling (float): Scaling factor. + """ + assert inputs.dtype == lora_a_weights.dtype + assert inputs.dtype in [torch.float16, torch.bfloat16] + assert lora_a_weights.dtype in [ + torch.float16, + torch.bfloat16, + ] + assert inputs.size(0) == token_nums + assert inputs.size(1) == lora_a_weights.size(-1) + assert b_seq_start_loc.size(0) == batches + assert lora_indices_tensor.size(0) == batches + assert inputs.is_contiguous() + + if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size) + assert lora_a_weights.size(1) == 1 + lora_a_weights = lora_a_weights.squeeze(dim=1) + else: + assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size) + assert lora_a_weights.is_contiguous() + assert output_tensor.is_contiguous() + # TODO tuning this config + N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank + BLOCK_M = 32 + BLOCK_N = 16 + BLOCK_K = 32 + SPLIT_K = 8 + EVEN_K = K % (BLOCK_K * SPLIT_K) == 0 + grid = ( + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N), + SPLIT_K, + batches, + ) + + _sgmv_shrink_kernel[grid]( + inputs, + lora_a_weights, + output_tensor, + N, + K, + b_seq_start_loc, + seq_len_tensor, + lora_indices_tensor, + scaling, + inputs.stride(0), + inputs.stride(1), + lora_a_weights.stride(0), + lora_a_weights.stride(1), + lora_a_weights.stride(2), + output_tensor.stride(0), + output_tensor.stride(1), + BLOCK_M, + BLOCK_N, + BLOCK_K, + EVEN_K, + SPLIT_K, + ) + return + + +try: + sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink", + _sgmv_shrink, + mutates_args=["output_tensor"]) +except AttributeError: + sgmv_shrink = _sgmv_shrink diff --git a/vllm/lora/ops/utils.py b/vllm/lora/ops/utils.py new file mode 100644 index 00000000..7c3e2731 --- /dev/null +++ b/vllm/lora/ops/utils.py @@ -0,0 +1,46 @@ +import functools +from typing import Dict + + +@functools.lru_cache +def _get_op_configs(op_type: str, batch: int, hidden_size: int): + # TODO: add optimal configurations + return None + + +def _check_divisibility(hidden_size: int): + # The bgmv_expand kernel requires that the hidden_size be divisible by + # the number below. + divisibility = [2, 4, 8, 16, 32, 64] + divisibility.sort(reverse=True) + for div in divisibility: + if hidden_size % div == 0: + return div + # hidden_size is an odd number + return 1 + + +def _get_default_config(op_type: str, batch: int, hidden_size: int): + if op_type == "expand": + return { + "BLOCK_N": 256, + "SPLIT_N": _check_divisibility(hidden_size), + "num_warps": 8 + } + else: + return {"BLOCK_K": 256, "SPLIT_K": 64, "num_warps": 8} + + +def get_lora_op_configs(op_type: str, batch: int, + hidden_size: int) -> Dict[str, int]: + """Inspired by `fused_moe_kernel` + The return value will be a dictionary mapping an irregular grid of batch + sizes and hidden_size to configurations of the bgmv-related kernel. + NOTE: It currently only supports the default configuration. We plan to + generate optimal configurations for different hardware in the future using + scripts similar to `benchmark_moe.py`. + """ + config = _get_op_configs(op_type, batch, hidden_size) + if not config: + config = _get_default_config(op_type, batch, hidden_size) + return config diff --git a/vllm/lora/punica.py b/vllm/lora/punica.py new file mode 100644 index 00000000..1aa9fabe --- /dev/null +++ b/vllm/lora/punica.py @@ -0,0 +1,629 @@ +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union + +import torch + +from vllm.triton_utils import HAS_TRITON + +if HAS_TRITON: + from vllm.lora.ops.bgmv_expand import bgmv_expand + from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice + from vllm.lora.ops.bgmv_shrink import bgmv_shrink + from vllm.lora.ops.sgmv_expand import sgmv_expand + from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice + from vllm.lora.ops.sgmv_shrink import sgmv_shrink + +if TYPE_CHECKING: + # avoid circuit import + from vllm.lora.layers import LoRAMapping + from vllm.lora.models import LongContextLoRAContext +from vllm import _custom_ops as ops + +def compute_meta( + token_lora_tensor: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]: + """ + Get the information required for the sgmv kernel. With the features: + 1. If consecutive requests in the batch use the same LoRA, this function + will combine them into a single request, improving sgmv kernel inference + performance. + 2. At the beginning of each prefill stage inference, recalculations are + needed based on the input, but only once. + """ + + lora_indices_tensor, seq_length_tensor = torch.unique_consecutive( + token_lora_tensor, return_counts=True) + cum_result = torch.cumsum(seq_length_tensor, dim=0) + b_seq_start_tensor = torch.zeros_like(seq_length_tensor) + b_seq_start_tensor[1:].copy_(cum_result[:-1]) + max_length = seq_length_tensor.max().item() + token_nums = seq_length_tensor.sum().item() + batch_size = lora_indices_tensor.size(0) + no_lora = False + # -1 means no lora should be applied. Use `no_lora` to determine whether + # the current step requires LoRA. If LoRA is not needed, the prefill stage + # does not need to launch the triton kernel, which can improve performance + if batch_size == 1 and lora_indices_tensor == -1: + no_lora = True + return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor, + batch_size, max_length, token_nums, no_lora) + + +# TODO see if this can be vectorized +def convert_mapping( + mapping: "LoRAMapping", + lora_index_to_id: List[Optional[int]], + max_loras: int, + vocab_size: int, + extra_vocab_size: int, + long_lora_context: Optional["LongContextLoRAContext"] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, + Optional[torch.Tensor], List[int]]: + """Converts LoRAMapping to index tensors. + + Args: + mapping: LoRAMapping mapping rows in a batch to LoRA ids. + lora_index_to_id: List mapping LoRA ids to LoRA indices. + max_loras: Maximum number of LoRAs. + vocab_size: Model vocab size. + extra_vocab_size: Extra vocab size each LoRA can have. + long_lora_context: Passed if there are long context lora in a batch. + + Returns: + A tuple of tensors: + base_indices: Tensor of shape [batch_size] mapping batch rows to + LoRA indices. + sampler_indices: Tensor of shape [batch_size] mapping requests to + LoRA indices for sampler. For generation, this will be the + same as base_indicies. For prefill, this will map requests + to LoRA indices. + sampler_indices_padded: Tensor of shape [batch_size] mapping + requests to LoRA indices for sampler with padding. + Same as sampler_indicies, but -1 is replaced with + max_loras. + embeddings_indices: Tensor of shape [2, batch_size] mapping + requests to embedding indices. First row is for embeddings + added by the LoRAs, second row is for the LoRA.lora_a + embeddings. + long_lora_indices: Tensor of shape [batch_size] mapping + requests to RoPE offsets and rot dims for long LoRAs. + None if long context lora doesn't exist. + indices_len: List of lengths of the above tensors. It contains + (base_indices, sampler_indices, sampler_indices_padded, + embeddings_indices, long_lora_indices). + """ + index_mapping_indices: List[int] = list(mapping.index_mapping).copy() + embedding_indices = index_mapping_indices.copy() + lora_indices = index_mapping_indices.copy() + long_lora_offsets: Optional[torch.Tensor] = None + if long_lora_context: + long_lora_offsets = torch.zeros(len(index_mapping_indices), + device="cuda", + dtype=torch.long) + prompt_mapping: List[int] = [ + lora_index_to_id.index(x) if x > 0 else -1 + for x in mapping.prompt_mapping + ] + lora_idx = None + for i in range(len(index_mapping_indices)): + # TODO index can be slow. optimize + lora_idx = (lora_index_to_id.index(index_mapping_indices[i]) + if index_mapping_indices[i] > 0 else -1) + embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0 + lora_indices[i] = lora_idx + if long_lora_context: + assert long_lora_offsets is not None + lora_offset: int = long_lora_context.offsets_by_lora_id.get( + index_mapping_indices[i], 0) + long_lora_offsets[i] = lora_offset + + indices_list: List[Union[List[int], torch.Tensor]] = [ + index_mapping_indices, + lora_indices, + embedding_indices, + ] + if long_lora_context: + assert long_lora_offsets is not None + indices_list.append(long_lora_offsets) + indices = torch.tensor(indices_list, dtype=torch.long, device="cuda") + prompt_mapping_tensor = torch.tensor(prompt_mapping, + device="cuda", + dtype=torch.long) + embeddings_indices = torch.stack([ + indices[2] * extra_vocab_size, + indices[2] * (vocab_size + extra_vocab_size), + ]) + embeddings_indices[embeddings_indices == -1] = max_loras - 1 + base_indices = indices[1] + sampler_indices = prompt_mapping_tensor + sampler_indices_padded = sampler_indices.clone() + sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1 + sampler_indices_padded = torch.arange( + 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) + ( + sampler_indices_padded * len(sampler_indices_padded)) + long_lora_indices = None + long_lora_indices_len: Optional[int] = None + if long_lora_context: + long_lora_indices = indices[3] + long_lora_indices_len = long_lora_indices.shape[-1] + # Contain length of indices tensors. Used to index into each tensor. + indices_len = [ + base_indices.shape[-1], + sampler_indices.shape[-1], + sampler_indices_padded.shape[-1], + embeddings_indices.shape[-1], + ] + if long_lora_indices_len is not None: + indices_len.append(long_lora_indices_len) + else: + # If long_lora doesn't exist,append None + indices_len.append(None) + + return ( + base_indices, + sampler_indices, + sampler_indices_padded, + embeddings_indices, + long_lora_indices, + indices_len, + ) + + +class PunicaWrapper: + """ + PunicaWrapper is designed to manage and provide metadata for the punica + kernel. The main function is to maintain the state information for + Multi-LoRA, and to provide the interface for the punica kernel. + """ + + def __init__(self, max_num_batched_tokens: int, max_batches: int, + device: str): + self._token_lora_indices = torch.empty(max_num_batched_tokens, + dtype=torch.long, + device=device) + self._sampler_indices = torch.empty(max_num_batched_tokens, + dtype=torch.long, + device=device) + self._sampler_indices_padded = torch.empty(max_num_batched_tokens, + dtype=torch.long, + device=device) + self._embeddings_indices = torch.empty(2, + max_num_batched_tokens, + dtype=torch.long, + device=device) + self._long_lora_indices = torch.empty(max_num_batched_tokens, + dtype=torch.long, + device=device) + + # 5 is the number of indicies tensors. + # base_indices, sampler_indices, sampler_indices_padded, + # embeddings_indices,long_lora_indices + self.indices_len: List[Optional[int]] = [None] * 5 + # these attributes are the information required for sgmv kernel + self._seq_start_locs = torch.empty(max_batches, + dtype=torch.long, + device=device) + self._seq_lengths = torch.empty(max_batches, + dtype=torch.long, + device=device) + self._lora_indices_per_batch = torch.empty(max_batches, + dtype=torch.long, + device=device) + self.max_length: int = 0 + self.token_nums: int = 0 + self.batch_size: int = -1 + self.is_prefill = False + self.no_lora = False + + def update_metadata( + self, + mapping: "LoRAMapping", + lora_index_to_id: List[Optional[int]], + max_loras: int, + vocab_size: int, + extra_vocab_size: int, + long_lora_context: Optional["LongContextLoRAContext"] = None, + ): + + self._update_base_metadata(mapping, lora_index_to_id, max_loras, + vocab_size, extra_vocab_size, + long_lora_context) + if mapping.is_prefill: + # Update metadata required for prefill-related operators. + self._update_prefill_metada(self.token_lora_indices) + self.is_prefill = True + else: + self.is_prefill = False + + def _update_base_metadata( + self, + mapping: "LoRAMapping", + lora_index_to_id: List[Optional[int]], + max_loras: int, + vocab_size: int, + extra_vocab_size: int, + long_lora_context: Optional["LongContextLoRAContext"] = None, + ): + ( + base_indices, + sampler_indices, + sampler_indices_padded, + embeddings_indices, + long_lora_offsets_tensor, + indices_len, + ) = convert_mapping( + mapping, + lora_index_to_id, + max_loras, + vocab_size, + extra_vocab_size, + long_lora_context, + ) + self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices) + self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices) + self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_( + sampler_indices_padded) + self._embeddings_indices[:embeddings_indices. + shape[0], :embeddings_indices.shape[1]].copy_( + embeddings_indices) + if long_lora_offsets_tensor is not None: + self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_( + long_lora_offsets_tensor) + else: + self._long_lora_indices.zero_() + self.indices_len[:] = indices_len + + def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None: + + (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor, + batch_size, max_length, token_nums, + no_lora) = compute_meta(token_lora_tensor) + + self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_( + b_seq_start_tensor) + self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor) + self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_( + lora_indices_tensor) + self.batch_size = batch_size + self.max_length = max_length + self.token_nums = token_nums + self.no_lora = no_lora + + @property + def prefill_metadata( + self + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]: + """ + This property provides a convenient way to access the necessary + metadata for prefill-related kernel computations. + 1. seq_start_locs: Tensor of sequence start positions. + 2. seq_lengths: Tensor of sequence lengths. + 3. lora_indices_per_batch: Tensor of lora indices, and an index of + -1 means no lora should be applied. + 4. batch_size: Batch size after clustering identical lora indices. + 5. max_length: The maximum sequence length in the batch. + 6. token_nums: The token numbers in the batch. + """ + return (self._seq_start_locs[:self.batch_size], + self._seq_lengths[:self.batch_size], + self._lora_indices_per_batch[:self.batch_size], + self.batch_size, self.max_length, self.token_nums) + + @property + def token_lora_indices(self) -> torch.Tensor: + """ + This property provides the lora indices corresponding to each token + in the batch. An index of -1 means no lora should be applied. + """ + token_lora_len = self.indices_len[0] + return self._token_lora_indices[:token_lora_len] + + @property + def sampler_indices(self) -> torch.Tensor: + """ + This property is used to access the lora indices specifically for + LogitsProcessorWithLoRA. + """ + sampler_indices_len = self.indices_len[1] + return self._sampler_indices[:sampler_indices_len] + + @property + def sampler_indices_padded(self) -> torch.Tensor: + """ + This property provides access to padded sampler indices. + """ + indices_padded_len = self.indices_len[2] + return self._sampler_indices_padded[:indices_padded_len] + + @property + def embeddings_indices(self) -> torch.Tensor: + """ + This property provides access to the indices used for lora embeddings, + specifically for VocabParallelEmbeddingWithLoRA. + """ + embeddings_indices_len = self.indices_len[3] + return self._embeddings_indices[:, :embeddings_indices_len] + + @property + def long_lora_indices(self) -> torch.Tensor: + """ + This property provides access to the indices used for long context + lora, specifically for LinearScalingRotaryEmbeddingWithLora. + """ + long_lora_len = self.indices_len[4] + return self._long_lora_indices[:long_lora_len] + + def shrink_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + scale: float, + ): + #No LoRA request, so return directly + if self.no_lora: + return + sgmv_shrink( + x, + w_t_all, + y, + *self.prefill_metadata, + scale, + ) + + def shrink_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + scale: float, + ): + bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale) + + def expand_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + add_input: bool, + ): + #No LoRA request, so return directly + if self.no_lora: + return + sgmv_expand( + x, + w_t_all, + y, + *self.prefill_metadata, + add_input, + ) + + def expand_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + add_input: bool, + ): + bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input) + + def expand_slice_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + y_offset: Optional[int], + y_slice_size: Optional[int], + add_input: bool, + ): + #No LoRA request, so return directly + if self.no_lora: + return + sgmv_expand_slice( + x, + w_t_all, + y, + *self.prefill_metadata, + y_offset, + y_slice_size, + add_input, + ) + + def expand_slice_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + y_offset: Optional[int], + y_slice_size: Optional[int], + add_input: bool, + ): + bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset, + y_slice_size, add_input) + + def add_shrink( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + scale: float, + ): + """ + Perform the ` y+=x@w_t_all` computation, which is suitable for the + GEMM of lora'a. + When `is_prefill is` true, it indicates that it is currently the + prefill stage, and the `shrink_prefill` function should be called. + Otherwise, it is the decode stage, and the shrink_decode function + should be called. + """ + # shrink_fun: Callable = (self.shrink_prefill + # if self.is_prefill else self.shrink_decode) + # shrink_fun(y, x, w_t_all, scale) + if self.is_prefill: + if self.no_lora: + return y + y = ops.sbgmv_shrink(x, w_t_all, y, *self.prefill_metadata, scale=scale) + else: + y = ops.sbgmv_shrink(x, w_t_all, y, lora_indices_tensor=self.token_lora_indices, scale=scale) + return y + + def add_expand( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + add_input: bool = True, + ): + """ + Perform the ` y+=x@w_t_all` computation, which is suitable for the + GEMM of lora'b. + When `is_prefill` is true, it indicates that it is currently the + prefill stage, and the `expand_prefill` function should be called. + Otherwise, it is the decode stage, and the expand_decode function + should be called. + """ + + # expand_fun: Callable = (self.expand_prefill + # if self.is_prefill else self.expand_decode) + # expand_fun(y, x, w_t_all, add_input) + if self.is_prefill: + if self.no_lora: + return y + y = ops.sbgmv_expand(x, w_t_all, y, *self.prefill_metadata, add_input=add_input) + else: + y = ops.sbgmv_expand(x, w_t_all, y, lora_indices_tensor=self.token_lora_indices, add_input=add_input) + + def add_expand_slice(self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + y_offset: Optional[int], + y_slice_size: Optional[int], + add_input: bool = True): + """ + Similar to `add_expand` + """ + + # expand_slice_fun: Callable = (self.expand_slice_prefill + # if self.is_prefill else + # self.expand_slice_decode) + # expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input) + if self.is_prefill: + if self.no_lora: + return y + ops.sbgmv_expand(x, w_t_all, y[:, y_offset:y_offset+y_slice_size], *self.prefill_metadata, add_input=add_input) + else: + ops.sbgmv_expand(x, w_t_all, y[:, y_offset:y_offset+y_slice_size], lora_indices_tensor=self.token_lora_indices, add_input=add_input) + + def add_lora(self, + y: torch.Tensor, + x: torch.Tensor, + wa_t_all: torch.Tensor, + wb_t_all: torch.Tensor, + scale: float, + y_offset: Optional[int] = None, + y_slice_size: Optional[int] = None, + *, + buffer: Optional[torch.Tensor] = None) -> None: + """ + Semantics: + y[i] += ( + x[i].unsqueeze(0) + @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2) + @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2) + * scale + ).squeeze(0) + Args: + y (torch.Tensor): Output tensor. Will be changed in-place. + x (torch.Tensor): Input tensor + wa_t_all (torch.Tensor): lora_a's weight + wb_t_all (torch.Tensor): lora_b's weight + scale (float): Scaling factor. + y_offset (Optional[int], optional): Offset to apply to the starting + column of y. + y_slice_size (Optional[int], optional): Size of the y column slice. + buffer (Optional[torch.Tensor], optional): Defaults to None. + """ + y_org = y + y = y.view(-1, y.shape[-1]) + x = x.view(-1, x.shape[-1]) + r = wb_t_all.size(-1) + if buffer is None: + # We set the buffer to be float32 by default ,refer to: + # https://github.com/triton-lang/triton/issues/1387 + buffer = torch.zeros((x.size(0), r), + dtype=torch.float32, + device=x.device) + + buffer = self.add_shrink(buffer, x, wa_t_all, scale) + if y_offset is None and y_slice_size is None: + self.add_expand(y, buffer, wb_t_all, add_input=True) + else: + self.add_expand_slice(y, + buffer, + wb_t_all, + y_offset, + y_slice_size, + add_input=True) + y = y.view_as(y_org) + + def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor, + lora_a_stacked: Tuple[torch.Tensor, + torch.Tensor, + torch.Tensor], + lora_b_stacked: Tuple[torch.Tensor, + torch.Tensor, + torch.Tensor], + scale: float, + output_slices: Tuple[int, ...]) -> None: + """ + Applies lora to each input. Similar to add_lora, This method is + used for layers that are composed of multiple sublayers + (slices) packed together. + """ + y_org = y + x = x.view(-1, x.shape[-1]) + y = y.view(-1, y.shape[-1]) + offset_left = 0 + # TODO fuse these kernels + for slice_idx in range(len(output_slices)): + self.add_lora(y, x, lora_a_stacked[slice_idx], + lora_b_stacked[slice_idx], scale, offset_left, + output_slices[slice_idx]) + offset_left += output_slices[slice_idx] + + y = y.view_as(y_org) + + def add_lora_logits(self, + y: torch.Tensor, + x: torch.Tensor, + wa_t_all: torch.Tensor, + wb_t_all: torch.Tensor, + scale, + *, + buffer: Optional[torch.Tensor] = None) -> None: + """ + LogitsProcessorWithLoRA always using bgmv + """ + y_org = y + y = y.view(-1, y.shape[-1]) + x = x.view(-1, x.shape[-1]) + r = wb_t_all.size(-1) + if buffer is None: + # We set the buffer to be float32 by default ,refer to: + # https://github.com/triton-lang/triton/issues/1387 + buffer = torch.zeros((x.size(0), r), + dtype=torch.float32, + device=x.device) + + # bgmv_shrink(x, wa_t_all, buffer, self.sampler_indices, scale) + buffer = ops.sbgmv_shrink(x, wa_t_all, buffer, lora_indices_tensor=self.sampler_indices, scale=scale) + # bgmv_expand(buffer, wb_t_all, y, self.sampler_indices, add_inputs=True) + y = ops.sbgmv_expand(buffer, wb_t_all, y, lora_indices_tensor=self.sampler_indices, add_input=True) + y = y.view_as(y_org) diff --git a/vllm/lora/request.py b/vllm/lora/request.py new file mode 100644 index 00000000..c4b26dc9 --- /dev/null +++ b/vllm/lora/request.py @@ -0,0 +1,95 @@ +import warnings +from typing import Optional + +import msgspec + +from vllm.adapter_commons.request import AdapterRequest + + +class LoRARequest( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """ + Request for a LoRA adapter. + + Note that this class should be used internally. For online + serving, it is recommended to not allow users to use this class but + instead provide another layer of abstraction to prevent users from + accessing unauthorized LoRA adapters. + + lora_int_id must be globally unique for a given adapter. + This is currently not enforced in vLLM. + """ + __metaclass__ = AdapterRequest + + lora_name: str + lora_int_id: int + lora_path: str = "" + lora_local_path: Optional[str] = msgspec.field(default=None) + long_lora_max_len: Optional[int] = None + base_model_name: Optional[str] = msgspec.field(default=None) + + def __post_init__(self): + if 'lora_local_path' in self.__struct_fields__: + warnings.warn( + "The 'lora_local_path' attribute is deprecated " + "and will be removed in a future version. " + "Please use 'lora_path' instead.", + DeprecationWarning, + stacklevel=2) + if not self.lora_path: + self.lora_path = self.lora_local_path or "" + + # Ensure lora_path is not empty + assert self.lora_path, "lora_path cannot be empty" + + @property + def adapter_id(self): + return self.lora_int_id + + @property + def name(self): + return self.lora_name + + @property + def path(self): + return self.lora_path + + @property + def local_path(self): + warnings.warn( + "The 'local_path' attribute is deprecated " + "and will be removed in a future version. " + "Please use 'path' instead.", + DeprecationWarning, + stacklevel=2) + return self.lora_path + + @local_path.setter + def local_path(self, value): + warnings.warn( + "The 'local_path' attribute is deprecated " + "and will be removed in a future version. " + "Please use 'path' instead.", + DeprecationWarning, + stacklevel=2) + self.lora_path = value + + def __eq__(self, value: object) -> bool: + """ + Overrides the equality method to compare LoRARequest + instances based on lora_name. This allows for identification + and comparison lora adapter across engines. + """ + return isinstance(value, + self.__class__) and self.lora_name == value.lora_name + + def __hash__(self) -> int: + """ + Overrides the hash method to hash LoRARequest instances + based on lora_name. This ensures that LoRARequest instances + can be used in hash-based collections such as sets and dictionaries, + identified by their names across engines. + """ + return hash(self.lora_name) diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py new file mode 100644 index 00000000..066d94ab --- /dev/null +++ b/vllm/lora/utils.py @@ -0,0 +1,189 @@ +import os +import regex as re +from typing import List, Optional, Set, Tuple, Type, Union + +import huggingface_hub +from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError, + HFValidationError, RepositoryNotFoundError) +from torch import nn +from transformers import PretrainedConfig + +from vllm.config import LoRAConfig +from vllm.logger import init_logger +from vllm.lora.fully_sharded_layers import ( + ColumnParallelLinearWithShardedLoRA, + MergedColumnParallelLinearWithShardedLoRA, + MergedQKVParallelLinearWithShardedLora, QKVParallelLinearWithShardedLora, + RowParallelLinearWithShardedLoRA) +# being imported for _all_lora_classes below +# yapf conflicts with isort for this block +# yapf: disable +from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA, + LinearScalingRotaryEmbeddingWithLora, + LogitsProcessorWithLoRA, + MergedColumnParallelLinearWithLoRA, + MergedQKVParallelLinearWithLora, + QKVParallelLinearWithLora, + ReplicatedLinearWithLoRA, + RowParallelLinearWithLoRA, + VocabParallelEmbeddingWithLoRA) +# yapf: enable +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead + +logger = init_logger(__name__) + +_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = { + VocabParallelEmbeddingWithLoRA, + ColumnParallelLinearWithLoRA, + MergedColumnParallelLinearWithLoRA, + QKVParallelLinearWithLora, + MergedQKVParallelLinearWithLora, + RowParallelLinearWithLoRA, + ReplicatedLinearWithLoRA, + LogitsProcessorWithLoRA, + ColumnParallelLinearWithShardedLoRA, + QKVParallelLinearWithShardedLora, + MergedColumnParallelLinearWithShardedLoRA, + MergedQKVParallelLinearWithShardedLora, + RowParallelLinearWithShardedLoRA, + LinearScalingRotaryEmbeddingWithLora, +} + + +def from_layer(layer: nn.Module, + max_loras: int, + lora_config: LoRAConfig, + packed_modules_list: List, + model_config: Optional[PretrainedConfig] = None) -> nn.Module: + for lora_cls in _all_lora_classes: + # specifying kwargs so they can be easily accessed in decorator + if lora_cls.can_replace_layer(source_layer=layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config): + ret = lora_cls(layer) + ret.create_lora_weights(max_loras, lora_config, model_config) + return ret + return layer + + +def from_layer_logits_processor( + layer: LogitsProcessor, + lm_head: ParallelLMHead, + max_loras: int, + lora_config: LoRAConfig, + model_config: Optional[PretrainedConfig] = None, +) -> LogitsProcessorWithLoRA: + ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim, + lm_head.weight.dtype, lm_head.weight.device, + lm_head.get_sharded_to_full_mapping()) + ret.create_lora_weights(max_loras, lora_config, model_config) + return ret + + +def replace_submodule(model: nn.Module, module_name: str, + new_module: nn.Module) -> nn.Module: + """Replace a submodule in a model with a new module.""" + parent = model.get_submodule(".".join(module_name.split(".")[:-1])) + target_name = module_name.split(".")[-1] + setattr(parent, target_name, new_module) + return new_module + + +def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]: + """Parse the name of lora weights. + + args: + name: the name of the fine-tuned LoRA, e.g. + base_model.model.dense1.weight + return: + Tuple(module_name, is_lora_a): + module_name: the name of the module, e.g. model.dense1, + is_lora_a whether the tensor is lora_a or lora_b. + """ + parts = name.split(".") + + if len(parts) >= 2 and parts[0] == "base_model" and parts[1] == "model": + if parts[-1] == "weight": + if parts[-2] == "lora_A" or parts[-2] == "lora_B": + return ".".join(parts[2:-2]), parts[-2] == "lora_A" + elif parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B": + return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A" + + raise ValueError(f"{name} is unsupported LoRA weight") + + +def is_regex_target_modules(load_modules: Union[str, List[str]], + expected_lora_modules: List[str]) -> bool: + """ + PEFT supports passing `target_modules` in the form of regular expressions, + such as `model.*(q_proj|k_proj|v_proj)$`. This function is mainly used to + determine whether the suffix in the regular expression is present in the + `expected_lora_modules`. + """ + + def is_valid_regex(pattern): + try: + re.compile(pattern) + return True + except re.error: + return False + + def is_subset(sub_list, full_list): + return set(sub_list).issubset(set(full_list)) + + # Similar to PEFT's processing logic, regex-related operations are only + # executed when the load_modules is a `str`. + if not isinstance(load_modules, str): + return False + + if is_valid_regex(load_modules): + match = re.search(r"\((.*?)\)\$?$", load_modules) + if match: + suffix = match.group(1).split("|") + return is_subset(suffix, expected_lora_modules) + return False + + +def get_adapter_absolute_path(lora_path: str) -> str: + """ + Resolves the given lora_path to an absolute local path. + + If the lora_path is identified as a Hugging Face model identifier, + it will download the model and return the local snapshot path. + Otherwise, it treats the lora_path as a local file path and + converts it to an absolute path. + + Parameters: + lora_path (str): The path to the lora model, which can be an absolute path, + a relative path, or a Hugging Face model identifier. + + Returns: + str: The resolved absolute local path to the lora model. + """ + + # Check if the path is an absolute path. Return it no matter exists or not. + if os.path.isabs(lora_path): + return lora_path + + # If the path starts with ~, expand the user home directory. + if lora_path.startswith('~'): + return os.path.expanduser(lora_path) + + # Check if the expanded relative path exists locally. + if os.path.exists(lora_path): + return os.path.abspath(lora_path) + + # If the path does not exist locally, assume it's a Hugging Face repo. + try: + local_snapshot_path = huggingface_hub.snapshot_download( + repo_id=lora_path) + except (HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError, + HFValidationError): + # Handle errors that may occur during the download + # Return original path instead instead of throwing error here + logger.exception("Error downloading the HuggingFace model") + return lora_path + + return local_snapshot_path diff --git a/vllm/lora/worker_manager.py b/vllm/lora/worker_manager.py new file mode 100644 index 00000000..724c308a --- /dev/null +++ b/vllm/lora/worker_manager.py @@ -0,0 +1,212 @@ +from contextlib import contextmanager +from typing import Any, Dict, List, Literal, Optional, Set, Type, Union + +import torch + +from vllm.adapter_commons.utils import (add_adapter_worker, + apply_adapters_worker, + list_adapters_worker, + set_active_adapters_worker) +from vllm.adapter_commons.worker_manager import AbstractWorkerManager +from vllm.config import LoRAConfig +from vllm.logger import init_logger +from vllm.lora.models import (LoRAModel, LoRAModelManager, + LRUCacheLoRAModelManager, create_lora_manager) +from vllm.lora.request import LoRARequest +from vllm.lora.utils import get_adapter_absolute_path + +logger = init_logger(__name__) + + +class WorkerLoRAManager(AbstractWorkerManager): + """WorkerLoRAManager that manages LoRA models on the worker side. + + Every request, the requested LoRAs will be loaded (unless they are already + loaded), and every other LoRA will be unloaded.""" + + _manager_cls: Type[LoRAModelManager] = LoRAModelManager + + def __init__( + self, + max_num_seqs: int, + max_num_batched_tokens: int, + vocab_size: int, + lora_config: LoRAConfig, + device: torch.device, + embedding_modules: Dict[str, str], + embedding_padding_modules: List[str], + lora_model_cls: Type[LoRAModel] = LoRAModel, + max_position_embeddings: Optional[int] = None, + ): + self._lora_model_cls = lora_model_cls + self.embedding_modules = embedding_modules + self.embedding_padding_modules = embedding_padding_modules + self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False + self.max_num_seqs = max_num_seqs + self.max_num_batched_tokens = max_num_batched_tokens + self.vocab_size = vocab_size + self.lora_config = lora_config + self.max_position_embeddings = max_position_embeddings + super().__init__(device) + # Lazily initialized by create_lora_manager. + self._adapter_manager: LoRAModelManager + + @contextmanager + def dummy_lora_cache(self): + """Use this context manager to reuse the dummy lora model + to avoid creating it repeatedly.""" + self._cached_dummy_lora = None + yield + self._cached_dummy_lora = False + + @property + def is_enabled(self) -> bool: + return True + + def create_lora_manager( + self, + model: torch.nn.Module, + ) -> Any: + lora_manager = create_lora_manager( + model, + max_num_seqs=self.max_num_seqs, + max_num_batched_tokens=self.max_num_batched_tokens, + vocab_size=self.vocab_size, + lora_config=self.lora_config, + lora_manager_cls=self._manager_cls, + ) + self._adapter_manager = lora_manager + return lora_manager.model + + def _load_adapter(self, lora_request: LoRARequest) -> LoRAModel: + try: + model = self._adapter_manager.model + supported_lora_modules = model.supported_lora_modules + packed_modules_mapping = model.packed_modules_mapping + expected_lora_modules: List[str] = [] + for module in supported_lora_modules: + if module in packed_modules_mapping: + expected_lora_modules.extend( + packed_modules_mapping[module]) + else: + expected_lora_modules.append(module) + lora_path = get_adapter_absolute_path(lora_request.lora_path) + lora = self._lora_model_cls.from_local_checkpoint( + lora_path, + expected_lora_modules, + max_position_embeddings=self.max_position_embeddings, + lora_model_id=lora_request.lora_int_id, + device="cpu", + dtype=self.lora_config.lora_dtype, + target_embedding_padding=self.vocab_size + + self.lora_config.lora_extra_vocab_size, + embedding_modules=self.embedding_modules, + embedding_padding_modules=self.embedding_padding_modules, + ) + except Exception as e: + raise RuntimeError(f"Loading lora {lora_path} failed") from e + if lora.rank > self.lora_config.max_lora_rank: + raise ValueError( + f"LoRA rank {lora.rank} is greater than max_lora_rank " + f"{self.lora_config.max_lora_rank}.") + if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size: + raise ValueError(f"LoRA added vocab size {lora.extra_vocab_size} " + f"is greater than lora_extra_vocab_size " + f"{self.lora_config.lora_extra_vocab_size}.") + return lora + + def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool: + if lora_request.lora_int_id in self.list_adapters(): + return False + if isinstance(self._cached_dummy_lora, LoRAModel): + dummy_lora = self._cached_dummy_lora.clone( + lora_request.lora_int_id) + else: + dummy_lora = self._adapter_manager.create_dummy_lora( + lora_request.lora_int_id, rank, 1, self.embedding_modules) + if self._cached_dummy_lora is None: + self._cached_dummy_lora = dummy_lora + return self._adapter_manager.add_adapter(dummy_lora) + + def pin_adapter(self, adapter_id: int) -> bool: + return self._adapter_manager.pin_adapter(adapter_id) + + def set_active_adapters(self, requests: Set[Any], + mapping: Optional[Any]) -> None: + set_active_adapters_worker(requests, mapping, self._apply_adapters, + self._adapter_manager.set_adapter_mapping) + + def _apply_adapters(self, adapter_requests: Set[Any]) -> None: + apply_adapters_worker(adapter_requests, self.list_adapters, + self._adapter_manager.adapter_slots, + self.remove_adapter, self.add_adapter) + + def add_adapter(self, adapter_request: Any) -> bool: + return add_adapter_worker(adapter_request, self.list_adapters, + self._load_adapter, + self._adapter_manager.add_adapter, + self._adapter_manager.activate_adapter) + + def remove_adapter(self, adapter_id: int) -> bool: + return self._adapter_manager.remove_adapter(adapter_id) + + def remove_all_adapters(self): + self._adapter_manager.remove_all_adapters() + + def list_adapters(self) -> Set[int]: + return list_adapters_worker(self._adapter_manager.list_adapters) + + +class LRUCacheWorkerLoRAManager(WorkerLoRAManager): + """WorkerLoRAManager that manages LoRA models on the worker side. + + Uses an LRU Cache. Every request, the requested LoRAs will be loaded + (unless they are already loaded) and least recently used LoRAs will + be unloaded if the cache is above capacity.""" + + _manager_cls: Type[LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager + + def create_lora_manager( + self, + model: torch.nn.Module, + ) -> Any: + lora_manager = create_lora_manager( + model, + lora_manager_cls=self._manager_cls, + max_num_seqs=self.max_num_seqs, + vocab_size=self.vocab_size, + lora_config=self.lora_config, + max_num_batched_tokens=self.max_num_batched_tokens, + ) + self._adapter_manager = lora_manager + return lora_manager.model + + def _apply_adapters(self, lora_requests: Set[LoRARequest]) -> None: + loras_map = { + lora_request.lora_int_id: lora_request + for lora_request in lora_requests if lora_request + } + if len(loras_map) > self._adapter_manager.lora_slots: + raise RuntimeError( + f"Number of requested LoRAs ({len(loras_map)}) is greater " + "than the number of GPU LoRA slots " + f"({self._adapter_manager.lora_slots}).") + for lora in loras_map.values(): + self.add_adapter(lora) + + def add_adapter(self, lora_request: LoRARequest) -> bool: + if lora_request.lora_int_id not in self.list_adapters(): + # Remove before we load the new lora to save memory + if len(self._adapter_manager) + 1 > self._adapter_manager.capacity: + assert isinstance(self._adapter_manager, + LRUCacheLoRAModelManager) + self._adapter_manager.remove_oldest_adapter() + lora = self._load_adapter(lora_request) + loaded = self._adapter_manager.add_adapter(lora) + else: + # If the lora is already loaded, just touch it to + # update its position in the caches + loaded = self._adapter_manager.get_adapter( + lora_request.lora_int_id) is not None + self._adapter_manager.activate_adapter(lora_request.lora_int_id) + return loaded diff --git a/vllm/model_executor/__init__.py b/vllm/model_executor/__init__.py new file mode 100644 index 00000000..7278c7fb --- /dev/null +++ b/vllm/model_executor/__init__.py @@ -0,0 +1,13 @@ +from vllm.model_executor.parameter import (BasevLLMParameter, + PackedvLLMParameter) +from vllm.model_executor.sampling_metadata import (SamplingMetadata, + SamplingMetadataCache) +from vllm.model_executor.utils import set_random_seed + +__all__ = [ + "SamplingMetadata", + "SamplingMetadataCache", + "set_random_seed", + "BasevLLMParameter", + "PackedvLLMParameter", +] diff --git a/vllm/model_executor/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..984b8c1061a008e0c2f9de139077ac1667ba871b GIT binary patch literal 482 zcmZvZy-ve06ou_HX$qpE2r)1)GGu72PB0*Z=-Q%6-Mm<)zG@=u#Il`2Ux$r>SKuKS zvoi4tOxP8u6xEiGbgu8uXSwQj_mN-2*SGu~BlHo>-xYat~RtZhkz9ri#r71=WB;sdGJ0h8(to`Y$Q5H2vx7@;VJf2Ls;R+m>=6K5I4}Lx!6}3yv9Hib^pHAhN{7ew;~yw1b3Kn3k8jY>%ET zrBnk|36Kn)U|ze*4D>J0CYPS?O0+*%%+s&AHnZsIu7cVX(ylR@lrb(PV=u^b0#T5! z5phZrUFg2!W&y5$q3T=>)gzI`J<%(dS0xFJ$u=@!tSAe|*g&tE&XvEcC0zSIL^ky8 Jp&)xga>@)q<}KF8b8*Z4fY1AUz@@I~kwuZT(9Cd9Io+v}N1<8V9i z=oMGclM=K!XzDJAl$?@tdO;{TA?KwFkj&s@hg{OFj+vKLmI#^KjQNqDGM2ZwuaZIP z4H-~yG>|->n`kT)=`iM(Ms6w*?B(V_?yKB-0ap9!lIB&$ykMXdW0&ObN4+#mdWoN` zrm!s1)dyiV4A;V=Uf@6PCC9K+)H@78BmuJF9+ihsC+aONJ0Xd;QIVt@@e}*KGHcWP?Y(OWQC>Ew-5wyHAj$0Fm zOt$H$Jp~;v&S2sZXiNl8$xpOHM}Iy!-i@Ul4u#`~NgzVVFr9Wzyyv7t;f!~5 z3Ua%(nlpZOhrSxmI-tb~ChH*RI-7g{ZYB5^j-NW-Aab4yCsTquM}9hVJe}V{s_Awr z^M=kqIl!?u?2gx+SUO`bogexSyUxQv#WOpCi%u$3>PP$2?UGC;u@b5~oq-Hk*oxE5 zNtA+bNEyo;Np6j!W(75OfHZ|%o0ZYZb*{%Y|4F%Affx0$O3e2g@fs66FXMyvrf77u zG=qi~Ggy<9l1me;ODb`n8`0_$G$3>X-^9oD_{s!xBJlp6?gm>-D ztw-Ol!?M&VQieJCJ_$TA}tgS3d~`shAq_JEQAv4?gMCQ8AQXh zs8xanKU+&2kKUOwb^Y4twD3}i2BCmUrN$Uk`7A)a!Pt-4AShOJftLtSxUc_G+Iw9C+}wxw-4E!Qr} zVHD-1C~~WKA$5^<`6;~buIO=5D|ONC2Kcd{3hG;Wb@H{K3Z83vGE051(np|A1PRZr j$C?8TeZ_B0yS{qtFAJ8w!sJqTLp8wp25rzqx@`Od9A+1D literal 0 HcmV?d00001 diff --git a/vllm/model_executor/__pycache__/parameter.cpython-310.pyc b/vllm/model_executor/__pycache__/parameter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78a8ef1d70abc3032a5191ed2054243469e12c1e GIT binary patch literal 13303 zcmb_j$&Vb@d9SUP>FF8H42Kej+DeNhwnPr8#j;F`T1+j`Hl{>bN;-Dh#jfg^9`>eJ z)g-6g%)z8BIf(@L5Fi3_pn@R491fLRC4AxireXMoZ?=rKSv5`O zt(Mict9IL|I_+FFCv|o!U(KVqMStOj;V%Tm>vnYo<&wXM@?x-p@+!(@e+lI!DX*ct>>on;P_P=TU3aR7 zQM2N&qGmOzSx3#9e;74~gH_CN1m$)A2+Bufj=NC4%fB1tyQTeZl#luwC~pLZ(f_D_ zELidH2{!zD@!c1!G>ZQH{sT9h>M`H-AG~2y@9`h83b`S*$h5NUIs1^mCu&ZkAt{=2& zJ+!u3$SALK2|X5@on};Pb+@-Mpct=y-wT7wZ@>LcI!k=`yjQ;z_}R+4&j;##nfF57 zYX#%p>+f)x4}<3RPBgAPc5cV(bb{8qeXkQujX3(c>h^oHT8`+I&vje#FzR&C!3oZVc*`q*fgzU@0V%3v+2^K(I=;rMyK0Adt_ z1EJAtR%mbVwEaMOMDDyZp)ZnP7)QCHP>HanfobePO^ zanFssOU=%<+w8dBXo4$35x3R#{6KBGA2wSpx8CicFJ_Wj-_?C&l4d9BN>e(wdu1o+ z=qlB{-1LJ{|71?L8McdFOyos7^NL?^$~?(ejpY+pImYF<3m+lzL(ol}=vt@bH> zJc6r1y${5mO833l+f#=D&M^Yb-K<%IjRU7-Gl3~y(n!RKBBJZu^61ArCpd>Mq>LO| zL!$-(h;3Vge-NyZ72d#!Y{!V zsgkz_4;)-Avx#^~ACzQk8Q$g;B(MVQ4lzn@GW=GgQg{chzs~tcL zr3lk_dXBvXt|#WeHEYzJ0;W#P0OpVY^UwOtAdLG~R(V)EFfwYjxKyKI z(Qh$duGK#3d#$7eV%qQ4QJoV-73a0RAi`7;iRwNiaj~bmkm}K%mI}gX6}CRX$y~8l9DMdsTl&<|ocX2p!bMb1 z+Y1?MAs6KRoS(m8z%nTKg`l{*(6Gc-a9}Bn$}}@+DU8ZAGhr!+=3j!eUyj#n+N#Oe zG2Wj*hKYziubP*=C~!rQxqi@q%!5ML>>~1+#YVp)HX(FHwCBpe<8ztpgbb@qkWmKE z&`5P%=w1vsF(-k37b75JfM~siYfIP_I&iTzo>0*Ok~07uAnHlv zfQSy^X^|ho*gPn2aw)EvH}5hB_s!y;Z`O^9%2n$fQ^}ckS&@}gig0YP{y`*z2WBoc zu@7pg40Z@5?YwjNLc&cu?;4DqU2~VIy7zx|B_3vu5XbqBry#)9BPi@UV`&^0qMo=raS2LR`Lz(H)%2bza^Axi zQU~VEWwVHX&h0e|I`#BS49H2Ft@tw7AgII!bw>CZlA(1IM&B1EaKM~}oD*U1krmku z>$;^};DNZwUDEmd&>lLz{i*dM>zad6#%|$SE-C`2`C;zn0(-ItFbGT~<^0eAR#|5b zbGs$rv;>^KBsk68TpVMx;%vobjhAkt5eAWZ2{6I}c^Q8sUn$2nOy{`N44a)W@;dcE z9b#h%NnGlQd&ncA$>4haZa<7_dd_CYhfN*UAZMVC>-0-B-GqW?IgL<|M5`3WL{oAO zQW&@l+}awASoI=DP4O%hMHOF2ZCA2NrZxI^+xbn|c1*`C-Ok&2X>o2BEmC*If@QKf zpSs_ucIm*SwEcS&X3J=s_6A0netE_vC3Eek+M5hS6FqgpUcR1z^Ca*J@MPu3lU3?)8OVXJCEGwaO zmd@rn+d$)l(l#jl$b3pqsKsSvR1Mju}_Lek0`d*@nwD#ru>DZ!2jJg5!+)Cwns^E?ZLl@ zf;*oE9A;V?;$IxFG-RYUG-lY#6^UKUF)3u|JhK9mjkR&cw$TxVNu!|IflWY}13z@q z5YTFSA#Kh`VjCMLI;pkQcbS|)0<~O%PnpbjP%FR9j`Otguh{rJlEKD%Q&FkkQX9X+ znOr7g#u1+3`hUuL(ZlN`u%yFlRID#VCwXP#9BPz(ei6|MwR#rh*QUwi> zGjbRaz)>bv`jSbdIIXA!nG_1K-3mI@0-`(gdqrof7f?UFdwKz%0#`;N$GO1c3^#)<2aYmP@)EvV zY|S2JM!pnh8T@$W!K9f!X^AK1Eh2(H3}I_u>>}Kg7__jtWeT7mJx(;wrE&{~(JU;C zc}{Y|cm(IK2yPw;cjq_);@o|y^XQ0u(1aRXpSzF5G7lLz4oAKQH>evzvEpU)HY39@ z>)};)qUOM3uzU87-PyZU?*fzwmKtx4F#Zi2Y4i!i-?*(M^c?)+|6yBo9SlEtrHPnt zc5kmD;&~Z(=|mK4Gxe3GDT)kU>jPt@^yx>-j69O!Fi!#sDbr~*x81N8)SC?=w6%9)E}7V^1@e!ya`*@eybIvv%%8*4Nqy=Zd}?#-Q&;j@&8RI5 z_>xmV?$FhY%}cR)S&lpudy#fs)4v){sP!baI!~kjBPNk>>N5Ro_x$X8u2i{;-5Te0 zU_4`O5GWhpIZj9`F~SZIA+QwGJM5)3*-biD_j<&a{s$x5g zl==WUT$4!DKn&-x{HEweka%RnBlrvBCi*AT%AG?Aj?l8R4C8okeD2<+24~2{Es+DVov=CgC=x`=){w#`!~)=7}w3glV1ztO#hKq!00zZCV^Bn zCJ!+A36ojNZ=&Jf*?1Aj;IXe=eC*Q9#0%-C=rQXhf1f#7QoNP1qNHo{pKSaCB!j2F zcFEb%5cLp7W^TghGIQ%>=#cZ0HrEjHxM>b)XIc@1K(^SQU>oLr90#9ap`-e^unc?D z8A3Z@q=P^bdZGQev^xkcX`7Uniq7BzkvqJcy3m5~7kQfcC~|@3Uc?QogdnLlw0Ld$ zq65;gWIPJjv_ec>kwIIPoOXME58F{f&xe_#Ktt2L%o7jV&lTY#es1>QDi2-$3!gZM z`9cY2b&a7YgrVTTJ~0pbGuXGXI@8c&C{Ypr>UWut0!l#U=^KaJwMRR)g9+DsJ9rVT zzs47mcW5b=myS8*a?vi@WqdHzZ;O_%qdD^{CZ$`u$V?g{3l?5p*nyZJVt^n=BP7Ue z7>&Fb8N8}r%3jsu+Fo9GAUMB>yqF}&m-V}j>XJCY%V3CAiCql-RtPzEf~oQ39(b%K zVU3Q0)~Mt|h*6YU#xlT-Tj^d`MI-y0Y?Xw>(Y3X=>0Sr|S3hOQg!^Rzy(BCPKZv|$ zE9C5Cz+|oDa-6Tgp??Jjsf-dn&+fy)NYEcvSFpO^2DWOL{Qk!K`>DF;1 zI#?~Y&nALKMh$^S1g;$kTsvbWEAH`rE?$<%9d51knuua2Fu?zgS8LOYk9F;S!uUQG z)#0Ilxp}JBfoQx%mdNMwri8ig*JwMJeF2gx-;7|RT;+F;OWeYc5&GjBB_VJ%(l;n_P&R?~>Svjur{W^RwTzAIhu(b)@56E7pD zDWmBt5T&9+X~K|i_-KO{9iKY5WVh&4zJxA1hA715+WU6=VC@@&!4uuJlpN1AMXuK{ z=P~T(79CR^^kY~z|38CgXYcJB*A@mjG*eFIu!fe77IO|0k)7XWj+~=X$aT>3S=Nm~ z7Bwn112yee_(IBmN=IwK`3k|w+X!9$CT}l)WtC8Zl;pQ!i)flFDHU5FzZiS>{tnof zE5cH@V~+gN0_D}KUc1+VkMR2W_uqBj5jm22EcAF_6FuWXF5>TVCq`n4h7xO@erhz! z_*v4d2VnGB@Ff&RKW`|hOSA8es0)k%061;~6 zdbk_rxxhhy-4xhFjBDl*HsVp4E3Z`pts;ljHglJlOj&4T%ac6kEo27w>a&p1G{23& zRBbNdG8r?o@G+tNPAX3l(VSgJ(^RfY(J}oo&yR9sESgn^MVHCcN<3NLOILbu){c^u z?pRQ&?!3i_{}C937PbnOl^*0>>Ss-jLhJfS?k@A;8ItzlaB=s zWy3>QVJvW#zEWiD!7kI)8`}~175ly;o{w(RGo|Il@_yp?nQg8Z&yV3V9gjSOo z&dH2scsA{Wr@l!a^JC*A!5a;F2J{3zF}|=&T!$iWePS~2mxge+ zRfI@4@^H#~7>P^Mq3xG8j12>jMp$i;5OL1k!qE8)`>mNWqKpyO#f~aUc?sn|7K;id zsLcGi)Gwp{b3G@93buhhC1GUFuZgoovf#^1?!Vn!Fs9M@vDG&bbE6)v&7lV0%uQG^z>a9 z+NBTGc9m~d6Ts`RrGt7j!u@f*77^G?DKXUtn34&TXR1E5mAuT8K@E<>7Wl{om*;b5 z(J;xe+tTOm0*=?WA+J-;$>68egBp9k%c#o zsqn-5U2F4nY&Q3;>T z4k|P8wE29i3)Tr=KxQ;rY_`F7XZClwK4R{zJ=gCBVdpWRC~@jkzyko1s&}xmeltUd zqOuqlKvyq9G;WoE+H{6;cvZh0#Cg1vz>~JPu;Yc&gQ~MyNYapade-u0H ztr784HVZb6%mn`}BIXrzDle7I6@&`p-aD`Fzsq>(Fv^v4Wix-f?358uDB*^j{dkj> zM~}R@hSY&r!$cXmZ=-_WIeZeAz=LfIk4xYQtQ!pyS%OxiG=h9D;g5C%s94AJ*Bfx> zI|N$E{+xRui>+Ad=lGMcjHrFcdIAY!VT1%vAOsftg*WRPeN!E#W&(VO8u3v%^Mtd4 zhgdf-s$$C2N+CZ5po&oUM}&caOTHZ_0!#W|{;Q%R?UnoDg+}uKF+xRP<9xK&Ba;wM z+F2l+@Z6X-?>Bl;jQB-mhwr{ z0_WvR0l8z83pu&IQoQMrD2q4{;vi@T<~kdQO96gj!*Vu-uZqv>N1QKU=SFxMNg2`2 oqE$A__<)8yo==`wd2{W)wc-l@y}0tNQUz$aZQ)r>@k}}Qzj$$nng9R* literal 0 HcmV?d00001 diff --git a/vllm/model_executor/__pycache__/pooling_metadata.cpython-310.pyc b/vllm/model_executor/__pycache__/pooling_metadata.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c03a4f1c274988316da0ad703cbb28eac8c4c924 GIT binary patch literal 2405 zcmZ`)&5qkP5GE-~mSuaBrb&|^O{+tI)Ief4rxuH($RYuXqU|C;w-;Rmtw`J2>Mta{ zi4Etpx#j_iq7RTmU!mx8=-N|XpocU+ossOd<8BFX#34Du`TbP%dLDss=eOS{f42zv z6PeAyfw>P;Pk;y_Xi72~P>O4ovdjvsj0Zfk13Pm9r{1^HR?vcTp1N5(XxDpo>Sdjv zL&?`fIHL8Oh}H=Ux{?C#iZ<}=6Lvy_p0Iu(eeW-Ljr3{N7LkeKG}5}yD{C*GRs3NR zn~HyzXj9n-(=wI&aIg2MDAFV!KZ;b8Y1p`&=&(%kFq2uKW?^(3C24e+$`}=I4i}jF zF!cctNdhWJzyu8}!6cV9Oy??*tr5FQt`cEMS30tdoZ!uwu!S>nMN7EPdEg0Kc1BFJ zg$G<$IN(gDI=8g$dubwcDEgTavSR#Eq5QHj-!Dq3B9j!k_LJN<6X~OwCG3rPh&}fgotiYAyex`&O-{RK;;3AZUJEfKsDnyTgFaIA`Hsds%$nO}_j&u}vl z`qoN$STPfAbJ+jD$!qR#Cb>L8-MKa4{=Qn96+P zeT*vXjLk%&QW?b)zq!~5R^Hm`%EdJ*tvVaZDo05(Rr!95{jnOjW-uSX)E@y^&;<#} zf&rxq3n&u=3WDyxv=p|T>LBR?f$HX3rlYOxmqeY1(^uaPO;!%eq`Yk+HJ0Y~7uj@@ z-N~K|)8uei&dj99hsSA}4YNYXG?Y(eJT-+HF6#=~psBn;Ijh_-O!CBpA)2TiAdcfu zkIt`bmFZx0F@|zI9k>RhXuSLc76QCHBEJA0z?Z@oJUwFY<-fMTTYG-MNgD0RT%ofX}riNr~Dmf{4KZK zH)na&2=sd>oJW}q!^#Uo2-`HpbvFzjPos2s(h5US#PEs@x)p}1vXa~=^j~4Rm4)O& zeV~m}7m>x3Z4fk7wSi#D^%Wp4;goeQF!t87*c*ql*Ua|*x!~2e{)15X*nJfJ637an zeuS_GmgGmabOgH#2qi4R0e$$93jj+-0J4Zk0I6-!Mo6u8mRUG}ZYY$VUtBIWL1Iu> zV|9MJZfeUaN66zOmZ}Ga#OV2IV0bV(b>)8!3qwxGGt1B;wxCCrps?Z#fYcZK8DEeE zn7v?V@CzpF7Zl8M=6{@a{{tlfVQS=q#y_SDFG!pM|0nZRGn-PeIvpG-{x mp~j@twO_wFMct?(FQ7ua+F!9DB7iWyOYRiY6`UX=%lnOFL?tYR*pg?#}GI zSluIXH$95vC2So`z(#^xf*8~xK!`kU34$DQ@F6(_$R(Gc5#-_khHgpZ#Clnq@2~24 zawrF$LH}J<{dd)0e^veU{eN{QmopXo&i>`!?|$hOMfn+J#(xXQT*T!+p(={2xN1{r zDYl|gnl;rHvzbUInu(TXYptZ6Z0WY%O4%t{-Q4yXGCmH4oXLYA?DOZ)uykSvPk_ zvzPHS?-ua1aLDWxMmf_Kba^mI@2j(JAo z_%>Tn@N@|!E4VLvDex_>D3Tjo8JQwJNT97K+j)1zUA>dEPY5ZGxyRA+aUtbNSG%fI zPQF9>tf*14%6WAkiL_e{sh{o)n=63>~)*oMukQB ztJPMwS#R%Lsq$*ekFtaO=6=_!B%ag71vif&Dokl-vm=!3MJeCA>C|06D){jzpo?M{ z40=KulsrwZ9i^dd=tm@Xytc=yL5D|a(jM=$xK_pzYLcf8=SFSmNTt+TCLXPWh`Gu{1Qx6?lJTC>?Y)9Sch)A4S5wH`!2GwgFK zo-OO$eN5htBjr7d%6*b8&kjya%&vj{)`z?g@6&UrV7yPffc~*Y!c}^zUwA?Jo{GH2 zEzqPWTZzkbk$aSKu5uOj=&Hhxp|VPfAIFWa5Sa&w3X^*G`7%8s%f=UpoFGD~lEgwP z$~jKA;|Ca%df+&}jUu7_yjoOsHIKjcvBnNQHM`pr?`~9zk>NP)YRhw+$aEZNx!0t0 z)^Tq3s!jQXC#l&)y&Xga$Emj4ouEp_z;_&O;1Sn|WQgd*nZyXycBAyO!pj4@NqCInw(w0kWF zGv0q-9BEP0DploAay6ldDMU~j7p=jbq($k3e7D#~2WJi5VDbrShH2&C9c9b2( zP25eR0!>cs&HHIy0>2nbUZ&J&=$0rehEohT^`)W;nMJ4XtNbzI71Z z+33`-sq0BFiItR9IjyP(6i2J<7NJ919rJ#0rw?)hF&VWNLgQV!C{gjlGP ze+2|^QSr$ZEeKFuvd(Nu$slp)G%D4O_@vABrK{|k`u#gb-e<}SNH2VN53PV@9e%1wF(&;?S2 zHeB5^-ISMc4O{@60CyQT3vie9a%nSd0_x@Pb{?F0+)dmI@xAC3+&R2e@`|XjfEwpw zFgcHs5;)VYvR!n`?!uiU5Xiin_6mToi|!I2Y}qxjU{|B!v<|E5(C}dOTH75g!FIP7 zSg=e2+xhF3Sj=^-Qd;XUOr~k!w~2+_^Q$|aRrjr`k5`+`&aJPCXE$!#5cl1>>ynkj zPUHF3la}21J=&qX=IQnI^@?BvUa1H4e1iWwqCEgp07D5T*MT~KBRv<^au4;!_~yE( zL`rvB)>V-*y54py^`QKm;3fG#Mg4+-3cQ^TuOFaNS;L@Kp^JeC5OrO=Nj(#ZOMJ&4 zRW`Ka{yB?~w$<6TwyFV{wV@_0Vd#BX4@oe)H|MO=v>8gQC;I8r<`84L%DVNuM0I3d zx1oVyhp5U}liR?9R)7H_8&|J3EunC$(+g;5rIsx@DkE_ou8MO*3r;?P_1Fcp4gI5b zfiFoCsk>2SB+Sz={!^ToDvw{bB=#pAiLS=gtXb6WI8(81?Rwms=p1!uyUw9q>=G@j z;&?%*^Vuucp1knX zm1`5q@;#1)RzDEng`|;7L{>W1Ecbf8C)*{{CYJ)b@jR`i8LbL_xxBzX(PaECWU2;M zl_qnQ3(F&>uQb*)8*Wv(i)TJe`BOd4y>`&tpB$VzmXvR`s`YkrA4VN>g*Jj#uNl-I z^8>GIxmXOY7hC>kB6FnBXmR!|k5(UUPom7&2u4f(?2(8TXX`4`<+|RerLg}Fev~F? zB47!jSi)nD5;9OVOn|ab15JcUP%YF!#~_N%2_tg??fh#*E)uy!4|nBXbxB@FU&zUW1Px znWF$o1P~bq2Yh}G?N8{5+$ho&K{)?+LC80TuGm`&G@~Et-_`o5zR^$j%|H!}0G{$& z%3E-r?rMGG2g(mrKnn5+j4*vqBhYuK_R~Q!Oob>3O;-b7Cd>eABscCSqNE#O%dm4Y zGT=yQIr-!hppv9X;}Xsr;W)zg*Yn?HeHD0Fg^Vmv8;}u_D0j6#U^juG;fTEeRsdy+ zTkr~z&x(SGt$!g>1L4-)XZ%@uLT$X!Eo39BqgTH z!SU&ePY)M^|K!2STD<8^A45f5es@#)}WF<%jUy5P&cS2```>dcus znxr)RHFDwxKTH?E#O*=YrjVCv{gvC6qme8?~0OTyp`>*(7XJv!crqAQ^qG35U|Ty4W(_m8(f4SFn;_53ZiLcE11 zI{LN3e=6Gy7t2%z?xF1e63dDEJ<`b>$(~m59J+!0KMPa$%E1B*z7ZZ{jpT>wux zoJZNxp$fW;pm^$Tx?c$Od&k29wRLkr&Ih!ziy3%A%qkt5VLHrgBYa*lsz1Trc12qM z2cRi|VnApL(oewl7-&7j#~hk=0SwMrmn;GZggId@h0`vfFM(F!gpQE&07J}tiNJ-0 z(C$<ME7u68QH3!>}kabkm2*QkuB1$b(xfARH ztzHqw8kF~;rdqS-dIEq;h6yYTfq-l4!lR?J5DCA1=`I1@)`Wy`dmt-^r#6yALK2cf zrbugmz+yZ&1_+)s&?>nYBUm+wo^M=@)h8Ey}(O@(wQF0MQ9d9062*b_N`rg(nf% z7QpWv!cv@q%$1XzphA@P(0huMd*Z+%()PT4U&c6(6EC4zaX^LeYSY8HFdbaT0}gSR z@d#CdLp;xwu$kb5ooQ9uI4%N4^<3MGrxZ>nqXfdk{84&E_in@MbP>X>ws!#Q#e5Nd zEKojhEbP$q;mMqMuiEY6JrRJdocJ}6VTjZ0qC7B^PBbLBEqpUOE#f&Mf01fe>vrkE zAr?xoO*y40s2-_}NGHC!JEjV8!n1&C^bP1jfnhhPT7sUOR#Cek|7;eG&aqEtTu>^?FQ z@{T8TU`9`m4DptMTv;utOfv;%o@t3Pm7p$`Fb$2k!HYNI6iZ~YgR~A#6Q_-$HneKQ z?kNtdkQ=xF1pI~D`pazAWZ+?Va>S`mHCz)vIvl--%QrAz2XXOh!2LEN;*F%Ii<5Vm zO5n6yrs%|7rs%v~rs%9)rs%X?rs%9)rs$+yrc9(#6hTMS+@OegBTe!0hUpd&DW~&# z{KZEUraR}B?r4aM=kNye^5SESM0`iY-7^qDPsVvYzNg~55#Q6e7r|%7_Xk=<1b)?> zcgv!sId=gFU@YqW*j{_8?QpW>BX`1lABnDUqa9L>)_8r+8prqG5(K`B~QIVYOYZL zwmvTU_+#p0P(;}r!X_txnaewBc%rNbEeh*qkP zSXO|O;cw9qDH|SjZlw6uLpFDiQI)2km9{;PWYA3jq>*M|@CcFOJ*DQ8}kM5Sc>D z1bqC;qfw&P^raEKNp*{;oiLmo%65q~i0l%n6WJruB+??%28puxtZ8Vm@hMpc2Y~3l zpnLe*vEv9m*y$nY8T)``pK4IMk-8nJoF7o7zel7`M1Vi~D8^pc4|64E~3 zCPvbVT@=M*#}rdzA1%(FV*8j5{@Yq&%fk8A)P=Rxbpui`<9C=kw z03V6(qUhTw`6m=q9Oh&hdxKCCp&8;QJY{@rCWsmd19?&78yZl|?7#MlZ2sUg|G$+~ zqwh8bKbl{Z7aQRVP(Yv@Vyg>{t{$ebHRSn8il-w&j;})+37BbiHz@*D6g3|&E4Fjs z&=4NY;Eo{1-2|CTY)^CjB1&`wgb)A{HE8{kC}CqI^P)r>D=CW-d;mFoYeAIgVBX`1(pb7MmM)Ic6mFp) z$_5rzw}%k!XhDf^KGcja6Gx?+amLQJ0eZIZ0o5L6ZzVmp-o)~ovFP}4g@V0+dU=sZ z1;kF`qq#3V5hVf7@Yx%ESYzvOgYf;Th}lLg=u}qto5Vj_*uTCZK4l|Bm%c}}(>Hc# z%L!~_V0-X|2(=y`7~1(Uvx@HnY(t_&N=clERC&tU0x6<&VcO!#VjQ+Th!zJG@8QL% zwa?#xpg*UD{u3nh65Nd8qN4K?{X>c()1Pw2hb3tWhCFnVFj#`WX1fv+1?PA0V2@NtCf8InoU@S48vo-fN%&*zV}>BSh9JC#V0~WC ze!yB#aU7h78~G7D$`AE8IIkP*Jw45SshjM7S%&?OmSw-ta_n7}FaEdKwD`BsUgeaK zF3Jo)7WaHkhy|X%+}Kez7b+B29zS&dKL-R7`*x z`C%!={cTe`?FDsgm1HsHLD*=?y-sBmHIN4)XDKC4I?hw-6(XM_LO!VM&J5k+J52c* z>;zF_VEVkH7^}3Df=dnuTRSq}SAL&; HuB`qai<(yx literal 0 HcmV?d00001 diff --git a/vllm/model_executor/__pycache__/utils.cpython-310.pyc b/vllm/model_executor/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96650f932251ea2b47925e9073bea2ffd3406426 GIT binary patch literal 1114 zcmaiz&2AGh5Xb$sn>0;I1r<_z@(o3(je0_g5NbumfkS&}xvW5|G)oKv1-Opb}_c(B8n=O(u_qc;}Oq9Bq)yYG>8J|MH*(cs0L$!uB%Y-flxJ7e zbG@qBfNhdZ0`se?3G+?Gk4d|AzWK~0sg;Am$gEI0mHLhDm##27-vEKMx##=&)Qf|p z?|l7e?2;nyrH7Z)-?Cb(Sf6P#b)zI7dVvEzG}@IW?^8^_HjEjx$IxsOoFg+br+`fW zf*zUx#@f{2_AiN9f$sZW$7SQrI2mudo*8Pl{U9qx*={!Jq{(S#3~Ls7=PXULPQ@MH zaCb`F(9U>@fG#$@ToqZ2Xjs-FMw6OV=zKWQ$#CR+bJ52==Zx*+A2%i-K(-3}2?R5$ zXh}?{=rP0}$tQ9^Ufg}oF4zTkbVg=u&dk9jvC9fNp+2*}1 zIFn=TB(9m9mQGt)PPs9 VDk?!XXq`3LZO(+C45-$A{sL8rD}w+4 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/custom_op.py b/vllm/model_executor/custom_op.py new file mode 100644 index 00000000..d0e90245 --- /dev/null +++ b/vllm/model_executor/custom_op.py @@ -0,0 +1,71 @@ +import torch.nn as nn + +import vllm.envs as envs +from vllm.compilation.levels import CompilationLevel +from vllm.platforms import current_platform +from vllm.utils import is_cpu, is_hip, is_xpu + + +class CustomOp(nn.Module): + + def __init__(self, *args, **kwargs): + super().__init__() + self._forward_method = self.dispatch_forward() + + def forward(self, *args, **kwargs): + return self._forward_method(*args, **kwargs) + + def forward_native(self, *args, **kwargs): + """PyTorch-native implementation of the forward method. + + This method is optional. If implemented, it can be used with compilers + such as torch.compile or PyTorch XLA. Also, it can be used for testing + purposes. + """ + raise NotImplementedError + + def forward_cuda(self, *args, **kwargs): + raise NotImplementedError + + def forward_hip(self, *args, **kwargs): + # By default, we assume that HIP ops are compatible with CUDA ops. + return self.forward_cuda(*args, **kwargs) + + def forward_xpu(self, *args, **kwargs): + # By default, we assume that XPU ops are compatible with the + # PyTorch-native implementation. + return self.forward_native(*args, **kwargs) + + def forward_cpu(self, *args, **kwargs): + # By default, we assume that CPU ops are compatible with CUDA ops. + return self.forward_cuda(*args, **kwargs) + + def forward_tpu(self, *args, **kwargs): + # By default, we assume that TPU ops are compatible with the + # PyTorch-native implementation. + # NOTE(woosuk): This is a placeholder for future extensions. + return self.forward_native(*args, **kwargs) + + def forward_gaudi(self, *args, **kwargs): + # By default, we assume that Gaudi ops are compatible with the + # PyTorch-native implementation. + # NOTE(woosuk): This is a placeholder for future extensions. + return self.forward_native(*args, **kwargs) + + def dispatch_forward(self): + # NOTE(woosuk): Here we assume that vLLM was built for only one + # specific backend. Currently, we do not support dynamic dispatching. + + if envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.INDUCTOR: + return self.forward_native + + if is_hip(): + return self.forward_hip + elif is_cpu(): + return self.forward_cpu + elif current_platform.is_tpu(): + return self.forward_tpu + elif is_xpu(): + return self.forward_xpu + else: + return self.forward_cuda diff --git a/vllm/model_executor/guided_decoding/__init__.py b/vllm/model_executor/guided_decoding/__init__.py new file mode 100644 index 00000000..368436aa --- /dev/null +++ b/vllm/model_executor/guided_decoding/__init__.py @@ -0,0 +1,45 @@ +from typing import Optional + +from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor + + +async def get_guided_decoding_logits_processor( + guided_params: GuidedDecodingParams, + tokenizer) -> Optional[LogitsProcessor]: + # CFG grammar not supported by LMFE, so we use outlines instead + if guided_params.backend == 'outlines' or guided_params.grammar: + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193 + from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa + get_outlines_guided_decoding_logits_processor) + return await get_outlines_guided_decoding_logits_processor( + guided_params, tokenizer) + if guided_params.backend == 'lm-format-enforcer': + from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa + get_local_lm_format_enforcer_guided_decoding_logits_processor) + return get_local_lm_format_enforcer_guided_decoding_logits_processor( + guided_params, tokenizer) + + raise ValueError( + f"Unknown guided decoding backend '{guided_params.backend}'. " + "Must be one of 'outlines, 'lm-format-enforcer'") + + +def get_local_guided_decoding_logits_processor( + guided_params: GuidedDecodingParams, + tokenizer) -> Optional[LogitsProcessor]: + # CFG grammar not supported by LMFE, so we use outlines instead + if guided_params.backend == 'outlines' or guided_params.grammar: + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193 + from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa + get_local_outlines_guided_decoding_logits_processor) + return get_local_outlines_guided_decoding_logits_processor( + guided_params, tokenizer) + if guided_params.backend == 'lm-format-enforcer': + from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa + get_local_lm_format_enforcer_guided_decoding_logits_processor) + return get_local_lm_format_enforcer_guided_decoding_logits_processor( + guided_params, tokenizer) + + raise ValueError( + f"Unknown guided decoding backend '{guided_params.backend}'. " + "Must be one of 'outlines, 'lm-format-enforcer'") diff --git a/vllm/model_executor/guided_decoding/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/guided_decoding/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59e35da15bd23144043052809da15b5fe6ff9139 GIT binary patch literal 1416 zcmb7EOK;RL5VoD{lU-W+pu7&@A*~j%8&wE#;UxhAq*5wXAhef|uH9`|CypiwrD!Vw z1=o+#i3Qd3;%Es}S45G5wBF0ioN%Zw zf`}6cx=ec<4HLh5{=^x*LHI-m}3ds2a^uS{j{COFhhz??QU(os@pt zS6oO(ycgXhR?r?J-Ol|wc^pv2w@UB-%h_Y-?|V6=y;NVhG7*mf6f{lH#hX@Qg)F@f z00UX|6Tmdrh7)DYn;8XQ~I5B2fa2>Uf% z&T4x4xSBpx%~$`UnxTaL^FqFPP=oskJc%Pp;~lN#nUoY9~& z15*g2)WNjg$n=>?r!31G%%YTDKM3ihNG_s%bO9lMYxC$^pF{N`mUS`S2(k5mZYG_X ztSTed5AnC|jrt83r~=(wvIK!XIgD#8!txy~n?#Cxx*J5=tVSjoGZo|+cc%L0?87+?4eoZgoGlfpb#OYsRS(oAF#Zhmt7}gk2^C? zvQfE^Ub!P7J^%;40v}|qoca}z5WMGgHl@U?d2fF2&AfT@{>;kJsPEvn`}-e@OW$$+ zB=F|fMeqoR{vHW;I1^66V#a9h3b*LQox+Q~!jJu;8+VId+-r0l5fot@GH2i69`|24 z+=njj^4?Eg+y{ozfQKjzFWpNP4|xA;XEJz=XE+mPL!O#66RFk{-+GVp%vkRr*Tx1< zE0fDI6*hdftbjKv|9Ocr+V*Lx;fdrRGBWh$=OK86Lq9_TPRzIyyU!iyfCuQ#h_MXK!G7IVlX&&zo-OS2OwxxLpa&K9`=U8NZ$RcZ{V(r&dX zIxUg2W5~?mq;G>HDboUy#P*YS0t=he&|LcC=A;GfC32lqRK! z>m*SWL+j0?6u&v@E-fbV*SAwsRMRT29-36m!94u1s29aTh0Cuai0%fIq_}oV$}@oo&Bap9=D193SQKp&BZ5AONiuAWEK#uA`+eD zLPRr&Y7IOxGE!hv4yVwp5ptfJs8TWmt)<#IRMgjMKx9Z{L}Y`=CJ_p0#*kcnE>5Tn zKEt68kQnEsbIdra8FIJfotAqo_leK_OZEeM$+&yssRyRZJGi6wi}NeC_K#=*@9ACa zev^tE`G*e0Y*% z;#yCGIEq)@oMf^t&2{~&cWLJ+Nh+yLl9xGdc^^fZ*h$?g!j&OG()P3c$fkRl?o?&j5u zTvzvz(9xP~_8rf?k5}2ovHA_&X2l_v@L>I1gH}{WwriFZzFMm%r8&QGmi|l{kK>?P n^0YL0);u^(FE{3H^27#vt7q$Dss~NgJapb;LwDPA``hea*mkB& literal 0 HcmV?d00001 diff --git a/vllm/model_executor/guided_decoding/__pycache__/lm_format_enforcer_decoding.cpython-310.pyc b/vllm/model_executor/guided_decoding/__pycache__/lm_format_enforcer_decoding.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04aec4d8e8a9337525a98c1ae8393616746591a0 GIT binary patch literal 2399 zcmbtW-EJF26y6=LH}=|2oaU!ZO6gDpicxE~5>jtMMQKX~Drq4tNGqY$WM^V;JNwg_ z*`|qH36Z`4*N6w;f>-E+>@8Qk0vD8X#*PybE|6GjX6KxlIsfNd=bcXAz;olbH>2yY z`W=arPaTL)Vdyt7afiFoN!`$OaZMyiYoTY?HCa#n&`;;WIZJ!8kv79-8iYaG3R`JA zY^R;Dlg@|pcE2tc(r(yI7sEwM`|@166fW8IoLo*qd_gP$uLHa;UwrKyy5U8BZrkZC{Q*8WJ-2E~ zRYoz3M*@ht%o*1`QZ*hHCdo4H{oE>MBG zX=)tu3Fis-wjI+Nld*S5Ue(8S$ZwskxQ-K0c3U+gvXVXig#8@eNf~M@5pp5DflQQ3`tF|CKw*;7{Z^*>-+~ zMo~@JgrXt+>y;mm@&qb)30QXeb9Z3UVo)T-HNzCd=i1G*9HlqXy@5=22F1P^<=Fs> zd64E@$Vj{tacOcjnAX-&xCc<~kcE_)lgb~uqhL5FoU<=LyOV;W#`~)Pzh}k zGa3DhfKxio^izM+Ue&i?aU}iV`5l&Jxq-lD+Nd%%5bTMd#Ns^rh!+fr1Wj z;Dy&t>$w2DO>68a0LVR_~pvxm#wj$#ba{3RY`4Vc^sim?|F}hOSVEG7A4fKoO*q z9zXhODg1s+iWDufP@~WrTdv`i)93q1W(3qNMn?A$RumSgH9l+`#WIZw0JGIR@WW+1 z(g#2+Bv>;2kSQz{)x`-*Kp~!}_i@62TJKIoz(Sp^(8eY}SSiIVtPlMmOl_y_c3n!k awKnMzQX?*bVhjFS|9~`L4HR*^Z~q3&mZ|Rm literal 0 HcmV?d00001 diff --git a/vllm/model_executor/guided_decoding/__pycache__/outlines_decoding.cpython-310.pyc b/vllm/model_executor/guided_decoding/__pycache__/outlines_decoding.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da22e84ae19bbd19a6a5dd77c7766d976daafe87 GIT binary patch literal 3451 zcma)8OK;rB5oR|zoDs#5h7xQ>aRPi z*Bt|%PyX@G{pStC_zyPTe(KQpHGIW~#4xzQNo;uiKk+EEX>2BzXAzW{aV4?6s^+b@ zmW+9h<}2}d;(BgU_v*TB#}mnvq*Je6GXE+q}GkxYBjT2_l^l38z-7#jv3lWn85 z_#I}j-6Ey6o^})XS9mwc3t((fgh4J^w5;rO^H`LX2Wgb0pqalf#f}W3RPdebP^8f_ zA#VkRXqjc>_U9YF$vTlL?#nC`MUlyJ`pfNm?NQ;(mgtD5BT^W9W}_Q%!S9GL<5AkV zAIKmnLUi)&2Nt-$)b9Z!43BUFmY8@Zr=G=4ufi?Q<`oFgE*lp|+|0Nrt%D*<%ZfxF zN;}-oqEM984$PGVvfZkdHQ!HzMEHK`_Fy_=BuA@pw7Z%gtNkop zeG>KSayVhK4GR?L=e#DxOn$93Du=*WXISj4Hq0#_>97CIth&tPd75*Ckz!so~ZsZ7Eb&rXhv(8$K0a zYViFFzG4T+ABZw~#wmG7Pw0u+qjJ1w7Ioajig40}N>lW}Jg_+B=ATJGRON;J(m1ht z){Ckls@9`D(ldL;!PratE`Z|~2J8;&-vC{PgA2A1JrOAjQg$yF>CL;#kb*o=(QYi5 z6py>2P)jV_7vUk>%OvZ^!iI&(bWlPGS?IxnlryMvruGF(1>!&;k6EmNf#pMF&?BKq zN6)RtT+#i%7Ay>4wtkoM9tGPyX4kag5<5#^YYF`3EK3KS_9LZ7IT9=h4h0KXu^+^7 zb_DC1A45Ngs9Ol_T>-nCbyF^6d<=NQKHuK#yU_j)mje)KJPDrqN0~f?;95>O?L%O` zcEN{P5gbI>=)~D>5c_Ihih%ohmc^wLW@*@!Ql!eO?sZ{>Md3Aew8eceSRZWivKkb} zX&7bR6pk#rX$p?`ahB!YB((e}Jp+VtR?pr~q<@ykvT+14iy{v~;lrxII;5?NyaEBp zMIdEG`&(MtSqq0gM<=;}7RxyI8N6t7eugsO3_|+(guj3;1rASt?~d0HZm3NtvERTS zIGZ@HWee07O4o+zIIsWgS%Z=r&s0k|702bs(w9Uva|ebJW)EU{Bc9 z+57`C|$X{=s!9P1m`V;jB#bJl_B z9@S0&Xux@L3h!U&V{lO?6z&18J}>?Ly`VG;B|%76ekSqs8O^$LBaRA%D6*E|1U3-c zaG^$?ee_?p(8>+HLd6;|!+x*GwO6F~inQcUp>u0e;tj8Kq9RJ+Kuh7sws;h((!tZ* z*Y_5E=*bzdlM_g0k)V5U6kJ82S+#j6Z!_v}IR{J-E!mAG$ zUuR+@WL=y-3|nA<+NMif*bNTs28f#2)PmR>=CJJ$i}uU;yl}CmL#(CYbD_4r0%?yb z;q(}c|{GFbXKav)0 zKfC%MJxsHs^c`qN%wVmtw=z2hj`TCD(}~v(olc1dR_T0&mx^^MGYLu5!663qw0=wQA4GW zm_xaQEn6Mu0Id>>USd6Zu987oz=4(kxXUVv*6<#-a`7IvGJ=OIXE-tnjD^mD)EUCO zcr$u9IAwCUxOD)W@jffZ(F24R`6-egBSE~>2qTw~V@NRkx6-A7O~ZBV26zaU(ltb) z;vtZlQG+_ecM9rmhPrghv|MYBnv~Et_|+%{5OZGBg?A(kl+01kEe>_~cPM@BniJpu E0GhIQ#Q*>R literal 0 HcmV?d00001 diff --git a/vllm/model_executor/guided_decoding/__pycache__/outlines_logits_processors.cpython-310.pyc b/vllm/model_executor/guided_decoding/__pycache__/outlines_logits_processors.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6c6685aa2de1b69d729c473043c7ac0bb238c65 GIT binary patch literal 7157 zcmbtZ%X1vZd7sxVW@i@<0w6)~r7|qBxO4eMGJ;NiVg|EA}CkJafXBKUhIJL zSamYL#k4xlYJ2V`+D{Pkn|(v1*fN9 z^Y!C<{l1U+3@C0(y4 zZ>4_V>$;hIBfT74*7Zj6&GhZyZPVay^KTuR!MFM4op;18|MtlAMu+QEBh&$2nR=lFYQ{~6lz>@?b^_4pNDWoOvJGn1X= ze%oS;?A#Lvt1U6}p3yx2CEfIT(=6(Yx1(O7Slm)*))LVRTT!dSo0Y;{jglnVN_bKK zsGRi?W))6Z-if6uD)+NE&+wshClU|wcDF^8rjd|MtEgx~#UzU*Q8`BU=B-=3nDN5B z#WOA<#fwVW;eE=t`Qx%)`H6@X2hD}8UYxK{&`{VGc^XP=FpX%xKa3>5n=_u!ybZxO zMHFY8ZRQVo7VmRG;~1*PS**e&-`?h8|DV?~>?h8){k*5}=F-U4q=qk~Z3e8p$^GwO*7V+ZvuZqTpc;xp`aD${Bl5TPve6*PT1k%nNUg6b6NHCw+ zaaonhc9l&q7o#;+1CPC8jt+k(-;R~s5P6GBnTw*bO*h}NKzjJ2Iz6j+$+K3cJH2$>r`|;fA4Rad-ubO0peQbIRPm@P;c`9!@Q4an|jrFlMr-%2qBo-h!*1 z$XYfi41YCL=+XD%!Wt>uP2W7k-+`?xsxkASBbt4aS&PPjQ;sv6S^tQ(o>%D`JNAx) zveK_C8hvo^nWbCybt79)RT@_lhnDzIxq5DmRTyy;JXPLKecNQUXA?9MHuU%;%1Ymc z40zAXgQ}Y9R}W1!{m>Gd{p!xl!9;du(02-b5%n>1(bzUIQd6_6UXFfld;u>1gQ@0r z=KI#pX*xmMRA+V;>IT|pS>u^~;4-uC_9w2R=V-C-l9lRPkg>`A8}}vmRfqdG@7?uP z?r+7B)a{Z>ko`99)t6D))fz`rGyYg%E;Mvm;5LmfU*fk;21AQc;Oa53C2*1`5rGF? zQ~0JAB{w&l)uP&kec=%01i~VMIY>e`+J*pz-CV{B21y39(9tb)B+$oBo-i(CQPCD8 zXvAqZ7b@&Vsw11GcpF>Tzjz<&+@*eKW}HbVX^SZNYOl+ESdc2?kKz_@o-3SABs)b# z^}4W(cEU5U2nxXzggd08mC7K_qqFd0NyMV8#l<8#1j!K&)ZWG{0kIb31cSz{Pgm~V zUHLTJSlRqoe2co~N92y}rf_IGVhR(As=g&DPEn(_iT*VqNWq~ztxn;zdr5*D?L=K( zI4RCb`q~!n5^*PDENs@q1vG?D#W$$CD*4ap+Px%_D%A9Yx~>lfU`9tb6HLK7bYk1Wy5yO zhB;+fR>O2&PT5t{F{@v@wukrc((7`mb=W?vn?)%PLl~$KmZb;(6Bdmy{5hnk91+vl zjR5p8!~_-RsGw~I)lHtkR*LUX4_UEe1)WNqLGd0QNt|$vDa&&l(|YOPKzi&olkQ?3 z{l2=1VrbI2&7DI48(v`++_lXoV8R_%Ax8~Y4W%nb+RiVal4J~?G=kbM3XV6hX+gs-RRoQ<@SOM!bxhKMhdM^EiEFUb zCRb_k3FsMmnA0XlSJj#YM+?nzwPq1?nkeRebnlb(R}~8wfncW=^kR0m19YnK9(fW- zO19o2+%f#oWm4P5BfBV|TxN->;N_>4edP4bqsnvRxq0AZ=k;h6Bj?Hy-8-Xu2^C!K z(=1n!CpJIFLB6oH*RJ)QzI|vNRFnsmsP`+R4p0Ug$8e&r;9mcbKF~>*O0e~(s6u%+ zCj@uJC)$2FQfdOvfaV5@I8s4>m;1X>223!NbjbM$N&^+;qt3A&kcJF@Z`PFI=<5s{ zk)tUaB@+q60ImCgEradvWhw#|X!&CT{<|#aGJ8k)z!DfdL1>`)3B*H0iEN%2TzgDN zh=1SL&J(dkvgb!WK*BamRvA5D;f(K6T%sd}6Q4gA?_nr)ABf98_57dx;PQjf%>DV* zD5J4SVc>*pCSsO(1ar@;R2i3*kfP<&Y5rR@RquQE{Y0apdzv7V^`Rx z#w;Z(36vq$DvhdU^u3a>?DAIJj$09Zy54LEg5+X}ia8X8vm^7YsIXqzmBFbJ^;6d% zsuQF*;ug*HK8uoG32Ovd0`WsC$jUXVrEU8Wb!f-)H>d?(c{ajUrB7O=aqW||U7=6+ z@knyuQ?oOGAPqdQEf!$KOV61Fw0MK+k3xo~_ln^YWOejz%YD-cORJ}i@yFC0`$WAR zy_4G*c#KCjP`Jj7Mb^!;&U`Isq=o0iZ)n@Z_t6c{Kf8MK)~n{p2C6LY;`cgwaRVd< zZe9F<>I6*TEIy?A1{L~3$Gg@WCj0#bp3;Q_h3#K_eGh+>11G%wCI^mq#vyx%h|?K3 z^Z)%r#%mB6_-({)8VNEWLDm>Cw0UANw@=XdO~Ll5p_} zaYB1d3NH$$m3Q}sE`7|UHwgH@4Y@R5pE38(zllo&H!jv_jWMIg;z^5tPIM?dDIM@R z^OB=C^`(zKL_`y5^jH!UBynlM(~N#LY6Pk2E&m0DGM)mTG3(G4?o%WalzC)5h2#4t z^dH&JQHGAHJT-O@!lDiz4M*oFcYu7ge?BnTgsLe|)hUR@82mTpjC(xg!lRQ{cbLm+ zPn-jU=AhQ-W38P@O?gsH>796owux2YBef`?%QFsCjZ=1z2O9wKGzQW| z!cBWPg0cRcBub-KFMF%>@RTj2Im1gwZb52?tSCRHJ;s0m!KrY#?J_{&>?TFP$SsuN z&Z}ZcB1tw{Qfx;enHZ^gV3IH#As@4C0(ap}fyHxEp6&xlA$zjOoW8XHanV_i%DPTl zKm}&*ZQNd4T?_B8f3)`Lo!jec`_=2$9{>5Z8^1CArV}h2XYFzuQAyZSG2*5BSUflj z3-X8~5uvv$@fzd=3J)0rp(Jqz8o!Cj`U@nfwEul!9<$J7oQs7Mq{fu~6N6fX8)n0r zML_LX`|rOR$t_<`asdFgNM4Fs5khQxgtsg$fcxT@D#OKheM>CKSMdI~pk zXl`Rs?dr6VL$2rVFof&-DPCh(P}G69%0wd<|BWF@q~PH`_&rwz;oj!MkH%Mcj1pKk z&MOd}X%p7v73vG?EoWB0MyoGvC1aRbT<@ng^^T*i5q zTt`r&Nc<5vy%l)+MN#M;!ZfaeSxV7p+!iH+I}WB!wo8md0nM>f1+)Y*V+87p2~eVi zac=Smf(AuMh4wIiOEnE82z2}sk0hySG#t~jJS2I}SQcVW3pmVrId6GiS3O8ABvylz z=CVGIxIrS}%&@uK!vBa6B?@c>A+}(4G=}u|a+ws@2|mgQl@ad@)hIg`oINr5 zIMyXp8@Sy8^49n`XdJg18cPN>N{}z}?2!x{UDkC6a;*6O3~`zkqIkThmOa6_&)@>W z&#r~{@2!QO+}|YAbT3?8S^apenAnX(Mu#B`tRra+IZM2u@t|hcxX|+HUN4hqH%Qau rU!riQU6V3&4fBHOf_)8h1~qz@f5_pXx9Ut;GvApxJM&F 1: + raise ValueError( + "You can only use one kind of guided decoding but multiple are " + f"specified: {self.__dict__}") diff --git a/vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py b/vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py new file mode 100644 index 00000000..cf2162ed --- /dev/null +++ b/vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py @@ -0,0 +1,63 @@ +from functools import lru_cache +from json import loads as json_loads +from typing import Optional, Union + +from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser, + RegexParser, StringParser, + TokenEnforcerTokenizerData, UnionParser) +from lmformatenforcer.integrations.vllm import ( + build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data) +from transformers import PreTrainedTokenizerBase + +from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor + + +def get_local_lm_format_enforcer_guided_decoding_logits_processor( + guided_params: GuidedDecodingParams, + tokenizer) -> Optional[LogitsProcessor]: + """ + Given an OpenAI-compatible request, check for guided decoding parameters + and get the necessary logits processor for the given guide. + We cache logit processors by (guide, tokenizer), and on cache hit + we make a shallow copy to reuse the same underlying FSM. + """ + + tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data( + tokenizer) + character_level_parser: CharacterLevelParser + if guided_params.json: + schema_dict = _normalize_json_schema_object(guided_params.json) + character_level_parser = JsonSchemaParser(schema_dict) + elif guided_params.choice: + character_level_parser = UnionParser( + [StringParser(choice) for choice in guided_params.choice]) + elif guided_params.regex: + character_level_parser = RegexParser(guided_params.regex) + elif guided_params.grammar: + # CFG grammar not supported by LMFE + raise ValueError("Cannot construct a guided decoding logits processor" + " using the grammar option with the" + " lm_format_enforcer backend.") + elif guided_params.json_object: + # None means any json object + character_level_parser = JsonSchemaParser(None) + else: + return None + + logits_processor = build_vllm_logits_processor(tokenizer_data, + character_level_parser) + return logits_processor + + +def _normalize_json_schema_object(schema: Union[str, dict]) -> dict: + if isinstance(schema, str): + return json_loads(schema) + if isinstance(schema, dict): + return schema + raise AssertionError(f"Unsupported schema type {schema}") + + +@lru_cache +def _cached_build_vllm_token_enforcer_tokenizer_data( + tokenizer: PreTrainedTokenizerBase) -> TokenEnforcerTokenizerData: + return build_vllm_token_enforcer_tokenizer_data(tokenizer) diff --git a/vllm/model_executor/guided_decoding/outlines_decoding.py b/vllm/model_executor/guided_decoding/outlines_decoding.py new file mode 100644 index 00000000..8a7ff38b --- /dev/null +++ b/vllm/model_executor/guided_decoding/outlines_decoding.py @@ -0,0 +1,133 @@ +import asyncio +import concurrent.futures +from enum import Enum +from json import dumps as json_dumps +from re import escape as regex_escape +from typing import Tuple, Union + +from transformers import PreTrainedTokenizerBase + +from vllm.model_executor.guided_decoding.outlines_logits_processors import ( + CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor) +from vllm.sampling_params import GuidedDecodingParams + + +class GuidedDecodingMode(Enum): + JSON = "json" + REGEX = "regex" + CHOICE = "choice" + GRAMMAR = "grammar" + + +# https://github.com/outlines-dev/outlines/blob/main/outlines/grammars/json.lark +# the main difference is that we changed the start: value to +# start: object | array, so we are denying scalar values as the root of the +# JSON. Starting with scalars as the root seems to cause llama to generate +# without stop. +JSON_GRAMMAR = r""" +?start: object | array + +?value: object +| array +| UNESCAPED_STRING +| SIGNED_NUMBER -> number +| "true" -> true +| "false" -> false +| "null" -> null + +array : "[" [value ("," value)*] "]" +object : "{" [pair ("," pair)*] "}" +pair : UNESCAPED_STRING ":" value + +%import common.UNESCAPED_STRING +%import common.SIGNED_NUMBER +%import common.WS + +%ignore WS +""" + +global_thread_pool = None # used for generating logits processor fsm + + +async def get_outlines_guided_decoding_logits_processor( + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor, + None]: + """ + Given an OpenAI-compatible request, check for guided decoding parameters + and get the necessary logits processor for the given guide. + We cache logit processors by (guide, tokenizer), and on cache hit + we make a shallow copy to reuse the same underlying FSM. + """ + global global_thread_pool + guide, mode = _get_guide_and_mode(guided_params) + if not guide or not mode: + return None + + if global_thread_pool is None: + global_thread_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=2) + loop = asyncio.get_running_loop() + + return await loop.run_in_executor(global_thread_pool, + _get_logits_processor, guide, tokenizer, + mode, guided_params.whitespace_pattern) + + +def get_local_outlines_guided_decoding_logits_processor( + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor, + None]: + """ + Given an OpenAI-compatible request, check for guided decoding parameters + and get the necessary logits processor for the given guide. + We cache logit processors by (guide, tokenizer), and on cache hit + we make a shallow copy to reuse the same underlying FSM. + """ + guide, mode = _get_guide_and_mode(guided_params) + if not guide or not mode: + return None + + return _get_logits_processor(guide, tokenizer, mode, + guided_params.whitespace_pattern) + + +def _get_guide_and_mode( + guided_params: GuidedDecodingParams +) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]: + if guided_params.json: + if isinstance(guided_params.json, dict): + # turn dict into hashable string + json = json_dumps(guided_params.json) + else: + json = guided_params.json + return json, GuidedDecodingMode.JSON + elif guided_params.regex: + return guided_params.regex, GuidedDecodingMode.REGEX + elif guided_params.choice: + # choice just uses regex + choices = [ + regex_escape(str(choice)) for choice in guided_params.choice + ] + choices_regex = "(" + "|".join(choices) + ")" + return choices_regex, GuidedDecodingMode.CHOICE + elif guided_params.grammar: + return guided_params.grammar, GuidedDecodingMode.GRAMMAR + elif guided_params.json_object: + return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR + else: + return None, None + + +def _get_logits_processor( + guide: str, tokenizer: PreTrainedTokenizerBase, mode: GuidedDecodingMode, + whitespace_pattern: Union[str, None] +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]: + if mode == GuidedDecodingMode.JSON: + return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern) + elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE: + return RegexLogitsProcessor(guide, tokenizer) + elif mode == GuidedDecodingMode.GRAMMAR: + return CFGLogitsProcessor(guide, tokenizer) + else: + raise ValueError(f"Unknown guided decoding mode {mode}") diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py new file mode 100644 index 00000000..fb8db44b --- /dev/null +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -0,0 +1,223 @@ +# Copyright 2024- the Outlines developers +# This file is adapted from +# https://github.com/outlines-dev/outlines/blob/main/outlines/serve/vllm.py +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import json +import math +from collections import defaultdict +from functools import lru_cache +from typing import Callable, DefaultDict, Dict, List, Union + +import torch +from lark import Lark +from outlines import grammars +from outlines.caching import cache, disable_cache +from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write +from outlines.fsm.json_schema import build_regex_from_schema +from pydantic import BaseModel +from transformers import PreTrainedTokenizerBase +import vllm.envs as envs +from vllm.logger import init_logger +logger = init_logger(__name__) + +if envs.VLLM_V0_USE_OUTLINES_CACHE: + logger.warning("Enabling outlines cache. This is an unbounded on-disk " + "cache. It may consume a lot of disk space and should " + "not be used with untrusted clients.") +else: + disable_cache() + +class BaseLogitsProcessor: + + def __init__(self, guide: Guide): + self._guide: Guide = guide + self._fsm_state: DefaultDict[int, int] = defaultdict(int) + + def __call__(self, input_ids: List[int], + scores: torch.Tensor) -> torch.Tensor: + """Use the FSM to bias the logits before sampling the next token.""" + seq_id = hash(tuple(input_ids)) + + if len(input_ids) > 0: + last_token = input_ids[-1] + last_seq_id = hash(tuple(input_ids[:-1])) + self._fsm_state[seq_id] = self._guide.get_next_state( + state=self._fsm_state[last_seq_id], token_id=last_token) + else: + # Note: this is a hack. + # Lark pickling does not work properly (silent failure), + # which breaks the RPC (which uses python pickleing). + # We need to find a better solution. + # On the first time this is called, we simply re-create + # the Lark object. + if isinstance(self._guide, CFGGuide): + self._guide.parser = Lark( + self._guide.cfg_string, + parser="lalr", + lexer="contextual", + propagate_positions=False, + maybe_placeholders=False, + regex=True, + import_paths=[grammars.GRAMMAR_PATH], + ) + + instruction = self._guide.get_next_instruction( + state=self._fsm_state[seq_id]) + + if type(instruction) == Generate: # noqa: E721 + allowed_tokens = instruction.tokens + elif type(instruction) == Write: # noqa: E721 + # TODO: support fast forward tokens + allowed_tokens = [instruction.tokens[0]] + else: + raise TypeError( + f"Unsupported instruction type {type(instruction)}") + + mask = torch.full((scores.shape[-1], ), + -math.inf, + device=scores.device) + mask[allowed_tokens] = 0 + scores.add_(mask) + return scores + + +class RegexLogitsProcessor(BaseLogitsProcessor): + + @classmethod + @cache() + def _get_guide(cls, regex_string: str, + tokenizer: PreTrainedTokenizerBase) -> Guide: + tokenizer = _adapt_tokenizer(tokenizer) + return RegexGuide(regex_string, tokenizer) + + def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase): + """Compile the FSM that drives the regex-structured generation. + + Parameters + ---------- + regex_string + A string that represents a regular expression + tokenizer + The model's tokenizer + + """ + super().__init__( + RegexLogitsProcessor._get_guide(regex_string, tokenizer)) + + +class JSONLogitsProcessor(RegexLogitsProcessor): + + def __init__(self, schema: Union[str, Dict, BaseModel], + tokenizer: PreTrainedTokenizerBase, + whitespace_pattern: Union[str, None]): + """Compile the FSM that drives the JSON-guided generation. + + Parameters + ---------- + schema + A JSON schema that encodes the structure we want the model to + generate + tokenizer + The model's tokenizer + whitespace_pattern + Pattern to use for JSON syntactic whitespace (doesn't impact + string literals) + Example: allow only a single space or newline with + `whitespace_pattern=r"[\n ]?"` + """ + if isinstance(schema, type(BaseModel)): + schema_str = json.dumps(schema.model_json_schema()) + elif isinstance(schema, Dict): + schema_str = json.dumps(schema) + elif isinstance(schema, str): + schema_str = schema + else: + raise ValueError( + f"Cannot parse schema {schema}. The schema must be either " + f"a Pydantic object, a dictionary or a string that contains " + f"the JSON Schema specification") + regex_string = build_regex_from_schema(schema_str, whitespace_pattern) + super().__init__(regex_string, tokenizer) + + +class CFGLogitsProcessor(BaseLogitsProcessor): + + @classmethod + @cache() + def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide: + tokenizer = _adapt_tokenizer(tokenizer) + return CFGGuide(cfg, tokenizer) + + def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase): + """Compile the FSM that drives the context free grammar generation. + + Parameters + ---------- + cfg + A string that represents a context-free grammar + tokenizer + The model's tokenizer + + """ + super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer)) + self._guide = self._guide.copy() + + +@lru_cache(maxsize=32) +def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase): + """Adapt vLLM's tokenizer to use to compile the FSM. + + The API of Outlines tokenizers is slightly different to that of + `transformers`. The decoder of outlines, returns a list whereas + the decode of vLLM returns an str. To sync the vLLM decoder with + outlines internal api, the decoder should be adapted. In addition + we need to handle the missing spaces to Llama's tokenizer to be + able to compile FSMs for this model. + + """ + if getattr(tokenizer, "_outlines_adapted", False): + return tokenizer + + tokenizer = copy.deepcopy(tokenizer) + + tokenizer.vocabulary = tokenizer.get_vocab() + tokenizer.special_tokens = set(tokenizer.all_special_tokens) + + def convert_token_to_string(token: str) -> str: + from transformers.file_utils import SPIECE_UNDERLINE + + string = tokenizer.convert_tokens_to_string([token]) + + # A hack to handle missing spaces to HF's Llama tokenizers + if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>": + return " " + string + + return string + + def change_decoder( + decoder: Callable[[List[int]], + str]) -> Callable[[List[int]], List[str]]: + """Sync vLLM's decoder with the outlines by returning list.""" + + def new_decoder(inp_tokens: List[int]) -> List[str]: + return [decoder(inp_tokens)] + + return new_decoder + + tokenizer.convert_token_to_string = convert_token_to_string + tokenizer.decode = change_decoder(tokenizer.decode) + setattr(tokenizer, "_outlines_adapted", True) # noqa: B010 + + return tokenizer diff --git a/vllm/model_executor/layers/__init__.py b/vllm/model_executor/layers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/model_executor/layers/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d3f145932ac021b781c09acce4df76be2406084 GIT binary patch literal 167 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HQeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_er|qBYEFD=MQUbb)WD3zye$V%Oycl)QDnat}QM^+EFChGD*`EESCgH1Ss31$m8X90~la- zX4x|ffLqH-%8>0GqVmBPmmOCDholOr#D|=6%Ab%bS8j7kC8<*BmMR`nkrE-l*R$VP z!cLV2=(p1!)6@OluYdjeO*lQBS8!eX`+wT_<#USi@01z7rjfaY6s@X?!W2hgD${(` zQSqz$T0?hqH7zq7L)MuM%dxs;nTDk)s?x|hIay-(`Notp)hIZH#^BR%T~D zQc?a4$`?>}c28kv|0?5L%xU;r$Z6~nJI9}8=ef>j7Zi2@zczlK;m@&)xHmR0A@?l3 zVXeSy_S}c2^E}g5l*;q%3wK&k95(D)J@&S1u@?sRS}Q;%45I2MWR2yDnq=Q@Qc2CP z=!wO=ZIAKf;yRDrm1|q=KHu2wO}i`{Pg;rQ25M^yf&x4(-FMg#Qog6 zty&O!Z8?xTVX)?{qqG>|O*`CM--z8>9E+$<;~KsUWNslv_aL|egeZ>2R7YnTW^Xdm zfw^WdgP9*1jx|5-;gSP z(T?~-*XKd6zW=!3aZ3baeVp_>tH@^)bvLonhwB;?-n(?%SV(t}qOU=8)J?2UU{5=z zJl0}$U+ZAyGNYsYL|fAqlmnIW9gP`jzNA3c=BF>e^Jp~`^^L0maInp7uhH~*g9mYn zPyVn4sA58~9fzd!otj`*D%DSpaD74psxAYkrc_~~zU~;L>_a^#d7MqXajWAf;_Wj0U3Ua>7>#aBxi+=49 z7t!Ki;Z&QCl9}#s+;r&Y@P>%MIitOxwigB%s*biBQ6s6)L?SglAwWxNQyS`HO(m6T zOovJhs8p9Zh3P+mGCTU3dPaF{#QLVuF;HV7S1M0zqlC5O{PE`)o zWXi2e25`e>B#3dv3{_}}$>K*%j^)m`y*jUC6N9uM7O2T7h{S{z>Klp48_oEUI8Wsl zsJuYxc54B18!i9P5_41|wXV}U=8<&@VJpUJa8n!U$VqFY;`+CDQupVa!P+7xHd#n9ZdUX$;RPF#@5FxeTJY}M^I_!5fcw8cfbqX7-3 zENseR3rjIIG>fU;EOwi%D^&kS5bg4Wfuq54C6{DfK&Qc7H_5w1t1X}I3$FXHRr9+w z8P{cD-F4+q5^Gg*uqadamq}De5V(f~=~Uz(ibxSzBdZu{IafB%5-XNm!k z0I0J`eu?*)_wy5&cZvT&#JhC&&HG;n@t(l2l7WvC^Ks&x*hiOvhoA2euMva+Po`?k zrU-YvMlI$eSSRiy5~dV=86sBplzsI8Zcs<p{My1~nAqQDz=#Q?dsnp_ZCy}*kB#cT9r{=J&t;&+7z z1v-=Kp31>8IoGZGwJ1VM2}}t`+9JMzB;lA>Ej6posK!xyeqixx{{;kDF$FO)W14A? zE2A68fb&MSoXxQvRKSg^T9fk zbD~xFEu=2%zX;YJnoW-l(%oTXCL^rBjUK)L)>p73tu0Md!T;G|{)ZN@SW)vTYI4Ih zm2A2*h_f`g{(c~7QNlx9M@1zso}v2wW-yTzB_;}_uikEu%*q@AtXDLNHYJTM;RR1f zyFJd&6YJ~w&r|N7pw*MPpU}~t-3v7O6J{s5`ee?h*@>@{APi5O1`+WJRlWw%E>9RT zSiepiaqpsg!t@=ai15bLSb??FJ-J%3PPOHBVQ5LdR?!ke@TN$Ss-~-M6qHGx@5Z9$ zQZpBQJ|Uhcs_oN50~)lbsM5}rX+aDkzCz+f5?3J-J&J{(g_~NuN_S&B&rNDK-Cgi? z?)$w(FGuww~ zBStO37mM|MIJjV1cw+nTxtMvNm6S6|39U%tWhv9FcPDr2pqBXtL0HnY~Z-zS+eaO*L>yvniqivLhC~7n6oSHvs zm%DR5$sHxFQ=*AKp>;D%OtH2PClu3%7msPvJ-L|rigKtA5lO`y;tO)107>#1UEkZ{ zNmi0Vz;_Of!5qXTG!xI07)7%rhb@#pp{W$qye0{uG(MGMY@nk#B{>no?C?Y|eAw(j z9U3^z6qF0KF!A0Y^g(PU+z?EFlMY3iC|4^ukxA3qarZ@&Z@it16oJ8Fklp)@wrelM+jIuwY@_P{N)5l(aBD{oT z{Z%|U0w9O_yH8^5Wdxd^Qu%uj?b5NA5n{ymMjw=2NPT@D738E3z+eIlR7wyTxyHee zd$;A)w|);A`VQ6{`s#r4i1^6%2S~IfNvp4C|L%grzW&+Gu9=-olaNKm{uNcG$U6Ssldb(e9{o$Cp0)L|0c%s~Q>`75hcehO|DR#8EKHnBPT%8i z-e18fRB(2?A1rKtUmj|1H=DkP&{m9qmw>zSX``$ zU2nS@iuJ|XgJ|*kwU=J4zVyn=FTb*VU3U4DZJ6NF$0IpNgvQ=3MDz{!h1_`f>|_eZ zisX<*>buj}o(jjmMSBUyqSn4VakeKxlLn?E5jNz3rlag*vB(`oMGQ6dk|r$$Vw&I5 zQ`ALXZC^V1-Gs{J%CYT(yt9cu*??*WjPU^)|24gmh+YN)jB|#d(qkARf~G6-A%-!8 z(6f$Uwjl$oCbw{`rvN~f=Qi^T%34;22PhCQ?M$&eN(%^@L&8Z*kNneY2G1f~-Yum- zx#66WNB<%WIiD0)>WK3V!|mZy0suQ+ykR%PD83reBQ}Mi8$3pUt{;l^A{+m$2q?rc z%#Okaw}Fj*xY7rz|2L1GY^Mv8DFI$3T1gw zKLDN-^KbWB$e4g?Kc&`8s@gu`3xGQL)8`S_L5K`jW9+&evG%dl8tNfShZ zuq8M|C6l^!H7a;OLiWyZ=Vaoh)S~n3F~z!wFpx@PkG?$!d2kp&_-lCjACV$j=J||; z@PQ$drdmgao<%Uhz=6>^Dr*S5j^XI{g%w5?Gf_uxG1F1(JoY?!j>1N_hC1Pqn6R>w zO!vJ1&(z=+MEmTqk#w6Rri4r~Ee*oDpF8TL{N6>-A&cBWir#^sSO*+$3`s1+8o&Yv z9MKH~+!66LI%Y`$T;P^tv$i_sf&N&d`i{1#Z)wqU-&KCBcIbox$aLYMr7k<2HCPSd zaA_vm@MQeMv4R>xJ3Ee951WtNo>j>e7eo$DUQWTKN2OiqGmw)#lJoAz*Ej2I0K}Cdw+82*$Z4|xQ>T+dv+@F?&Nav@-dP~Mp6CDssj0a)p z3o>xZmNITlU=q^z-lV$?5;P?7EfNxRQb-I+pX*1c_y;5jKVndPic^ix3{Cr7)3yKA z4DEli=FDg1@hjaUF;XF6!;1w3QOw!j;RO79qZ#!C8)!E8Uj#ACZQ z`D59GM>d7CtL@Ujn;3j7YtIo`2Vra@(AW$`%vp80{q18MVK?yc!hXQ*sLAWzn%5(D zx!#9;kikdIikX;xxP$0r$-RAN^^Ny#ufFm2vU~6D(iIJ`eZ3n561k(remX9zHal?09`K3qz>eT;l`0V12tsv0nE9A&{(1z;V28X$CJ zXrhG%6=QiBu)ZwAF5)}Xj|Iy0g7pD-ovegRH3$R&-I?!YtG(&gd?!D+mtK%ll%F=3 zhl7vz0AD0oO7U`?6F!`0WP{PqUDf_qB0)HEzBH*#`bn%hysHXgEoWg;>sIXf-CZJw zMl$9jkFfISB0XB^l8At6e?XE;=2X{S1(EtDd literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/layernorm.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/layernorm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e77cc9c9ee384733f0d75ae61ba3d09040cd2eaf GIT binary patch literal 4661 zcmb7H%a7Z}8Rra#6vfr9*X#8wPExi=8q2YzY?8)}n>cM9!-uw;z;=639|Wx#d9|iU zWk~HqkWrv>t5v3j{tC=&1$bb+^B7D6LlOAZ;k{(>%WU zX6AeTn9k4p7M_*g|79&WZCQV#&f!O(^Ig2z&p@QrvLur{WhRfXgDg*bL6YiT z6nBSF-c6Grj<%J~8jrBQRksW8y*>>`apBw^^kY>xKTgnJt+T=_*T2_q6Yt??e%#6S z&`?&3;U-+NmR;plTb`0a3F#=DQMpQ-rPJ9R?_;aOq$jHn=35nTotD1x)~j+3JoqOr zWk%E-_37>KvzB>XS(Eb*oK~%BR~O`&r&b<=4ChNFcydu5hlKgByz`PgfprTw-LVdr z%kty{(OUf25?1Rtw0yRx-2Bn4RcLtl+nZ^c2R#)fLH5BQ(rQ-m4c`y&TuXcXL9Vi3 zE4Xwu*bFY9(Zm9spTGXotzW$Q^Wfz`^|N}gxuzguGw5bPzNUg$MLJP3*i_xSYk9-S zx~V!!2YDK-<#|85(riY0tGm%i_1$K4Cu^>}x$;WmmCLWcaoKQx_Q^l~wDa%3{_*N+ zF}K#0QYB&59jd~{cEz!cNQZ|B zP&9tPoH`hxQ-|q>(z+{E!IPw@z8~qRr*g<~@>I9iS|UwUR=B0M>Y@;t ziaSQMu0p=1D1FF^Y8bZTD9gg|q4oI>nt89^?05T@@<`uR`K8Of!CG&nx7m!lcbfg} zd@W6y8*$ug_EM?hP;II90NXXokShNu*c$z9y?_mlL4#C&4R7`@kP%zA?nA*lEa&U?F1yc0h`a&Q&%gwHGJ4v= zICw_H!Lde2#L|5+5+iHfxo7KU?n; z?pgW<_#K)(!q1SS*he@)Nr~el-m$TlGjeu3>FzSbIH&yd3BT`-aOxd@0v|At-qij;3s&k6~CZgVZbfB(e!JS+M$NX7HdcZ!9HKZjA=UyEiEi3=9;Y~$4l8n8X8A-s5QgoHv?--XrTRXj#r zypNg)FQ@qzM`B0NJXSdC)Ki>}H z?mdNY9<$cnBj(c=NvMqtK?eCzTh%5CZ4YGhE?BekAU-Tr;~p2V31=>B0NXtFDrB|E zh}aYEaQUcRNX$%1p)5`YFv=$w@;{VsA1MEYk$qJ8My2B*#Vj)XA$1cBnX20B_SKfr zbM=s(7f8>Ia5AS~$ANT%2>HHl5-}Qn1+BU`s>>CcKjQ6k6Af%<{#Okni zDDAv$oNwq}q(c8T?sRZdlqu9UbKg2QS1nj}^m@M9%5@ZK)z{lNMMkz&1jnlg^zxX$hSZWJIl5HHnrx*#kQ^D7(~$9zdV5g79@c1HMn{3YU=j1bou$A8R&S*}xOjQlo4 z&phN_^kjbrnMJemgtUQW1+Xm8Y{xYimaz7G%T*O=P%6-<0~AYGwpJooAW>(GVrx%f zSzpe}g$Jm%z@jejSYmv&5u7K^&3YqG&rd7c?3$x<;4eiO?>+?V=3v9snS4i1d4O8KiOv+Y z0*u(X*g=ulEpe%95Nw6eX&Eq{rf;3n-wDp?B_bx{sdZc05}#*)W6p@1+vEOYFoqQpMU_^ZwgAS^-*HF6fJl-}zz~I*=$lJNC8)*a#2b=uHU3KAw~TiB5x6Sn}{(gecBYY zQlI$xT1#bV-FJw?)QM?TFdCbdhVK!}^pw*I@O@1D5^wfH5K{qs@nn+!xmegU6@l@6 z8WYI>p2_yL$NbD}w&%6O{iQuklIt~dhr$I$?z9dZ&wN2Pl0;L$x1KWq tW%i4&aR3@MMWHDxM*g8InadfU_1-G$>s3M%=6g_}V0)YZ$)deH_%E=*C>sC( literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/linear.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/linear.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1cc88f613aac2fbd57d4ebf0cf2d89873e69eed GIT binary patch literal 28115 zcmeHwYm6M%m0nkMb@gLNxvF%5@IaM>$ zJ^dKos^MeOr>3 zK}om*Yi-7`bVUK_gs+=IS-@2y+6?(^JpzH?4ZcYM5%!snsi{iD_Y_^(r` z|G~uIhgVPF>wnHjrQDQjv{G%OY8Z^0Ewi1jrrTE4YGcibzwB`@v0$FseAy?d93J{PJkmUhyanXl;qFJ?es2Nk zV@Mxx??n1eNk546LDxar@gBtWhmgL@J%scjNk5GA-R@ze4@>z+-1N(-%01ua3{{M< zR6pkjs=gGoy$!#>gt-^igGRSgZ-tq&8|y7^v0{e#Gj&x*X-|c- zuXGxnM$o9Y8k?RQ=U42|b{iKOt{2|9;sv$9>-b$&Yj<6*Ra?j1trjAx-dPI|?|%Ja zSGC-lkD9|f{Pk8NuBYDWc2;Vwx*ycsMmuzNt{xAmt<-~6H1ifEl;>_Nc^HQ=cBIzo zuB>Gk&y6`b0swijO< zu5DjZ+ry6z-9CNx%q!1!g5f&LEIw3h&*>+ge(L$>M<2fTd9EQW%Fzqc&z^be^^ra; zrMbqVkA+`;pTyTcf*?rsK#JVdmGoA+mv&PdDOKtjJ#*Q7IfdW!OsZ!xWV&K5hUK=5 zVl8O2JXw{Pk1SR%=K;GW!*$&b3;t?dMXR=`>Rt$YPFB)J>y;SmNtT}I@YRR_LPrOoZ*G_ z%i(n0ZEj#AM2(Xt-pwxN5EP9Z{>r90h=iI)5RRQ+yHL|D^23Q%x9)mqt+ct}hPLv8 z4b>SMI8tvDUm3Vli1ZBCm`i=s+)8g*959Jzw(MT2nQ9s?(p%YHrkBM4rsq>zxt<-z z^Syj8Cj*+^Jatl9?qG*H=La^M6SdB%cU*_N-*=X~%6Xy@_$NE=6PE*INP`=lm4z@9 zbc1?J;b6feg_c_n>R|?#*S$(XdaN8=Qg<=nxP(TnI@VQflaRerGfr=g?R*{~Q}zdlV@>#1Ej1Uqn#M6pf;ZuaPs=y~r3E z@u8*K!c{C&(@ne9TUe@Cwq{^lF;%&j=GbqUoucHKS4@9@kajaY^XsX%%`3()8v*90 znYn`bT1%@38RyKo=9=jr`B3VY4Ly@vMh|t)VP0is>^e@t)M*U)@&;&%gB7^Z!kFLp z;)h?AIeGrMGZp8e=d9K*c*t<-O4To8)|{5tSqWAP4nB*R)_UNqc3ZCRJmmN!Z8D8s zeQDJ>1Txz|)*+`|4^-pQg7b1$)`9FHsi5KSP_?tT;GD&BiI%M#ML$774Qwq&jNPPgN@4S#LHdB#)T5w_oNHI}^OvCcWrahK(!hWOT6!UGq8s|e}Ei^7{PQKDv>vS)6oR_()oM&AJ$6=x2H#&Y$?<{#?8s}}8 zZghgM@M^ua;XS2P7a15mbr0&QSbC?95A`z4Nk7n-YK~jD;a>7bSKt0<^>MW83TwC~ zGgy-eI*zLSDFnrA!7LaBqhyqj3addUw<8em#TtPR!!Haraj9w zy-d??X5F-xb1g6L<*=$Uh}-yPajmdibaQV04GY}Um}lh+ZaKbZ4A+Zp(HrM4?vvl; zoLh1y-oRR|mfQje?qpaVJP&xrZXRdoEaBkSB*ASgFJm%0fpcHGe)(KXMm)FTXbS0Q zrp5=I5Xu?uYjoB(up6;=YwMmOGZKJ*>2|`|?nb~%agG%7F>~=9;aF6Lw}(YptbWZ6 zF0Xq_SqxJDL(?=Q##<;^G6ihn##F-J-q*L#R3iQc8yYa3Xe zE3y|xN8MQlZ`f6r^}1&z{3L@iJdDSkmoYZ|21OfG3S;-Rxw**oIj;}bw)4>Iy5ipH zLb{q|N9agIMutXAnioAdd7Sgq!pg!CXXgjW_6Bv+T^|}S=_|$>Q{)GrG@YQ1eJlfY zvEc@*?CERgk2tTb;d^1X{w1R|JU#;hqiI#t?)sAIVvU1?mErPt94#)w;p&D*hy?6i z)i=0~k2|Muf^tRo=TNSa*&k@dbt9Z!>;})aArZ8(|2$V{SS1%)zSmk-@57o`?`QB= z8Ibg-4={KPK{(}Ig1|Av#*R|5(vobGsEmIWxA~;A*7WAV(FM9NTy`-uE``>)M%`D3 zaK{q&U;o26Hzrvb^*Dlxv3c?2`g-fKp3!JiHadEv@HF8l>S@waMluyoUBb4<79M^) z?~#M?oac2M5aJfjlye?97>sB~s?*z4eGt#TZmAP&iwu~w>J*dY5eK-6Or1?^AX5)Q zc5`;yAV`tx#$sh6%++cg$Y-@$Sg29v!s(58u~q}`(u%HRYc;pKRI4eDDw!y?v>KY7 zPH2O{#trI0Hi6?6W`w}0Cm6T2grPFLdYVDfREd!bQtw6ZBEJ5^2y!XQFmd3RhI!LA z%^#cmrp+5xx_HB!F)izcjVl&%bCO%WA@^@Pj>tBhV5O@F29F~gELIc85e}m)4x^lx zmBT2D!zhcx$ZF>Mhf%c<9Ynf34yGmqdauY|If(GvKZvU1(J{35l}^m_4UYQe-8vN= zG0w#XnD&ieh?QNq^-dT%H1g<{jV})kpFbqwRmSK9@AHOG>{;xQ~N8~aAtDuQ#_e9tbkGD-|ovZ)TCx^2{e3z#HsX9%31@*9?6RbF)v0FvsA`NC-Gi-FVaE!s-^RK>1%{sGK(_S zjg*s0HMo`=DbT}8Mz}s1h0r(~8mFsvOT;_%6xuX8FkBf|@bxK{-EV> zg&SshbibEwDBinmzmxss$9EA7?)SWzPvg%5w3Y!YYU2x*)6D^P%Qne~<`K)|x8Q*d z{d{WK5ZG?KItErV3)Zv5aM^Mv+%llK@y`PmsFpB4Q(-}GX|kF1It~iqH}z!b6JbF! zSHq0e;OQQp=s-eUK-I_q(~3=i71TS#S+%1JX?0#UX>N>e*W;2?3pVe{5qM@#=sBS`Am&Kx+x9Z)3a*|-yXAoQ{ZW(lC# zap%QB*?~2GX_MxHLpLNtk$2rMX(F2mfnZF!%ljn{ zI1s354bP~I0cO~M2*Kl4pu^e#pYt)4kd0NHB);i-%k_;GemQi~Em`wn)Hf}pRLd-? zd#iIQSxrL|{z>X36QxM6q#@wN^0k@>ML5orL--sDd>vnZ9zh{*r)|R)IEbNT=5E?) z(I=#54K8iw>i5&?QDHAxmZnvdr5zit`q{@sPW58Z(a$^bKW@t#Obotx+hs z(@eWFQ7*L#R&@gD!m=qk1*=*X(l!awHXoK>@&KVPLH2a@(AWPrrZUMKmO+0(=ITF7 zGDm{1373|=$%My8gUI0!IY>=`5CI%;p?^67qP*1F5IRjc5OfxTmcLCn?Vlz0c@S`k z)p@N25itCP2Nwtwt*dUcBDbAAt+&?zojQeus&*hMOF6FN=)*X8XmS8;EAelN+VdyN#+6sxwTY z|2=B(N!fBaO?DXCKP=^PM!~S6Kl_JP)-=k7O$iEl#UGb5n{&zL)jK*N-z53!5d=f9 zGUt!qC-L>!Il{(GL*QXk6?^7Y(>1RdTi|K0rmx`|L-4+{8Y*sDR{WhbaI-HI(wX?u-tgO!Tt9S||Fe^(t2@R`oS%giu-?h)Y}G9xTG+ znv=HlrjdzW;^X-MaPlV@m-g^dXlte}#tq4EjrN)6)c?g-82U0~n=<}##^%0ly^hYw zB_6^3507LEUzyWK5plW8aPYrv0@jDxfIVR9eJM$^U$RG&JRA?!;ycAgr?w1e60Dj3 z_&ZZERF>prj|ju~cqKhTK(Mn;;K`ve+PAQvksjr(tM8*kq?0NeGg!l$cWmomQo51> zv5J_1sa=@?wSr1-!wbmPJ;QRclR4VzB7;>138&&RDGUK|mT%#kpcm!-&IC&UtYSB| zVCK+lem1LkKi9<2XF5vxzlF#))&SCs{AJwzJFo{CmyCgyZHf_#Fpl4XXU(M`=ajs$ zmR$Vl(yVZKy#C70OD#zlyfqXe1@d6ZBO3u7Vhr>S!=sgrtJ}Gy6SZz6HzokkBkOtk%qu?jFZc;{vEf5H^e%YnGP40v;kv++ z98Ud+-Dej)5z1H*hT6jT&R-?fqES_)Pb}Es@102%jv?B zq#YSlpfp;yYezVaz)m~Vc4#G}EHUYl<7v$KDEo#=JtZa@{sO@6xP5)DDEj@&CG@gU z1B(kQMwAO!QPG_9+~hu>4GGvl45z3wML%@}g|Qkistz<3yLFhnq*QvbyU`;4EQ)mJ zkz{rMg|eesVY3TA8jcU`k`eYaEDjExNWe>l0aD9&%&dL~2@M7avns4=FMo}!D#d<) zt5;ILA`Ggqt^3Fb1j2%TA$0{uCx8Jko@88WU{Gl;0=@gALTUlTzLCS#!l{(7syP6% zG$Hvw&s;HJ^8i-2IVQ|6Sy-m|OZHNArkWmtHOYitSTM;|;2rbLc+J7p4E zxVibzPG?8VqAonv>VoL`A4H^I$Mx*3;c|U~#|*t%LmMRNnr&hR$3AAf{}U42fv{ThJRMFCXVe7h<6;f>z>Ee^?1)8>2&GzjDk)MeHOInMBkfp6B{8{FIj=|@7u^@16 zjaF3Z4rb(mNT`6;CIh^(thpvGe~4@DK16J&A&TiD)(z}R(o?W0vCSgnH=3VuRh2RY zs41YgIke-78rn9y%BOc>s&1v~qQvN6yn(#6J_#(P%NPa-uJ_!&v!D@6l;Qdx z8Pb{1j3# z2X6o-9}$hYh7~95HuCW$6uF77e*%F3=JS2|n7`KdMrC9Z(0LBfxt%qCP{^6zH}j?M z%{(%5S1vbOQmk#8v^!FVR9W zgV0RizeDg}BnFYHrEDZ|#G>$m^V%xxR(7mL+aqE#tZG3kF~0*L&{lMi0Ac4MmWmKv ze2)|&@LDh)ljr{I^`_f}r?@Mz;n*PfkO)AOF$W~@pQ)_Q;nO3#n}8O}@|@0hE41%c znaJ`mIMt6#y7&tzUE87zsKp1B$+uQbQ%e5}6_RDPhh=0D0OXL_Nj#uhdWrH*|HC*r z%QU_s8i*nsd^4rqz!Y)_h!}FLNh$V9q`2V`3Fe8CFx4;szO|AE5^nm4?Ie#HF3mwgx`6_hB=S|nZ8EXpjyQt5`zlPFEWEnXsMY7C4w&HcX zOY7=`zeDbU2bcNj`PAlncPFc{STZ15_A9xb8-c8M6t6_BZ)@?7hj^IA#vG6=UqfEs ztVz!N+fT<%g4#9qXdmjb^R_V$J_z=21Am1dY~KAYdYK4c;@dD}*d&$X0Z-!VAHuXy z#-|SfBqdAYV3H|&LBcvn1EyYjE_D@V5>)PhrKa-7zzRq|Cd##*1tOk-3}**6!mQK) zNeeY#5=bcV@V5mNMF*TYzE0I&DtS#8Orm?zgZ>YY<0Ot+1&8^c5gzlk-njZgQ0_t6b8}bH@LCyz z>SwGu2^nt^GTv82#vAMRp3ib~TcwrM)HJk`0Zc?NCR4o$+)?>xdTV-X24gbS%b`F28J;gjb4EsKCW_4jdw@&jdo$9m z{1(L-j@yh#sK9@&&f@x>-t@IJw$2~QIHWO3(yFzT|9=Fp(sLs7BYif$VyKq|E1Gtp zT>UIOJ22jypBdiCxz@8a)MjZV?!52av{UXZ$av58W^vc!9L*k#uh8EpM%O7kacrPR zWAXaHJ^o8kZzFlUv(PJ|rWM}r>YB9NE_Jy5Q5oyZ_r_7q*K-O%Cz{hvNm?}~t=ik$ zizj?0*mrf#Ey}8nXB_!o+&<@(@kK>h__P*=DP5=v0=NbGHCW@+#j$8l$MamjmiSkQ zXN3?miq6Dq>5-^q=~82*1FPQ}Okdz6xw84`=d=TfQ}1(VeUs5oMDi4F{X}Bc59HU` zoe;q0%ZL4?_FsuOKCpV^JbYRg5FjRBk+5b9m3e^}1TGM8Kz)HdoYambkqj<~gNQJK zNwhf-Ut;hm%i6Fe^g6DH&0&dSLNhPg#KRCOa(8REa7J4Xrv|W!TE~NdnHy%$M=~@W zPPDc-?@{h1mneic9OE$7w9kQdTO-h*NP9zDduN8(XhY}VnUrS5IK|fSw&zZz)XnJ-7G>_rmZW@0Dz{@U(C$m z59ZV;nY&pAzM_%5Db!JrvN?;MJtpeP-LzowW!)^J{FKyPFlJF(QA+g72-XLq0wOhh zCQPsyns!>J%}?9uf(aKwgC!xl-z-|FRmx6ffalz|`wCr%R+Nm78bWorzKC?l0rgcx zTw^N@lFP#hI7Liy=~QTeGy~7**%8S_3?w&wKHsxIX72}?&2Htk^01SJyITNc17P9% zKwxcr3GFeSquU{@O-zt+!i45k5@Q@(LTF)k0re^k5d0EhRe3qCL#wi&+tZF{&r;5!c!s*fAx#u> zovxOjoQ6whH(X!d=q%AiDXzz*J4U;WO&4u=aWsRQ+PTYUARMKG$e9Oi)V4t4w+Gsw zJrWWtB9e=Uc<5;wa0HT-%vKCI6m=Z$Qk`BOv|g%fb0aObq9%@rcns=iCZ0#DL9p&0 zKXz<&V`T*`!{+uDmb%B<4Z68>mxE(xRvSkjIf@Q7jy}3}l-8(fV=0LH+qd75{mV_a z2w3y);aBr&R~ZwKbegdl2C_-H=|kuw#X(kY=0?YbFRfn#c?7Vxw1(zr@<#My8(6BY zv#x)@;2$#h0E2kz3qq=Xhk3)hJUk@UN0Ikk)uE_FjILq_VZVWmi^})X0aa(YD+btJ(vUJgRx`y*4_d0vv|-Nl$3F!4i2mVx4$ zdWG{$h@7OEoW2cT7ZP@;STt!iNtUUY;a}m#<^#8w>miZ~?6_hJCMZ_3+A`@DxFDsd zpTcA&85j0vce|Od;p!K8w7t&W!UMpZOXM2s`-p4*A9(oy0RGV+Drn|P-#7PJH&}+A zKsa=6*q?+l;ih?bFn8*P;!~FJ9r|$J^W*<1f^B%A1&;v}rnh$v*w$Dyy%pS0cMstDv{>TIKwfsjNhtfzM20bu8O}q{ z6AYT+XlGJVbZ9Zb%;3bh*Qb#Hd<5zp9~2K988Rb!zVodSRyKS`-&FPd2JA?VIBOmP z+NE5e_p(sw3-E;NDxWOtPy_Vwqn);+b0#vh8G$9itM(0jSiyE!@+@-YT%kn@VZ~t+ zA$ja7eHdc=Pvc-QoC!Ab92{P8ZlV3hr0EWCK%-$;)MkD3Xn@fP8<3QiPD>|sX)&}t z;z~BKDC%khHa8tn%^%13!-d(=S&=a@jY;|w7*DIG5|ohLif-?h73bP`6-i@~^73kg z8V0B)19BV)Cf>j>Fk;D#1AW=)epn~pwu+I-T7b_>%ohNm+xA(6sZ1gpv1QWK{-a>X zG}8X0VYfwue}Sj|e~+gwo;@vxWjHplZ^C@M2f~Sw?WX7*Mn#T%{m_0m*^z1oo_t2ZKE$mm{FbY4x`dtrXQiW6tj}_~#6MpMkJcM5e=h zOhdvVqy6CxX9iNWh)DXHA*-{Ba|kNqTK3=R5xY%~X4eF|6;9q|Wx~1>be3wrM0V{N zr0jzBmxAmK=8RG}JMW!d>coOeS})IZ|wUt;hEg1*aBA`J^ha(iS%;yy+RsP9xoED zghK`KOY_Ip4@-qzA&^IR*i9nW%ev^l=VAn98A3j|m56=_-S|~Rem=;DYh8dxm#7$0 zP>w-SL>nxqK^V(K=tinZvqUIXv9U4{!U4-8nWcY%rYS-jLZ)C~BzS|+=;0B@8_p7t zL*C_9qsN~XwA4Rk4I{|Lzhd4WG5A9Sk#Sc-eKdZ4OFZMxIC+P0Gw}=p*9n$UG)eqy z0jPNZ7JjsOX#1p%LKs6+M9pP9MV|waCn&Xx=JElVK0`gAUxWdR=(K=G&~*_^x{XG& zEgH?NXf$*3l$>bqpufVT<)P8c0*oqv%K?7`PKuhjJoujiZAUSQKbb}|4*wy&Hn}FY@iP%O zgtohI*>fks5zRm!YCx|#6+ctY=d{cxHLd$_eNS&n&anR?kSF!3*tBl;LJB&=r)5kz z{>E2KMbc4RGtusg2~Nk;g;$p}a7T;_O1@uX#OZrqyDYcj&~w-{lqkL9cuWyk0dF2>{a zXT99kR40Rdm1^FBbuk6~xZYt{1^&(9eb#vdx?WSwiB6cd+=Z2toBf7yb-xhPSv~{3 zniUy_JF^Jwa0z$ZqjyK79mej6w8K-ecG&bE2@YJn)14OWaJ+ZEV)~D5-#wL?A?>hG z=~&v)%;Z^6u5XhwP1K5`%=YJb^=$_Kf&tZon-`z^$(6#}XljCY0J(}_Hh@~hv3Au3 zAKXb7to{xgCK4KXS=DABr@kCKSD6%W{2F85VDNVt{3e4~r#s42P|2WGx%V>GWFV&* zg$+4~!i?x;)NeC`jPC6;yVWc#(#fN1k?za9k29cEGCbQyRJ>Y=lgvPTkXFm`{?CE8 zqKZbRIyRs$+EtTlW5WF-SPseGRsqXg=Sgm zMtxE3d?xORoqI5W$K4*yoiYRKwk`35@?x`(w-!uGe)g= z7OsoV_~5n$Jgh2)33lxLtCbpib~kng2p%alo?}e>AVJ5Rpn6Nrdr<3L})N2iADo$W}o0sn`0a86zflyw zVZLTW`wf?%m6(Y0w)+iJ1G3$3coMI!9?@3VVCDu@4q9srhm{1(|9N-}k9zu2ZSlHR zHsCc!J9-o+dg};WK8oL?u`H`IdvG*<$mwm%UBKnRTzV(>9LbGp!{gzw{Ce!L{5l+# zZ(9xT2hY_0mLodMHqys9_0bwXu4*0i1jHIABC^apw1S1cl& z=#`qhnf}qDze2@sR~jW{MrL843dgChz{V+Q?w?l^@N;l~-mUeofeH9a_0F$hlV$;8 zPl2d~liSun09l?n6FYSF|=U+hB5#WyB z3AxL9W5&w^<;|@87HC`DP(<>gAQ9ywQe&6ybCVPKD+~vm8vqLgEBc zQ{vV;jd$blEjL;J1-$9zgZ}Sd`0w}++TX2WS2yP1@>^@eQ)qL;3kN!iI;tgdQ;@%jYE8u_7W@6 z!~7EJ)}X{1W1OeT9RnuOf6UxJVZfo+rgp+Uev(N}g?O0RPxswphR`)W8m*#NZow=C>Vt3Dh4$WkU!rbwuP~p2XMx z4uZa~tQgAxX8`s=o8nBsxb~Q}C9XYYvl53!Mr^Tj>K@2XV!e$Q-DTBN&=eV1*d}7) zHw&l0JZ!!{03;&=D`1%0FeX;m(D3x~uoE`%x+R-tKakdokmLDcJVSlFXZs)T6@Uff zKA<7#-k4~SO!&rTfJ2OlM{Nmr!{kRiYMW@uMC4J+`)~!;)_CnCmVM!BSzCNl&aS|t z_GR>3Y$i{O5w5nuP1YVot;(ccVzRpCSpEwPeuaU6U^&D>i_YZQGxm#2X9E7y63rDJ zW>FDl9%ih?po5^VNs4DkFJoE3al?{!Rf`dV1pxsspLmni+`&Mkj)Z-b^~I%3U4^{i zNenzGCd7`iWXMGyva_@7!R_m52t6BGN?650_qdeQuQB)vfd8*cERQ5d>Kl zaty7=8eX>y`*|DtAd5R}YA#jvLn-`gF1aW^OQfW)t%O8j!wZ=hqFlSiTxug@CLBlrCrkC?O2 z7=euU1}}^Z)J5byi-A&qllAT2J_tiDxZqyS{yAK56RA|rHjzc;yqjXn-eACqyNwI( zzeSO6VZ-~U5D|fNef0y#)qEUPRVgFy6<_Fd7Pz3{Mu8XdNxw-*s%8Fz zn+4cHNB_sb)eM7dCL!~(qj*HhKBA(F@JpH2A4K195 ziFNcsT2*F+lf3=FPPZp5l_{a%txbyeFh+e!b#ABoHt>Iee9fIa!;-(v;0l9-3}oUT zWb9rB0+zj(u@@OU&tQ?kM;J61Tw?Gw2H#}B85&r+vWR)cgt^DGh4BZ0L#6yD5EN7U zz^oLZRddWD$n}1N^c=_mf|vq^!xxxARrBNAM0s!d(9HgFrTld2Q{^JQtEo?=J~i{~ F{{sMnGYS9z literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/logits_processor.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/logits_processor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f996ecf11a7103e0b672aced14bee485c09c6e73 GIT binary patch literal 3905 zcmZ`+TaO$^6|SnT?&;~d?CjcEd+h*2M1VobEOrD2%MO@}jZm@)wgplP)asq8+39iL zT-D?Cu6kZrqdf5oA|&q%QjmDb58yZOi24y8P~Lcsah7wcXZ99TqdI-+)T!IKd{vWm zIv#;??XQ2E{QVLk|H8)U;y~laQ1xGc2;vh#1&gUq;cLY#v3!fxGVXKJXD5#D7#WWn ziR-&b({Gx#9eYX3ZyDZ++eydo7`_pAlNEo3l6yqBqWO%7rgWyg1Iu3(p5)Ut(UKj} zMsDngj_5w){spn}rJkNxg`SIoJs^X%{o$Q{9PUZgFD7BpA7x2CD`Y>;cA`S}$0|$u zlSl}e2D%6fJ^T#LN45sEazDt6C`-e5z^Z;BQ=O?G$%Kr9JXB#E0|PYJ35$tT)$6a; zsC!|y^$e$6c|as55#AJ`c{)QttM|WaY@*c3r7OoAfh21;mQVB)RqpkJ;BE9kI0ycmS{ig z_%4jNEIP6&y3(7rz^ZN016|-zd&!MGu`1S{**;j`=;4T0>AB*9xCmb8!q}BD6R(MN zX!SlKocOEYmyK%WJ>!B8RW_0ujJyBO1-tZ3n2LUw=keZBkS|pie$xxX^^=9=hMw1l z@^BLAh0jGMH8@`O$5|X_yI|#;M&;VDf6q8>IkyP}E57L-1>Lzw*UO-Zsk3}hzNAFFRN~*c7juFTjOfgN*$#^ zsDmhlu&rF3jf-Fu=9QylF;nRX^Kw~`XHYGu`WBFa9FXVqkWf-uB?(wbfm$U8%1f57 znzmiq&~}Q(bN7(63Ghup!9#5SfjK)`^p4umwc)UZ#cc3q|Bhk6to?B~Dl$m2Y~eFt z(C+mK>eJ)6t$|nBdX_`VxB;9hDuO_@VXq4DM`2j27LqQK1`ywgbWr58%A0AqNVkDg zaV_I;rIm~bz`3bF>K`@F#t%Kkb^7;MEBFQ)ND1W;M)s8IQzLCr($+cv6Gmf@5 z^Sxq{rJGOUIN7XITt1bf8JuU+K!@I3IN~@0hxwjbg>^QdYFyCs911_?F^Bco{^fJQ zJzSo(YQ#w}k)fzMr(j3wYIPaS!7`ewu3W-IupNbZWWzekg1*M~Poe5R11ZUrD5qp| zJN-sMOIBJ3Y|ctH1yq0xrjNL~Qj+Hu^w5$YTAN0xfe$XY){y*BW;7tn*K;#-Qs zZUIvrA)iDeS=pl4gK%h~v||FGx)|hYmdfC)g={?ub6K?l@Ly0nq_V+Bqe*4UBroc!ML zN(_M;l2i=<=G{<gh>>hd1#GnabH9|O+fZ2!{vxLC|F;xK+$(67h7 z{RK3@BV>v`rKKe(cm^_f%G7EBxqLw9Jmm$ixsn?{y#Ze)FedCjurb>pFoJ{iEx6cx zj=?k>IDQ9i9T~!{CL#PqWgF9|HCRyLb`;`>NydR)8^_qiSP1Q+zKP_lQB)sfA3)U? zfw&Du8FjvJERVSjm#tAw_phEe!6|=^FgBJ2q3qZUKY#{=Dg^n*$iO z7tHRyU{|3Jo@D&Fc>XHPo6kiR~t^oXa z#Psx+#Bi#By_cMe4*icIe{B)kyR(4TUq;2 z?stu%TzbQIaRz({`@L)i7k{zzme76=1_RWeNUBDZ>KvR~HFdO;hVUqom1kCzg;ct- zVww8ui(#+>;I@}bsjr#+*Fc7N?_U~cvs8TfdhmN^H!p?XJU+DgJ`7bikRVL?PHh6e zyBMRgU9iYg-#cmhD|x6F$AHHjoNV><$!gKLH8Z8~QFkgMO)|a-vR$b9T_7AH)We4e z{2*50I^wi_qerM7RIGb?GNj_;&RziIY%(n{NYJ9T}R zk{g6Kxbuu~N6A^jbsaz^xIklE<3eJIPLu{8Ki^L6?m&Kh@7p1N`$S-8Qi|~engTm$- zOmzZ?AU@>;Y@@!#nQwE;Z*UtlYgBWyTXzbvdl8irGSqRNd4pVfJRS^$6q)ukJXc;m z@QPY(FBN*2bJdr4(_19#J*R2~K^CSW2r4%S(wvtGt~)`nU53f@Nizs|9>KAFYjxvZ zWv$-2^@hkJAiwPLgyPE)Yo| zo{Ff{xm-@d9U;{+YA9sC*pW?;IR{e#5=e{K)TS+JPXFUC2RLx3Z-5;3mjHi>#=w3~ z29(>p@vP%J5b@@y#haoHv33UzZ*%vV?aw_YHt{>)*}2N8;}gCF->iB@H%|{kZ9J$v zRfs4a#E}rjp3W#H?-tE;Nd$lqsCYvd};{7lLfC}lsdmm1X+wdyT z5}i=LSalBGUbR24N0xy^O>oU%;=lR~h$egF6@5)88PhQd*q8yeIDz2P^`2cdR9Og# z84+i(4g$FdFUe1U^sLHOA{p$TyrES)2%;oZ3K&f)EFkwWAfLlj2v3(f)TO(oZ~(q% za;t6)n4ktswJ@^=GPJHNjx<3!WKJ6dVtRaw(tih{!5A=sj@WB9vL@D;jBMQ)Iggk; zr2*?>i&M^CFm9b76B|snPm)O^J2T~L1K(v%C(ag}j;-tD!O=-`+<4^zrvty{aq=@e zNnTTM7wj&PiL-q+Yw0;|ql90{OSVUU0!NNI@{aC~=0}}z1FEaVofp=mQPId&)id2FsRNJk5h@2Vt46g_873wXRtJ2T=Cvw z{Gl$>LY{(Eghg*Q@KFQi224|Px5$`0XicaFcpyyn=r3#=x`Byqk4a>%91T4{X^_eK zUsX0@vX1@-2vqJB-1H<1&PhiKQ@3O6MPI3C~4wx9EobKfX9Un5~v$BKoKUg3#4j5m5qkJ6P2kdQ#p@k zPFNHow!x(tiwRcTN-uG{>D=}*%eqcy!E3$(G-Ba0{pr}b#c^dY1WR~bA)W?y}r2J7>;Z&aj!Jf6$VGWzHcg$k{FnjTDiBEg;l`WuI zOT=A$Wy=JopJnn4ocX7*L-r4M$o|Jsuk|yq=o)@C{m?M*tQ9x*Ik!zOH+LU!@;*#k YSL$wm6_Y|CGCHKiI&^`-&suW+1$uyn)&Kwi literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/rejection_sampler.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/rejection_sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6362a4a530cdd7e90c5609f3c8ef229720ca2cd GIT binary patch literal 12057 zcmbVS&6C^4mB$xB4u?aHMq_ZtcOh)ZW(iTKjw507!5~mJ&`8Lo^!Q zuV26SdtV(bEHo8-F8|w~Hs1QRqWl-#On>O*BW_ z7=xN!8<@5^sN3~H!)}OnBbc-2(ANl>`TzMr%WjEhwP0b;w%g*j85|od+KZ~E#TYs zYThZdobGF96n2cB^_y&w9e-fhXP9wY=`4MX)hHb`JJxf18y<6pJQ{jD*+bJ@-w&A+ z54~Q;$c%UWUXmF%{5Z)Pza1ui6uLoH`*<`AyqoAe?}vWk1kw7s$8r1OZM^ob*Nd2U zJBfN5Zk+hNx82yg?GEq;x10UIjW_(TFS?&a4?h%hjRH6Ew>>AOj()h_BLb&C?ffQw zTU8WKu~nwn8dGh3PMg!&3(W8gR>Q)LGfLn1gVI--$?6XlY!gpkWDT#r*Z;3k{q1hV*H?$Ezxw)>ORsk?UA}VpXI zAV9mVh&!XW>(tzNO;O(Ao||}(dP{+3>g(aVv}x zD~T*9Xg}fuE8cL2-YWC9{T}`$dqdBH4)?a=%3y57iPy!(b*+y#{MhQb3HsuH#5wjp z^y1j^<9LLowNYZZj8#VoqY%8^_T9jOf_hiqS#7=2_PXB|adOm}McYY0KjwMh1))l!hsjkBksFvfmdCo;N4^?SXUH6Q9v%U zuqRgH0q8*4aKwjE?8TF}J1TEM&~EPka4{zJN<+ zO2p0#AaT^uGb8qbz93XQ{|Z*XaeCN<*m1s6zPY`c42G*ie|Rw=1y3%%F&J$OE)RBA z1AlE5#BN04>UIzeR!Ic|$J>Q&V+E^$yXWzEwOpr!cc{T|j~@deFX0<6;nHq5RFnQ5 zo4TpB)wZ^OW)_~h>cHjHML#mjKKKo6cZbA@ z4R@OW7+;~VIb&n+D#vB=?{f27js`Ut0pfvuWz%6J^#s+s3j0#sF9x3xZo-Hh467Va zSPfjFZDA4$d0L0>7lMo6>@LjE?50kyVuGzSmN16aFRjnk$g(-HzYp9GM}vw}neXRo z*5wszJLqTlC}WztA9&>g-4sPXdL3^h3@s_YrGk#+8CBE z4@W?#d`*{*mYLEo&T7IjzWMcId^(GE%b9Yn?K2K{eS`^qAKp{$3gZ`;2)~NF>#Qbr zE~|NiVY0_x#Mqhc!xa|Z(7hdi?dQ>FI5-efKvN-g?wtt8b!D41VReJ7I;*p{_>MVgt(j{O6p+Pj?39E za1;Lq7jxBA4OLT(?^{Nrc3L%!7Vev>@mSMMv}@lt9xpXds!OVI-1wg*YN5aH%*RF} z|E(I|8TI|sw~A<0dMmt}PnETDnfSF!Ryl=X$Ly_Ys*Snf26@%gQo^h&rFTsOFn|ahlYHPXU(*cHv0$@JG%P-68cdd z!WDaaggPR!kbX`93s2MjYgR(p7($?A2aw~1F+7`UWO?bTbqxkC3i8L)d1u2fyeVOr zgmYC)O0G_^uT`xjjXTolZhc zBikwA2gDr>DbGS2e*yc9m}j0ePZAdMzeAf)l}?SH$M0;xL7Y>nakeaOOSccOf?Vs_De~e3O zsi7NC6a!c7aSLC~kSYO{`mRyazHQX^uN+c~r{l|nOaXlrm6^lD4Se4rD*hdpRE=8+ zWl^9gOhvGRyvaiynGt1EtG116>&nuxW2l&VU_gmfC|85&$h}-l&4U`X zsLH+PX;dhd997wbf}xl3um+Czg>b<#VniJ<0R z4Mizs?0spUkpqFZO>QKqM~;scynTi|K`toBDNyn%$}^$x2T7kz?P9|uXAynixghG@1 z59*PB`|Q#R-WQpYyHl>q-N{JMD4^rEk7yWbh!+bW$Wuk}Fbaz(t6U!39HgM)$tXqK zle;b;wGzsS;jO#Ew03i*M;yxn&&oA%4}`E$&zE5z<*k5iAPIcU~KkvDaRLyANT zzQEfjR!Wfqwv}t7H|cYj710`{WMjEil*GD=5D;-`E`B0pI5f;*E)c^GgM8%)-w%MRUpb@mSb0klb##Klx&x`M^c&hgrzKu7%>Tf z9g?oZaj8vDGn|D7L<&VYZ`n&wlsgp4rqWg=~s%Q-w_w#(n3S7)r%Ptf*fd}A_j#!JiUG7MA; z&Rs)osir0!KTTc!&Zrq8{YQ@A67Kdd9JX3V``i`VMCrQHHvJJd)9zfN!XDL9brbgF zAu|0;Ln=(9!Nv{o%q*Ub=U}%QNWbZQ{fq)9PMx}==0rUZZ zmB55jFh1rZhU7gJUfYW01xRfnYZU_+10>oeg_()-R1WyQNWfeLB^y`o z-n(!IS&lDu$IchwYnR9Au7kwM9Tp|=inZ&69e@(48A;J3J68^s4^kQ8u2s$Ogsrv69x|GtpWO2OyULooz}Kozeg}n+JAv0vE)aA3-0gkwG`^3^nWuq0|NX_&I5ED%{q>Ec zGa*d)B0g1SsrG>q9uSp9u;^5A0#=~JL27Pf%qC>b@KshgM^{!@U%O&JNO z@?Tx}{SrP+%JHJ2O>sXb5XByD;k_K$`2*epO7}zcmIn1K&Wpw)nUxB)L5*pM7K}&wIR&vmU0~y6AWILPg;+ihvb8Bvi`C zP|5(HuoM7Dm~@^fQCq;)^edHMvV2VlyQpAt2s)K4N!g(iR2R?X8L?ilm*WEPxJn^h zDBxB(-gE|=hAM+S>jJs1a1}R@6+vc1_+TFD*>Pt|D?Xlq+v&Jj1k99LmgE*$5Lw#5 zR6siq7s+ZU>L=k{qBU4dG%a;5?O- z;yqM5q`9~=aCa{tv=ifvzqo@OqqA#OTPHN!_B^XNR4QkmX*|D#WYzkn4_=>ipUsJfRYqRM4e6pI2=ioT@X zf=5L~Dv2U{)#{BTKqSQ{wuj;1L=2mHac%G7zQ-fW;~c3un8b*VCQt&M>h4`99$*$6 zA#i|8@JnzEN}f$;dVG8I_Kw#GkbKsNAH`-2s(* zap=bNgLnc2q%lI0Z~^Up9KK2=*91xqhmXYPnb=cO4mS0J$W5-iz9J409X&#bLJ3Y0 z36@RRI1Nf(*xm8Bd^k70D~^28-RkYpntYXt4V0T8r4$RJvnv>ryw=Hp9YeZsHNYw! zQ<=dwV83y)QA+EK6kbOMholh4wv={jzjIjn3U173XGgn^6OcDMr|1w)h_J$OvZh0a z&_)6MZaL09urz;CcN`Y=&}@)f%gJ*RCtw7%nJLo_99c<)zk*9P_kK|{OfU%nJbndY(}p=?B^)kwC!xm!OA|1?XFQsNoz@V{G1gladRdIG-2kh8F5i>gEE&k=BQ*pjUlF zHKn?YU_|{>fcm=R&0l^6VHf!r;!UTWncnV@5<;S~(WD%ezdl0AHZ!(;Z$~OjQrLE zx9H7U3^yU+zoOqdhBz{3^!Iq0kU~<l`Mi8sVO;kEi9@GqV895}%Bb`)F zB9GKo&uX0B?3~5<h$lFy)`cSnONFC~pwvjAWiR8_ zo|8$)O+T^c=ww@$j@R);dNeQY%F literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/resampler.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/resampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3807358b84c58e47d9cda87194a3d5007f3113e2 GIT binary patch literal 7105 zcma)BO>7*=b?&bI@A(l$Qlx%X+Lm@>Yj!EpqMbNVHrka}E629PUGHjRqt`}nbE<~Z z!=9c|b&n`ALm-F*_!8w1#774tkc$opf&f0+z6Cz`Xdpl%2M0MAdkB)7*VabL_o~Su zsg;8CpkKeLuD`DO>b>vP3>pnz!*%%|e%k%j7d7oaC~^3zA@LUO{9|3yxE5+$=PcDj z9Zw@=eIqn<$}>~5Z-rLh4(+}ZI(;{ERX$6-&{uV8VNE^j{RY!?t=|lfsS;LN>o>xN z((QDre>^;{bSIte&xA9&CJfQ&&2pD}4_SDE`=ZvH<2B*ZQ_OeRDUH`j@7TP-n-9$J zq<4&;=Tl;y9~XY_6rUE3m_Jg|0>CC_-&>IK#6DfB&m(54!} zY4Or?`tkW0{t{}R{kq1_@^cR~eh%Mq?!FPejGUKwfSf?QjC8=+2U_cutzG|vZY%{4 zq{!p`FcmTw3fUIPU1*soHU@G%809FCWI=zF7D<1=<1~O2X&%HG_oYbV0tLDvMb$j$ z3}lceS$mKN!$F=D$sj`o(O(suC)rxQ==<;7Q+@5VawYiohRA}8BrpBh&DKJ2E6I}f zuzz=X@k{#-b^wP%e9C9`Qx46{L4RG#Y;mB^U{e|^9W-u-D0hX@?=YtwiLxk zW^G2J*t<+T9o%o@&i@Xg&$~{f*SC@CncGHTjrBsi&*a=#AL|{qt&iCl zSU#n#Gx@60fhVPFA83W$bI^*rtL=E4;Vt9FuDL^aHaX+gu36QEM6C|vHg~xDuqt6u z8I&f}Tf3}R9~(fhj#c(oBsP=wi4sq7ePU3OJ}GoollBPrjh0cG>WfWU+OQF)$yye* z2QtH&KGI8z7n?)TYLrHj6{Xpai!LhMk%gQ_JMsiX=_gqCyoj^5DD@j*4JCyf4hSBl z)kz0&Q98g`Hy#SCrjY#+UN{}RFGN{6YWtkkV|R2i7#22;Cyko zS=P`%w72MALhJcU5VMx8`?{;2)J^=(=}n#CXMT3lvLDL}DDa5M<9Mh^Ej8 z3F074@*)^?Chzi~7zC>#5HK5D@GCGNTOzo4YoQe^1sDHxA^1*9RYxyZC_A`5c|p2* zcd#gO7E`>2o36KhT|NK%&14E5N4|ojJO;7l^cVQ0t9>q;NIlX+GrpJPTV8jAvb!6+ z<-&*zOFNcvwx)mz1b=y15=cT@0O}(wIv`&fFm`3Fja3KKGu$m#zDoW49*HkNl-8gx z*5cB@7laPx4am+ah3Xo}QTVRJd8%=qzslsl`} z@pG9Excc}_>!#iOY){kWGBs#Yog=ukkxkZ2;WCBtd0bYgtSru$-8W#4<_PZ7MK68S^O9k@{?kGI^asvw|^0}p($0<`EO7#ReGVr{MX3*Gq!DRTNySgIe=C6|EdbSGU!SoPwKJuKqk@<_Lw zt)HlUH<^|z;0S707hMrhE+|2=)-8x71RDu2y45~a9`%aSC!oF(d^Bn2wTZTHU}3%= z45X?;c4zy@Fj=%cdqMtC?R{l0q{5vt>|`QZ4S9;*&yi3F ze2ui%AwstgCq^BY7Q9`19V~;#QpTG|lXFuRLZL>%wzRf{9ONy(LhtWWJAXhzLAa*A zpsRotR*8)&VOMX$>bW5a_N6zOvS*P#Pxj(DNSCOVQ&)byu1~WmHU&%3#61gpF^#7Q zjp_DN)2tKP-$2ne?jslNN4oNR5JCeV*FwM}G&mCgZO7oIu(*ZW#_b45@MIUP)mqbq z+w&A_^@I=q!#d3$XdRu`c>O^GEWLsEb(L@OCO-y7-2m^kz;BN8X{1ov)WTzc+)UZ{ zTAYjb_qpoUzbZ&v3zn}R(AG;67e67e8%fa(2AK%jSO>6H;u}O8gNhUctD6&OT|5BP zYs;-51y9U~X$E&jLMB2{-e8#@Vz^|L7ghatl@H7-XcI=S1?VUz_|X7pTPS>x=R4BR zA)Hs(T)AD=vQar?@!^eCq*VxE`w`dvjie4@9J`igRz56DRr(NGS0qD|i?8(6J2 zan=2cbh-`P`7a^BPPH9<7ZDOnlok=|)FQ<_BD1h|?84b`!DW%ZOGZJ9JTe4552k@V zWT<6wZKTV~Ff44{ke7=ZX|Vq0L-3cVo&goyegK~l)zMx9?Hjw)M-zO6-KQ3hO|+@8 zjr8%cE#Is1GT;n3)5sAt8sa`P8pbq3?0A>WYHCcg9PC+@R9eej`SqXA{^QsGed}-E zgkLxEg5LngH&>H5KZ0zcz+}K9nho0NU__XUv6f&q{$BrU^xmO3NndF(Nz)d(-{1tD zVzb3c69kl&VuPN~?nd{GH14nR`077X+vF9dh($ixx_AV^i&2DFr--7(H_|~nPV=i! zs)}op6Yn^~y8NsZS?tW=`6>lnY@+N6Cw$ghKOr-x2tzqR2`!&7wP{1-X7#eoa+^PUs3Pupy2T)I!Ta3?S0Z zv%`6d`npDf>LPmZqNtt5c^*a8Jdtab--P%P?tB5l_iSeCwmt{vI|cgU8%>kwiw&I_ z_7mdd?5Pju>^|9=d;T1jdjCkEw!ge@Qzw4`QTgK@gSY%bVT^V7+dUW`dUFB!mIC#w zATIWfqtXB^jtLaZ2x>&0$+!1hv~~5_a+9>l&BOIt){3OT$m=AO|M?=cmRI>2g)qAc zPStvq_J;DHl9*m;5>i4BG?QYiA}YO`6w|E01OT45smbq<2quE2xolJjt6Iu`hOB%V zf_%Un`GBW&c!XJk=9y=vRUvb)Ua7Afp^jugCLzr%5(q~S!0hTfU^m#N`mTYE!&a1o z#b)tFSq>nIDyANaxB$IVW4?vT@_QtLiBJP~A}Ofg&yk(eTF&ZQC!QH%RiZUTTO$Mb zqp0+w2%+C7CA}WOgvROK8z+kRpiM=tzjNc-hc|CWH&-eLPewq}C#jtzLufD;gF&i5 zTH3c%gYqLPLB;};S^>H((sUox0loc@go5{HzX1#ed~byin_=AIO%SFg8CQoxInn}E zB+NN^m!ea<7!0vCYHvHQ6{-+ zX$ZCgkD`v(4oi|5)cYMe6uX;N1Y_mKZK>$vtH6L%mj5l%u0fP;%wrse;rxM& zZsq#P=8 z(x@Nj>o1NHpP(;TOKpTmWbYV}j&(4z2giu;RTk!#$PXP2(Gd@kGDLAb#F9N9cTGiu zR@Nx$C<&5vpFR5ytJEB$)$idDavQ6=bBt0uQ}BQwXgD|m5_-oeGR5JLwlj@(W++a? za~A0nh!q_oZoS6N9F1?D2%~jKH@Nc&?q;v~iMpq<+xKsVR*#_z1W1 z5moTt!+gM7*j(IA+CrWDReqAp9{@F^$LY#w&qJ3KxK~Gz2WV2egMcL0NxV$r4v7;a zE|4G;$rTdcCUKU;ACY*EgmOvml19c}exHQeRPU4a0R)0Mg}RI~1|(=?vQI)GRF9NT zEA=~)P8L7s+hF^^8~K0XT~1BA_AI!Ly$7?VVZt%n20-`J)=dDfY0m0CqhM_gAsd0w z{ES&9oCh%@ANgC;hakDvJRI;!Yf64hBZx_?k|5tAISCcrKF9ePG2 zP!18(L*MfuC6&kPLi+{o{5r&e$HP!ReX-ADpE9rc8+ML9$b@Hf;Tc_cM$?*pEU87T zI$4kwyHP#QSL3JEFTKtvYZvg-xnz{K;XCjjL*>fVcM(UCM88W**^?@kCvb!_iX<;) z*`jI%@lu>tEZ;wrahi%YCPj%8DQ*^?r&H#|qo>S^RY19zsy`SM-Bx6xn(YydMNRny z(*BUdGKsH{P*bj~VTY1L0@U{(n(Bb^XsS#1jf&%3)sf1nf_d7t>o~M_nG4EHr-60W xWKBQ_k-ZM$Ja0ZBIKh%V^)b&T^PjM@&&fNXpoLP?dpyNgtKmPMer5Kn{|kc7&GrBQ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/rotary_embedding.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/rotary_embedding.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6eecd9e649903f8ff1010add48de1b3532295a8e GIT binary patch literal 25494 zcmch936LDud0t=BGt+bI?19B%u>c1ENnt2(L6DM2Nd)ghl&B?5LQ*s`JseE;VrO@b zWp^)t9nD6GL4}kO%9fQxwhn9I*yW&N#Y&t~qKe~`Q{}|59ak!;q?Arlj-ym1OgUAS zsfaQu0{OoG^_;s4LP{#L)o)+Fe)s#|f4|qxwCf= zQFAA~+$FO-jrS>c8t>EIG~Q?MKI87g`#yQ!m!3`UcOP=+yjk}Eo_TM-dl3I^Z~wC6 zF1UwYOqA#1Jcp5f(~f*eZ`PY#HV$aZS-0pOLH+~oQPgq_4{N*`J)Q6O@)q}2^zxwl zuzMWe*^1?s+}luU0X=)CdjdT>Aw4^U_uJh?yf1p0)x+rfP3|L7>SSEfcJFXcp=1#` z?{x1%&by@K5xl?4eK+3UE$>J1{vP*kyx%SF$Ld=7X3z4H-ZcB|-QwPJuadQ|<%qgXqmWT;o}-^jPQ6 zGwrbIZx)|!2lcStZdDt_N1JCo*R8kC1&gl|z?MpOWIov1jI2lMwJ@@ttOsF~dTL!| zj}m7#)*D`w_(%)IOL~-gcq0hg&8OBQ{bFSMUbx}6YC0>5erRB9R9ziRCjbL7%*JpHpKcaij%|rOA=ElX$d|DP&&53Icr5K^Ms`ux@aucr@ zTPU|}xb|fovyxm|^s}hyRW5>`VUXkNRm@b$^b3e%io$AbElMsos$uB0{3+(*oTy-y zdHN$cw;ry(11UiY!IVCQkvO`)z&vKH9YkhOZ$-I!AVaPM;bz0DS@_!fu~;7fzJZsn zzKw;xtZ#F%jmtU)X^f>9UDLlQG*>N5h=Ji-ub4k+Y*|~0))Z0_u8qIm6fsW%SZa`x z=vrNUM(ZZJ#*8+nZP|$N-QxRxZ7Ye|pG576uH8)npjbZu3Muw*%65{y#FVBw*PJ4L z5BleV_eqRz7vER>L~Em23EOL4D>zYPe8sD}i1J@0sCT^Q`nfSvWBL1;iYHGN?;PaB zblzajkvV@~@nb7qxZ?T6uwC2;ydvVoB|zcrvLwZ}Uwr)OGq-;*epSHR?;Z!VK0pe6 zm<|>ti+=#?CSWyysKq6J4ymP7l-_6sFKl>T$BT@x9VOgwbKQ#}t+uk!TJtAae4fEU zDI)vPKg5K?3~pjTlIG7KC>w2T)b^d@AURp7uCF&XWsCWn+<1$~9S~fn@>9B{&+1dg zlwQD}1vs5E{9Ev;hG*oLkKcmh*dm?=f{7Mgy==OMXSpUG3r_-%?E$bndle+mO)uwP z(w24CaWgMw%PA5*AYX@=%S|Cq21Fz4=0HfY@w*X|zg#*EqE%oX5I}}g5>9)0{+IF- zjPYT8AaW#!iE`n`C^_ryN6E<_qvV`>041l;dypV^-aUx1%)F#oT6rHf!J%kE5syB> z=-l#@Uq4qT5mC^zD+Db@={}@Kj>=d8rjGVEs~0Ohm>XcxAhOO@15bg(Q7n~DMrf&) z99qL3N+E3t{sMwfyQFRFmo**8rK?pyD-fDp6QPA|uT>I20a}IITdO32EmN0?29n$9 zF0dwIEUB$Jpg&0>oL5Y+K^jJFEk!wOU2Nu7rB`vhAN-{Z2U}%E@63&@?P6~%W_=oOcJ z4~U~x+g$wbs*OLr-xsJIb$VDp%T05S6k;inb|0@+-O%r(6c*{L#8-Nq0$N~ z+=(`NGv3SLqENslQg&9iu338Onq_p3?3jb`awEi>0NJN&&)@}MMIZvr*wRB|6~MP@ z;eW!_L86V&*an3HVCt96Er8~B5+G>dKS42pd}#n`qDPHDv+SMI28bzETW%51Sd=AS z^zTN)Z=~xdR*SdaSFEoN&~wT0-;Zw!{7Civdl(2vz8A5`_Uf%*qdA0yiKVY3)s()E zS@aMSjTsGwGTp^mMtKGQ4y3j41VpV_py0fb(o;aemfpE_&siOT|0vx<1woVmO!%9T zx#n#~$@cPc;DterQ{DSbAkv@uJOHL^x%!6gr?C4i5O_f4wuWiNf5IksOYHttV;dMA z({FHcSApqW;hw-)B)3vsV^!ZugAoCya0$OQmI=!Rwt{R*GUibortsa>N?$I zH-*&nC2PwGoz)CdfXP+;+u3d!AOV&JEo8Ca!z}<1##+cRWu%2XTF4Exkdqd2-TY7s zS+rmRxWF|6Y?7VZA1pq-d4?SD?X7B9KkpUmP4Mkape($$TgCQrak=eZsQT{l5-53p z(+vPz*sVM1t!jqRK z6l^`nJP$D-BJm$#9%sNxMadX422dAe1SBeowNI-T(2~ee(RJXwC=Gatb0it%_Y@=i zItH>)J|2BVP0uF!5ZsC&l_KOidLGbg8wG@9DCS1uDZO)e9I8jF8-Z3z0Hk4pHA6&1 z@Yb68Rzj!_QH23km#D&gg_O$xJZpgGQa7<}1E#UAga|yuvjS;d>oP?7-ijlivoqgi zdyoT^Ocog;Tc%PInXS@Z4+MXdGG&W$j{g(roUF8>Mz-snuh%@m!qME>Y6$x04sl;f zZd8jeOHPsy6)^9RF$Wa=ioN%;o9x3VHi@#mxvXut)i0vT*T^Ggla4_|owD*kD=FR3 zt+!Ii{M8f?c*^K3?6n}Q(%dH~ucalW*!A+-us9YlPa74d|oX2k@dq^)Ev{14!y$*1l>H z-MR!+18>INf_?>+n#e^2Y%!Ng2a>XvI!>Q6Jl)Bih`C5Iho?L9CyIOUho?Kq6GgI! zr#poc#St0kbSW`}iblDwk|9tS={w-f@h*mbwc>m0{_kL5BInM?rU&_&AD^H><}3PA zX_kCntJ?G`mB^`7peWdAFrKYcUf8HM;xEZc#ckJ+Y}Q*)shn-M8^Y75Wh8&*lbi4r zT|NVGvh6GCP0H$X-hE;b|7ivvV^Br_(L410pJXf-Gk!I%(daRM0s!w~W?=(~-9CgT zSVxf3EZyMms$m*$8J6)wBVoL0*v1=H()e>DmH)xQY{4-K)|(dcSjb}`PYQYRw)2K@ zY>)JX-P33H`)s|2XXHl`!C%lpa3hi!)MF_q$5OG>l?KNSMdWfpB(sd0g=Cg-(@=tC zK^t>!9uiE>%OhpNOL>;aR)#y_7GAV2>1C9HY%>W^oQxKpthcANM&IDLZ~Uo2>2VV7DnBZ;`+jWan18vUZV){rr2ovRcWSHZLvVV zC8>;Gi(pEt*qA~k*iLU2kPHQ*a2-%hge%ohDylYuHd^r3MIspKFIf{wjps9-ENKe; zJyksJddpSJXsK8Yi`7P>7}lE}bQO~0(Tg5<9jZB{nhWjm=_+14ed8-7mPw zf}8z_iBN!r!yAHRE~zw5?H1U|bugmjZbt#72MK48Nma~xMPfqODxUR<)mqI90_>Rc z^=k3?xOhcvN~rWIs@U_z2cLc-2CIjC4^SMiCN};Ixu1}I_lqYtPOeq!)B}QJlJ4zE zhN6klwgN>1l#YY4b??VG9}`bNEFNlIJJc^3kaFyEI%ODgilKQ)+lIPRcti-0-K21R z&}UG8;Xj9hr9|Y^feWDY1*Q=qY{V&8hOj)e>vA5F@4^zl5qFZ0;h5#6x zb`x7pYvN^?CLlOwAn@Toly(WcWxJVf76PX8vIeovxvZ1HRsx=`y^s?!WV^{1Q(=z1 zyre&(U7|uNA5N(E5cOWt*NmY3LG7n?Y2SeMS{XD_OTv!(_hR4!IZW905nhb5L!N*N z(}28j9dq(r27)ZaOn?d6eHn_d1KJmi7a$G_Cs@*D;b9*!7Vn7kdXyi9LnvD)hbgX+ zOB*@&x-bAy1#;gk)=1bQTeFSqHTzoU@$0W`PjM0pa!Pf!cyFT(`WW1|IHHgjY)ycUF7Q5jLc zQFa6+R2opZ`3)pQN%EY?<=@XH9$+wNmYR>qR*0dTftoFhu|zrB0yj|Um6hGmWLeoF zcOdpAo`Bm8>L%SX9ocs*53>W@ zSpdU6UeNs)kOIh%85xro=JuM#Lj(qj1Gdq5_jUWXR}G~p|EEyiuP}I?L6yN-1~mr4 z41?!V9Pv1-?D7L=HA=1fZ7?U{rq3Bfe}E@Afk61*h28kyJosNL zW&FTM=U*>O8Apq{&ywud$d4e(UvLJ&jqt9}t0z}sM;PN*)8I~%tFR-){AxzH(k!^r ziKzIZC_Ltf65(2>^ma46JDp0S zmA*9mS&jzV3J@1?d#hOu;Rn%ZpNow3I;<%aw#S+sTP6_E&8pw3EQ1w?nacOc zWw%?@%zTK$XP;A6R=2MiYWc4DpF&cUgrK3+t}FF(D~#2+OL{+|bv`nl2N2zi#o+Lgf2^mteRJE zL?$RRf_kfz^#2mF_X=$WQ0^vdf)_H~qieXzM+vWGe*6T#krsS!0sWqG4jE2n*A zl<4?`%&G~AGfTE+HCQ9#5jdy^?2A4RcE-~E&-OY2Gbt#Tw}Uw$j0-%Fk0o5M=|1da z&w~3FSXa<(>?ce^h0TTb--F=Xmwti2Z{2^zkY)F`kO)@huQJ1c{1fR!1=U};_*9zG ziza)@>4)+d`hEX%i0&Zp2aw2L@FIdy0xv!(FzO3%tlwG{8pNQ#>BxA-;en{J+lNZ!q|q z31O6uoL7wo3{p!<{cU;7VrPuzj4LWIbMHYydMd?Sh9m&sSppg|lC#%~@Oqg7R$7 zHvn#aum!!fv zq>=@;Yp6})H@Yl1J6#s8Bz9F|lrl=r7oZ<-nOqyXA;OpAt~-as{4XM1d4`Fse;;4R z=JXDv{R-!l)Hf?yC+>v2SZ-msIdvoRHlm*><$qkRQFv!|Xm8f>XYe&(cOODTSjP!4 z3iz{MyDkgq7~eNC#{V|5#`lbz@tTp(|Jls^+?3Dyc91NhX8wY2BN!%GbR%{Tjrq3q zh*knbi#~1+Y$R!Tqh;d%S*L(V-zjw7Zz3sS3giw`U*5yC)UDgl6^%e|% zuIs?JLGyp2o7;kyjKn{NIDKZc3Kgtc1gM5D zlctZJINi^exGJ3)wQoqWnrXxf6PYYWZh zyLHXBdOR5DH25z%p)dy3F6lOCJ9QeM=#olW?jR-z|Eg^<|KOdp-@ax`OU~8KZEp*G zeRhrxqaxA7s3H&!d56y?8NG+`1SF&)RZ_MDF2F9J?yZWYLhDfi z?2jottG#W<@f7mImADJ11a*=CZiH!AUA}~>VOYCSy_%ySU$&LZj_^P-Mfam7{{aT0 z#)#cmA`J|_%$;LHl_v8iZkYG{2y-`zDIR3QASK;(#!}!IN{KR@r1h-}xdv83y6venv!4c6zwIg3 z0cKkMD~HV`W$)|nCpojDv>nt#l&1a1xB?RlWHSj$lr3jgTbm;Il+()_@cwDk*SyFg znu|<0UiiPvGDjFZ!{AW{#}M$e*1A^>{UTHLGdPFqy@FK=hBVz_n zr}Xx~8`oblmX?&!`|WH@LVb?1(^MlU$&p|z263O_lG?g=Fh*UG|LY8v84Ly|Vmf7V z|0@W-f%OZ{A|eca0d(Sf*JJMOwDGzP2HysQZ)fx0n|lNfSa7$pW0x!|;k;=~@8(i9 z^@g;kkQ=77|6`ov5p2)VjjGgWk{DO3`F{=B{8tdbMNTNFV4lGkIlO2>6v6`)+6lDO zSE0@7{x^`RhG*nQaNsZaJp{w@2oz{UmJA{Me+9?6pd1^h&@9Lz3CJO~n-w*hQlZ&O zg@&AvIe3P~opxuI7ep$dbU}3*%S^}WwEgZJB#un{Zny{Bd8*UWGwu-_Cxy+;U6AmQ zgonKx|3d=dxg5m!#1hC&ZV?j5gh(I-DB_MrhaZ9CQs8-OyDQ`TpT&UH_We9#+}Ch} z-A&QM$&T`hov*-^iOk}dFfk-5khxLjt-b2_bZHoAzXn zgEK^HU|f^EcP$U@EOo8(j_a>*p9LFXw#aC4l4bi520&MK(Wrs4P-oWDso|WW=+sE( zZcL|U@1j#nzG98&)NrObCo(9EvPDM`PC$Q7GylTpiU2%7*asjQs+GQFI`v z@uQqcYP8_UKqk$wuilVG>yEdZ*>N6jZ>^S()`fk877Jv#f9wX6tBhjQYZV4y3Jc8Hl5)1Ov%|5;1QoH5gTj zop;~pbpL2QjY~Lzi3Whm#4y<=-8!lL5DS|_|Hfs+STb%rqy&Q&;;JC;H% z15WMW6c3m`uyAxWr`^lI*@*kXgq)2~Y#f}O$ad3kPKqLon**H!R@graW-X0VHZ~YI zle!5&3N#Np(lV`?F3#@sY8d3)k*4bXWSDK~VGjC_1R1_A{)5fS2`-Clw!wu%#Fhr^&^~I)|0UqDT1d^>|BThvL}yGh@Q|u2IPU>C9m`HcjrqSF+6=j2zJm ze*p>afRXdS!r?LTSa{+i(@-5GJsg?F1J@?7awZjSvHF-!`8+;t;rO)a0sKYboB*q3 z2_r}6d^qv?u+U;+8O+GD$7(h>A95 zow!4KKgRp46B!do`&vcYX=4nnOB#;%#l8!)deIqoU`e!Tr5nC zIN~yH(Hsa@6u2<~ACjpNHiuuJeO{Cj&=|9nuqxCqA%=W*3$g$xZE`z>X|vW)I*yxb zFgYS$`XKbxT^r@VBB^&^OT?j7rh61rFrdtE5fB6?JP!F79D|kl6*Y`Z_h9O%sr3IM z10uL6aRJ&uU@(@79iqu`meW1!T2O}eFoviLKy$#a<;ej^^^#JI(2KNe2hDanTq&np zo+=iMYa7|E-{4BIB`Ui>#9`CTLyHIu1|1@Gh740UkVLB>4&uCJIh}Xxv5MoY(@syQ zvuH?pSqfGcka{oEPBKU`7zn?6O1s}gzOS<1+zu%Xm}t+VM0BMzULWA2{P&K}_$!CgkY69@EujQ0`Y19<7-19B>~1uKLn=uz+k zY)Vw)Q6&a$Z(f7r`(Zl16D0~-{3I%YOIjZV`oD<;oNL=?_O{@4do0<<_Y?v74+)zD z@VsMz1F&IHpA|)N=fL;`_EF!M%4ce5nmw#nLDPD;RKNP?Ey{z?x7p+AnHh64HiP*i;#Hj6lVKBbg7^ z-AF!o6tjk1@+GN&<+*k!MU`Hkwltn-hYvE!WMZxxM;>5N;{k_s?~Hqh6YIdORG{W? zqCL$PkwPD383Sx{DWL#7(Vn4Kz)%Z3(LUM&&b*JckV6aEp%${zLbjV5Y9WIb`W$e{ zseTZX{^K#d>aY=QbE6mdFr{ah9lFCD7!YOnA7C&l*Z3b~+J_iC$>75bmKZ$6U@!{l_OB!DkI-!qb>=1< z7#gR*B;e1OqRWtF|IkVFVm53)dt6*Fx7S3AT8KA1050P@t)hIrSv}`f&iQq>60Tf` z^5^TW*B*LT-v#I^b+`Au|Gw`$&Yi(85GT*z+R#s)j+1e6XOK7*tiy@0+Nj_h58CAI zSy2W@8@cbuOx37c>-{+Agdai2Q-IZwzR~bp!|^y=gs^S$@(-lZgN2#gXIr5vyi2Qod?bd`hMJ{2ebynwr0pJEp$q-;&=|H(DGnaCxF0m2Al}C zCS4P|Vrpv|*K{ClKfDTYI_=f18Tx6T*Zhye!N6Fv@Sk3WW_JP?knC&OA>17whnZs z#FcmoebDdG;PS9dtp3aM#*lj4O|Eam4=?jv6>Mx1G5RQn_DiA$98}ICEq%_qpz2%zGXRB3SZZ zO9Sbo4J_OzOjtz+F|Ag8HM8(j%SQHHeMoT@hFwCx#; z=w_q?%DXp-QUJbIfc6vMUW~d<4fWEf5J=&@t2Pha=O>|S;LDA z+6?Yt@E8KzN_24@ol`f@%rlu9B%i1?vg)DNEZyS&4paU)gMY!`cNzRk2EWJP_Zj>F zgMWpfJUxujD#|^fJTror&?Q`R5|ndssv_cLM`1Tm;Gpa%%qQ^zuIJ#zWx#lWoE?me zAS~y6uUW+{VNwWn#jjE>C}&h%GWc>vRVFWyUMZ(=tBQP=n0%B2HIcp&=@-g|=fZ^r zZ7MuG6ki#NUnr+~s1kB#h|XOsC-ILr_LP$=y~u?)l8Xl+=kkMSMsfZ%**(w<@@AJs&zeX8sS7EyT z9pYfX2=dZfKVnRnh(6o%0@sS$*4A+A2X4;cJYc1+bx!X(3v#M;Y`u5m$NWD;oj7|T z{V`SZ(vd+Q{u6t{jzn{u-`#3nWAAKa$=eXu-s+TYxKBzis|(74r^HXOtqOzZ8B`gZ zWl&=vXp6J0u2mtAJ7n@_`(o&aKPFFT)zpazr5E`R{P>?4EHfBzXrg|TY5vFf@cUTU zfVw5o*gDn^_R@_w@i9~AVuvnS$D$|}iUoidH~t%G4BGe*@r z?s8HGJNgGNP&Hh_ly2~0{12YTki`v0^8xbPxQP+&QVDUlSpY#ZUc_ar+s0*$$Ki48 z-t4CMhIkr!KU!{Jw*gyo90$E{10xS+@E9fHO0NKEbU6w3%n1`sB$O53)zo?2_s~BmlIY?C z57n&eAkQU~!R?&zI!(CnOa{xBaC7)y0HZh59`0t#cPF}qORzV!eidc&DEo6T&<^tC zy7r|czWzMEPT=chlI?zvdtX0;uLXSl?Or=B*ujyRn+#!jLb-xyTTpH}iKG9MzpQ@_ za(Jtb@>3}P$NlotJIZITQ~n8*pMf6Zf8zhXH3Mu-0X5BH#8ddcAOENOHP7v+87@>Z zBGos$JJp?DP7n2N9AL#oE!i%I{yl1C>c(;2fJBbASIs38LsucS~mmq zxv(^XK7lUebR;EIIgly0AQWN=HRk@o`xkJU5S3xMjp6$cOvT0gN0i20(Y(~UC%=B3 zN&5Mr{ViL&h2+l=M^W&g;5tjpxj=Nry)%Ru`}ZcumWwQ2hf6CKRBMbz$@umfT+GhI z`>=U%mYp73d<8eL5!q_Tm2I55hVf%a-~Ux)MzS2#D<@rbKs>VOp0i_m`U3o66dJxH z0w)Y9a6pXjO$yNW&mof<^ndHe1_Xq|{}zV-nu7?35AW8iwgHd-*b_z$+3nc{|4&d1 zjr{$7)q6GayJ*D!U(EI@ga5(c7a4pF!Q0gja=6XBml;Us_iEdC@ce>&$yZwO~vG|2u5-EQ3E{a4UkyL~hY` z%Pj(O7Ff=1__VVrCoW=Y|5p%{W`^{$|AncyF%aFtA2W88!GA?io)T!H>o59O5%7f_ zmJrrU0~;QQ?xEbfbQh-`%Gus$xuUtj`ug^e^1j}YhY^EVl$8@vl{H)zi7NX`F}6RV zRBtDI5;Gh8dsLlrc$XqHJn#&GR}Jm4V7{o2( zG8K#)|HR_?X3DlD4i@t*l#|-%b;7UA%a&}L78Cs8TC)_dEK}c1elu(`Jy4h6f{iJ# zor6~7D^k!uoWDwehO*q?4;Gj|&tp6|)B0Tu4RCgmlefvZF47Z{s3%i#>kRx5-@}i` z6$8>m9D>s3I@ZhQhF4csaEA`v@0R4NKv_zzuU~AnBBRwRACfBpm1!63x~8%6^z11jd$Y2`CC{Pan_Q%>rCe@u=~38=16LMqn(4Y`MlH! rAMzQeU{9T%>lW<7T47Q9)XXB-C|P7MX~T8KrSrHCFV$zZ@~{6NZj$VY literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/sampler.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4fa31a305698f73df98e1d2d6ca988d4447f9fa4 GIT binary patch literal 31666 zcmeHwX_O>aR$gXg!5ief6{oZ%)ckg@I9v&_v@OS)2e||at9}|f` z$NL#Gk#G`r!bv(tBWWk)+prCMry8lIX`4wtGaKn<#?Hufx{+<>?3`R@8u@0yE;NVi zp=Qx8HizxuX2~weGovwLk4S!__NaW1HOKAo@YzIj(w>y-Y-6f9ZBI96?3w1QJu7j! z#=ho$d%s-g8wZ*P?Ssuj_Mzr2_ASj@?OU6cZ8dMRZ^qVPcXH)@>av@0 zhPIOGvXpqIeHXrqjXRrn+lP~hmf;xoVRy(K#;3e5p~FREUt(XvzQ--N`}>~E?o03q zKlc0O+33mt30pI{Z7ewpK~YPc_e)2 zE_rr$Uy7V_*eN5$2f`Hh^gYWs?{|*i*$L+;`WQXw%!NTi1ZS`II~3oN`Y(r}3F}k1v*-GtOD0eDXb{ zeA+pOlutR&NG+a~FdCin&U1Kn%ITFf@7{wFI7^G}t(={>vcg%Gb3W`mkMyUVkGQkW z3zE|X2`@_cqJ%G@XI^&8sN*Zn*C6#7No`A>12g}x&0*B}sPigvI2+ba_xx*8%GXLb z>lo#^WcgMs?j)mcLx}Ew9?_G+YliuGMm{`j+2bbz7db zsM<}5Y_=V@akpo+*Zj_!@6COYTyegf4DwFZuhtq>&*RNvz2#KAj$6Y`s@_^Grvvj; zz2*nz`MT!^xes^zdb?F^1nG-wC@@IB*us4|6^veB4eG6>=Ul(aEQ66K<{~Q6R$ef5 z!M(KRwrcKK)n4mdY|C>O1r_6z>m9e838r2;fBw13x%rn)o>bE~+n_h6ewT6FE zEqNEwb-{>K%B?h`F_>uLalO@U)~gL&W7i8N!c*%33WFgF?j3TF~8Y4 z)~R=n`c<{$`bQsbt}Qo@H?JIP)EAC**8Syn>)7Q+qj^k7q~cz6Yir2vm{6B@O!~!D zbDecHgZv)E$15QyByvVB$$zEfcv7C-eEcCa^o_rhv@rwRG)6eX@pn>oHXic~X8>Y` z+?=CRaMV+F5h;e{jjY_4`rgP%%t&8MUSdWOGbZm8f|3i-t4mjDZu)hIix61n%}Zlg`xZ341@*=>teJi< zq%eCA2E%$hl_tRI7<%VSTUl}^Hkb_!_b+_SwL!$zYd%0e7EnhRrIe8(-wl%Asj|kQQ=K)jKyk?^)Q+`g* z_#hY2V{p)M!L5`FHZE&IB_eaoelH|*Jk(j$9k6Act}m)m>#PO)xe8PqVTzS2!WGF^ zge?v)*Ri6tDjrA|3}vdL+{JpMQ5lRL*93(IUa2Z%UvIiW7^jKM)p4o1bG2rGq@q7w z^Yauw9*Ml0SV?q~`w|xtZzSj6NUD1fS2hCEa~q3_6xpM9br`q7P^AJ=ulsJZQn`wV zFEP79a%11X#GC6&RU@yC{&>xgRDi_ZHFOm~0jPv}2vO=`1m%=^gx9P?kiJ}Pthsws z#fz%)bx7v1Dy2QD5~r$x9O6G-5A=o4-bijG!)l-#QVq8yU4ovG+QiIHcSg2SL5fP{ zJEz z8t5xlG>?xA+dM8fXijnsi-cpeQoLJ9ZnFAN^AI~8#u|t<96nEdKH-?%B*JvpK$tfEGMZ|b)V{&IaX!x;>Lx{a8y*~C=RTUfta8CVDcOkt;8!}AQIMi|`AfK3t!K;6l!yBHi{ zaFhYpYu(agykbk$B!lA&?qhI2f*`%vXjgsp0IwnR`sxFG^eBT5F)pK*7sYuMq_7;Q zr}?Zmh2@HjN0kxqq4z!ng@kDqv*W4J)M$DDj`nZxL7vqVF+nH z`0Zj9GLUB%s!CPYD_l!z$ZFs(8{T;fxQN6`3KgG9ECSAOQm^N12$F7kCF7*sEL4Y# zlg09z>%}<`Ffe9k&m_i9)zgvRu8g7ufH3}T??RIv&}V$=&Q zmrj|N$iO;WpL6GqSk*>DvLcCvI*@y{=60YFK{bMUQ*DSSP;Y5L0aOKDp{(ihzB!9J z#v0@TrpB5|O5QCZcslw0bJodDr?GBf{qv~U9D%;2#g}yp5(X&0L)H)J*6O$@*{{3d zs}Dr4Vu^wzTNUvFEvE*hGk-8n(e51WNOn*>j;>meMp`uynRUX4<}5uTQs+M9Pd|Rz zRjAhgdSE`p#BILmua%hQ`fZhwaHExHVbg>TA>&Cc;(Q8`q54-}Tt`=k>L_a#UhXuiHCH{3 zG9jZpi7(AOg=IdE+b`hbk+&C;nKa}yjV3_ric8Muma79uG$5;S!RMP#;p36@LLy1L zYHX%9p<|;xN#EE`L4L!szQRxXy7{WPncmE7W;b))G*fi3%vUnqEW&IzhcM?ryebO^Q zf?KN&WXERKEMbu3bG7AaYsrl=k8{(RLz&Q&gGq@Rg^E+iVOZ#VIj_#5OM5rp0DR18?Z{uzgGjJufNkKEci5a+(+06M_=CNf!RL!U} zehzo(l{`KLe1`BT;xoLJ!Sm#H$sbu6y-u+b2-EtvOTI%} zDdbk#9P#hy4xyYi4Q(iMzbGCYKfFy;Fg@S&Z1eptZglXMl2U^=k zhCAPiUlWY$|5}w#5gmvtu3HZ-643$HDk6$~^^fZfc}nn!IN~1bGz5_j(hD}Dmew>h z!sV_Gr6+_oG#W8jWSoZ;tz9QXQ#|ppkUTZ6^Rb?gpydXRDbdc}2s1`64JWL_t+l2G zHh|0YcupwXF(v<^rp6+}3G2BKFO?cc&-Dt*H|D>{rSlCS;!eH!#T)o58_GeHdYSQt z`Ut*)i9MD+Xy;??7NmOOJK<941vB1iJp_H)i>@*ZTCIy?C!18S2&5wKD$ zw*^y`g|#|_o-o&F7)N)kf$F1t|5XOBG5A^r6$T$e5Tu2>h~63;YF5`5T&jV4VR-fd z{aBY50s~$+uhcR#8wp1@Vn4A>Ad1+zj%L9giO>COyzTi2iZc+xQY9>x#WeMH{jXq{ zsT@HW{qOB!ZZcT}0FL_`W~!JRH^&VF`n~?gRE4)P`63oz(=<~iaxo1fiKQ4hB8*c4 z5lCj<+R(-#2z+5}b_uSeKv{w*>tU&(keaOiG@@T#1{4dP1~Hrx6E%r&JS;uFmL^Pj z!mh3^)Ejkw9a;`lkU9Wb=w~rQ0xuNVwcc1i8g_G}M|x|}kwgpav#P}G#JWQHH!M2N zjvB(1bIUiijZ=B(YmPCLxrNNh({#t}^s6h?gm zu7VtD4yDIm56p#jyCGG90IZSBnhTV>(pTFHe3|m7BHs8}d^`eExrCW4P8xa4`FD*} z;T>bYzB1m)n1#2E=4?{ZlZnwxE4(4daruH2@9h>s)5w$c6bhhh?E}e#cmz>&1}+%5 zBS4*U42W&1ZfaXpA3w1TG3|QtdJ2Ld)H)~eO@;&U^_vX3O)#C4S~WM#ZmMf8(x`0C zD9CxO|ldaHe%DDwO)5&KFn6-nxH*9EQOEvzYr`X0P!Z#XlRG060g~4wh7objnIJfZ9)dmc2mCD&2A+T znk(robnV0JCUYg@-F7M4%{nkad?wM&tWa~ztmHQHU1*A{rmA-Hs)`Z{KBV-mB=VmM zWnvRDdl8*Ij68=|;GfgYttQm@ZehFF9ioJO>EUicReQi)w{R-4mE44ky*=!gwnrcp z=dT-*GTSv&+aGlR&JBSD`eXih*Z5YXeiXxt;h$jrRt@i})-5og>5Y~KsJJTXsTKlJ z>miYYwRjM++MS5Wfqaj`-c26VdnqP(L5Ij?7pUq+b~(pdceRyErpo~DhO`bCF_Dp*)PfE3JS(3YikQc&In!HZ!t$gyx5 z8|~?Sag50-0t5#cNI7oH2{Lf*;CCLzE#-qjq2{OxEDXL9nJev!V@7JD-VtfCT@=7@73a(xuaT6CZbC4LctYH*D=D8ag$_pZYK!!+;;%NyV;}fPaZkSAS z1J|i(VCrU4F|S}U9FuSYXmur+?C;T7SZcBwEe34_HA)rHUlA8EqV(Okp|}Bh24dtx z%LY!A5iJ$;k9#A&FKj|LkJv0(>^3+~H+$XKOgq$=^WC8*6C;U76FwMp7dHJ>gx&N3 z6vRjx;Jd%*OR2(-H#gV!-yZ&Qc@Bi^3AwE^ZEg1G$d>5S(trqB1vOdyL-d#WF$P>j%2_?` z|CkY*2!gz5-5gd?W<~IWX-0#-_GrX$+pyp^+c1mV%{L_wR(}PzKg;nYZ!P7q8R24P4n9zAssHovO11jPy_0o{;L4;#d<_GtMD3$C3I zRfz7RKg4BHdqa7a80uW7r$vQ%hu~K)H(lhBGV;>vpyU=-1a=5mH{I3}>}u`BSh}Mw z8sum!N2Y4dd4#=UAjXPwjLlUf`8QeLMm zTsLk^JQc-E7&qoIX$kCe6tjBnWrsid+7aucuaz|sVyY*E+&0Q*cg%|rlMqR|-GcF$ zj%g@suAHSh$kr$d9qRiThy*4Ar^zQlTG1XoQI z>dRQsA;?K8mP&OjWCTb7Mqq#u(5W$1oe3F%Zsy-QBN*)6ubv;o^PV)+M>m>f{C|Ex z8U+QwCE;(#59leK0@%Nkx~X45o%3iaoOXBhk}gP&tCxD`=V$ zF~zM*$orRA4Bf5)uLPbJ}*RgnBgG{7S znYHlf@8FTTjsO__Hm`e<{I}uu918bxxX2_7qi`d0W8>}$=U}uqM6d9B0&kyUZZ!tf zdT-u}|g>UF;2FW^Hio~`ySjm@BN%ietRUy;63Q@nzs{Sg2!Q5Yo za)-5{C#}87ggtIdxlR2dgF6ufCUmG(Ve1g|#FVb}@RyiEl=LhNqJYy|>PHcUoJGhZ zW9{SkZ}PTR2AMs;b@u+1y5ld~- z0w%~dDrh%x6?ATN1PO-2@^wmH>8L>WD7_N z-2#S|co5fp;CdL^(=xPKjq$-jFxMU808=wq0j!Hu85v^4V~BH??uQu&u2+;(woFJG z#m(XF@OI&P!uzB@L|H;42%wDq@Cr>lXbaFteK(3bxQ~T*W6XywLJXyqaVL*9bE*{P zzi6SeG^T@&(p(j+P`JsjBVyy&8E#%Y$IXk-mbpQSj>y8Cfi7B$)jALy_^(30Um%`@ z9gYQK$fN%$EM z$q7Q?-Q?w@I&^6olRL97fyphlP#+5Qs-foKtC;Ji;7yos?FWRKS;^xAf%Y=Y{QeN0 z0eRb(kmm(?Uc_5Z`@?vWX6&aE^k)R=Jf7BlXh&xUtog%a--M@{g zFHQ}MaEh1tjSYXmv$0=b(Bn^NfW6|0D}Cthr%|KWjnA>38gLWrd+9K;vyMK_1eUkn zf-AfBNV|y#B77hKdc~EZ*Kg}GZ2gWEZPvzgKFTd|A_jwsu(`15I(53zMfC}t{3XS6 zoKU;G1_zT8XNx?CELhs>4Xe6{HNukF2)Ige>h%k}COs&#@cj^TF8c)AtBEd3!hWD~ z_ryBU>tK;p#W$Ls9|N_g9|);M=eR?Xk5ZynbpGU1GDcGVz|;lNh(Uh>-?Fhtq{9Nm zTX6qKgEaa@T=?t3r9ebD@WuUju}7|oa_n!C5+gabL^<|2eGod4S7+qd71EQpmqc5h zP!zVs*I6Cp)jkG?7)X)BxC%07UpRgG)YoVrEc3$2`BNW$E*R3^l?!;Jeu>Gq^AEFZyA5pV^UyKg&f#d5zMU!upi7#*cdqqn`te0vDM!!$gg6DuXW`7J1I1Uz*brDH=D#Fdfmn3Gyds&4?8FiH@iJJy}X zn`!mU;Dg}U-RbVg7EY-_napJzp9$f#zwe|sXS!3UX-WEGQvI;*5qN;lpjW2BX=g5d zkA4ef|EQB{CA(9bv;K^f_^a>=NACM*1l<>Y@0ahH?(9|?KG|qNYsfzkrb4VUbSduI-F^DIT&kYr8o>SkN873n$rtP-1ep()dt0{qndwcK zws4*I=rSIrJr^-;0QoUH>&aM}zBKdOn}h!i%G}F^`!2l)4W|IbY*})SoVGo1);_Q7 zsH)N_l5iKZrJe8sKHfNjI1*~XyhPYY;ZdMC+E?e#$TKK< z05V7xZfx8Yj*u3P^j7wsqfY$VyjF!4@S^$@f<0>~OTsjs%94=nAs&5;0Tux2hS6eh zfY)(FWic26lM#2r>BL`CI~rod0jXzeRey#IHx9fA(WBFWX*^odz=JLF;1d>HGNe#p z+OkB*ViB(s<=d646#L;&=PqF)Z_pYlYHj3Oq`sH6mS%jGSO0~<_cIX5++191G_<5n zHA;P&foKDd@#^;({96VJL2&RZ?$r7Nru)wf-emCO41SS;$AGeCpUnMRyzMafLj*zL zC2oU1trRrY8ODkWHCcUmzZTa8(U9I4SIyKY*G&A&ynoiiM9-$R4XOk76B_{L0o125z)vwR`y94I5of`hT%Ymp%X|x^ z=c!bHHs65;pUS|us*r;H4x6_Yy~V>lll`j#cg#3ELqrM;SxXRr1UfSO@dwU#B8FFNt~{l}}jror*S;_4mA-L2-Egi1l2L4mfcAf8y|G&2!MH zoweF?95l`NL=}E6dL#seT@lUR%n5sVS9Q*Z?kWAv>ltmNk9N)G{uZ6Ej-T4AA!WEF zbKCRY)$P@*zk#QDY7qQVxcQ9JpRLqW8YC7@Qy``g#gL#kL{iEAe0Ux1s@Tyi$x<6+ zb`Bn25*Y=cxU97=W6Eh17fw`Mmu=v!^x4q*Tbu^|dvJ*}={H|C)&E37^(6#AW#K48 zpH1q)JLm$(Yf&+i33ip_QN$bjOJx6(0Mkd4>VGqzKSnSBf^&%^^n+LM4kj@@D(Ecb zzyY2Ix^OSS79sAJpg_UR75))Wq!6amz1H{66=t2cF5@0r5;i+7~ zQ3}9~Tmkq|uXMA>jZU%jp9F>p*LkEC#R!Nj6r&+3M)8{kxyr2!NolWyr431G#V(xQ zeOO4hXaij!9($>%@8L#kC=abE0;mVh3SW=n5-9ruPH#c^*h(tjqnp?qi?nI=YeZee zVSWTwU4u#-?g@M8`w8%a@$SSDZObK}cq|l^?gZ*F0_!i7GTe>mZ`9=rf(bXrU=p7xsF~CF%nXc6S&xf93H6d& zVv#FUx#_L!tEtV|&1tB6liO3>S@xhmjWpSTR5_&DC#j~8Y9CWw`faU_5=pL3reQ}G zcdC%(N4!R3Wq@$tmk`H=980{hWV1REH}C`DdH2(03&#rzbsfbqUG(dVI0Lv&w9hum zO}r?NSUsK@Eh0F{qvgajN}2|+YT@*YD1koOFtS2N1wMtU^7xJ&b>?=4NCDcJRIhST zg#gs`CzTouiYHN#dr2tO8aBvyT1@PhQsyk-QX-V-Cfwa~SOW`*O1H4{NE^;^sK=n| z-`!T<+ZY&dNs&pwU(eMU+w6<3bqG?qcHz`^n0ii7h|a1xtSJ`$dk@LqH3 zmtvwJMJ8XdKKC85=H~RcOZSB4hrS)aoXv60J-Ljj`KRYdgagJd57B8PWsw-QN8lcA zj02t8bGRFn-}>A<=Pc}LjNCfpxW&E>(}&#YYVE9;oZu3kc-MD`3#8=*%q!F^lpK4$ z%LDlkF^tnoQ;%*HuiP9&{4na zEc(5MAZ`jgsBgid+FPeX=ZPR64>LD1X=P+!Ba>hbzkwB5d=xgYm#Vj~cu$dhJ9*$0ooS&n5h&L{`pws3TM-KBYF4GqlYQNH{ET$r~3 zj4%l)6tN{M15iMq3D_zL?q^`bETp+Ti>?~jB+jQdrc*_^0zd&LfSf5CdmyD6X{vIW zcMa`F`0lKkOJ!tdN8w$<4agyBrX&wI60<~Xo<)wvyLdMV2NB!>rjWa03OhhqQ<)74 zt{T{^${@u6#`T0sGXO%KsAZ8Z`Xe9_qZ8A33dxU-90?c`;F1BEHm5$>&8`?c$V7b- zGFhH?@C|@@6f!T20+55#*hGNsoZ8WY{sgekbCcKZ_+iM(ND1lsSuKx3ek*oMkRkzd z^JwcjWy6B_IJ6;`X0S)75PByw-!U3Zcf^hRS)5l0sd|RS)CtI#INQxh!!(kIYzXOf zD;3Mu`EB?e@)=UW=n%(eoD{qjGM65R>Wn*XQiJF8BK-@q8y}3MDT-fue#U}|94uYk z#X%Qw`ZQUUE#3M=j z3t;zcYv*QS^{2@AB#QbH2Rg%(dd0f;6}eU)*NyS#fvQt|imOhYN~)g6prY#CBm{Qs z#_g*(FlXWyZ=^In4ks$jhwRai@MMeg&8mNW>(RPne@@E)o@>w~%U1^kqDrMn`xyE;0pd55zV>8WRCtf=_U} zAOKHnvX}Fn#3vbtEt=+y95@Y64+ZbQkrUBX2`**iD*!>?#r8vkVG%q(FYGY`&C=q54z`tYNa`hw;ObTIQ~MK8=lm&GZS)O24Pf_7TK zj(qR~ADfKoqWNCBrF98>1KRWeKjD4@>Zp6gkH}typ>B~svVeR)f*OqCGYn2a!`UeH zQ-;D>#=ERTQhgh~>G<|-;3nDb*h~VuAo6{CASS@O_KKhMhrns_A*Y#w;j$QVnnHJi zI%FTGLF+wQsEGvz)LERha2~OWm)B`VhySg7dOPKVtjiy3yw?w zBumFHhR}=YJNZPYTVx`U%7m~0ctSf4n9;e+nPZLVz_EIGBTfqzfK?9<4^Wrq7*OeH-lHIXU4_FhQ(| zd!8l}!@u$IETM=QDY-mKMid=6PDVrcq8vpqI!_9(TDSKVa69$4<8$|N9~_i1{ExI> zYhhjT;sL-nJIuww9`F`U8;1iR7jvl&Rez{^IK~{R9dE)zTqHA^sX{QduMz`i=7{-0 z_RiIO9Q*;$N8Y>x+A9v%Y4hY7QR_&#XgH&^R}?^q9lIqzq+h*^-+jPuROrr91ViXv zs4?64s8}3VLBA!sOoh4Ns?Xm}DsdRg2{xORnnidvYA+uu*GDa}`}34opeB7MVTI?S#s|B0 zEbg5~8Nf1lRxD4I6NzC^xn`aN)xmJq6%=}{%JR@toU= zS}#aH#off?D_K`P1n6aU4`&RzsE!ls9m;{xoR3RkR{ZNfQE6vEXLsy_j0g8)(C}bj z{TTbUy}-lzr9i1|d}QjbqL%AR%REz+2P8M@%k6gLwRUa^j-wo^9aBsuOcm{*oj`t8 zfWLKuqTrCn2GXJr^PMN?>aNp02kRON5!yqk4Na#+ zBL>UXZPhP=7%@*!gdR5~MOJ$QjEe5ut%{E0vE(@M45$%m;ko#!088OXKGyzNVI53*$v_V_z((Cj& zY^;aeW9Kx9PeP?TFg%LawZoz?lBnzFq_)u%^+M8?3Jj1+TmvCi9JGao@MPwY%S00~ zjE~HsNF-TB!=nd_gZQ5ML3SB)P&66Frv~ul{;9<&d3U1`%?xaHj%RjcmXaC0>tHge z9F_wf%TsM!sjhpnOlXb@Pp2BP0w|3G1i)GMSR$fzgrgMFLflb(LnXy5^iKME4-14) z8PUCF8z+p3$s1EuIEkhN_CX&%8R!qIcibA24fZ~l$Nm?OOYHzFg8cSsBD#l1o`rL4 z{=*kf!-m~}du2<`KI2)@q+Z$cj)Y*!fXRKFWq^r^-ougqfP|uBy3q!h9P4nBmr{4~ znKh#C{R>jaTEy2%*gc%j!xP4NXj-T0FNa41OMYw{nJ^BqrbBbQX7gAD5CGtEuf7CI zj*tF@(=Lvo=7HS822eePqui7xW)Q%d{B}wttqC7wkD{G;4Zlhvw1g5BO3+5sJ~v|I zMrQ@3*?A~^{{u)%B2dG0DFW*zBLqgIn6*>Z{&U2AFNHsfT%&WaQ!?g^`Z&`HT=@xJ zEi<6A(10_^fD-mccnY-(5QQqysl-oi8QzenT2Loi=OMoFY~D$UNy+toUPo!wZM*}> z1Fun-Jpt&@DP_xq8a0Pgq2W6!21^(@0q0P&AKomXyb+`>(Q}i^&TFFpD`42$9Eb6a zhR~v@T!3A+upI@awI{k6#Al&|6+$oe_hah`!6c_^;@yeO$<3)24jzfF;WeLuGDQzj zz&h8`B5J-0Df%f4zx?=2=vx^6&W#r@i{a0CaWX_S8_w}To7Ju}Na3Q=e|`aHEDk3b z{2iLO6q%N?z$5699S*Y>sv>S4m9E8^UGz$?5RqF4vchb}(dIl1+dqnI=_k&Lkh9HZjIE4DP%H-sp{=^mch9#w5m(%-%FMNj4Va%z5l9tYLk!)f<^26GH# zJ|D#u4!=3ch#3Z=ZVAvXs+y=}6MQhjpo{?8m(1q_jM!sR!)r`d02Q7Rs#r!(VY8=X zZ+W;0JceeYX#H*+trfVRV-7!y0B1nJCZZ2s@s$j zz*eLBh~dU(XmDp8qmnW(o;Z+tj!! zafV0vgaw4zL3U_Xm+CkmWWBwn1)sPe$Zpgdg`mX5pldZ}eS|J^`bIz#p`g`P_`5Z` zngNfkz6wZoJ1cQ9R5+_R2ET!2km*B7eXr~yMNnQl`TRL8G|R6iNOMIgl(m}65Pzxx zS+atVPa>aB>bGDkp;cFEg{^5zI6+3PcR3gJPR!X=BWZuszfs3ZE*p_dZ=%d^a2>!e?#^y(HZ_aI9_H0waJ0(Jt`uOMk=AsoUws7m84;*3< zMHVqBZ^g;9jv!9i_ON~w&Iu*%)x!9R@2fEe;|#bW*t0q@DjWWK+8m^4{%o53_Kl)^c;_rmQ4dVl>LG%LcRBTIq^6p<`iTuh|D3YT95$}g2Tt; zxLSSSDiJ(6olj0;lLNeDUz2QM(i^jXi%EW$!JjbLV(=FX=-8>9)V_{a0=vJ>D++|N zGU%z*TbSb2DabvF5k*xV2QehPUq)yqh#nT@-$5}o;@ZHu)F}&}5|G1^VWwwOJhr5O zo#qFVMsYM*ERLrNdYQs5^YLdBUvE5!MGIyEp4Dr-10%uP1`b2fF($CY(lOrA%wXa< zK*+aErkNRxMa-dI49@%=BOb}OvE3^ql*{Jm0&kx>WOhx(f{b`hlcM8mXpq+hr z8jtcRPWkTj0-l;uBKCge3V&HT6PL-le!%38FzMG4$|#_YMbrwW+%Fqh{l@;s{twrJ B3W@*# literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/spec_decode_base_sampler.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/spec_decode_base_sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae776c2dbe72d5c7f02ea2bbff11fec924f911a5 GIT binary patch literal 7330 zcmcIp&5s<#74Pb=>6x9~9q+Ec_;LdVGGKPYM}RoSGGHK}unE{CQ3urO?W*z4Zcoop z)nnsPucBm)fCGozA|a&Rqa)>-TXKo=2jr4N-=av7ICF{#IKNjtvpX|h3lk+R)tjo< zRj*&w`_+4|YPz$tH3QERzyH(vuTC1qKd3SO*l3(XQ4)k3oFztw1&mSMOw5iISRFgC zJ5Jzeo|U+Pi+5I18UA{mYEWgys|L5ZbIst6sBYA@tzZTmmsh~4h#9o&XnVYhc2&1$ zxpl#4);`B~jV3GRqBWJtD9$<}TkrCwRoLg-aaMRg?q%(68YPAEc8bbsOasQBnPEfy zoEe5N0>+KNgz@uLk3Yx{-_q-d&?ARmtMQ}!*fl3O zbcel<^SjV{;SPJ>&F?|)#XIbMFTW4Hm-q=$;S=-1s(9Sz_hVLvKQwG3I08eREDm4j ziTJ#TyIi~&DRCj{^b#QlZ&SmMlSnDQ)s?;~`O5Dr!8M1*^^+)TUlzU|-%c<2mxXM% z+A+A@wB`>Vm5JW6ZoN{>01f~^*unQ=bIjmWr(&U~Qy`hKZS1gJgBdx?jS#j6wYUjt zYwB&o)Gq{s{Mk~+Po6LH2=Iqm;!9S9ZD$6V8H=4V~M zCnRRx@zr|N6DwR?ZpZk^uJ*7l>muG%6T(O!?@L+)y;i6!`xn>S%ExM;KYqJXia1LJ zmb9%@AFQ>0=0}|0Dk<%zSf|VFDDknxB8{)^PvmoK2xC%5UxitBQ>3a`Bu*5^qL&FiVlEIXq??Cu2 zk5;2_OS=`W6j@)w&rD;^fy9?cbOMqa8*q_*Lsecke$6tLv&`H!HviK)oEk=@Y`IaR9H%4Pe$;9XN5HN2doT&TDY_oxaNa zHQ~3@jBp4$p7ODhzP9ysvv4%} zENnt-;b5iX^}q#|6fu)-dP&ZbAXCocM+=n;RMTXUgf}##F7^6hIZWLTuDT0k4r^aW zSzs=+naRAbU5kF}Uk@HW*do)SV#4p}p*j^Z#1}3|W^AwvfY>V3yBF=C(n`8f_SBQt zS*h-nrZYX+T~nc6pJft4W_HcN{Rb;E*88@4NFO33&3L4PgY|vpY54&0r@lX@_g2{E zr3z0{7X)TwaPnXUCc78jXgd~5s*i}-&ia`sHyvcEVt#BFmW;6-O^ldeJbK8iv_mzl zU&OlR=7#Z(x#euRxlJy0%LY8zrnzHsbC+#cTb10+D?=VYz02l}Ge+iYxZpE_^e5(y z$Js91@^UY)ppHJ9##VK#Wp33lqMADxV{Fup8ZEf$jTt>2TVcl7n#pIl!<`RIjy@ll z6s))EnX^;RtGR{W+1%P?Gsdo!*C+dtF6cGlx;o6en^taf7rjSP!9Qc<^_{uAHgD{* zo%!6{Y2>wC=#3zPkhgItXFChK#<71o@E^{$8w6@COJ^k3XS<$q1nG<5>wHK<9gPW1Mu1ZOFV-mO z2~~R_PWhYuWBzm&^6uV19o(b`e+A!8p-mnES^fMbp5|fXH4Li zF)LJIMd{TtK)8{N(n~^12&(;b*gvqlNVp`OCgejPg>wa2o(QVM4wLq#z%i(wgn^Ab zEQIuaQU+E_^*cI1N=iJMvt`(MnB+V{SK8V7WSZudZ*GY+VJrnw!mxb(48EeC2B{tQSi@XmN31zE z$84*PxNI};t0mjBYi#M}5!Yson|0e`_RR(B2qL#_-Yz;#exf*9CUoOE>Hu2Il)FPW zCL$0;BMRd>0{sw`+}NQs-pt{%cWv^!*Y9Vwyq9!+F0WwRh1YQx zkC=s^eJb^)=1v#v@bunYbp-lgfjAPfj&=7y$>3tXCLdPzcc3VRnRC0jrEz|KXH|+ z47==WG6gaD9A^D7ih2~p_S~Du#jP54KeBVsIkaq(xSB(r;imni?R*$-{ES&1>Yv;KOpWWuG^;*~d7k z=!+;^OkDFmGKja$l%C`e#}fw(S~a?yyFfzeAax1zf48~?XQ($2oZyD>8}mKfY3MbO zWL6xz&E@YFESs8pQ9F~seZ}2Q@9ZB*U=74I2m$t2t~Ge|PMpV-2V6drz|$q_EGi?} z2hM;Fi0T-Ko`JW;KElR@vxZA3^>4B=vXi~AJ1wW@P`w_8@Aac( z_@)wuyc>gU)A>PP$w=BC@;DK~vOED&c<@}fk;<-)ZCpCS7w$!U_aMJVU1??I^F+vl z>0KVv!_aI0pL$bC%(ST3runsPnP1tq z`ET1XzqH-j4fAEVzA?TBzDfLJx8Xl@8~(y=`19tNbkkY*Qj(T2%EUv-=s6Tc{%rhe zTj1<1aO%bdH&I(tM^l$5uBCCgeGQQkTd0a`s8&3%5Bc)~xn&3WJMIr+x(yxOYxIxnKcxJ-$3o_rY1TbwK5pRoP~VIjlD&}hW9;^JyI zdyTH4JIKrg*VkCh@vLh~`s5BND!8@15=s6yh}G`I_6CoB^D-?Dhpjd}c^*UM%S2uw z@+uK>O!}xHU#A*{y)j^F9W*e>Rs0o2-2+0vG;Jh!W56ik*tj97=Plr%r0pDvdJAL{ z990wlW!wfD_&Wywe~2nzQlsk;{4?Qc$kg>eArAfjp>N9T#fkm3xR7<@^=KL|>;E^r zsJ-v$&1yw57;| z|3oQ4QXM?<4GS|uNa;1}cY+e_<}CU)dC8*ssz`;5a2I=f{Je>de$z{?x$Ebz)|nieLC`-b-V p+F@y6YPn-8wg>RSJ5xr8vt$NE{JOEkJkvvtxnLQ;Y`nJA_z#>mRW1Mk literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/__pycache__/typical_acceptance_sampler.cpython-310.pyc b/vllm/model_executor/layers/__pycache__/typical_acceptance_sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93151545afe866485118316d2e5760d235c460ea GIT binary patch literal 6706 zcmcIoOK;@H5hgjD(de~L#g9Ck&H@Q`;?>C7HDEwU7E%0&4S0>fyFrkN0xpN$qoKqh z$K9k}9w7&>;av8d>+aEq9COXT$c2YooQsdS1&-~_SIzfGYsU#fK^#7s)z#HiUwu{4 z>gtk#WAjgc8Q=eoVSGY2^~1)^&+xBa!igK)42(hcYwBMsuqLfR%QSvr@D{fp7~B@E z4_X78n|F;~`%`>nywfw&Yj>x@drf!|7q5v}$cZ2ND)zmfyGq=3C(}SkkH*#ytr;Gu zJ)DFwFu5_XxH)JoTKp<+2^(|Sye-Y*D< z$8Nf;V(ELaGr=}`!8Ess^U=`-a4ZUElA$CH$v*1tDS$l!j zS0OscTqD8aUdGgzQg`CBfK5 z{41KebERWmF;9MwdpyN>17P}v>gltYE)K=JGmOGSIkdehy_ntTt#y2^(q%5)QCz&u zZmPE(DdI$i9<9DO7I8t^yo!JIE>1IJ&yY(qm7BT(Lxv+vFBI4?pw3 z%WSDChlhtrv4I;xl`9TKr2TyLDKg{dORwOUb4 zkc)7uhmERX(OX&M0gzP;j8K9d0(%fZL6EDVHvrL#hnd=f-$2j-JlJmHN;ix}EkdFY zA+KZg(1!?wqL}z`Op#bBozMAovt@>(*~li^SDKXDH+2O9^ohoA_EYxTA%U@@{1b75 zg~`NG;1mg9785y{y>GCC$a9B!%aAqa7 z*|YDF8A4#L67MBCf!C${hbyWnspj;r1v%*sKbpdgCPb#lsS*H06$+X1L|n3j1|(`) zV-6)UYgBuut$?gke9qZOMw2Qy&KRSLGG8E2$l%5C4YsegGR>$?@lm zutN~PBGmtD3$PcpXE*(zu;e?RKEcsz4_2HS^%*?xNY_`w@kdB|IZ?qCER{a*2;vi9 zB6)HWJdc$0IR}Km0jZ&|Egq9m2NxVq6Ei0>cW+THBSpwdbh=EZD|EVwQ@T=zhqPP7 z%;4&IaxcwqpS(bV^obH=x&Rt+=t}-W+^Fy3)VbaC6>XJ%|Z z>L4+;pEVwvYbb)a{m@!APCIzMfDv6JzZP#l1l!$PTr===Y1UcBSllnK8Az6t8W;A9 z)Yx%9twF599Gxv+|lc@9$6=e?OA`t{=L=>FM=i>KT;Syn)lr8*GP1 zv7&3zYJKyx$8I-we+P{KwiAd^e4Xuxsq%v;q|ZbUyX?n!bTrLhml_plxc=t#^VB#q zj>hkJP=qx!3+7epu^lXC9GAbPr1Byod0W~b_s^O9epV-{DQrn;BIl18HLn&^Q=hL zWxOKL5iGYSbPh{770eH#C=hO#!M%?lM9+Z&@^I`IB?|Q+!e)V93OYlSa!4-+dquqauKl#I- z|M~aZADa@*Py>C-PS!T+tw`!>rEP7@L1!0`DCA%<2^Gk`I1z(|{330sWRkXmXg6J; zCg&uX3>Hz|DJSsvg+$&r(iTunc4>l~T0MtrO{nxNgNxX0?sH_JtmaP_a%3D_uG}mS zhIFxX=fP@WPlu?(9J`XwV3Ob9U;PND&N_IN*#(p8Sk{+z+eTNS`^2_Z@YH&8$+Exb z;@Qc~r}rtIMl3xv_R51881-IW#lIrj7#n+NF;Gk4p>=A_vR>fd%o-b_fGOEy9+*^4 z>cV(L)iF%ZMm|;ENeC!Le-2PW#dEp0c|#z_7yYi5G%F%heo)e&_43jGhX+yl>%uM0 zV?k%&@VEhN6-ehHU#j{IBK~V5ox00p;JGa*=hE>`+o92kN?c?VbY-vyS&5i68VS>D zb)XJ5`8_y5&ywHA&!B^G6Zfds%8Cn$fWa~zpl|^K3N`P&&tkwI@vq38E?l;)lg+35 z=2I%0p7};^mAZq-1Hy6AC5JlMNkG>tj`LpP2Kk!>$KjEO$L&!Nxp8U}QtD1#u6;&` z<+q`X4lGIfrK@>Y5smx6acWCgp>b5>NQwvwebv_&h_)sBa|^EDJ{j4?FCvxDRS&zQ2LrJ)3 gOiGGUEwH9#Rf%H{_uIZ|t>0xhr zRZVglJyzmTEIWXZ9XZ$zNWeMhkXz0<1jsGd06_sc<)pI_z>6s6`~O#6-94O<$A<*z zM!kAlz53t({`ar1I;E1O;J5OR|Ge??2a57vR2ctNPOG906oaWXB_F}YrEW}PhB^=7V>*A!K0SxzBoDYi;Zne#?-s#S3+t!ZbP%bDg( zYu1@<%{gXpr4}+J6F19K&@SmyPR5 zT;;%|HoEE>-c|Nb_$NO$oM$k`Q~oK8aSCnu&C`Irin^!$XHfS{P(}G`zJ61wRzIiJ zsj6|__ad*>^un-e#KyHo7{$5wx>2Lk_L_0-X7FJzXxD-`bF0^F2CG#q&foBa*9xLQ z#PjdB8|_Ba@S2T&;HUK%$n+a`8h#KTTMr^P3ff^uxUG&KG~F(qZZ?q-Ui)_Z)c);v zJEG~kAx4hvy+^<}t_b{IEvV}8%r|>pJ8JYj0{x9nyWUt&>w@b+w9)Zj_d<*X(2;jH zywGj5FsD|~jso8eT5EytH`?pK$u#x6bM4yo(UjD<5@ITMgU0$sIStekTv|}kuh=oKCbX>B#{!Sn_8p~^c{7e4NPDC6xj6jPqpn_ zWDIh?;b(mFr`nFDDg*U?#?KCN50Iy}L6&O$9I%k@pL|0EUKH4J6~oC{+C0VO&(t`> zFuh$>ost0kH$)J+>%#Lt2kTMu!Pi+vn~yo2_$lLk9PPS`Sd3 zifC^`j6irRlRO3rcHy>7vqFPq_)iJ8tMRCiM zqpZg2z1Uc5cwuacAnJ*B&A^z$ALTQ+-oO=-7>ShoL@#&0`(MBMhi^M( zv(XMb!T6{e;v{-YY(7J|)07a7WO53*IOBD@&8?@A4K*Ycqi=r&;M_Sj<)bQe8|_HY%)}8&2urbf zi@CX=$qIst0UQIG5lpxhvq*@H!xxd{6hpN%kjW!c)4tT`(zS<%ZavhdjfW=c4b

-jDSwu(_}v}nU0<>}%zpiMLYJ6M3Ov55O&TyqN8fYyKd zhUeSSMqu-xHVtY=9j;k#+zHzD9k1C7mc>yF!4s=$f~Fu|8di)EP??|^LAQb~yi#+z zxQu+fIKd~!dd3DzK(kY?L$XW|6OlWiohM==Gf)T0oU#oLI8YXpZ3B00K^Yj7)`#RJ zLCuKY!vo@FN?t+YWE*WixaZb~ndaD9@fN5`=mE`o;MPT_HObn?Kz;$^g~Xb3Moz7$ z;#Fkh>Gfu3&1;Tzi>;CDp4iGXFS>YM#5^KeBJk~PBT_LNb)XY(r$t)*Ol9zadXfkH zs%l1jo$@@l*Cta?d1pGr4DzdoH=>aW{P{FzT8Q`Tns1r7xS-l>9Q+)i|-=O3=CA)x{p+O}UhG2c0y1j>_f9ZfgO+J4xu->A#yh+Jk zU@h#PD^wOHGd%DM;!%cXsK%rIdp|Ij9>-oXx=)XM3rYXw z1A;h-#NP?XZ_>+F_XG0K5SMon-H#2S26g9Ee(`_+j&>HU1#GE09{(yMgHNgHr{v1OO8O?1q z%zZ=o0W<|D#>%FSawRDnC}&V!Ov)z8S(NRub_M0rW9=Hs=aX^<{Z*7-Ov>40{MV9l z4&^-B-%ZLE$_12fP}xtBSCqS%_cd=A-a5n&C1+$7#LS2(l`+j^u`jwyuG{T|BuW~Q z>-I2GXd|JmpMyS0X+^K76=DUH*C;=y_Rk++Atr`itxk#a1n3GS7btm-lIM}cQ?Bc^ z+ntDA4gNFR4cfK7sBGt;zL|3hncQ=vq~-_aAp5$qZ4E#UgZv^Xur#JMwOeH^ zd^MC#dBj&*UU=K7jBjMFDasnDN;^g9?+xf1tXMmV=|KI)&`cPPm&p4>4bV=cP3J-P zpo%{uBs`5ImxW>olf{HA(U5A7VCN#oh758!t*Bb6aL~1eYvMPRl)`^OW~ewyR}eXE z$c0hQz!j8%8wd^z_UYx|;DHhtWN^=+7Rp|bN8SpodifItUPw|EOHLsu!V#2#Cotnr z;i}-8#x;X$HYnB0?0qD4v5c}ApwE&5Pod_|`wMUkRs6+Z+Fyc~aHejs-_dYp{iFU9 zPzdMngoU2R{Nw0(=o1*W&OB5^JD%Tr-+2F-QQ z?1{i`M0O*@%kDJ7##%G5*R~iCFzB^Fkj~2Pj}%z@og43m2}D=hc1QT=1*503jww@Z88ij^rex0w~@?) z7J%Vc3-GQNU&IuHn%4tXfTMatgb~vTfq{{Dn3eMvE}pT01AFCp+%bQ=rY*^-!-G`i zoOc7l?hdkdT3EGar*j)L&;XGUl=V)t*|`g7F7rEX-B>welT~k@w@?^UbTP@0>j=>!`=I_dsX-lXX;{=*JM%Is)Fy zZU^QYP!@G=2W_|EU$zG}K%{hq(&go4q$}sozeH7M$Lf*-BXZPl<O`~jV1UU z&~0BBA^qY@FHqs)OD`r9!|l?_3Q{b`_+A5l8mWnnMsA9cg- zpbr1rSYLUGXkr}cw!}K4{fbx*NBCxgV;Ip1FH2TCl8ch35rL$V$b|sDEEy_<&qf0r zx^2Vf5e1{pBs+$|naehXXh={I%i|V`{y|s{>5lZX-Ed3_h7myaY2xCDn6Si3crFc; z$ODcWT^^Psm=ukoce5zqp+@iKu$EinfZt`COSUYrkBT1Sz(xve7h4issZ{H2PU89+sM&4JN9#N{ldn^`F2`H1W? zvenETScI-D=LdOsj)!$*spWB5u-2ZJ*4zC8m6Lk1{18n;YCM1e;1;3sD{jdrwchlV zo+>^Vn44v&^;vjsHB$CbtBCJLQ-IWpDtK~wM~!BB%61`|McLX>chv3Tpdhve#c(T{ z^EFw!dFUqy_5TbpKG?8$zdH|aFpc$bv~Yj%0eQb+#S%O)I23m&I*dHQY4GfhAXnTx ziuA~BE&OR>i554X;QC|y%yG`!oIc4lCy<}tJPG)f%@VF7>x4bZS>$Wqe9AXI&C>Az zsK1%jsw&p7D1GY|sET9ecK;o_)q}c`hLV|3pGisMnM6@MV~0WTRiPy~x+%ODmy!@O z>6wA3o{BTjKoNRHBu84g!a_Ge)>@p6x=d8!yHxc(sv5JENG+Bzh;dISDihC9%x<3) z(r{Q^bjd7&)FzK_Tw*0=SQKO!Iu+_5=~zb7Wst@oJ`v|#mr;X&68j0s3W{e}JJHoq zwEZn1pqP+B9hYRt&P@iQ7?d+josxbQRis$u?9Pe)Q21zh>uS0%gN}4w6Rmi zt7B7V+N9(29*<;?yrIsCz23QNc(M@%S$@@$bm{U^5SDm3g-f~bQQ}cT{WAj?48I^_f~FF;RJ7fIS(gv%rQ_Ff=fY?ILn{&XthEhDTi)xdP{=A`SiW zs4onPgVLZpnA)LZ1X}+e-U%WSn6L`Tjvy;8*)1$`q`@*Yf=6=q%*b3ICXosxtTu^5 zT(sb+#Vq*?BC}y6puC^4$?Ai>DD7z(sKowfa|2+c!zDanSzG~LsyfHCV!hR9vqPv_ ziS=4D6cHNtMYwZhZ_4NWcVH(T6Xl39X_;_hOS-diSww*;7W~LCDE0=bd5AoY5g3mw z(51W+EhYai`X~3Js4;Fe;VC5oNr?n!z=+2qYIj=^SsyXeb!A({rQy=C=xEzf-&DT4 zj_9g#pFOCkW6vq>ARy}Tz}%J+(SS3Hy{I58eSutCkh}MD7{|mvvk4iyqdibnxW}#{zEM0|n#kRZ4Zu z&Uzz~Cu-nD=z@~hM><;C>&+gKvOVm<{scJ`ENeLcIe`#DWHTCm5R$YaXiC@WMx&b! z1$C6cIs59+s+ArAX_7OL?9NCGcQ4#~gQer88*SNsw{thR69{+->dvrhw;vYk9u(nXXGh670Ou27G3p73mq?D`P( zRf(q|6OxB$uxRY^YJ}-nQpOT`J9n|H1e#Ch9=vF@R-c0wlvO_aP%Sh2jWc{SK+pyj z4N90%v!G)}{UPdN6UUOk38lxIo!vL6E4l7ty&1Htd6tjP6tS_9tP6+U+>tzvvx&ki zCBSVOp}-)x%ygZ+EZ}Xij=Apk>K-A6kT}$EPQ)ceA5L8+>}QDq`54hEfU&RuB%w!4 zhQ-$xR;J9Q934M-gySh_l{t;Q?{q9h`w}e>lJuMg4=~p~gfEy6t!S$CsHn}Lt$$%3 zmh}|?#u+xSpcaw-=O-tvCQgSCPs`wv!VeRIkW+ocS$1@AGRkY5H#iR+dPf(>aa>9T zSCc$RyRSb$Ki43=Hp&dt2o{2PEy@nGO>hYCIMTdT@jOJQzC?U8kK;4Qzl^6gt*9`_ z&`G?_BD7;Vb0_um_vsy*IT{1}JKY;$4C4QbC5es+!%HW?%B3A-WR?8ihF7I=b+J} z*8_bUWBO1OHgi6l;(v+*v}3rTd2zizY&qm-h?BDoV6KWMh#sx3SEHx_OlfBpy4(7C zuRc7INBXKbj9&7n;C!6Ldee!~*r2nvPNvrBZn?41h=P_=NYs5+axJ*3+n{kAxD{j( z#zvFQ<7Pn27?M;WrvNWxAmoW#94@3ox$=LdhpcoILM8I1ukSmNX04#`rOH*d^^rCH^*T zw%3p;s`%>^D1mN>e@>lI=hU1rgL-I_XsJL8KpRecY1llJ%W0N6rM^zd)IJ+& z!d99%3ufZASzNqHNZBFr4dHKyBM{Og1hq{v@H7_j8XJ}x#5~nxcT8xC+O|%_2Vzk+ zbN68+548K%1LXk@E1^b?b#Y+8T-NUweQYdWe?W&$OVB833uQ>ON=EwmBm5*aNS4a+ zBC%`9CS!AK7x39SP(i1??i7#EpD?ac=>@20DDygvl?ng)SeaBZV^jv-&DH+Yq*eGz zzj9`ziqpP*r9XWJ;fwT45)9*4`iqD*Q3OAAs3o?^D}4)b32AM=f^%pj^4J#UeI%tK z?N3 z-njC*o5Tv-n^&E;I6f_pYLa0sIR*Kba1uE}WPox7HZ&6Ro0dchdRidyeM)E>2|D`6 zcCPr8a(|8_M)aMZtp#8n?%0Gw>T-$_@&Noju8__f(GgIsD%E{#_g|3#KnFqT(5Go= zUuYTaH>RomTFaJyrCkNdQTy=XL!-qn=a0^v&pkVTh)1qr(23u4QVMCu?~1tUICu|b z-8b;BXpXIjaFFNzrKAvli>5KwNnA57M2%qhjm5vw^hp_< z!?an{9^N(0vCjL?jhx2cS0l`b{OdSh-9IO>^bBCvWoX zNigx3)Q1Q}x|#ouax9zvnsQ7-oQ#Cg-=N{=1QngyVqr8h6h@$-Uk=3)eV34fI5HuQ zG>9WJtNlvLm48`YtmFlafiM6@QL4|zh4kaOAQVJLu?a;UE)MzC1kGM#C?R!}zprp+ z(-SWBUb`m0vlAE5>{w$J%TzJbxJTX#;C|Q=I{7p!p5;F2aLg1bFee2zk&IKJ?|hbh z7!jhe*25tid6=Gvfg=d4J37yCFXsC0@N1xD_Cbfs?B!3ss#|8$T?W?#X%T`xbP?p> zcb?d5j1fp=jyOwu_31?oxFeZs@T$+!khDM?y5MvF?821C*;oys!&c%FB^+R&B^g_9 zUUc3@xtB2~eDU%styXvkNl{tE9)j6{lXHvOoLZ50d^DH)AG@|TqyPW_ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/activation.py b/vllm/model_executor/layers/activation.py new file mode 100644 index 00000000..43056786 --- /dev/null +++ b/vllm/model_executor/layers/activation.py @@ -0,0 +1,252 @@ +"""Custom activation functions.""" +import math +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from vllm.distributed import (divide, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.custom_op import CustomOp +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.utils import set_weight_attrs + + +class SiluAndMul(CustomOp): + """An activation function for SwiGLU. + + The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. + + Shapes: + x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) + return: (num_tokens, d) or (batch_size, seq_len, d) + """ + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + from vllm import _custom_ops as ops + + d = x.shape[-1] // 2 + output_shape = (x.shape[:-1] + (d, )) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + ops.silu_and_mul(out, x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + from vllm._ipex_ops import ipex_ops as ops + + d = x.shape[-1] // 2 + output_shape = (x.shape[:-1] + (d, )) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + ops.silu_and_mul(out, x) + return out + + +class GeluAndMul(CustomOp): + """An activation function for GeGLU. + + The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. + + Shapes: + x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) + return: (batch_size, seq_len, d) or (num_tokens, d) + """ + + def __init__(self, approximate: str = "none"): + super().__init__() + self.approximate = approximate + if approximate not in ("none", "tanh"): + raise ValueError(f"Unknown approximate mode: {approximate}") + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + d = x.shape[-1] // 2 + return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:] + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + from vllm import _custom_ops as ops + + d = x.shape[-1] // 2 + output_shape = (x.shape[:-1] + (d, )) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + if self.approximate == "none": + ops.gelu_and_mul(out, x) + elif self.approximate == "tanh": + ops.gelu_tanh_and_mul(out, x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + from vllm._ipex_ops import ipex_ops as ops + + d = x.shape[-1] // 2 + output_shape = (x.shape[:-1] + (d, )) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + if self.approximate == "none": + ops.gelu_and_mul(out, x) + elif self.approximate == "tanh": + ops.gelu_tanh_and_mul(out, x) + return out + + def extra_repr(self) -> str: + return f'approximate={repr(self.approximate)}' + + +class NewGELU(CustomOp): + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + c = math.sqrt(2.0 / math.pi) + return 0.5 * x * (1.0 + torch.tanh(c * + (x + 0.044715 * torch.pow(x, 3.0)))) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + from vllm import _custom_ops as ops + + out = torch.empty_like(x) + ops.gelu_new(out, x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + from vllm._ipex_ops import ipex_ops as ops + + return ops.gelu_new(x) + + +class FastGELU(CustomOp): + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 * + (1.0 + 0.044715 * x * x))) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + from vllm import _custom_ops as ops + + out = torch.empty_like(x) + ops.gelu_fast(out, x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + from vllm._ipex_ops import ipex_ops as ops + + return ops.gelu_fast(x) + + +class QuickGELU(CustomOp): + + # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90 + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + return x * torch.sigmoid(1.702 * x) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + from vllm import _custom_ops as ops + + out = torch.empty_like(x) + ops.gelu_quick(out, x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + from vllm._ipex_ops import ipex_ops as ops + + out = torch.empty_like(x) + ops.gelu_quick(out, x) + return out + + # TODO implement forward_xpu for QuickGELU + # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + + +class ReLUSquaredActivation(CustomOp): + """ + Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2 + """ + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + return torch.square(F.relu(x)) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) + + +class ScaledActivation(nn.Module): + """An activation function with post-scale parameters. + + This is used for some quantization methods like AWQ. + """ + + def __init__( + self, + act_module: nn.Module, + intermediate_size: int, + input_is_parallel: bool = True, + params_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.act = act_module + self.input_is_parallel = input_is_parallel + if input_is_parallel: + tp_size = get_tensor_model_parallel_world_size() + intermediate_size_per_partition = divide(intermediate_size, + tp_size) + else: + intermediate_size_per_partition = intermediate_size + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.scales = nn.Parameter( + torch.empty(intermediate_size_per_partition, dtype=params_dtype)) + set_weight_attrs(self.scales, {"weight_loader": self.weight_loader}) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.act(x) / self.scales + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + if self.input_is_parallel: + tp_rank = get_tensor_model_parallel_rank() + shard_size = param_data.shape[0] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(0, start_idx, shard_size) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +_ACTIVATION_REGISTRY = { + "gelu": nn.GELU(), + "gelu_fast": FastGELU(), + "gelu_new": NewGELU(), + "gelu_pytorch_tanh": nn.GELU(approximate="tanh"), + "relu": nn.ReLU(), + "relu2": ReLUSquaredActivation(), + "quick_gelu": QuickGELU(), +} + + +def get_act_fn( + act_fn_name: str, + quant_config: Optional[QuantizationConfig] = None, + intermediate_size: Optional[int] = None, + input_is_parallel: bool = True, + params_dtype: Optional[torch.dtype] = None, +) -> nn.Module: + """Get an activation function by name.""" + act_fn_name = act_fn_name.lower() + if act_fn_name not in _ACTIVATION_REGISTRY: + raise ValueError( + f"Activation function {act_fn_name!r} is not supported.") + + act_fn = _ACTIVATION_REGISTRY[act_fn_name] + if (quant_config is not None + and act_fn_name in quant_config.get_scaled_act_names()): + if intermediate_size is None: + raise ValueError("intermediate_size must be specified for scaled " + "activation functions.") + return ScaledActivation(act_fn, intermediate_size, input_is_parallel, + params_dtype) + return act_fn diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py new file mode 100644 index 00000000..031c2ca8 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -0,0 +1,26 @@ +from vllm.model_executor.layers.fused_moe.layer import ( + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported) +from vllm.triton_utils import HAS_TRITON + +__all__ = [ + "FusedMoE", + "FusedMoEMethodBase", + "FusedMoeWeightScaleSupported", +] + +# if HAS_TRITON: +from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + fused_marlin_moe, single_marlin_moe) +from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_experts, fused_moe, fused_topk, get_config_file_name, + grouped_topk) + +__all__ += [ + "fused_marlin_moe", + "single_marlin_moe", + "fused_moe", + "fused_topk", + "fused_experts", + "get_config_file_name", + "grouped_topk", +] diff --git a/vllm/model_executor/layers/fused_moe/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/fused_moe/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b6e09f7b6ad28355c1356669d699b74368758a3 GIT binary patch literal 743 zcmb7BL2uJA6tM*jev8Q%xKPA0A&H!j$z9AyOy!KsWJ*$*kxb6XTpp1ljMx~m zqFa1{AaA@xdHfTUP#(JERo}7tO1*TC{+zCu-KzRU)iHOv=h%D3o2^~0E5Vk%*2-8` z=h#hOKVQ<-yEm)1*LmO;8()jc$_U<4shB(Nc-si}fXXA64Pop?Gh;h9*=IrcbU0e2 zKf05KS-Mv3hBtJ>p+;Lt=F-NfUW15_$zwByE==rU;=$W&5)U&EDZq8UAdf!@Aq%N$ zCMdgM>)t9;h-$}7R~*y_!oYlx@dGg<0CIenY7MulrM=~%GZFAErnC}*Qu5@VYVTDJ z&fI^T6#r8|v7x>DZV!!qL1ym1?cvv|v{ILv>oZ#!aPsUz_FH)_KbL}kDD@5|+?HOr r{M(gsaHTxBQc}uWZYeGF&LvMFx)TW{RP6(+e{F898YR#(doeM_XomaND;Enbut`c#OG=`&TyBP3 zuPtz2;-XJ+QJ{}~v4Ol8X!{2W^au1O@U<@$5HxL*UgFkqzcVCPyYj|yfS@QBdpI*^ z&Y77rGiSatbc@BDfT#AG-)~(yEeL<4%;0kpnX~wNKNAJP6zYO0no?7&i+HD+l9Q^Z z9Jww#>3Uj>cuHLXPi|%$wVvf=>1NK!*Yi%HUT})_A}>>#C8u03Bb~7-rfQ9C$Y$2e z{Zy`xXQlX;&lb!@vuG*X6K09iGN%83UWUFeWHl$er zb*QyzNX?MU#+*58&iyo9pX|38wM{KQ9=Cjmw>Hn|VNQ>58uj84p4K=$+Hdt(KXtsH zI?+#^?57^>ryktT6Xs*)0{S^+N|%MjCw32??|7D}U2@N99p7%+zHNEhhRd|q-d=lW zX^$pwbuksDUTR$ny3`*yN>R*J=Dr=kTTtO+dE;^aV_0w z+M6wXz3Db~bkE+k!a1K^(>E<20>14SO`U2r+}4J@33UapVKfb<``6l*7pkp}qp#b( z7b;tZr|-5K5_KnjC{6!Y@b?}E2}G6;uyA%HPrWF7B>KX(cv*-_*f??+FlY+HmR9#< zNG!@Mi&s6}v0MJr^$dTlpQ&dlab~@?a%NFtw5VAg~E@ZM%KiHyDO; z`dO#5<W&~@m;puG_G09TaIjmaVIH*HSHD| zVyS&CER%5t%s@?gDu^r=L`AGhtc=Vam8^xuE!#A$mhSn64}*!_P$5wyi<^dTw>EXR zxE_aip7|1CX*9pZBO6P zH#*H`s4&a#uvVjhJ|xdP5>xo|+4vJScpz+JE@p)w6}>SlrHg_oT^BzRIWGmC2p-<1 zTou4S0)CqFNhaY#2Y9EzQzF@$;w`~@8u<**i#-1V@@lMG@>AO~WL~9J5ai746~Vj6 z)9BOtQm@b}@|K~wSK_H0`jRtqbHbd^EBomHy$<9ceO*TFN^i_pwlls;HTi!m>YbSn zvSz_7-pKUEgYlqnUF=N+a4*nW^|QfPPz_4gQ}3sGlf9|lR8Tg{b9|(|Y4k4_5XE}k zE|?W_?1tQ%2~;!R5qq=zEwEpMUo^*2LmH^11`28k>~|3tv^2m~f(+;Wfpf(J+>Boi za>49%sW%5sWqa%=V zzwmR+k2!IJTu9aEU@zq~JZoDvd`G9%Apf8>Tn9@J-o)RsG^f+_?RL{{U|E9aZfHYG zSL@pTmS$aTTg>OGx|Y4U<$D@;PIgPX#5W1KsZ%J!%GQz^UCn5j8rJve9W7e)nqxJ# zjF#;=OI+6l=xKLY+x09B9YmLCYj82sTJLObQjR*wcP4j*I~HqM&7~Zdf7@V&WBJ&- z&-4801ASk>=5A~)tywMXrN!rA2DH<{*wGXfp>te)GKLNmletc7U2T-CXZd8H$QsEw z6Om|tkCgkcW3+sGH?nY4FOeGZ1|x+dM?vk9C1OLmm=CUw_m}#-ux{aSur$xz@Ezmo zV!r`$CfY)BG`u%xk`G{$grOM?_M*gEbZI_z}d4Y9})#pL?5 z1$gv$YH+Hoqj?%BUo?@Ou`XQAvzk`JPXq_NCC2T#O(?jqp{=owbxIp#lEorj$)Y4( z9fwrb5^b?(eVCz<3)b2$y3BjEZbjRGPUUW*?tp(DP)mpOd!EluX*jb#7&3k$)SpgL zNxhHOJGLwx#GPo)la`WGxNiem-}^?S0~&LAV7Y38i!t6u@A6}#zljEyNVI9X*-LPVcOd=AX0?t{cf@aR1DMH+F@p+=^B3R8KzKSmiuZfY#jV;gZZ}6 zWCSc&hKoc<4GXlN^?0L&D$m&_ z5DNjPXaNJJv>KrjWx~Qf{M4PWLVkv0R4f%ZEubbn9EU5@yDr=D+D5~QkZQJ{ua`Ld z#HQ5wrmW`&@L7Ev=77_&OdA+XZy1d&i{r$Il&Je-bYL~Dc)rOrG;q%cnR-kT<$Ibm zFIM=kAm*f;I4-J^B&zomSr%nc6(#;t?x-168kY)^f>Ie}%3U?B0u}D(MyGRWRVqM# zL6q-KrP&4Ql+>-^zY?n58poR!QY%r4oka!q64g>TO0322m9RMIrfb95NLRFKNEc5v zXx@|OQ6%~(@#WM2N=yS&MM&@AQ6eEsLWvUI3f^T?Ap{5H_vFO{9g^N*B>5tEF;e8n z@jUX`0VKJKe2(WO&i@he`2)!ECmct06$Rqu@jFlN-WV`lu_LV1Ox<+yKz zY=JcU$ab3BB}UgkmziyDiQcTA-d6k!N3yV7Qf?N55=Ww$-W(9%B+%@k;E*{EG&>LM zn+fKFX^v(O_m1?A_*qj4W~xFkbzL$iVqE(O5MnMMnm4QFq&Wpts|6Fk5Lpgo1#>#a zvc;HJ7+G2j#vt)r!~-RKn@Ff5OS8N*LAy!axXl9^^1#65pb89pw4XXg@<73{g)pj_ z4Z_?->aRI-Bh@>O8vA(pFvrO$pkx(zS__T_lZiEeq^E<~U>=s71ZF1e?>KOGq)!Io&H{65!7=n@3OGE`HOcY!NytnFCxO3D#`t@VdoKDzo$fjS5*ZIK!(>_quu#vx3*U+Kz4JxDF4XWBl|C%-e{|_pE z$o_r{?Sp9KzUf}$0Yh7xy@$#5jZG*4POe6MYR}ga<>p(J_tl>@g~!dtBVPQng zihZBTp9fidGzwiVQo$QU-XwB~2oGY;Qi{MbTP5-qk++GwL*!i|mx&C=H1!Gv6)jiq z`lkO+^ZUZxb?!LnD0oFj6c$ZJGi2f;A+ z@7QZYxt03F{jto*!rG{sPaO<zU zT|JG}JM@!3otdVX?%{s&r=@AMdyoAX7F$$S@$fr@ zwZ$x@@xQQvVypeu`fi@S6d?%jy*_KCqbhw-mnHLYZ#M_$I3>(X}Y<0(`~;Pd=uBr9vM# z7!Q-x7tonb(|QprMDTtKA`7ZmMJ!y9s$xMJPf6u{`^jG*0;cuzm7J39%ChtyDJ}h5 zQY1vl(wC|#{Y%YC|19OCe@J=h?@;lDT+DqgeYme7O39xc)FVTW3_UXR$gv(3dQ|99 dp+_xBpQ|P5Z>6&ISE(ZXMH-VnlgCSc{x9jUAJG5+ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/fused_moe/__pycache__/fused_moe.cpython-310.pyc b/vllm/model_executor/layers/fused_moe/__pycache__/fused_moe.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fd895963b6cd5342f3f85a91194bf399b00a14d GIT binary patch literal 15811 zcmcgzTW}o5b)DDF&Mp><2bTZ{QRI-~Ybg*UKuV%0N){iWtOp^Xe zFTswhk_x2l>FMddef#$9dr#kPHb+OZ68;u`_xIP-x+MK0z4ZT`#LJ8L`G2EIk|SA? zBRfh>wq$-PmcmcfQt?!4T0Lc@>bj-Z(^k4}SVlc#Wn|GNYh}6ah&95`Tz%Ba$KS^4 zok=&l{(Sd+wW%RJ^5AC15(9srkv^bHS4f5 z!*vgEI?L%}u0hWOl5?<|sS+3fdBfN1Z&rjN!N6 zQ7T8BC!AyG^9koT>Yl*wBa%hPBQ?uh==;Iun7 za16SYyz`86@%@x_%6XQhaEVju{W*L!oXgJh=&6{IGqUrUz3MnGpzf#KLzSHKqH_h5 zPG=Uu)m!K}3)zjKPZNFR#j~P2SDkC9^(1P6drlSP6z5h<{W^27#A*MG-GCH^N07|S zOQ0@h`_tXr%iY{7Nv<;Myz0D$-sj@px#s~c-rj?Y0hu^^@o^Kpy?|a0VWgjmpUZga zJaVBuKP8WtX^fAOxwVJ(Bb0h6~wH!po>#d;LY}mCZb*tT~xkWh|DV5uP(5#o5EuUVijcQP;HCI<% z53l3pw&%Hxpwz0_L8a-{qukZkUcdHg>8+QorP9rTx8;GiSEJlZpLzZ5HKQGITw4!|(-K;o4o`=<4XN{Fcru(WfM2c&zNV!xGbMHA z;*uF~dM1>@(~jz>Ur-#C7IFC4tQ3x-zcW-y(jB|=6uxWWeE8(RNC!ul4W9~U3et?U0ng^7 z9Q}dXd79gu8yZ_*&A8q(;i0d}-T~CkginXh>}11(yK;EJNrT3PMj?D0UymnWyX8iB z0i1G+UF8|6a{(OAgpf4y^Wizsmc-%3@IrX8e^k)2 z3(U>u@MW3Pge|CdyFA*uCAjp;laD&gdyj_|zg3!m+uU!J#1LlPdS z5oSoTo#!x$k?<1uc+N2%sBq)Z>0-wId`gnCCjMTo!(`N94SX|LgCAhqBNnesX2pbo zFm3Z@^?y4Yy(14aao3hFP}*uUdBf`7D2b z)!j1Rusyr(2CnB{;<7X5l}qL=^jmGfT9|>`@X?*;wmjDdPathFS>N2O25Y9jX184P z)ag0%)#9AFx#ohSlc!IbWxK)TXh172c-V@o4p3ULgYuf|GmEd9)p)?J%>#3)5Q0ww zZdSUqT)Vs`7?_XQy(-xC1G~IIM*8(7bJMM^u623u=W}V!T<&Le37vLnle8P{`ikq3 zP=&~R)J_-^5~&QcH^=NR4^wKk0}|jTswaG36MVFA`CKQ`#qLfl<)S$^6ora0h%J3 zu6PN)J3P7XQeQDuA$>A8vC^4Xzc5kgrFysOS68aFYOsX{Jl=kQLzaW&g@HC)sDf{M znAaxk6+j7o7zr$a4`Wou$Qq^)kq|2r8xnrz?`CNY4*LUzYJ)WatJT|6H587s6<-b_BWzb6KY1-O|{GAqU`O*=VC5Wf|`{@ zH>0*xV*3iv8-g3koSw$d5)ewYQ|K(PpLMcg>!a?Y&xAVGK!etTdOeG%TGn z8_)<6vTu#xy(DJEw{)Iv-^%sPt#75{H!B^_zcm8wYeV~OXiFq-M6w;pD^|ANbYmTe z`-51v!th86_vr$dj`ED6bymr-qWOtUOXsmQ^$*i+Y9hl%lJ7d6vo*{YScn zoUA|8wHaAQt8pc#g!YrT;ES49q2o0>|W3tb`yG--qk^ga}`3As{Mz0j3}?v z;QoZqU`2#sq=mN9W;hue%Xp+@cHx=`Z^Er$Pb+X9%Y{uX$%Y|l?9~QMZ;UKIi7FEj z_n`qeECC;H2LxbZ>%&N;v~gz$bFTH=IY^z^=yN&@9ul`KJYeGHd($b8ig;=SLU9w!59;tDHx0*{A4Fps9W z!qAn;(pqFir1g6nJUu;UJ}bI8Cm6(oJDoG1K@KCJH~jz9oOzL6V%DksE!x)mD5Fh| z*=RP-xB#mVM_`^J)CS>;*`m)-K;TFR)86f$6xDA7W4+g=HhZdbpP-UeJh&o=$gmlV zB}gc~xtT2STF-C~tTvV_kTSM3gX@Y)ct?XU1r7H@g5)149s!6*cig+M_eCR0LB(A} zSOG=KYf;Lrw}P!Gh5f?$MeiJ{TB%C4Rx3p+BCb(^01>aoQg5{s4evB6dQVa^M@{zE z?R%wuYsVNEP*iMbF^SfAQl%~^0W$~BoT8qdM)EEE{1ZqDrwl+J z4bUa8=9DC_tD-DNPk3Kc0~Nx9tUI2NT3d?}3sZ6B1qSCAbL3w`T?(Ov1fbYFf)QV_bVUWo#T>9I+OaKSb45l^#AEx;f7OZ0H=9#39Nr;7Gq9#F9MlnNNP>d{Lf;L^rffSsA=~BmlgH(fu6bFqE zLAA|Q#u+{0=87*MOS#tcT?7jIJ^>X|C(M9;9fJt%DvDm(78&`H$1*PJhtCrJLtGSL!!3qOAr zNfioqC-XU_BX@8f63X|XY`>wrha)_((h;hGcPOfnGJh^oH{7jAg}{cjJ$sp^i?%;6 zTqPY>KzUZ$eu5SOIgyy{`O5^oYyJ!Kz52`B?})KDZpB7yjt>k){dw~Z5xKJ)W!J28 zcpBn_7e2!ZXV;FH*I3_7T4L~@h#WWD5Hs}UxL+LO1r{lY7-`s-twq{;6CtIvQ}xIa zM26?uHU40f#3C(4I->$KfJ1FWx;Si%v}&W$M03~;kE8~d8Mt)>qBd=h0<%SZ*jD7J zZg1sV<6=mng!_~$!%}2Z)F-YdNt7nK%jI=t3O+4|t+|e^x%OyEH9yu2+NNs{HPtH- zPx4Lg2t74&Yf+}^BVyszs*Q3QrO6-JJTku5@bha(U~pk_$ufLH>(JI%-jMx?Q1@m+ z$x$|BZwALzWR~@A4GqUdMu;Obd_4i%l6KOUr7(>{M#C{sb`BP)TjpeX`C@kY!xz!# zhbARt=!)82VX6<=^UUKrf31?reh5nx>tx1zmp=8ZGY#9bDIq)Utx)nJCFBW*r93GN zJ}XYri;}N{u8Yz#5)|0U`>vYXZ)g6 z;fZ?RaMTAlaTP5B1=zv>nvfg~r?7-F^!sV?t(aOKSo#8ubKtG6sQE8eS{U=Hdj%*>K1$~frj9qD0lck?CLe+Z(aV~v5;{ZY7 z^jCRI>fk+0SLm8XdYNPQA01d84iVI=KK3}}HSa9O@Zlr;-TFX5q`g|V(i^VZLZ+DY zXq4U}CFd!5jFNMdT%d%&tR+yk->d}K1Vkwg-AAh5t`{?&PqhL{Y)aacyh{mBZWpIo z`^A1R*)!15SJm~0S9Vt5Wio$Xra8U@N>e(`X;#&-nlallMLj>6(^BzTm$i=-RG5?v z#S76q;@ExMFNe1ygr#v-!1*%awd9Xfmfm^dzlY3Ens}Z<+;ZdV2Q)pbB3QWkj*2P7 zMDMCJ-T02F?-;1BV`5V%p-I%7G)zDykP*zJ>mJ!KyJG+pXPopdFdW`8pdjOb9<-P` zZl8tPthB4{jKChNfY>AGD+l8O6gUPLjglPqk_*Wja4%5J(M}#J)WlY0bUlyqjFV** z!u!q`>W|~fLY{LIyE3cPh)^l-E=1sMBEhoX!s7;({4pw{mq-`8Esu?S0VTJ*dq|6u zJ(aT3iPC6ytL`SwgP=b;<3ykJ#me$=P^nhka4iE{B1AX5l9g#Ud~DKPM5E!&TYy~_ zFg@F8YlMRM*u>E;H_}#`&6@YK#M2>4SbIrvfgJYI%nnuhIV9MJAw1>c9H+Z;wUWb+ z_(O6+a?i?&EFWWJxgDP*;CQ3n!bX(n^zF#ljB&zdOi<;6;nv2tNU5)*|EV-7bzVh4 zp4=52l7>LPf;}&l(=UraeirJUla+^tqCA{LC|{^}_D3*uI56c}=D<0GvS-kxt`II`UEWZaa{<4_D2*!yv|$tOm=5@aHx@Y? zE;`P^LsPCyCt=Coab$=w9`v^{{4nj~3YBLBSdna^M;xgw`{QAbo?ZdviRs4rWwS1m z^MzSI3bUT#HtIT#7f4k89LjWI>UWe~$-jX2G{4LIei`ouzbpKH9q$=Ye_ai<^%QJGT)@8cCZfEi~xInbFJ{9cg4aR+v# zq72lU9;`(@615>(K$twnG*s#jf{FEXLqV!{4geJWNhoz@8yNl0WJuT3W`lh+vRUk5 zr-4{-LF;!qkA;tgxFCixKSrgL4@d6H$MYe9VS`9wkCa;fL=-dNKn9q zz5Pxe+A(%t#S9(oJb`O``@=)n%TFQnlL?QY#!+1GJLZgaWxXF405NCI_ydH5f~lS9 za5|jYnQ3a)tN0Qq_pCLJ{#Y^$vw2xr!3@RNTP5({S&-kg08;2+-m za~gfIu)O_(aGAov`3P=P6cXY7@M(rvyl=<;P5_SWedJ){L#}S2DXPY33`b71`KA*$+^$+}H{1U9(Is(RD~{(a z{1kN-_oxFyH38m)p(4uw_eJm%#*BdtTQQg=u{9y5Hcl_SHstbdp*FiY6>zaQ8|_kUHKNH&_}j4W}{Vc}`ZSYh0>@N}fPjk~^e;AdWY?Plr5 z+i&5K&x- z6l~=HUX0-UYOP&@A1xsa>iq(JB&S(CF1+Lyso+bLbOD?9WqM`Q-V@9(Q`xUj@)b%R zAhEJt2kHyV*WJ`v*}gHoOLcJhDlS3r3nJI|tWg|=^u12fjpV_m)|iOz)8z?4 zWRxp0h@{~6MDI1$T#v)5y{PPvqZ}Pra(_!6@-HxSW7NQ5y&)e|3wY9h3kP}H9IFCn45p$tD$~DHn9$;_^}#0?+?>pdC5 z=fcj%;s0> z$E0v)FA~WkJ3~zD)s}A(5!-a7Q8=XdPHKFlg_etZw)9=t0p}+-9oW6^NkwezsdEnF zibMbIu^RJE2WzKRT%7mQHB8>n7LzkSc3P(>Rw_gR!`3}>@7=uM=(IR9NE-G95iZi06|c#n?yFVGZJ&{ik3LutzkKS#__s%7gj%&rg6~QrUd=s0a-u)IL);2b@T>ifzGi-nQ<_Mn< z^~b%q*fc??I9JB0JXEO{k?2YQ_{zNmF}p3_bvNz;gU}5zy4LCskcs)Hj=K=-6Z%1Y z<15azajzUV(yL|ChS*-PpsS6B+q)}^qhqIChGn36=vssMA3y8VeIMxi|7UAKav$CJ^n*OCA^S*q~Q7E5Cvn2 z?|Q$D#zkfM1^_pMa!QKIjT=Sd#`29D%lIqiki#>6n~&0^lG7}gO5U&27;aPYCXz_U zon=giM==mjrG(~S!|(s2hcXf}ZB_&Hz0#pJfVikL%^^B7cT==SG4fpzbz zXl!Zp?xl$iheb#DyEtF+462dmlH?z=`FuPb;be-=s=)1MsgAl{4Xg~TCM?XomdD5b zmM(m-_Zksr?EXG-h_MEq*ic-h4|LcZWolkqz+yf#;=sGu6Hv5MXa@yXSUXs_eL+^9 zMh;6EmtnNpHTMM)xBtgTHA&n}JqTcE04Ne1WMHTeAk+>i|DYXK{$4YcztfH=f2$o; z{ziL3`D^W%@>kk%X$I6pX(jPL<-BN0k{h@LcUmhal z_6WCWA1eEL(3WyIaljXVU@@wJelQ}8#K1_5tnxi$MEOr6m;EkeGd4&gdDH%7FIq}G tvV1HR9kg`N(sRmpl~LtCl)Uoq#+dRQV_f++V?y~?V^aASZD0PM{~H6^_dx&v literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/fused_moe/__pycache__/layer.cpython-310.pyc b/vllm/model_executor/layers/fused_moe/__pycache__/layer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9684cff7b5604d5d8272be89c7568a3701cac23f GIT binary patch literal 15762 zcmb_jS&$pYdBz+V3aFo>Sr#S)i$ z23&H3wH1+$lvpg=Ny>?1xr&UBm^mvCdCf!elFCE!mWO#&s#2;v$)_%o`ToCWa4dHv z+7+PYM^8`pKiz--{r7*NT_`vyeD?j_KQH~FmrA`wH^ZMaZVuxLeg+^?LTjX&nx|=0 z)*E`$@C+^+4YQf{(oM^=ni((Cv^|^a%|_PCqHVh2G;?0AnfLO|f>+>YR-@P)^TxQG zX_T7d-Z+=-#zb?{n`};bQ+%IoY-vt=)6K2kR=#%{GtF6VR!jL~{H`+_LqzUWWkekbn7L<#pLf2Tiw(e&;>&A6CA&4j-bEq394QcU4~%Fiyl z%XgygE@7TZmAAZ&d8f)+RIHu}LRqbaO+Q>}3)Gm$Tis^ajO?S;Mx%PB;Ya4v^&pJw zXF6fM-KsXC^yzM=;m?aAM)z#~e5EZLq7u|s z{m9)I1MpO&FS<1!qsrA=^{~=tFE08LxAxI)5Vo7objo@(@m#ms3hS#?8s5=%YoWf_ zuk%mxv>vYpK3a|+Z8tm84+1ni(U#3>7)=d#J_SmI{^A9^pcG(cD}H@(DXdh(PzE(3 z*ziZg&0$=@41k~VG?DUjp?QYTJyRH7nhBkbcAV%2zBt+TUu0~jYSo5+s@v(bW$25@ zQgin*wWVsS<%773vfb^>m#xUIR9e-hU#Ub+g{a$YP&r?ze6?F`#80f#$LCKy^L&&! z`s9)M`QuMV=_j6l=7ndkrQ~*uioyHO?G2lqy`6gJfv_sEAP+p;>@GF;HCOgF>Sy+L zE@0hTd(Sl*&An;~{qugU8@A=%M)iU(gS`thehk1@6?2^nastSwY3??Foia63vo)^0 zg-l~;ahRGe0VHJd4Waqwa(dYkx}OmSE)!Q8b@qZIERnfnVh3c0+Sz#Kh%7?~ZF4^M z2Cf3tEEGjijB%g5D1pDnqluX8#|In`rFlkC4h&xL&v$$o22pOQE`;AwoH$-@g}!Y1 zq7FV#Re9p*W>68~g^pjdFp1=oL4lay3%Gu60)(kowQJh^H7%N)Z--Acu}PXf5Q^hc zwq-dJnStL}kawV|+(p18a3_Jg0HO*1JQ#3<>jujxcTkgJP5Lx6#hC%2`ne8ZnyYu- zI^ns|){%BTDuBB%McPSq%sw5qJC(ChPWmm{4-MLmC9HRaxe&{a=Vv2J`e9eLYBb~I zlg$1oD&*Y+?ji6AfO1CerZP|TlT_M6-~j@21ojf>GpL$by4vY9E-;S!@pyG+gXwUS z`Eo&SAq2#Law`F%08K^KmLk2?ima2oTOtF?8KrsUWSQDnr&(4-=9zZ8A@@<=!7#W) zRF*pds1VRd?3At-HC@wh(q-!T8+y6~hFAAouj@BVy>x^7-Le5_tcMBN9xzL?0hF8o z>(Uk&uzyBasG}{A5%wk1%c_sqj%}jaI>hc^eu8;yg95fd5!aZHyIDF4)bBWc3%H-4 zmJ3sS45Yo{Cin;gVN-Ys=f~mOBBn2;y)m&>%v?%&rT;g)vtk?Yj*Bcd@s4Qrg%*Sz z33p$N^24fovVGiDhnBlihYah6Zrqj+g1Jri*~li&Vr7VxX%hz>fH1YJE$e_r57NA6 zPNjNAFD>-TkmIb0;O0-8@RwVdkdJxwiG&%$Eby{ynK6Z`4&AQq2_1=&V2 z>~Y|bECG+91hpZv(3UGzDay_enKzr49PPfI)+C+?Z7>C36%=I$(0X@u*XFckQ8_aC z0t?T5oy`PmG0$wc>(^vY=K19 zyrdHl6E7^VmJpw+WjiV)=W2-aDo`f?%l*MIFgpFy;OHlSEYer@;o2YNdDY{SF0%Qq zE&?x8Z*>|~XoXfZM{3L5sM<28)Rs9lAgC>ak~~59?L?A9MV!ZeElhNQ0o&tMaT*K|muQZ?2GtM_ic z%)>})v=qt5Ktad#3@+BvLag^i?NtqIy}I}GlJ6dU_67G$wRYBT3Afgj&;Y{51y{8F zz-_fdH&7}niP<@IsvAs`$b=FZMAlib`C_nsmbw0bI^7<psm}PShr!I4_+2`PF zpJ8G~luzP?@+pAGWHqR;h?4~Z26Cp;4Ox{bn;xsqAIpQ_Lr7v6RDnE2X!{bjock?U zjZv&)us-QAxL5EBwoCE_Tg&s8IBJNeZeuZk=bAA;KVP0$r*B1>N=3A5mCE1*lAoeU zeVPEBVytb+hpBXs0C9md39!yZOevoy@I?ZHiExPznn&<}F53cd#LTPS}{HkZwb>Aw1xXT^_g9;kg8sA)N15RMc)%n{rPt)dRN%{ta9du+80C zY)t#&0cFH;HH>Pv(XI+?l-tw5Xpi?buqntb>`iSJ2eZ&IAp<3?m#SzZwI*uyJdW>* zq3TgGQA*EJUAkuqYbv!!{j8yKCbXBc%MM@;x*eX7Zil>q3Nme!sa`_x%I_$EoP;Wr3tLH`YK!%0+!lvJ4cZR1o! z?}1)g5EH%ZRVqX0m2dUmv!!?IJ7}c0Vw)VnHl?uj@qpH(Rp5S@&JMqMOV6PQk_g^w%* zwi0IMqdc28m74N!Sx?tnepN=9L_vb@0;f_1uDtp=wt#}Dpq|0luL|h@hp^M+X8=yf zN2y3>(A%UmC?(E_vI>Q$HzP+Kaut;2@1jXrmtUclRRC{G%lAc43E;}AO1KCrRVk|% zPJ2`JpaPToVB*4{2Jj~g>=*9hN0UiA^*+^XjG7Hqu>BE@^$ni5ZxIzkt;*BUxN>XZ z`4apsm1ec$+2qqCPo9?%7vL(Z*P>~>Gk`8yul2iAbLr8nXolW6`9v_{%1T{?OF?Dd z0k5>KYX8uQ_c`i#m^zv?XL*F~#|e}O%+iz00!SYo*zctm>aB%#WHjq7N&Z65Y`6Si z0$jW;pCkp1|Ak>(%=|;_O;yNuEBjxh<)zHP+nOJS6)+`H*!)Ix}mVq;`HDXAtYm z#{7xZlYRZrT{&1i=)&G{>*sM0f{Es^DMML5qwJB`0BP3Ao!6)czXsPhTyEq}t;7m4 zOv|V72Bi!xQK_#H$}@E96A%Oz0B+Boq=74=asvgCRWB3UJ)k+-+0{EgilS^kmQ9dy zzFb_V|H_w>CFx1;;9+P9IO_a&hE9lPKqAQwuv@OD$-uixs@Qn8uT>z*tx=iW&Tl ziCN!Vu*Ej99WK9;*nxNM2Aevl#2pQudKPAHsEJfUhy>=XMhr4c9CHr#i?6bHnofP1^YLwp+VdieME1>(RQQT zY>~f@!fbAQ`n&hRHuWprPNgH;%kEy6QQ<@_yH(it&$m~S9!hN(^b_rs7I)g~uIw+z zV?g-AKjfZN0*oYG;I8c3g9dvT-O2$TI=ym_+wG{zKB77IMW0_^Yj-ZV;Svl2I)Yn^ zuoznB;AVht&Rz1Q&$tlw5)CR|aHWs4yF(4@E%Z}JNOwuX9d6>bUXt0#33FBQFY)o3=X;T-R2ozy6puw_E*I(ROtGBVDwkcZq!=2jeNmW;Bka) z_)+>D6W15q)`#4uNG)?!l}j3k?^4v0-i|rCI3iksIz{^ZZ;3(?7I<*uz|=_pcn1Bx z?Cy=jRfpVT)v)SP2g6Y`gL@6E`8$+m*hW z$oz@ZQ5;ue7o&?!Hxyhej!tqY5H&b8Nf(SVVMj&&EQE>|>gOf-U~6Ll$%phi(i^a$ z5hB8g8m2C$u4q>g4TRSpJ||fS^()5J)LI(8(o|QIlf8UP56z3pI@Q@OCF?YUI03B5 zZB&n#0ALm_=v2jl8#-0V;aN^TLY5+MJPaENzriEQw}t2AmrYd)y&_WKo;g;p;<*l$=>28K>Y(WHfUc;L z5GcCJ;mBZvD%qx&Jpz!}eJa)!IrRW1LRf9pd@r^LNqSU6Ni6CWFbPT!P%*P8eXiQ* z;@qG|Hp##^_3qaNy>XXmFf#Qc{MUaa)JGl5CEf7OWN68}WB4{>|gLsarQ4L}aJ%rE<|IYH$N8 zHIZ!&U31=B9Z$~Mxr7-aJ6J+!rcxJCnx$Y=V8b{$UZSj$rD#y2{GjAR&MDgJwB7+l z5PJGDZEa0x=MaF#ua0d~5JpQA=9&f8hXV?ZDC<#Xc|50Wb*HLVb-Wd<`D;4+m=Wzb zpq$O1L`qf&klPtVGX(^A5>G!*4=xZ`CGdLyk;5~oP&0(P#rYv;D)~{PooxIwBP(Vb zvCUs0(ihRrGHtzNlr$TLB3$+N4v5F+FYSX3x<^?>QF0jMV&d^gNQP1Z&`7A2oWMj& zrXz7i4_P3fObJ4Qf~*L(QV0s6*0d59cy55g?5^49IZ^Q%9QaJDv}E#J1YSj+OH}#> zfgS*|e{4kxM3%B~3F?uhIGww355~gu<-P-(g~@rJ>Zu}5v3CWzPn<0=LfLN-Wl3h) z85JZY26aBP$cMLc$&TSeY&<}|~6xp=XMI;Xm6A5J5 zy^buzfE!^MAqFuQZCqxsub_Zv4>l`KH*H%A8`Hd^Vd_>d4Zp8-)mSq@EZVv|KrB2F z@Fk;GYZXv$!tZ()pK1IIz4#egf_Mp8%jtRLRO3hyZ{9V$_L_l%XsO!qJ*x%JSbIf& zlWOCvn9bdxr5M=KsJI(budvnU%KSY@g zsWLK0mDY0^edMRw`p)R9a~ytpXK#)WJ}FHj#folhBz~%nHd?;te$5hfSqUPR(i&C)4@= zO4z_QhWAX~0vk-Nn;dtDaqNUw}+4(J@fle9DiVti!8M0oDlqCm-|ROlQ_UDVbrK|ujW=x-X7bVKUoY--I8a~D%pAjFFE@sgz!rwQ`z}VGr=H7&s~W~0;}E&ChI}Kr>*+J8E5)nGsy%DUH!!cU zgnH4{Q$X$2s5Xa@dR^9i}A~x9gOi#@OyrOXC(xUh<_=6WU?<;r|5a`_hJjT z`x%dVI&_xDmnSgtd@rxc;H!TFj!9ska0%hqn&99zQ_B-*6QRwPP1{U!oBu_dt!Sf0 zI&(RuEao-avoIHSXo_t#9t0{jp4ImFDXI4xzK_p~9rNCP?CK7-5Ryc@WGDz;&~Uwc zOcFlrB1_$sXB5^ILQUC^krqY{`LbZGx4S{)e36V2PKu2(utF%mIvP_luD{sw>(p~v z)xxMb*bmzvro^enlFY+M58II?{B!l19~nqym*1w>euuy;0a|wX27zq^{)oT~fmZ>N zJfFd&|Ck=JTqgC}n||^5Q%^j3x^n91k*AMWPQCE#v(G$#`uH(#>!~M?Jb$e6)UnFx zXDVDieynorsgrU*}9TS#J@i{RaxRO{ZoaVHW*{ZBk&yxtWhcEl4=k z91fts*nyd2-OMAY0me}prV!L7+bHVOoEu>Q2QWaHqF?g{jHoo}nHDl6h7SwtLo*Fs zA7^!v55lcJFt_C2Syxlcli!`c)^#(3(E+!UR^}Grbtnkb|5_iImigZ5^u{}ev%5GE ztVS!BeDWyJ{ZU*2EmS|p$8PFtS$H!Hc`y4_v^dAdVvj?d1J{s?OuX(3@9C^ zlrar2UIz{haG|4!JXN8(1pV>el``j^@eL4B$*YCJL0o!OKUp$C-h% z9FD3?>=GiILlGZI8+b~tA$EJ=7cQ^m4CPy{vBOB-hx;s)a#P+HIz4C>s5js?GAI|5 z%&AS2MAs)%k`;~kD(^jxKCvHzsmXL*N74r2X2%67vRJa+RPh9XJpkS$0=S6fC7xZC z)VOX>A4kJih~{^pP}HqBvyVck$=CXIDMvImZ<@wxsZZZHC>TxF&UV5IyYk78&;S33 zEBo$sRXNTrV}Czajrc4Q{hC%PAi?)q>NV{(eE}A?&@b5`Y{QEhX<1C$kt3BBl-y&J z)RpqH_(&FSADwR5*3^RtyPN;sxHFL?F_OpjpFy!9tZuXu>?!p%34!%B&5;l`RbH z!Mj6Uuwb1{VF}Yix-gs3u`de?3|vpZGRWZf2!3tost=1S{9%+X&u!rk%f$Y$;lAV& zcIM@uV`R$j^*t*61%dAa3=|B?BvGCT<{FmC2I~m&C)D~1K-pezCyt)x%3}O|YR*eA zV2!^--A{?u_oHCpw45M0cr!Us)i0f>dHBhcrz~$EDWU%-mDuj?YcC_4kruGO2+!k& z7a^_OgfO#+`0O$yFT}Eb6%Zx~;s-1U?+)?L1N9t>028BckWjw1sfbGF6#ETF=?5i{ z)F}BQ0zW1&AXe7s?q_Ri%Q|TcB2MLhb8W~9QQE8=;>JAY29D#=kyh?zim{R)Q zpb|N(2N_n4N8Xj5KO};s(dk%XgEiUsM14GJV7~Bpr14Gvx&m%!(y=v1se56(j!fN*tpR zu-A~Rga5$^wjoqBh|Ya4#|;ph#NSLMsWsltjlw$dKTg^mnWr`fHfqCB(O@o)#i+>i z8T=15PT*q>mPP7*_;$N*5IMqMUU15c0S%{}d9xegQjNx`(H4NniYGWhH9HABLf}gT zcoTh!O2n~4g0e3yBl}T!sqt^r$4CmZvXf8cwHX8trr=`BYbC8@AW}mC1#4og)JlDQ HX6F9@@FBUX literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/fused_moe/__pycache__/moe_pallas.cpython-310.pyc b/vllm/model_executor/layers/fused_moe/__pycache__/moe_pallas.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..afcf2182b6c7e61dd265caa82037ce8f26f5eae1 GIT binary patch literal 1860 zcmZ`)OK%)S5bo}I?Ck8zyEchU0L>vn+9>hLMgl292!sR@QX)%492%n0*xkE3_Po;F znkRM+FH>Zn;az5#jG)sCRr=$oMRSo4J3K?XPX zb^8bgJ|LD(F|%}&wN}w{ zYc|k$-e)b=hRm=gmpsu(_rY%&JO;bV^A)|Uu^yaSpm}3{OE0qyBiesK#?Y}-Y-H^- zvXED9`jLenL|B6E-ey?a&=rUTn$GTOT{$*BdKoJ2LLac<=eM;B6#xfA(mVdu4`6*F zCh}p;8R^E1JPf|RcmGlaqww?+cd!A*tV~0GSa6}_{UFX17b#~kOjjrY9NY^w|Id5# zpWc6;3x59*>gwDZ!@9`tS{4i3**UDo0)`TuEde>+dNSp~R+J=>3_gGKDA?m7<4GX+ zvoaQ31}1{a_jo2DXEXmmf^8l|L0Tp%E)pK(<6z_cV7pYoBv-+ZR2GZV%3hv>DbX+pOO|OxR9-d4Nu(6dXd{9<%!Q(EB%*A>XbmlBN~@jM_3zO4@+F)Cu}!CZCm@AMgOV!gbTmcwHu}WRflQo+I`XfzozD@6|`-xB0S*nWUAza z6lVM6tnlHFNW29#y!>jU(qdG^ z#RCP=xq9$HT29lA^k9_4+oR%0P4jHDpCsuh%^6Q%{`pP`mo!SEBQE4${8>+Agd2L(dPunJc5Qn@XE_rD;y1)MgJlznt literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py new file mode 100644 index 00000000..5964d5a5 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py @@ -0,0 +1,313 @@ +"""Fused MoE utilities for GPTQ.""" +import functools +from typing import Any, Dict, Optional + +import torch + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_topk, moe_align_block_size, try_get_optimal_moe_config) +from vllm.scalar_type import scalar_types + + +def get_scalar_type(num_bits: int, has_zp: bool): + if has_zp: + assert num_bits == 4 + return scalar_types.uint4 + else: + return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128 + + +def single_marlin_moe( + hidden_states: torch.Tensor, + w: torch.Tensor, + scales: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + g_idx: Optional[torch.Tensor] = None, + sort_indices: Optional[torch.Tensor] = None, + w_zeros: Optional[torch.Tensor] = None, + override_config: Optional[Dict[str, Any]] = None, + num_bits: int = 8, + is_k_full: bool = True, +) -> torch.Tensor: + """ + This function computes the multiplication of hidden_states with expert + weights used in Marlin MoE, using weights w and top-k gating mechanism. + Its purpose is testing and debugging the fused MoE kernel. + + Parameters: + - hidden_states (torch.Tensor): The input tensor to the Marlin Mul. + - w (torch.Tensor): The set of expert weights. + - scales (torch.Tensor): The quantization scales. + - gating_output (torch.Tensor): The output of the gating operation + (before softmax). + - g_idx (Optional[torch.Tensor]): Optional act_order indices. + - sort_indices (Optional[torch.Tensor]): Optional act_order input + permutation. + - topk (int): The number of top-k experts to select. + - renormalize (bool): If True, renormalize the top-k weights to sum to 1. + - w_zeros (Optional[torch.Tensor]): Optional zero points to be used for w. + - override_config (Optional[Dict[str, Any]]): Optional override + for the kernel configuration. + - num_bits (bool): The number of bits in expert weights quantization. + + Returns: + - torch.Tensor: The output tensor after applying the MoE layer. + """ + # Check constraints. + assert hidden_states.shape[0] == gating_output.shape[0], ( + "Number of tokens mismatch") + assert hidden_states.shape[1] == w.shape[1] * 16, "Hidden size mismatch" + assert gating_output.shape[1] == w.shape[0], "Number of experts mismatch" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w.is_contiguous(), "Expert weights must be contiguous" + assert hidden_states.dtype == torch.float16 + assert num_bits in [4, 8] + + M, K = hidden_states.shape + E = w.shape[0] + N = w.shape[2] // (num_bits // 2) + + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk, + renormalize) + + # This might not be an optimal config for a single MMM + get_config_func = functools.partial(try_get_optimal_moe_config, + w.shape, + w.shape, + topk_ids.shape[1], + None, + override_config=override_config, + is_marlin=True) + config = get_config_func(M) + + block_size_m = config['BLOCK_SIZE_M'] + + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E) + + max_workspace_size = (N // 64) * 16 + workspace = torch.zeros(max_workspace_size, + dtype=torch.int, + device=hidden_states.device, + requires_grad=False) + + has_zero_point = w_zeros is not None + if w_zeros is None: + w_zeros = torch.empty((0, 0), + dtype=hidden_states.dtype, + device=hidden_states.device, + requires_grad=False) + + if g_idx is None: + g_idx = torch.empty((0, 0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + + if sort_indices is None: + sort_indices = torch.empty((0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + + scalar_type = get_scalar_type(num_bits, has_zero_point) + + intermediate_cache = torch.ops._moe_C.marlin_gemm_moe( + hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales, + w_zeros, g_idx, sort_indices, workspace, scalar_type, M, N, K, + is_k_full, E, topk, block_size_m, True, False) + + return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1) + + +def fused_marlin_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + gating_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + g_idx1: Optional[torch.Tensor] = None, + g_idx2: Optional[torch.Tensor] = None, + sort_indices1: Optional[torch.Tensor] = None, + sort_indices2: Optional[torch.Tensor] = None, + w1_zeros: Optional[torch.Tensor] = None, + w2_zeros: Optional[torch.Tensor] = None, + override_config: Optional[Dict[str, Any]] = None, + num_bits: int = 8, + is_k_full: bool = True, +) -> torch.Tensor: + """ + This function computes a Mixture of Experts (MoE) layer using two sets of + weights, w1 and w2, and top-k gating mechanism. + + Parameters: + - hidden_states (torch.Tensor): The input tensor to the MoE layer. + - w1 (torch.Tensor): The first set of expert weights. + - w2 (torch.Tensor): The second set of expert weights. + - w1_scale (torch.Tensor): Scale to be used for w1. + - w2_scale (torch.Tensor): Scale to be used for w2. + - gating_output (torch.Tensor): The output of the gating operation + (before softmax). + - g_idx1 (Optional[torch.Tensor]): The first set of act_order indices. + - g_idx2 (Optional[torch.Tensor]): The second set of act_order indices. + - sort_indices1 (Optional[torch.Tensor]): The first act_order input + permutation. + - sort_indices2 (Optional[torch.Tensor]): The second act_order input + permutation. + - topk_weights (torch.Tensor): Top-k weights. + - topk_ids (torch.Tensor): Indices of topk-k elements. + - override_config (Optional[Dict[str, Any]]): Optional override + for the kernel configuration. + - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1. + - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2. + - num_bits (bool): The number of bits in expert weights quantization. + + Returns: + - torch.Tensor: The output tensor after applying the MoE layer. + """ + # Check constraints. + assert hidden_states.shape[0] == gating_output.shape[ + 0], "Number of tokens mismatch" + assert hidden_states.shape[ + 1] == w1.shape[1] * 16, "Hidden size mismatch w1" + assert hidden_states.shape[1] == w2.shape[2] // ( + num_bits // 2), "Hidden size mismatch w2" + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.is_contiguous(), "Expert weights1 must be contiguous" + assert w2.is_contiguous(), "Expert weights2 must be contiguous" + assert hidden_states.dtype == torch.float16 + assert num_bits in [4, 8] + + has_no_act_order = (g_idx1 is None and g_idx2 is None + and sort_indices1 is None and sort_indices2 is None) + has_all_act_order = (g_idx1 is not None and g_idx2 is not None + and sort_indices1 is not None + and sort_indices2 is not None) + assert has_no_act_order or has_all_act_order, ( + "g_idx and sorted_indices " + "must be all not None or must be all None") + + has_no_zp = w1_zeros is None and w2_zeros is None + has_all_zp = w1_zeros is not None and w2_zeros is not None + assert has_no_zp or has_all_zp, ("zero points must be both not None or " + "must be both None") + + M, K = hidden_states.shape + E = w1.shape[0] + N = w2.shape[1] * 16 + topk = topk_ids.shape[1] + + get_config_func = functools.partial( + try_get_optimal_moe_config, + w1.shape, + w2.shape, + topk_ids.shape[1], + None, + override_config=override_config, + is_marlin=True, + ) + config = get_config_func(M) + + block_size_m = config["BLOCK_SIZE_M"] + + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E) + + max_workspace_size = (max(2 * N, K) // 64) * 16 + workspace = torch.zeros(max_workspace_size, + dtype=torch.int, + device="cuda", + requires_grad=False) + + if has_no_zp: + w1_zeros = torch.empty((0, 0), + dtype=hidden_states.dtype, + device=hidden_states.device, + requires_grad=False) + w2_zeros = torch.empty((0, 0), + dtype=hidden_states.dtype, + device=hidden_states.device, + requires_grad=False) + + if has_no_act_order: + g_idx1 = torch.empty((0, 0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + g_idx2 = torch.empty((0, 0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + sort_indices1 = torch.empty((0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + sort_indices2 = torch.empty((0, 0), + dtype=torch.int32, + device=hidden_states.device, + requires_grad=False) + + scalar_type1 = get_scalar_type(num_bits, has_all_zp) + scalar_type2 = get_scalar_type(num_bits, has_all_zp) + + intermediate_cache2 = torch.empty( + (M * topk_ids.shape[1], N), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe( + hidden_states, + w1, + sorted_token_ids, + topk_weights, + topk_ids, + w1_scale, + w1_zeros, + g_idx1, + sort_indices1, + workspace, + scalar_type1, + M, + 2 * N, + K, + is_k_full, + E, + topk, + block_size_m, + True, + False, + ) + + ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N)) + + intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe( + intermediate_cache2, + w2, + sorted_token_ids, + topk_weights, + topk_ids, + w2_scale, + w2_zeros, + g_idx2, + sort_indices2, + workspace, + scalar_type2, + M, + K, + N, + is_k_full, + E, + topk, + block_size_m, + False, + True, + ) + + return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape), + dim=1) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py new file mode 100644 index 00000000..03ffc2f2 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -0,0 +1,693 @@ +"""Fused MoE kernel.""" +import functools +import json +import os +from typing import Any, Callable, Dict, Optional, Tuple + +import torch +import triton +import triton.language as tl + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + + +@triton.jit +def fused_moe_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + a_scale_ptr, + b_scale_ptr, + topk_weights_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + num_tokens_post_padded_ptr, + # Matrix dimensions + N, + K, + EM, + num_valid_tokens, + # The stride variables represent how much to increase the ptr by when + # moving by 1 element in a particular dimension. E.g. `stride_am` is + # how much to increase `a_ptr` by to get the element one row down + # (A has M rows). + stride_am, + stride_ak, + stride_be, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_bse, + stride_bsn, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + MUL_ROUTED_WEIGHT: tl.constexpr, + top_k: tl.constexpr, + compute_type: tl.constexpr, + use_fp8_w8a8: tl.constexpr, + use_int8_w8a16: tl.constexpr): + """ + Implements the fused computation for a Mixture of Experts (MOE) using + token and expert matrices. + + Key Parameters: + - A: The input tensor representing tokens with shape (*, K), where '*' can + be any shape representing batches and K is the feature dimension of + each token. + - B: The stacked MOE weight tensor with shape (E, N, K), where E is + the number of experts, K is the input feature dimension, and N is + the output feature dimension. + - C: The output cache tensor with shape (M, topk, N), where M is the + total number of tokens post padding, topk is the number of times + each token is repeated, and N is the output feature dimension. + - sorted_token_ids: A tensor containing the sorted indices of tokens, + repeated topk times and arranged by the expert index they are + assigned to. + - expert_ids: A tensor containing the indices of the expert for each + block. It determines which expert matrix from B should be used for + each block in A. + This kernel performs the multiplication of a token by its corresponding + expert matrix as determined by `expert_ids`. The sorting of + `sorted_token_ids` by expert index and padding ensures divisibility by + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix + multiplication across different blocks processed by the same expert. + """ + # ----------------------------------------------------------- + # Map program ids `pid` to the block of C it should compute. + # This is done in a grouped ordering to promote L2 data reuse. + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + # ---------------------------------------------------------- + # Create pointers for the first blocks of A and B. + # We will advance this pointer as we move in the K direction + # and accumulate + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr) + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: + return + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id) + token_mask = offs_token < num_valid_tokens + + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am + + offs_k[None, :] * stride_ak) + + off_experts = tl.load(expert_ids_ptr + pid_m) + b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk + + offs_bn[None, :] * stride_bn) + if use_int8_w8a16: + b_scale_ptrs = b_scale_ptr + off_experts * stride_bse + offs_bn[ + None, :] * stride_bsn + b_scale = tl.load(b_scale_ptrs) + + if use_fp8_w8a8: + a_scale = tl.load(a_scale_ptr) + b_scale = tl.load(b_scale_ptr + off_experts) + + # ----------------------------------------------------------- + # Iterate to compute a block of the C matrix. + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block + # of fp32 values for higher accuracy. + # `accumulator` will be converted back to fp16 after the loop. + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + # Load the next block of A and B, generate a mask by checking the + # K dimension. + a = tl.load(a_ptrs, + mask=token_mask[:, None] & + (offs_k[None, :] < K - k * BLOCK_SIZE_K), + other=0.0) + b = tl.load(b_ptrs, + mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, + other=0.0) + # We accumulate along the K dimension. + if use_int8_w8a16: + accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) + elif use_fp8_w8a8: + accumulator = tl.dot(a, b, acc=accumulator) + else: + accumulator += tl.dot(a, b) + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if MUL_ROUTED_WEIGHT: + moe_weight = tl.load(topk_weights_ptr + offs_token, + mask=token_mask, + other=0) + accumulator = accumulator * moe_weight[:, None] + if use_int8_w8a16: + accumulator = (accumulator * b_scale).to(compute_type) + elif use_fp8_w8a8: + accumulator = (accumulator * a_scale * b_scale).to(compute_type) + else: + accumulator = accumulator.to(compute_type) + # ----------------------------------------------------------- + # Write back the block of the output + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[ + None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +def moe_align_block_size( + topk_ids: torch.Tensor, block_size: int, + num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Aligns the token distribution across experts to be compatible with block + size for matrix multiplication. + + Parameters: + - topk_ids: A tensor of shape [total_tokens, top_k] representing the + top-k expert indices for each token. + - block_size: The block size used in block matrix multiplication. + - num_experts: The total number of experts. + + Returns: + - sorted_token_ids: A tensor containing the sorted token indices according + to their allocated expert. + - expert_ids: A tensor indicating the assigned expert index for each block. + - num_tokens_post_padded: The total number of tokens after padding, + ensuring divisibility by block_size. + + This function pads the number of tokens that each expert needs to process + so that it is divisible by block_size. + Padding ensures that during block matrix multiplication, the dimensions + align correctly. + + Example: + Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], + block_size = 4, and num_experts = 4: + - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, + with each expert needing to process 3 tokens. + - As block_size is 4, we pad 1 token for each expert. + - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3]. + - Then append padding tokens [12, 12, 12, 12] for each block. + - After sorting by expert index, we obtain token_ids + [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12]. + Tokens 12 are non-existent (padding) and are ignored in + the subsequent matrix multiplication. + - The padding ensures that the total number of tokens is now divisible + by block_size for proper block matrix operations. + """ + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + sorted_ids = torch.empty((max_num_tokens_padded, ), + dtype=torch.int32, + device=topk_ids.device) + sorted_ids.fill_(topk_ids.numel()) + # max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size) + max_num_m_blocks = topk_ids.numel() + num_experts + expert_ids = torch.empty((max_num_m_blocks, ), + dtype=torch.int32, + device=topk_ids.device) + num_tokens_post_pad = torch.empty((1), + dtype=torch.int32, + device=topk_ids.device) + ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids, + expert_ids, num_tokens_post_pad) + return sorted_ids, expert_ids, num_tokens_post_pad + + +def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor, + A_scale: Optional[torch.Tensor], + B_scale: Optional[torch.Tensor], + topk_weights: torch.Tensor, topk_ids: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + mul_routed_weight: bool, top_k: int, + config: Dict[str, Any], compute_type: tl.dtype, + use_fp8_w8a8: bool, use_int8_w8a16: bool) -> None: + ops.invoke_fused_moe_kernel(A,B,C,A_scale,B_scale,topk_weights,topk_ids,sorted_token_ids,expert_ids,num_tokens_post_padded,mul_routed_weight,top_k,config,compute_type,use_fp8_w8a8,use_int8_w8a16) + return + assert topk_weights.stride(1) == 1 + assert sorted_token_ids.stride(0) == 1 + + if use_fp8_w8a8: + A, A_scale = ops.scaled_fp8_quant(A, A_scale) + assert B_scale is not None + elif use_int8_w8a16: + assert B_scale is not None + else: + assert A_scale is None + assert B_scale is None + + grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[ + 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), ) + + fused_moe_kernel[grid]( + A, + B, + C, + A_scale, + B_scale, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + B.shape[1], + B.shape[2], + sorted_token_ids.shape[0], + topk_ids.numel(), + A.stride(0), + A.stride(1), + B.stride(0), + B.stride(2), + B.stride(1), + C.stride(1), + C.stride(2), + B_scale.stride(0) if B_scale is not None and use_int8_w8a16 else 0, + B_scale.stride(1) if B_scale is not None and use_int8_w8a16 else 0, + MUL_ROUTED_WEIGHT=mul_routed_weight, + top_k=top_k, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16, + **config, + ) + + +def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str: + device_name = current_platform.get_device_name().replace(" ", "_") + dtype_selector = "" if not dtype else f",dtype={dtype}" + return f"E={E},N={N},device_name={device_name}{dtype_selector}.json" + + +@functools.lru_cache +def get_moe_configs(E: int, N: int, + dtype: Optional[str]) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the fused MoE kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the fused_moe kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + + # First look up if an optimized configuration is available in the configs + # directory + json_file_name = get_config_file_name(E, N, dtype) + + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + logger.info("Using configuration from %s for MoE layer.", + config_file_path) + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ("Using default MoE config. Performance might be sub-optimal! " + "Config file not found at %s"), config_file_path) + return None + + +def get_default_config( + M: int, + E: int, + N: int, + K: int, + topk: int, + dtype: Optional[str], + is_marlin: bool, +) -> Dict[str, int]: + config = { + 'BLOCK_SIZE_M': 64, + 'BLOCK_SIZE_N': 64, + 'BLOCK_SIZE_K': 32, + 'GROUP_SIZE_M': 8 + } + # A heuristic: fused marlin works faster with this config for small M + if M <= E or (is_marlin and M <= 32): + config = { + 'BLOCK_SIZE_M': 16, + 'BLOCK_SIZE_N': 32, + 'BLOCK_SIZE_K': 64, + 'GROUP_SIZE_M': 1 + } + numel = M * topk + if numel <= 64: + config['BLOCK_SIZE_M'] = 32 + elif numel <= 1024: + config['BLOCK_SIZE_M'] = 64 + else: + config['BLOCK_SIZE_M'] = 256 + return config + + +def try_get_optimal_moe_config( + w1_shape: Tuple[int, ...], + w2_shape: Tuple[int, ...], + top_k: int, + dtype: Optional[str], + M: int, + override_config: Optional[Dict[str, Any]] = None, + is_marlin: bool = False, +): + if override_config: + config = override_config + else: + # First try to load optimal config from the file + E, _, N = w2_shape + # configs = get_moe_configs(E, N, dtype) + configs = None + + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Else use the default config + config = get_default_config(M, E, N, w1_shape[2], top_k, dtype, + is_marlin) + return config + + +def fused_topk( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + assert hidden_states.shape[0] == gating_output.shape[0], ( + "Number of tokens mismatch") + + M, _ = hidden_states.shape + + topk_weights = torch.empty(M, + topk, + dtype=torch.float32, + device=hidden_states.device) + topk_ids = torch.empty(M, + topk, + dtype=torch.int32, + device=hidden_states.device) + token_expert_indicies = torch.empty(M, + topk, + dtype=torch.int32, + device=hidden_states.device) + + ops.topk_softmax( + topk_weights, + topk_ids, + token_expert_indicies, + gating_output.float(), # TODO(woosuk): Optimize this. + ) + del token_expert_indicies # Not used. Will be used in the future. + + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + + return topk_weights, topk_ids + + +# This is used by the Deepseek-V2 model +def grouped_topk(hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: int = 0, + topk_group: int = 0): + + assert hidden_states.shape[0] == gating_output.shape[0], ( + "Number of tokens mismatch") + + scores = torch.softmax(gating_output, dim=-1) + num_token = scores.shape[0] + group_scores = scores.view(num_token, num_expert_group, + -1).max(dim=-1).values # [n, n_group] + group_idx = torch.topk(group_scores, k=topk_group, dim=-1, + sorted=False)[1] # [n, top_k_group] + group_mask = torch.zeros_like(group_scores) # [n, n_group] + group_mask.scatter_(1, group_idx, 1) # [n, n_group] + score_mask = group_mask.unsqueeze(-1).expand( + num_token, num_expert_group, + scores.shape[-1] // num_expert_group).reshape(num_token, -1) # [n, e] + tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0) # [n, e] + topk_weights, topk_ids = torch.topk(tmp_scores, + k=topk, + dim=-1, + sorted=False) + + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + + return topk_weights.to(torch.float32), topk_ids.to(torch.int32) + + +def get_config_dtype_str(dtype: torch.dtype, + use_int8_w8a16: Optional[bool] = False, + use_fp8_w8a8: Optional[bool] = False): + if use_fp8_w8a8: + return "fp8_w8a8" + elif use_int8_w8a16: + return "int8_w8a16" + elif dtype == torch.float: + # avoiding cases where kernel fails when float32 MoE + # use fp16/bfloat16 configs + return "float32" + return None + + +def fused_experts(hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + inplace: bool = False, + override_config: Optional[Dict[str, Any]] = None, + use_fp8_w8a8: bool = False, + use_int8_w8a16: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None): + if use_fp8_w8a8 or use_int8_w8a16: + raise NotImplementedError("fused_experts has not implemented fp8_w8a8 and int8_w8a16 yet.") + if a1_scale is not None or a2_scale is not None: + raise NotImplementedError("fused_experts has not implemented static_w8a8 yet.") + + # Check constraints. + assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch" + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.is_contiguous(), "Expert weights1 must be contiguous" + assert w2.is_contiguous(), "Expert weights2 must be contiguous" + assert hidden_states.dtype in [ + torch.float32, torch.float16, torch.bfloat16 + ] + + num_tokens, _ = hidden_states.shape + E, N, _ = w1.shape + # We execute the fused_moe kernel in chunks to circumvent this issue: + # https://github.com/vllm-project/vllm/issues/5938 + CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE + M = min(num_tokens, CHUNK_SIZE) + config_dtype = get_config_dtype_str(use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16, + dtype=hidden_states.dtype) + + get_config_func = functools.partial( + try_get_optimal_moe_config, + w1.shape, + w2.shape, + topk_ids.shape[1], + config_dtype, + override_config=override_config, + ) + + config = get_config_func(M) + + intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N), + device=hidden_states.device, + dtype=hidden_states.dtype) + intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N // 2), + device=hidden_states.device, + dtype=hidden_states.dtype) + intermediate_cache3 = torch.empty((M, topk_ids.shape[1], w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype) + + compute_type = (tl.bfloat16 + if hidden_states.dtype == torch.bfloat16 else tl.float16) + + if inplace: + out_hidden_states = hidden_states + else: + out_hidden_states = torch.empty_like(hidden_states) + + for chunk in range((num_tokens // CHUNK_SIZE) + 1): + begin_chunk_idx, end_chunk_idx = (chunk * CHUNK_SIZE, + min((chunk + 1) * CHUNK_SIZE, + num_tokens)) + curr_hidden_states = hidden_states[begin_chunk_idx:end_chunk_idx] + tokens_in_chunk, _ = curr_hidden_states.shape + + if tokens_in_chunk == 0: + break + + if tokens_in_chunk < CHUNK_SIZE and chunk > 0: + # Adjust the intermediate cache size and config for the last + # chunk. Note that in most cases we only have one chunk + # so the cache size and config are already set correctly and + # do not need to be adjusted. + intermediate_cache1 = intermediate_cache1[:tokens_in_chunk] + intermediate_cache2 = intermediate_cache2[:tokens_in_chunk] + intermediate_cache3 = intermediate_cache3[:tokens_in_chunk] + config = get_config_func(tokens_in_chunk) + + curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx] + curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx] + + sorted_token_ids, expert_ids, num_tokens_post_padded = ( + moe_align_block_size(curr_topk_ids, config['BLOCK_SIZE_M'], E)) + + invoke_fused_moe_kernel(curr_hidden_states, + w1, + intermediate_cache1, + a1_scale, + w1_scale, + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + False, + topk_ids.shape[1], + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16) + + ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N)) + + invoke_fused_moe_kernel(intermediate_cache2, + w2, + intermediate_cache3, + a2_scale, + w2_scale, + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + True, + 1, + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16) + + torch.sum(intermediate_cache3.view(*intermediate_cache3.shape), + dim=1, + out=out_hidden_states[begin_chunk_idx:end_chunk_idx]) + return out_hidden_states + + +def fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + inplace: bool = False, + override_config: Optional[Dict[str, Any]] = None, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + use_fp8_w8a8: bool = False, + use_int8_w8a16: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + This function computes a Mixture of Experts (MoE) layer using two sets of + weights, w1 and w2, and top-k gating mechanism. + + Parameters: + - hidden_states (torch.Tensor): The input tensor to the MoE layer. + - w1 (torch.Tensor): The first set of expert weights. + - w2 (torch.Tensor): The second set of expert weights. + - gating_output (torch.Tensor): The output of the gating operation + (before softmax). + - topk (int): The number of top-k experts to select. + - renormalize (bool): If True, renormalize the top-k weights to sum to 1. + - inplace (bool): If True, perform the operation in-place. + Defaults to False. + - override_config (Optional[Dict[str, Any]]): Optional override + for the kernel configuration. + - num_expert_group: Optional[int]: additional parameter for grouped_topk + - topk_group: Optional[int]: additional parameter for grouped_topk + - use_grouped_topk: If True, use grouped_topk instead of fused_topk + note: Deepseekv2 model uses grouped_topk + - use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner + products for w1 and w2. Defaults to False. + - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner + products for w1 and w2. Defaults to False. + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for + w1. + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for + w2. + + Returns: + - torch.Tensor: The output tensor after applying the MoE layer. + """ + # Check constraints. + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch" + + if use_grouped_topk: + assert num_expert_group is not None and topk_group is not None + topk_weights, topk_ids = grouped_topk(hidden_states, gating_output, + topk, renormalize, + num_expert_group, topk_group) + elif custom_routing_function is None: + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk, + renormalize) + else: + topk_weights, topk_ids = custom_routing_function( + hidden_states, gating_output, topk, renormalize) + + return fused_experts(hidden_states, + w1, + w2, + topk_weights, + topk_ids, + inplace=inplace, + override_config=override_config, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a16=use_int8_w8a16, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py new file mode 100644 index 00000000..69ffb0f7 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -0,0 +1,717 @@ +from abc import abstractmethod +from enum import Enum +from typing import Callable, List, Optional, Tuple + +import torch + +from vllm.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.logger import init_logger +from vllm.model_executor.custom_op import CustomOp +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +# from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import CompressedTensorsConfig +from vllm.model_executor.layers.quantization.compressed_tensors.utils import CompressionFormat, QuantizationStrategy +# from vllm.model_executor.layers.quantization.utils.w8a8_utils import create_per_channel_scale_param +from vllm.model_executor.utils import set_weight_attrs + +logger = init_logger(__name__) + + +class FusedMoeWeightScaleSupported(Enum): + TENSOR = "tensor" + CHANNEL = "channel" + GROUP = "group" + + +class FusedMoEMethodBase(QuantizeMethodBase): + + @abstractmethod + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + raise NotImplementedError + + @abstractmethod + def apply(self, layer: torch.nn.Module, x: torch.Tensor, + router_logits: torch.Tensor, top_k: int, renormalize: bool, + use_grouped_topk: bool) -> torch.Tensor: + raise NotImplementedError + + +class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): + """MoE method without quantization.""" + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + 2 * intermediate_size, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size, + intermediate_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None + ) -> torch.Tensor: + + return self.forward(x=x, + layer=layer, + router_logits=router_logits, + top_k=top_k, + renormalize=renormalize, + use_grouped_topk=use_grouped_topk, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + def forward_cuda( + self, + layer: torch.nn.Module, + x: torch.Tensor, + use_grouped_topk: bool, + top_k: int, + router_logits: torch.Tensor, + renormalize: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_experts) + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_experts(hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=True) + + def forward_cpu(self, *args, **kwargs): + raise NotImplementedError( + "The CPU backend currently does not support MoE.") + + def forward_tpu( + self, + layer: torch.nn.Module, + x: torch.Tensor, + use_grouped_topk: bool, + top_k: int, + router_logits: torch.Tensor, + renormalize: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe + assert not use_grouped_topk + assert num_expert_group is None + assert topk_group is None + assert custom_routing_function is None + return fused_moe(hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk=top_k, + gating_output=router_logits, + renormalize=renormalize) + +class W8A8QuantizedFusedMoEMethod(FusedMoEMethodBase): + """MoE method W8A8 quantization. This class is for compressed-tensors format loading""" + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + self.strategy = extra_weight_attrs['quant_config'].target_scheme_map['Linear']['weights'].strategy + self.is_static_input_scheme = not extra_weight_attrs['quant_config'].target_scheme_map['Linear']['input_activations'].dynamic + # assert self.is_static_input_scheme, "W8A8 int quantization only support static input activation for now" + + self.quant_config = extra_weight_attrs["quant_config"] + self.weight_loader = extra_weight_attrs["weight_loader"] + + self.logical_widths_13 = [intermediate_size * 2] + self.logical_widths_2 = [intermediate_size * 2] + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + 2 * intermediate_size, + hidden_size, + dtype=torch.int8), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, { + "input_dim": 1, + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + # WEIGHT SCALE + layer_kwargs = {"weight_loader": self.weight_loader} + if self.strategy == QuantizationStrategy.CHANNEL: + scale = torch.nn.Parameter(torch.empty((num_experts, intermediate_size * 2, 1), + dtype=torch.float32), + requires_grad=False) + scale[:] = torch.finfo(torch.float32).min + set_weight_attrs(scale, {"input_dim": 1, "output_dim": 0, **layer_kwargs}) + else: + assert self.strategy == QuantizationStrategy.TENSOR + scale = torch.nn.Parameter(torch.empty((num_experts, 2), dtype=torch.float32), + requires_grad=False) + scale[:] = torch.finfo(torch.float32).min + set_weight_attrs(scale, { + "needs_scalar_to_array": True, + **layer_kwargs + }) + set_weight_attrs(scale, {"is_int8_weight_scale": True}) + layer.register_parameter("w13_weight_scale", scale) + + + # INPUT SCALE + if self.is_static_input_scheme: + scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + set_weight_attrs(scale, { + "needs_scalar_to_array": True, + **layer_kwargs + }) + set_weight_attrs(scale, {"is_int8_input_scale": True}) + layer.register_parameter("w13_input_scale", scale) + + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size, + intermediate_size, + dtype=torch.int8), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + + set_weight_attrs(w2_weight, { + "input_dim": 1, + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + + # WEIGHT SCALE + if self.strategy == QuantizationStrategy.CHANNEL: + scale = torch.nn.Parameter(torch.empty((num_experts, hidden_size, 1), + dtype=torch.float32), + requires_grad=False) + scale[:] = torch.finfo(torch.float32).min + set_weight_attrs(scale, {"input_dim": 0, "output_dim": 1, **layer_kwargs}) + + else: + assert self.strategy == QuantizationStrategy.TENSOR + scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + set_weight_attrs(scale, { + "needs_scalar_to_array": True, + **layer_kwargs + }) + set_weight_attrs(scale, {"is_int8_weight_scale": True}) + layer.register_parameter("w2_weight_scale", scale) + + # INPUT SCALE + if self.is_static_input_scheme: + scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + set_weight_attrs(scale, { + "needs_scalar_to_array": True, + **layer_kwargs + }) + set_weight_attrs(scale, {"is_int8_input_scale": True}) + layer.register_parameter("w2_input_scale", scale) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None) -> torch.Tensor: + assert False, "fused_moe w8a8 use ixformer.contrib.vllm.layers.mixtral_decoder_layer_forward" + from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe + if self.is_static_input_scheme: + return fused_moe(x, + layer.w13_weight, + layer.w2_weight, + router_logits, + top_k, + renormalize=renormalize, + inplace=True, + use_grouped_topk=use_grouped_topk, + num_expert_group=num_expert_group, + topk_group=topk_group, + use_int8_w8a8=True, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale) + else: + return fused_moe(x, + layer.w13_weight, + layer.w2_weight, + router_logits, + top_k, + renormalize=renormalize, + inplace=True, + use_grouped_topk=use_grouped_topk, + num_expert_group=num_expert_group, + topk_group=topk_group, + use_int8_w8a8=True, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=None, + a2_scale=None) + +class FusedMoE(torch.nn.Module): + """FusedMoE layer for MoE models. + + This layer contains both MergedColumnParallel weights (gate_up_proj / + w13) and RowParallelLinear weights (down_proj/ w2). + + Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We + copy that naming convention here and handle any remapping in the + load_weights function in each model implementation. + + Args: + num_experts: Number of experts in the model + top_k: Number of experts selected for each token + hidden_size: Input hidden state size of the transformer + intermediate_size: Intermediate size of the experts + params_dtype: Data type for the parameters. + reduce_results: Whether to all all_reduce on the output of the layer + renomalize: Whether to renormalize the logits in the fused_moe kernel + quant_config: Quantization configure. + """ + + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + reduce_results: bool = False, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + prefix: str = "", + custom_routing_function: Optional[Callable] = None, + ): + super().__init__() + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + + self.tp_size = (tp_size if tp_size is not None else + get_tensor_model_parallel_world_size()) + self.top_k = top_k + self.num_experts = num_experts + self.intermediate_size_per_partition = intermediate_size // self.tp_size + self.reduce_results = reduce_results + self.renormalize = renormalize + self.use_grouped_topk = use_grouped_topk + if self.use_grouped_topk: + assert num_expert_group is not None and topk_group is not None + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.custom_routing_function = custom_routing_function + + from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import CompressedTensorsConfig + if quant_config is None: + self.quant_method: Optional[QuantizeMethodBase] = ( + UnquantizedFusedMoEMethod()) + elif (isinstance(quant_config, CompressedTensorsConfig) + and quant_config.quant_format == CompressionFormat.int_quantized.value + and quant_config.target_scheme_map['Linear']['input_activations'].num_bits == 8 + and quant_config.target_scheme_map['Linear']['weights'].num_bits == 8): + self.quant_method: Optional[QuantizeMethodBase] = ( + W8A8QuantizedFusedMoEMethod()) + else: + self.quant_method = quant_config.get_quant_method(self, prefix) + assert self.quant_method is not None + + self.quant_method.create_weights( + layer=self, + num_experts=num_experts, + hidden_size=hidden_size, + intermediate_size=self.intermediate_size_per_partition, + params_dtype=params_dtype, + weight_loader=self.weight_loader, + quant_config=quant_config) + + def _load_per_tensor_weight_scale(self, shard_id: str, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + expert_id: int): + param_data = param.data + # for per tensor weight quantization + if shard_id in ("w1", "w3"): + # We have to keep the weight scales of w1 and w3 because + # we need to re-quantize w1/w3 weights after weight loading. + idx = 0 if shard_id == "w1" else 1 + param_data[expert_id][idx] = loaded_weight + # If we are in the row parallel case (down_proj) + elif shard_id == "w2": + param_data[expert_id] = loaded_weight + + def _load_model_weight_or_group_weight_scale(self, shard_dim: int, + expert_data: torch.Tensor, + shard_id: str, + loaded_weight: torch.tensor, + tp_rank: int): + # Load grouped weight scales for group quantization + # or model weights + if shard_id == "w2": + self._load_w2(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + elif shard_id in ("w1", "w3"): + self._load_w13(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + + def _load_per_channel_weight_scale(self, expert_data: torch.Tensor, + shard_dim: int, shard_id: str, + loaded_weight: torch.tensor, + tp_rank: int): + # for per channel weight quantization + if shard_id == "w2": + expert_data.copy_(loaded_weight) + elif shard_id in ("w1", "w3"): + self._load_w13(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + + def _load_w13(self, expert_data: torch.Tensor, shard_dim: int, + shard_id: str, loaded_weight: torch.tensor, tp_rank: int): + + # Index the loaded weight for tp sharding. + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim + shard_size = expert_data.shape[shard_dim] // 2 + loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank, + shard_size) + # Narrow parameter and load. + # w1, gate_proj: Load into first logical weight of w13. + if shard_id == "w1": + expert_data = expert_data.narrow(shard_dim, 0, shard_size) + # w3, up_proj: Load into second logical weight of w13. + else: + assert shard_id == "w3" + expert_data = expert_data.narrow(shard_dim, shard_size, shard_size) + expert_data.copy_(loaded_weight) + + def _load_w2(self, expert_data: torch.Tensor, shard_dim: int, + shard_id: str, loaded_weight: torch.tensor, tp_rank: int): + + # Index the loaded weight for tp sharding. + # down_proj: "RowParallel" so tp sharding on input_dim + # Narrow parameter and load. + shard_size = expert_data.shape[shard_dim] + loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank, + shard_size) + # w2, down_proj: Load into only logical weight of w2. + expert_data.copy_(loaded_weight) + + def _load_single_value(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, expert_id: int): + param_data = param.data + + # Input scales can be loaded directly and should be equal. + param_data[expert_id] = loaded_weight + + def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor, + shard_dim: int, loaded_weight: torch.tensor, tp_rank: int): + + if shard_id == "w2": + self._load_w2(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + else: + assert shard_id in ("w1", "w3") + expert_data.copy_(loaded_weight) + + def weight_loader(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, weight_name: str, + shard_id: str, expert_id: int) -> None: + + # compressed-tensors checkpoints with packed weights are stored flipped + # TODO (mgoin): check self.quant_method.quant_config.quant_format + # against known CompressionFormat enum values that have this quality + loaded_weight = loaded_weight.t().contiguous() if ( + self.quant_method.__class__.__name__ + == "CompressedTensorsWNA16MoEMethod") else loaded_weight + + if shard_id not in ("w1", "w2", "w3"): + raise ValueError(f"shard_id must be ['w1','w2','w3'] but " + f"got {shard_id}.") + + # Special case for fp8 scales. + if getattr(param, "is_fp8_scale", False): + self._load_fp8_scale(param.data, loaded_weight, weight_name, + shard_id, expert_id) + return + elif getattr(param, "is_int8_input_scale", False): + self._load_int8_input_scale(param.data, loaded_weight, + shard_id, expert_id) + return + + WEIGHT_SCALE_SUPPORTED = [ + e.value for e in FusedMoeWeightScaleSupported + ] + # Fetch the dim to shard the parameter/loaded weight + # based on the shard id. This will be whatever + # dimension intermediate_size is used. + SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0} + + expert_data = param.data[expert_id] + tp_rank = get_tensor_model_parallel_rank() + + # is_transposed: if the dim to shard the weight + # should be flipped. Required by GPTQ, compressed-tensors + # should be whatever dimension intermediate_size is + is_transposed = getattr(param, "is_transposed", False) + shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id] + if is_transposed: + shard_dim = ~shard_dim + + # Case input scale: input_scale loading is only supported for fp8 + if "input_scale" in weight_name: + # this is needed for compressed-tensors only + loaded_weight = loaded_weight.to(param.data.device) + + if param.data[expert_id] != 1 and (param.data[expert_id] - + loaded_weight).abs() > 1e-5: + raise ValueError( + "input_scales of w1 and w3 of a layer " + f"must be equal. But got {param.data[expert_id]} " + f"vs. {loaded_weight}") + + self._load_single_value(param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + return + + # Case g_idx + if "g_idx" in weight_name: + self._load_g_idx(shard_dim=0, + shard_id=shard_id, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + return + + # Case weight scales and zero_points + if ("scale" in weight_name or "zero" in weight_name): + # load the weight scales and zp based on the quantization scheme + # supported weight scales/zp can be found in + # FusedMoeWeightScaleSupported + # TODO @dsikka: once hardened, refactor to use vLLM Parameters + # specific to each case + quant_method = getattr(param, "quant_method", FusedMoeWeightScaleSupported.CHANNEL.value) + if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value: + self._load_per_channel_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + elif quant_method == FusedMoeWeightScaleSupported.GROUP.value: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: + self._load_per_tensor_weight_scale(shard_id=shard_id, + param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + else: + raise ValueError( + f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}") + return + + # Case weight_shape + if "weight_shape" in weight_name: + # only required by compressed-tensors + self._load_single_value(param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + return + + # Case model weights + if "weight" in weight_name: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + return + + @staticmethod + def select_experts(hidden_states: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + use_grouped_topk: bool, + renormalize: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None): + from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_topk, grouped_topk) + + # DeekSeekv2 uses grouped_top_k + if use_grouped_topk: + assert topk_group is not None + assert num_expert_group is not None + topk_weights, topk_ids = grouped_topk( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + num_expert_group=num_expert_group, + topk_group=topk_group) + elif custom_routing_function is None: + topk_weights, topk_ids = fused_topk(hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize) + else: + topk_weights, topk_ids = custom_routing_function( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize) + + return topk_weights, topk_ids + + def forward(self, hidden_states: torch.Tensor, + router_logits: torch.Tensor): + assert self.quant_method is not None + + # Matrix multiply. + final_hidden_states = self.quant_method.apply( + layer=self, + x=hidden_states, + router_logits=router_logits, + top_k=self.top_k, + renormalize=self.renormalize, + use_grouped_topk=self.use_grouped_topk, + topk_group=self.topk_group, + num_expert_group=self.num_expert_group, + custom_routing_function=self.custom_routing_function) + + if self.reduce_results and self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states + + @classmethod + def make_expert_params_mapping( + cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int) -> List[Tuple[str, str, int, str]]: + + return [ + # (param_name, weight_name, expert_id, shard_id) + ("experts.w13_" if weight_name + in [ckpt_gate_proj_name, ckpt_up_proj_name] else "experts.w2_", + f"experts.{expert_id}.{weight_name}.", expert_id, shard_id) + for expert_id in range(num_experts) for shard_id, weight_name in [ + ("w1", ckpt_gate_proj_name), + ("w2", ckpt_down_proj_name), + ("w3", ckpt_up_proj_name), + ] + ] + + def _load_fp8_scale(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, weight_name: str, + shard_id: str, expert_id: int) -> None: + param_data = param.data + + # Input scales can be loaded directly and should be equal. + if "input_scale" in weight_name: + if param_data[expert_id] != 1 and (param_data[expert_id] - + loaded_weight).abs() > 1e-5: + raise ValueError( + "input_scales of w1 and w3 of a layer " + f"must be equal. But got {param_data[expert_id]} " + f"vs. {loaded_weight}") + param_data[expert_id] = loaded_weight + # Weight scales + elif "weight_scale" in weight_name: + # If we are in merged column case (gate_up_proj) + if shard_id in ("w1", "w3"): + # We have to keep the weight scales of w1 and w3 because + # we need to re-quantize w1/w3 weights after weight loading. + idx = 0 if shard_id == "w1" else 1 + param_data[expert_id][idx] = loaded_weight + # If we are in the row parallel case (down_proj) + else: + param_data[expert_id] = loaded_weight + + def _load_int8_input_scale(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + shard_id: str, expert_id: int) -> None: + param_data = param.data + # Input scales can be loaded directly and should be equal. + if param_data[expert_id] != 1 and (param_data[expert_id].to(loaded_weight.device) - + loaded_weight).abs() > 1e-5: + raise ValueError( + "input_scales of w1 and w3 of a layer " + f"must be equal. But got {param_data[expert_id]} " + f"vs. {loaded_weight}") + param_data[expert_id] = loaded_weight \ No newline at end of file diff --git a/vllm/model_executor/layers/fused_moe/moe_pallas.py b/vllm/model_executor/layers/fused_moe/moe_pallas.py new file mode 100644 index 00000000..563ee18c --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/moe_pallas.py @@ -0,0 +1,62 @@ +import torch +import torch.nn.functional as F +from torch_xla.experimental.custom_kernel import _histogram + + +def fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +) -> torch.Tensor: + """ + Args: + hidden_states: [*, hidden_size] + w1: [num_experts, intermediate_size * 2, hidden_size] + w2: [num_experts, hidden_size, intermediate_size] + gating_output: [*, num_experts] + """ + orig_shape = hidden_states.shape + hidden_size = hidden_states.shape[-1] + num_tokens = hidden_states.shape[:-1].numel() + num_experts = w1.shape[0] + intermediate_size = w2.shape[-1] + device = hidden_states.device + dtype = hidden_states.dtype + assert (num_tokens * topk) % 16 == 0, ( + "The Pallas GMM kernel requires num_tokens * topk to be a multiple of " + f"16 but got {num_tokens * topk}") + + hidden_states = hidden_states.view(num_tokens, hidden_size) + gating_output = gating_output.view(num_tokens, num_experts) + topk_weights = gating_output.softmax(dim=-1, dtype=torch.float) + topk_weights, topk_indices = topk_weights.topk(topk, dim=-1) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_weights = topk_weights.to(dtype) + + topk_indices = topk_indices.flatten() + topk_argsort_indices = topk_indices.argsort() + topk_argsort_revert_indices = topk_argsort_indices.argsort() + token_indices = torch.arange(num_tokens, + device=device).repeat_interleave(topk) + token_indices = token_indices[topk_argsort_indices] + group_sizes = _histogram(topk_indices.to(torch.int32), 0, num_experts - 1) + + # NOTE(woosuk): The GMM Pallas kernel requires a different weight layout + # from HF Transformers. + w1 = w1.transpose(1, 2) + w2 = w2.transpose(1, 2) + + x = hidden_states[token_indices] + x = torch.ops.xla.gmm(x, w1, group_sizes) + x = F.silu(x[..., :intermediate_size]) * x[..., intermediate_size:] + x = torch.ops.xla.gmm(x, w2, group_sizes) + x = x[topk_argsort_revert_indices].reshape(-1, topk, hidden_size) + + x = x * topk_weights.unsqueeze_(dim=-1) + x = x.sum(dim=-2) + x = x.reshape(orig_shape) + return x diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py new file mode 100644 index 00000000..eb5be2d1 --- /dev/null +++ b/vllm/model_executor/layers/layernorm.py @@ -0,0 +1,187 @@ +"""Custom normalization layers.""" +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +from vllm.model_executor.custom_op import CustomOp + + +class RMSNorm(CustomOp): + """Root mean square normalization. + + Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight. + Refer to https://arxiv.org/abs/1910.07467 + """ + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + var_hidden_size: Optional[int] = None, + ) -> None: + super().__init__() + + self.hidden_size = hidden_size + self.variance_epsilon = eps + self.variance_size_override = (None if var_hidden_size == hidden_size + else var_hidden_size) + + self.weight = nn.Parameter(torch.ones(hidden_size)) + + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """PyTorch-native implementation equivalent to forward().""" + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + hidden_size = x.shape[-1] + if hidden_size != self.hidden_size: + raise ValueError("Expected hidden_size to be " + f"{self.hidden_size}, but found: {hidden_size}") + + if self.variance_size_override is None: + x_var = x + else: + if hidden_size < self.variance_size_override: + raise ValueError( + "Expected hidden_size to be at least " + f"{self.variance_size_override}, but found: {hidden_size}") + + x_var = x[:, :, :self.variance_size_override] + + variance = x_var.pow(2).mean(dim=-1, keepdim=True) + + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = x.to(orig_dtype) * self.weight + if residual is None: + return x + else: + return x, residual + + def forward_cuda( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + residual_alpha: Optional[float] = 1.0, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if self.variance_size_override is not None: + return self.forward_native(x, residual) + + from vllm import _custom_ops as ops + + if residual is not None: + ops.fused_add_rms_norm( + x, + residual, + self.weight.data, + self.variance_epsilon, + residual_alpha, + ) + return x, residual + out = torch.empty_like(x) + ops.rms_norm( + out, + x, + self.weight.data, + self.variance_epsilon, + ) + return out + + def forward_xpu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if self.variance_size_override is not None: + return self.forward_native(x, residual) + + from vllm._ipex_ops import ipex_ops as ops + + if residual is not None: + ops.fused_add_rms_norm( + x, + residual, + self.weight.data, + self.variance_epsilon, + ) + return x, residual + return ops.rms_norm( + x, + self.weight.data, + self.variance_epsilon, + ) + + def extra_repr(self) -> str: + s = f"hidden_size={self.weight.data.size(0)}" + s += f", eps={self.variance_epsilon}" + return s + + +class GemmaRMSNorm(CustomOp): + """RMS normalization for Gemma. + + Two differences from the above RMSNorm: + 1. x * (1 + w) instead of x * w. + 2. (x * w).to(orig_dtype) instead of x.to(orig_dtype) * w. + """ + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + @staticmethod + def forward_static( + weight: torch.Tensor, + variance_epsilon: float, + x: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """PyTorch-native implementation equivalent to forward().""" + orig_dtype = x.dtype + if residual is not None: + x = x + residual + residual = x + + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + x = x * (1.0 + weight.float()) + x = x.to(orig_dtype) + return x if residual is None else (x, residual) + + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """PyTorch-native implementation equivalent to forward().""" + return self.forward_static(self.weight.data, self.variance_epsilon, x, + residual) + + def forward_cuda( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + # if torch.compiler.is_compiling(): + # return self.forward_native(x, residual) + + # if not getattr(self, "_is_compiled", False): + # self.forward_static = torch.compile( # type: ignore + # self.forward_static) + # self._is_compiled = True + return self.forward_native(x, residual) diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py new file mode 100644 index 00000000..789a77e6 --- /dev/null +++ b/vllm/model_executor/layers/linear.py @@ -0,0 +1,1103 @@ +from abc import abstractmethod +from typing import Dict, List, Optional, Tuple + +import torch +#import ixformer.inference.functions as F +import ixformer.functions as F +from torch.nn.parameter import Parameter, UninitializedParameter + +from vllm.distributed import (divide, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce) +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.parameter import (BasevLLMParameter, + PackedColumnParameter, + PackedvLLMParameter, + PerTensorScaleParameter, + RowvLLMParameter) +from vllm.model_executor.utils import set_weight_attrs + +logger = init_logger(__name__) + +WEIGHT_LOADER_V2_SUPPORTED = [ + "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod", + "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod", + "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod", + "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod", + "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod" +] + + +def adjust_marlin_shard(param, shard_size, shard_offset): + marlin_tile_size = getattr(param, "marlin_tile_size", None) + if marlin_tile_size is None: + return shard_size, shard_offset + + return shard_size * marlin_tile_size, shard_offset * marlin_tile_size + + +def adjust_bitsandbytes_4bit_shard(param: Parameter, + qkv_offsets: Dict[str, Tuple[int, int]], + loaded_shard_id: str) -> Tuple[int, int]: + """Adjust the quantization offsets and sizes for BitsAndBytes sharding.""" + + total, _ = qkv_offsets["total"] + orig_offset, orig_size = qkv_offsets[loaded_shard_id] + + quantized_total = param.data.shape[0] + quantized_offset = orig_offset * quantized_total // total + quantized_size = orig_size * quantized_total // total + + return quantized_size, quantized_offset + + +def adjust_scalar_to_fused_array(param, loaded_weight, shard_id): + """For fused modules (QKV and MLP) we have an array of length + N that holds 1 scale for each "logical" matrix. So the param + is an array of length N. The loaded_weight corresponds to + one of the shards on disk. Here, we slice the param based on + the shard_id for loading. + """ + qkv_idxs = {"q": 0, "k": 1, "v": 2} + + if isinstance(shard_id, str): + shard_id = qkv_idxs[shard_id] + elif not isinstance(shard_id, int): + raise ValueError(f"Unknown Shard Id {shard_id}") + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + return param[shard_id], loaded_weight + + +class LinearMethodBase(QuantizeMethodBase): + """Base class for different (maybe quantized) linear methods.""" + + @abstractmethod + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + """Create weights for a linear layer. + The weights will be set as attributes of the layer. + + Args: + layer: The layer that is using the LinearMethodBase factory. + input_size_per_partition: Size of the weight input dim on rank X. + output_partition_sizes: Sizes of the output dim of each logical + weight on rank X. E.g., output_partition_sizes for QKVLinear + is a list contains the width of Wq, Wk, Wv on rank X. + input_size: Size of the input dim of the weight across all ranks. + output_size: Size of the output dim of the weight across all ranks. + params_dtype: Datatype of the parameters. + """ + raise NotImplementedError + + @abstractmethod + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + +class UnquantizedLinearMethod(LinearMethodBase): + """Linear method without quantization.""" + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + weight = Parameter(torch.empty(sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype), + requires_grad=False) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + if (x.shape[0] == 8192 or x.shape[0] == 16384 or x.shape[0] == 15360): + if bias is None: + return x @ layer.weight.T + else: + return x @ layer.weight.T + bias + return F.linear(x, layer.weight, bias) + + +class LinearBase(torch.nn.Module): + """Base linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__( + self, + input_size: int, + output_size: int, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + + # Keep input parameters + self.input_size = input_size + self.output_size = output_size + self.skip_bias_add = skip_bias_add + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + if quant_config is None: + self.quant_method: Optional[ + QuantizeMethodBase] = UnquantizedLinearMethod() + else: + self.quant_method = quant_config.get_quant_method(self, + prefix=prefix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + +class ReplicatedLinear(LinearBase): + """Replicated linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__(self, + input_size: int, + output_size: int, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__(input_size, + output_size, + skip_bias_add, + params_dtype, + quant_config, + prefix=prefix) + + # All the linear layer supports quant method. + assert self.quant_method is not None + self.quant_method.create_weights(self, + self.input_size, [self.output_size], + self.input_size, + self.output_size, + self.params_dtype, + weight_loader=self.weight_loader) + + if bias: + self.bias = Parameter( + torch.empty(self.output_size, dtype=self.params_dtype)) + set_weight_attrs(self.bias, { + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + # If the weight on disk does not have a shape, give it one + # (such scales for AutoFp8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param.size() == loaded_weight.size() + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + bias = self.bias if not self.skip_bias_add else None + assert self.quant_method is not None + output = self.quant_method.apply(self, x, bias) + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + return s + + +class ColumnParallelLinear(LinearBase): + """Linear layer with column parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its second dimension as A = [A_1, ..., A_p]. + + Args: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make Y available + to all GPUs, otherwise, every GPU will have its output + which is Y_i = XA_i + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + output_sizes: list of output sizes packed into one output, like for QKV + the list would be size 3. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__(self, + input_size: int, + output_size: int, + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + output_sizes: Optional[List[int]] = None, + prefix: str = ""): + super().__init__(input_size, output_size, skip_bias_add, params_dtype, + quant_config, prefix) + + self.gather_output = gather_output + + # Divide the weight matrix along the last dimension. + tp_size = get_tensor_model_parallel_world_size() + assert self.quant_method is not None + self.output_size_per_partition = divide(self.output_size, tp_size) + self.output_partition_sizes = [self.output_size_per_partition] + # If QKV or MergedColumn, use output size of each partition. + if hasattr(self, "output_sizes"): + self.output_partition_sizes = [ + divide(output_size, tp_size) + for output_size in self.output_sizes + ] + + if output_sizes is None: + output_sizes = [output_size] + + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size, + output_partition_sizes=self.output_partition_sizes, + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 if self.quant_method.__class__.__name__ + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader)) + if bias: + self.bias = Parameter( + torch.empty(self.output_size_per_partition, + dtype=params_dtype)) + set_weight_attrs(self.bias, { + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + tp_rank = get_tensor_model_parallel_rank() + output_dim = getattr(param, "output_dim", None) + + # Special case for GGUF + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.weight_type = loaded_weight.item() + + # Materialize GGUF UninitializedParameter + if is_gguf_weight and isinstance(param, UninitializedParameter): + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + + param_data = param.data + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if output_dim is not None and not use_bitsandbytes_4bit: + shard_size = param_data.shape[output_dim] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(output_dim, start_idx, + shard_size) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor): + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + param.load_column_parallel_weight(loaded_weight=loaded_weight) + + def forward(self, input_): + bias = self.bias if not self.skip_bias_add else None + + # Matrix multiply. + assert self.quant_method is not None + output_parallel = self.quant_method.apply(self, input_, bias) + if self.gather_output: + # All-gather across the partitions. + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size_per_partition}" + s += f", bias={self.bias is not None}" + s += f", tp_size={get_tensor_model_parallel_world_size()}" + s += f", gather_output={self.gather_output}" + return s + + +class MergedColumnParallelLinear(ColumnParallelLinear): + """Packed linear layers with column parallelism. + + Similar to ColumnParallelLinear, but the weight matrix is concatenated + along the output dimension. When the weight matrix is loaded, the + different partitions are sharded separately. + + Args: + input_size: input dimension of the linear layer. + output_sizes: list of output dimensions of the linear layer. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make the output + available to all GPUs, otherwise, every GPU will have + its own output. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__(self, + input_size: int, + output_sizes: List[int], + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + self.output_sizes = output_sizes + tp_size = get_tensor_model_parallel_world_size() + assert all(output_size % tp_size == 0 for output_size in output_sizes) + super().__init__(input_size=input_size, + output_size=sum(output_sizes), + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix) + + def weight_loader(self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[int] = None): + + # Special case for GGUF + # initialize GGUF param after we know the quantize type + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.data[loaded_shard_id].copy_(loaded_weight) + param.shard_weight_type[loaded_shard_id] = loaded_weight.item() + return + + if is_gguf_weight: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + + output_dim = getattr(param, "output_dim", None) + shard_size = loaded_weight.size(output_dim) // tp_size + start_idx = tp_rank * shard_size + + loaded_weight = loaded_weight.narrow(output_dim, start_idx, + shard_size) + + param.shard_id.append(loaded_shard_id) + param.shard_id_map[loaded_shard_id] = len(param.data_container) + param.data_container.append(loaded_weight) + if len(param.data_container) == 2: + self.qweight = param.materialize_nested() + return + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + # Special case for per-tensor scale to load scalar into fused array. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (qkv/mlp). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + current_shard_offset = 0 + shard_offsets: List[Tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + packed_dim = getattr(param, "packed_dim", None) + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset) + + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + assert loaded_shard_id < len(self.output_sizes) + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + if output_dim is not None: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size + shard_size = self.output_sizes[loaded_shard_id] // tp_size + # Special case for quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + packed_dim = getattr(param, "packed_dim", None) + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", + False) + if use_bitsandbytes_4bit: + shard_size = loaded_weight.shape[output_dim] + shard_offset = loaded_weight.shape[output_dim] * \ + loaded_shard_id + + param_data = param_data.narrow(output_dim, shard_offset, + shard_size) + start_idx = tp_rank * shard_size + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if not use_bitsandbytes_4bit: + loaded_weight = loaded_weight.narrow(output_dim, start_idx, + shard_size) + # Special case for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_offset = loaded_shard_id * shard_size + param_data = param_data.narrow(0, shard_offset, shard_size) + + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id) + + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "MergedColumnParallelLinear, assume the weight is " + "the same for all partitions.") + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter, + loaded_weight: torch.Tensor): + """ + Handle special case for models where MLP layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + + current_shard_offset = 0 + shard_offsets: List[Tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if isinstance(param, (PackedColumnParameter, PackedvLLMParameter + )) and param.packed_dim == param.output_dim: + shard_size, shard_offset = \ + param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset) + + loaded_weight_shard = loaded_weight.narrow(param.output_dim, + shard_offset, + shard_size) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2(self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[int] = None): + if loaded_shard_id is None: + if isinstance(param, PerTensorScaleParameter): + param.load_merged_column_weight(loaded_weight=loaded_weight, + shard_id=0) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_merged_column_weight(loaded_weight=loaded_weight) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id < len(self.output_sizes) + + tp_size = get_tensor_model_parallel_world_size() + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size + shard_size = self.output_sizes[loaded_shard_id] // tp_size + + param.load_merged_column_weight(loaded_weight=loaded_weight, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size) + + +class QKVParallelLinear(ColumnParallelLinear): + """Linear layers for the attention's QKV transformation. + + Linear layers for the linear transformation of the query, key, and value + vectors in the attention layer. The weight matrix is concatenated along + the output dimension. The layer is parallelized along the head dimension. + When the number of key/value heads is smaller than the number of query + heads (e.g., multi-query/grouped-query attention), the key/value head may + be replicated while the query heads are partitioned. + + Args: + hidden_size: input hidden state size of the transformer. + head_size: size of each attention head. + total_num_heads: total number of attention query heads. + total_num_kv_heads: total number of attention key/value heads. If + None, assume total_num_kv_heads = total_num_heads. + bias: If true, add bias. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__(self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: Optional[int] = None, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + self.hidden_size = hidden_size + self.head_size = head_size + self.total_num_heads = total_num_heads + if total_num_kv_heads is None: + total_num_kv_heads = total_num_heads + self.total_num_kv_heads = total_num_kv_heads + # Divide the weight matrix along the last dimension. + tp_size = get_tensor_model_parallel_world_size() + self.num_heads = divide(self.total_num_heads, tp_size) + if tp_size >= self.total_num_kv_heads: + self.num_kv_heads = 1 + self.num_kv_head_replicas = divide(tp_size, + self.total_num_kv_heads) + else: + self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) + self.num_kv_head_replicas = 1 + input_size = self.hidden_size + output_size = (self.num_heads + + 2 * self.num_kv_heads) * tp_size * self.head_size + self.output_sizes = [ + self.num_heads * self.head_size * tp_size, # q_proj + self.num_kv_heads * self.head_size * tp_size, # k_proj + self.num_kv_heads * self.head_size * tp_size, # v_proj + ] + + super().__init__(input_size=input_size, + output_size=output_size, + bias=bias, + gather_output=False, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix) + + def _get_shard_offset_mapping(self, loaded_shard_id: str): + shard_offset_mapping = { + "q": 0, + "k": self.num_heads * self.head_size, + "v": (self.num_heads + self.num_kv_heads) * self.head_size, + "total": (self.num_heads + 2 * self.num_kv_heads) * self.head_size + } + return shard_offset_mapping.get(loaded_shard_id) + + def _get_shard_size_mapping(self, loaded_shard_id: str): + shard_size_mapping = { + "q": self.num_heads * self.head_size, + "k": self.num_kv_heads * self.head_size, + "v": self.num_kv_heads * self.head_size, + } + return shard_size_mapping.get(loaded_shard_id) + + def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter, + loaded_weight: torch.Tensor): + """ + Handle special case for models where QKV layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ("k", self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size), + ("v", + (self.total_num_heads + self.total_num_kv_heads) * self.head_size, + self.total_num_kv_heads * self.head_size), + ] + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if isinstance(param, (PackedColumnParameter, PackedvLLMParameter + )) and param.packed_dim == param.output_dim: + shard_size, shard_offset = \ + param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset) + + loaded_weight_shard = loaded_weight.narrow(param.output_dim, + shard_offset, + shard_size) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2(self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[str] = None): + if loaded_shard_id is None: # special case for certain models + if isinstance(param, PerTensorScaleParameter): + param.load_qkv_weight(loaded_weight=loaded_weight, shard_id=0) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_qkv_weight(loaded_weight=loaded_weight) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id in ["q", "k", "v"] + + shard_offset = self._get_shard_offset_mapping(loaded_shard_id) + shard_size = self._get_shard_size_mapping(loaded_shard_id) + + param.load_qkv_weight(loaded_weight=loaded_weight, + num_heads=self.num_kv_head_replicas, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size) + + def weight_loader(self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[str] = None): + + # Special case for GGUF + # initialize GGUF param after we know the quantize type + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type and loaded_shard_id is not None: + idx_map = {"q": 0, "k": 1, "v": 2} + param.data[idx_map[loaded_shard_id]].copy_(loaded_weight) + param.shard_weight_type[loaded_shard_id] = loaded_weight.item() + return + + if is_gguf_weight: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + + output_dim = getattr(param, "output_dim", None) + shard_size = loaded_weight.size(output_dim) // tp_size + start_idx = tp_rank * shard_size + + loaded_weight = loaded_weight.narrow(output_dim, start_idx, + shard_size) + + param.shard_id.append(loaded_shard_id) + param.shard_id_map[loaded_shard_id] = len(param.data_container) + param.data_container.append(loaded_weight) + if len(param.data_container) == 3: + self.qweight = param.materialize_nested() + return + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + + # Special case for per-tensor scales in fused case. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (qkv/mlp). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ("k", self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size), + ("v", (self.total_num_heads + self.total_num_kv_heads) * + self.head_size, self.total_num_kv_heads * self.head_size), + ] + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", + False) + + packed_dim = getattr(param, "packed_dim", None) + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantized Weights. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset) + + if use_bitsandbytes_4bit: + orig_qkv_offsets = { + "q": (0, self.total_num_heads * self.head_size), + "k": (self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size), + "v": + ((self.total_num_heads + self.total_num_kv_heads) * + self.head_size, + self.total_num_kv_heads * self.head_size), + "total": + ((self.total_num_heads + 2 * self.total_num_kv_heads) * + self.head_size, 0) + } + + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( + param, orig_qkv_offsets, shard_id) + + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + tp_rank = get_tensor_model_parallel_rank() + assert loaded_shard_id in ["q", "k", "v"] + + # If output dim is defined, use the default loading process. + if output_dim is not None: + if loaded_shard_id == "q": + shard_offset = 0 + shard_size = self.num_heads * self.head_size + elif loaded_shard_id == "k": + shard_offset = self.num_heads * self.head_size + shard_size = self.num_kv_heads * self.head_size + elif loaded_shard_id == "v": + shard_offset = (self.num_heads + + self.num_kv_heads) * self.head_size + shard_size = self.num_kv_heads * self.head_size + # Special case for Quantized Weights. + # If quantized, we need to adjust the offset and size to account + # for the packing. + packed_dim = getattr(param, "packed_dim", None) + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", + False) + if use_bitsandbytes_4bit: + orig_qkv_offsets = { + "q": (0, self.num_heads * self.head_size), + "k": (self.num_heads * self.head_size, + self.num_kv_heads * self.head_size), + "v": + ((self.num_heads + self.num_kv_heads) * self.head_size, + self.num_kv_heads * self.head_size), + "total": + ((self.num_heads + 2 * self.num_kv_heads) * self.head_size, + 0) + } + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( + param, orig_qkv_offsets, loaded_shard_id) + + param_data = param_data.narrow(output_dim, shard_offset, + shard_size) + if loaded_shard_id == "q": + shard_id = tp_rank + else: + shard_id = tp_rank // self.num_kv_head_replicas + start_idx = shard_id * shard_size + + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if not use_bitsandbytes_4bit: + loaded_weight = loaded_weight.narrow(output_dim, start_idx, + shard_size) + + # Special case for for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_index = ["q", "k", "v"].index(loaded_shard_id) + param_data = param_data.narrow(0, shard_index * shard_size, + shard_size) + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id) + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "QKVParallelLinear, assume the weight is the same " + "for all partitions.") + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class RowParallelLinear(LinearBase): + """Linear layer with row parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its first dimension and X along its second dimension as: + - - + | A_1 | + | . | + A = | . | X = [X_1, ..., X_p] + | . | + | A_p | + - - + Arguments: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. Note that bias is not parallelized. + input_is_parallel: If true, we assume that the input is already + split across the GPUs and we do not split + again. + skip_bias_add: This was added to enable performance optimization where + bias can be fused with other element-wise operations. + We skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__(self, + input_size: int, + output_size: int, + bias: bool = True, + input_is_parallel: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + reduce_results: bool = True, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__(input_size, output_size, skip_bias_add, params_dtype, + quant_config, prefix) + + self.input_is_parallel = input_is_parallel + self.reduce_results = reduce_results + + # Divide the weight matrix along the last dimension. + self.tp_rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + self.input_size_per_partition = divide(input_size, self.tp_size) + assert self.quant_method is not None + + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size_per_partition, + output_partition_sizes=[self.output_size], + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 if self.quant_method.__class__.__name__ + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader)) + if not reduce_results and (bias and not skip_bias_add): + raise ValueError("When not reduce the results, adding bias to the " + "results can lead to incorrect results") + + if bias: + self.bias = Parameter( + torch.empty(self.output_size, dtype=params_dtype)) + set_weight_attrs(self.bias, { + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + input_dim = getattr(param, "input_dim", None) + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + + # Special case for GGUF + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.weight_type = loaded_weight.item() + + # Materialize GGUF UninitializedParameter + if is_gguf_weight and isinstance(param, UninitializedParameter): + weight_shape = list(loaded_weight.shape) + if input_dim: + weight_shape[input_dim] = weight_shape[input_dim] // tp_size + param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype) + + param_data = param.data + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if input_dim is not None and not use_bitsandbytes_4bit: + shard_size = param_data.shape[input_dim] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(input_dim, start_idx, + shard_size) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: BasevLLMParameter, + loaded_weight: torch.Tensor): + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + + param.load_row_parallel_weight(loaded_weight=loaded_weight) + + def forward(self, input_): + if self.input_is_parallel: + input_parallel = input_ + else: + tp_rank = get_tensor_model_parallel_rank() + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.tp_size) + input_parallel = splitted_input[tp_rank].contiguous() + + # Matrix multiply. + assert self.quant_method is not None + # Only fuse bias add into GEMM for rank 0 (this ensures that + # bias will not get added more than once in TP>1 case) + bias_ = None if (self.tp_rank > 0 or self.skip_bias_add) else self.bias + output_parallel = self.quant_method.apply(self, + input_parallel, + bias=bias_) + if self.reduce_results and self.tp_size > 1: + output = tensor_model_parallel_all_reduce(output_parallel) + else: + output = output_parallel + + output_bias = self.bias if self.skip_bias_add else None + + return output, output_bias + + def extra_repr(self) -> str: + s = f"input_features={self.input_size_per_partition}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + s += f", tp_size={self.tp_size}" + s += f", reduce_results={self.reduce_results}" + return s diff --git a/vllm/model_executor/layers/logits_processor.py b/vllm/model_executor/layers/logits_processor.py new file mode 100644 index 00000000..e985c1b1 --- /dev/null +++ b/vllm/model_executor/layers/logits_processor.py @@ -0,0 +1,156 @@ +"""A layer that compute logits from hidden_stats.""" +import inspect +from typing import Optional + +import torch +import torch.nn as nn + +from vllm.distributed import (tensor_model_parallel_all_gather, + tensor_model_parallel_gather) +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.platforms import current_platform + + +class LogitsProcessor(nn.Module): + """Process logits and apply logits processors from sampling metadata. + + This layer does the following: + 1. Gather logits from model hidden_states. + 2. Scale logits if needed. + 3. Apply logits processors (if any). + """ + + def __init__(self, + vocab_size: int, + org_vocab_size: Optional[int] = None, + scale: float = 1.0, + logits_as_input: bool = False, + soft_cap: Optional[float] = None) -> None: + """ + Args: + scale: A scaling factor to apply to the logits. + """ + super().__init__() + self.scale = scale + self.vocab_size = vocab_size + # Whether the input is logits (default is hidden states). + self.logits_as_input = logits_as_input + # original vocabulary size (without LoRA). + self.org_vocab_size = org_vocab_size or vocab_size + # Soft cap the logits. Used in Gemma 2. + self.soft_cap = soft_cap + # Whether to use gather or all-gather to gather the logits. + self.use_gather = not current_platform.is_tpu() + + def forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + embedding_bias: Optional[torch.Tensor] = None, + ) -> Optional[torch.Tensor]: + if self.logits_as_input: + logits = hidden_states + else: + hidden_states = _prune_hidden_states(hidden_states, + sampling_metadata) + + # Get the logits for the next tokens. + if hidden_states.shape[0] > 0: + logits = self._get_logits(hidden_states, lm_head, embedding_bias) + else: + logits = torch.empty([0, lm_head.weight.shape[0]], device=hidden_states.device, dtype=hidden_states.dtype) + if logits is not None: + if self.soft_cap is not None: + logits = logits / self.soft_cap + logits = torch.tanh(logits) + logits = logits * self.soft_cap + + if self.scale != 1.0: + logits *= self.scale + + # Apply logits processors (if any). + logits = _apply_logits_processors(logits, sampling_metadata) + + return logits + + def _get_logits( + self, + hidden_states: torch.Tensor, + lm_head: VocabParallelEmbedding, + embedding_bias: Optional[torch.Tensor], + ) -> Optional[torch.Tensor]: + # Get the logits for the next tokens. + logits = lm_head.linear_method.apply(lm_head, + hidden_states, + bias=embedding_bias) + if self.use_gather: + # None may be returned for rank > 0 + logits = tensor_model_parallel_gather(logits) + else: + # Gather is not supported for some devices such as TPUs. + # Use all-gather instead. + # NOTE(woosuk): Here, the outputs of every device should not be None + # because XLA requires strict SPMD among all devices. Every device + # should execute the same operations after gathering the logits. + logits = tensor_model_parallel_all_gather(logits) + # Remove paddings in vocab (if any). + if logits is not None: + logits = logits[..., :self.org_vocab_size] + return logits + + def extra_repr(self) -> str: + s = f"vocab_size={self.vocab_size}" + s += f", forg_vocab_size={self.org_vocab_size}" + s += f", scale={self.scale}, logits_as_input={self.logits_as_input}" + return s + + +def _prune_hidden_states( + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> torch.Tensor: + return hidden_states.index_select(0, + sampling_metadata.selected_token_indices) + + +def _apply_logits_processors( + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> torch.Tensor: + found_logits_processors = False + logits_processed = 0 + for seq_group in sampling_metadata.seq_groups: + seq_ids = seq_group.seq_ids + sampling_params = seq_group.sampling_params + logits_processors = sampling_params.logits_processors + if logits_processors: + found_logits_processors = True + + for seq_id, logits_row_idx in zip(seq_ids, + seq_group.sample_indices): + logits_row = logits[logits_row_idx] + past_tokens_ids = seq_group.seq_data[seq_id].output_token_ids + prompt_tokens_ids = seq_group.seq_data[seq_id].prompt_token_ids + + for logits_processor in logits_processors: + parameters = inspect.signature(logits_processor).parameters + if len(parameters) == 3: + logits_row = logits_processor(prompt_tokens_ids, + past_tokens_ids, + logits_row) + else: + logits_row = logits_processor(past_tokens_ids, + logits_row) + + logits[logits_row_idx] = logits_row + + logits_processed += len(seq_group.sample_indices) + len( + seq_group.prompt_logprob_indices) + + if found_logits_processors: + # verifies that no rows in logits were missed unexpectedly + assert logits_processed == logits.shape[0] + return logits diff --git a/vllm/model_executor/layers/mamba/__init__.py b/vllm/model_executor/layers/mamba/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/model_executor/layers/mamba/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/mamba/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bff33db5c3da07f304a07729f238a353a13cb81f GIT binary patch literal 173 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;x_Oeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_er|qBYEFD=MQU&ryk0@&Ee;!?%F>)vJCIGqOhAH#0RSEFD|Y|@ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/mamba/ops/__init__.py b/vllm/model_executor/layers/mamba/ops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/model_executor/layers/mamba/ops/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/mamba/ops/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db529d1875dfae4dfafefdd9e6141514567c69a7 GIT binary patch literal 177 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;x_Weo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_er|qBYEFD=MQU&ryk0@&Ee;!?>e8H4JCJR~OhAH#0RZ1AEc^ff literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/mamba/ops/__pycache__/causal_conv1d.cpython-310.pyc b/vllm/model_executor/layers/mamba/ops/__pycache__/causal_conv1d.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8096759a1daf09e389e16f121dca1485fd45a16 GIT binary patch literal 3037 zcma)8OK;mo5avr1B|qZCNg6i=x(HA-LKG=(ff_KHAVASWF7Baefhq_Din}t|kX&Y$ zl&pfjwEZFRF^B$z-g@n=7hZevKSY_&YfyH7_|gAm zL(_i6&iQf$IuGHIKSM<|s)t&n4|E;dMraHSc$=XaS%ZeIwT$X#(W|tuIIC15Yxb8vbgNK5;{Q|xFN1eS+VBvcv7}y(RRHwn7DYT=OBoA zX~BC9o=@PB1E}&Q)hD`eQn2=S`nP(nz0j$7qEqXcDYl_!oM^{dZqP=ii<`ND{aB}s zV3Wmn8_Wo+aW=me)vI!=&=ttvM39|#1mEl1SIic>Py1M zB!wk4@SxTPC+5b;B&Yzu;ab@d9>J?EJ(38PFqoiZG$Z$V)!C>YJ948WWcy?VGr14q zgy;7jvsiL*`^mlTF8n|0?(RM9bV()Ss3(0lVfU3$cK1sCmt1V2eTS^5u3QjoFO`NK z7lO%zgI~snM0sMV&AG$E;2D^L@8iwQJrr=(9rNT_x=V7H& z4Q%Q&FtQ-h=UE5_27^CiB4!~;U4VjpF9Y0hI!EkE0Ihi{kEJ^C z9DKmj>-D1yHW zYbd8W>^yjBC<$T{I-N$beZbR4=yVa-jnPkq;9_9QRDcde8_baghZ)agVFIrWE_-e! zft%1wyL337(oRb>K$5VrYGQ@NB#@B=rU-$&B-Wt<30eA56K_HL?5lnnCH*8w?gAGc zvh?m=l=;zaH0_7MsGrPIpU3@47)JewQx*={5%V&Lfqv-Dn2`O*jYe)CAoZ6J^^#e! zwKOsu$A~~)hsq(2zNtI>xN)b?(IrRxasG7FX)21yg&d>7t;T zsBg`_)(ZPt)SPI?O=RH{4LI09E)_e-!?batA8Y4W*UX#1x%Nr(xCLwOg7UW#rEdSRDye zMKOFz>Zz&k(U`o=6p@3U9!>JD66I$29(QEaMNSVf^mc#+lPJezTqSE2CeoZJ(WAX;&*Z&;ADWpkEvS literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/mamba/ops/__pycache__/mamba_ssm.cpython-310.pyc b/vllm/model_executor/layers/mamba/ops/__pycache__/mamba_ssm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9df7c51bc0b63da80b2d94073b52820c916e8b55 GIT binary patch literal 8948 zcmcgyYit|GcHSK>mn({*9;W5Uah%P|HWOR2?8J_o#MgS*v7J~>V_jTydkcc*t|VF{ zmv?q)%jys*8W(X;q)l3&Kz|gl`Y*uk-~K9YTcAK6>HCqi+jno!RDIm0O`Eo9Z`|*k zC0CSe=O+14QZswz%$YOi%*^ij_E7%7Kvu(N^5?&`{JTdq?c3Do{fwe<3ME+3HOV%Bg(biJ)Z7~D?6zyiZN|bxDU8{Z={`3S6tGH4{gwT7xgH;&}!Dac~F_Fo(!sf z17v@x(h5SqUh_qY*+CrH{2_8okYPWQSeOl2&?TWaS;@VK!y6%cy zs6nOLn$j?HYW@QZz~{+R8>FY0jzn0C($|(f=|x7p94<$xn!gmKyw&F0dL-QNZB6DO z`N>x&!g_O}S#2H)%W}yJ4;`mW6LsJ9Y9(*gtF%I2 zPSncR;oC<)$kNfKM@13WEB0^Lk_j{$kjywSr+x5*VoV2}74dAnxt;qv*ZU>s##e z7JtROOhdMqmKj_mEt~8^6{X?iWK;GNBS#ha7*~Tt(fqqm*diMiMg3=|FO}vNN;BuC zFWq68fGF+9N2H7PVhndnOUH_f`KaFwOP8zVpwtW{Pl1Isu!h05Nh9mo8WzbdpWI8T zNM}xyWJQ$b$?rL>+yNn)Dm|A>B6otfZ3b5MAo1@@v^t+d>)Q9$YA0HEk*${&rWa0^ zW~LWr&z8>3&z+k+eeXp}tlt%c<&b{9xnN*SQ{Fm0$bGIP;$9@cPtKYN9}!lyY@7v5se1d#clVa$U4fz^5t1 zAnivHk?1L_Mlk*aafrv99i$86o`J7ZYLr^-XW`?N8e{r7`1XtNZA#tG?UQO!{;JBr zzsK0W41C=>#{+lFv9~kFSFjc*)I+T26#PG-9%XtOI==&*Q)(Z#XRrnzV-2QMk=wKI zeM%iX~hG4yw-w$G^XH}&9$SmL?6 zN<6O)zp2Z!>Uowqc{dsC`Sa=!q%MZ0dj3_t9m{^@i&&XhbQU|9QJAlH=L)z1+vghN ze2T@oo=C^m{Ulk3^PSr4No@L=O`9NLY)Z6~nPTni-zn@0)^6J>o#(#kyY{`HCgJ(J zEHQJp)oH(|jzHoAmYBV(#0B1kPt^G>XLH*MG*arkdcL5w=hbO-4ySBho!PoZIJ>mh z7S6}%&$;%QH&Qx1OZqV4h2GfbvXgB}e@T|>rK%elHNWgSW?7aSOCG?9?_?Ii3XdM)n9LD7N&_DOucU%j-N^*i zrMh2ri!V8Sar{7-b<(R8vK$kkm!fU~vpMN$=A~!i%PjZoMvvS>-5I|X3sdxs(j0{% zv$5=z-5^T2)w;AvtPpcbjV0-~nn6jH!>aF$1c(=|SL%si&hQo<>2Zd(u%Oe~v6aK) zoqUJ68plFsAZfFfT+$>BgPj(k42qn|XfRm3I~wiY8A+Z{3Cnct?=nfSJ8n7_>CQUM zTDr5MwbkaCZC1>bc%c$AXSNTR-8Nu$+kn~bfUZ4rTkV-+d%8=uwr$DQwhFJYaCaMM z$Ky5YxA?>Dgstcx>f80ph^g`1A_nMj|F^tni1LwhDlL4F%tj2amU#v=0G_q;H-*ZNYOVq4Z=Tm&^Hm)h z(0fx;`Ob)6;1OwP1s`(X;&$xoJIai;FH+At+!Ooxp2{S4yvO|5!}pcdGva;j$+GVs zsw~aBs6*3-#1B(=eMF+6J@}gTn*N$tOt~4?x?#14;p$@>0)L+x8bc6P+FQEPQ3Q&Gl0q?1 z(kLcM24xT>`<8wavNs`jljH}8R{7mDUW)M8$IC0ag%l zM8&;C#l5_1SifJvX^dBzu8>+3SS91)qfnx zw66h#8k(WK3V_z=o7CPE!O?5r)A##PR6jVUpfO7t+kWg#mu7Z%Y+}pm#zJYj@2#W?=`NN$h+Ye954;}J1qN5lV*Bm zxQjP23QIgqESvpu!{$2n#jf`lw(0X)kLGIq^Q;l$tj|{bvKO{wV{ugo|TTpt%U}f8q4P^xG+U5KK8v)gh`lv&Km^N@d+f;WWvpwsxQg zT`)k3;l=o~HNR4>1t&qo`LJS&UD~WRC_h&e8#XQ*h|ZdPh8gk%(a#e16f-tB<;RR4 zW(Ie11@nn7PomnmjnBrXsG`0`t6sWRmdyaKC*(!c*J=dcXnEe62Pm~%ZsLO14*>0~ zS8s;bOSS5X7nv~%kBmyoE#tkzyNc{pltvcITV;HK7nN8^ai^T1c@I-Xh)hmWg;cyI zk5cs{Rg@iK2uK*+p%h{_QA)22J!mpclR16M9Vj0}dMSQW$#nKu9)qsPSi%d6)3=6K z6naw$cnjqWO=E>sue3}km-lLl=BEvdM(@5szl6Tv1P)nljNskfN&)^EHHXB9*0A^&Yeam<+9CegDu{owc8Y(r zM#Vo^yTpcdpZI%gOnhMN7JrxBV==VEeD7Nih;Lha#ot;FiodaJ@z>Tv;;*cS#a~*F zh`+EN6@P9$CjQLchjih7@u%hi@h4_cyk|Ww-nE_(e{3BTew852v;WDQ9_pB z3ZTLiP~l!ef@qtBC=;n7VM=frQ6}6c2~&cb2F{_evp_YQW2i;FNQB)Iz`sBy@@18U zq=u`3)Eq@8qtly4Z<%{_T&dUO-gGjy*%{l!tnt`1DAG)0U+Ii}g?njiIz;?Nq7R4} zsW?h20=*((mR0>K2fQ|bNJ(~U@~0JkT2UE1o-*a@c#3f(Xmy^tcIb!YTB+tWmcr$t zy@+^twJg!>jf@72@;aqsWLQEt zfe_;&`w4rpXjdD^CnamTu7~=y2_8)o|H9Jf3Kvv*@js>-Zt?guTsb`#C5?q#0lE z8i6kl{J`OZllVJ!aPs&M77xZ}lupFzILD>ucr_=F$16}NSC+l*&nR90xBEVAhpaj_ z%fk@6;?-()HL%Ne#h20xnm+7qgqyQf8_iZ|mm97PltLta%8L_UvG42}g`;Yt8dh=J zalG2J;Ii-66894WntqdZpu0kT!-EN!W6hI((BmurE(AHd+S!(5@^Z-gv}tvFp2f25oqc zhAd0O&-EaYju)zPQa|%f-vZyaJdAp8{h-xs`Z6Tzo3%2|DvlRj>5U+nv{nv)8VD=c z*bSSHNb(xM_g&}pV0_c9?j)Zz0j>PzmS2PFc)>h3_Wcwh7lYAx`81IUBrTfq3>ZOUd hwD@&nYV_B%SHxZ*HU4HlMytQuqO0*yejxwR{{b{U4ZQ#W literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/mamba/ops/causal_conv1d.py b/vllm/model_executor/layers/mamba/ops/causal_conv1d.py new file mode 100644 index 00000000..ed7241af --- /dev/null +++ b/vllm/model_executor/layers/mamba/ops/causal_conv1d.py @@ -0,0 +1,89 @@ +# Copyright (c) 2024, Tri Dao. +# Adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/causal_conv1d/causal_conv1d_interface.py + +from typing import Optional + +import torch + +from vllm import _custom_ops as ops + + +def causal_conv1d_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + query_start_loc: Optional[torch.Tensor] = None, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + conv_states: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", +): + """ + x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen + sequences are concatenated from left to right for varlen + weight: (dim, width) + bias: (dim,) + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended by 0. + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + indicates the corresponding state index, + like so: conv_state = conv_states[cache_indices[batch_id]] + has_initial_state: (batch) bool + indicates whether should the kernel take the current state as initial + state for the calculations + conv_states: (...,dim,width - 1) itype + updated inplace if provided + activation: either None or "silu" or "swish" + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + if x.stride(-1) != 1: + x = x.contiguous() + bias = bias.contiguous() if bias is not None else None + + out = ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc, + cache_indices, has_initial_state, activation + in ["silu", "swish"]) + return out + + +def causal_conv1d_update(x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + activation: Optional[str] = None, + cache_seqlens: Optional[torch.Tensor] = None, + conv_state_indices: Optional[torch.Tensor] = None): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the conv_state + starting at the index + @cache_seqlens % state_len. + conv_state_indices: (batch,), dtype int32 + If not None, the conv_state is a larger tensor along the batch dim, + and we are selecting the batch coords specified by conv_state_indices. + Useful for a continuous batching scenario. + + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + activation_val = activation in ["silu", "swish"] + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + out = ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val, + cache_seqlens, conv_state_indices) + if unsqueeze: + out = out.squeeze(-1) + return out diff --git a/vllm/model_executor/layers/mamba/ops/mamba_ssm.py b/vllm/model_executor/layers/mamba/ops/mamba_ssm.py new file mode 100644 index 00000000..08b016c2 --- /dev/null +++ b/vllm/model_executor/layers/mamba/ops/mamba_ssm.py @@ -0,0 +1,395 @@ +# Copyright (c) 2024, Tri Dao, Albert Gu. +# Adapted from https://github.com/state-spaces/mamba/blob/main/mamba_ssm/ops/triton/selective_state_update.py + +from typing import Tuple + +import torch +import triton +import triton.language as tl +from packaging import version + +from vllm import _custom_ops as ops + +TRITON3 = version.parse(triton.__version__) >= version.parse("3.0.0") + +if TRITON3: + + @triton.jit + def softplus(dt): + dt = tl.where(dt <= 20.0, tl.math.log(tl.math.exp(dt) + 1), dt) + return dt +else: + + @triton.jit + def softplus(dt): + dt = tl.where(dt <= 20.0, tl.math.log1p(tl.exp(dt)), dt) + return dt + + +@triton.heuristics( + {"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None}) +@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None}) +@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None}) +@triton.heuristics({ + "HAS_STATE_BATCH_INDICES": + lambda args: args["state_batch_indices_ptr"] is not None +}) +@triton.heuristics( + {"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])}) +@triton.jit +def _selective_scan_update_kernel( + # Pointers to matrices + state_ptr, + x_ptr, + dt_ptr, + dt_bias_ptr, + A_ptr, + B_ptr, + C_ptr, + D_ptr, + z_ptr, + out_ptr, + state_batch_indices_ptr, + # Matrix dimensions + batch, + nheads, + dim, + dstate, + nheads_ngroups_ratio, + # Strides + stride_state_batch, + stride_state_head, + stride_state_dim, + stride_state_dstate, + stride_x_batch, + stride_x_head, + stride_x_dim, + stride_dt_batch, + stride_dt_head, + stride_dt_dim, + stride_dt_bias_head, + stride_dt_bias_dim, + stride_A_head, + stride_A_dim, + stride_A_dstate, + stride_B_batch, + stride_B_group, + stride_B_dstate, + stride_C_batch, + stride_C_group, + stride_C_dstate, + stride_D_head, + stride_D_dim, + stride_z_batch, + stride_z_head, + stride_z_dim, + stride_out_batch, + stride_out_head, + stride_out_dim, + # Meta-parameters + DT_SOFTPLUS: tl.constexpr, + TIE_HDIM: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + HAS_DT_BIAS: tl.constexpr, + HAS_D: tl.constexpr, + HAS_Z: tl.constexpr, + HAS_STATE_BATCH_INDICES: tl.constexpr, + BLOCK_SIZE_DSTATE: tl.constexpr, +): + pid_m = tl.program_id(axis=0) + pid_b = tl.program_id(axis=1) + pid_h = tl.program_id(axis=2) + + # If HAS_STATE_BATCH_INDICES is true, then the ssm state's batch coordinate + # is taken from the state_batch_indices_ptr Otherwise, the state coordinate + # is the same as the batch id. + if HAS_STATE_BATCH_INDICES: + state_batch_indices_ptr += pid_b + state_batch_idx = tl.load(state_batch_indices_ptr) + state_ptr += (state_batch_idx * stride_state_batch + + pid_h * stride_state_head) + else: + state_ptr += pid_b * stride_state_batch + pid_h * stride_state_head + + x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head + dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head + if HAS_DT_BIAS: + dt_bias_ptr += pid_h * stride_dt_bias_head + A_ptr += pid_h * stride_A_head + B_ptr += pid_b * stride_B_batch + (pid_h // + nheads_ngroups_ratio) * stride_B_group + C_ptr += pid_b * stride_C_batch + (pid_h // + nheads_ngroups_ratio) * stride_C_group + if HAS_Z: + z_ptr += pid_b * stride_z_batch + pid_h * stride_z_head + out_ptr += pid_b * stride_out_batch + pid_h * stride_out_head + + offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_n = tl.arange(0, BLOCK_SIZE_DSTATE) + state_ptrs = state_ptr + (offs_m[:, None] * stride_state_dim + + offs_n[None, :] * stride_state_dstate) + x_ptrs = x_ptr + offs_m * stride_x_dim + dt_ptrs = dt_ptr + offs_m * stride_dt_dim + if HAS_DT_BIAS: + dt_bias_ptrs = dt_bias_ptr + offs_m * stride_dt_bias_dim + if HAS_D: + D_ptr += pid_h * stride_D_head + A_ptrs = A_ptr + (offs_m[:, None] * stride_A_dim + + offs_n[None, :] * stride_A_dstate) + B_ptrs = B_ptr + offs_n * stride_B_dstate + C_ptrs = C_ptr + offs_n * stride_C_dstate + if HAS_D: + D_ptrs = D_ptr + offs_m * stride_D_dim + if HAS_Z: + z_ptrs = z_ptr + offs_m * stride_z_dim + out_ptrs = out_ptr + offs_m * stride_out_dim + + state = tl.load(state_ptrs, + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), + other=0.0) + x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32) + if not TIE_HDIM: + dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32) + if HAS_DT_BIAS: + dt += tl.load(dt_bias_ptrs, mask=offs_m < dim, + other=0.0).to(tl.float32) + if DT_SOFTPLUS: + dt = softplus(dt) + A = tl.load(A_ptrs, + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), + other=0.0).to(tl.float32) + dA = tl.exp(A * dt[:, None]) + else: + dt = tl.load(dt_ptr).to(tl.float32) + if HAS_DT_BIAS: + dt += tl.load(dt_bias_ptr).to(tl.float32) + if DT_SOFTPLUS: + dt = softplus(dt) + A = tl.load(A_ptr).to(tl.float32) + dA = tl.exp(A * dt) # scalar, not a matrix + + B = tl.load(B_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32) + C = tl.load(C_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32) + if HAS_D: + D = tl.load(D_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32) + if HAS_Z: + z = tl.load(z_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32) + + dB = B[None, :] * dt[:, None] if not TIE_HDIM else B * dt + state = state * dA + dB * x[:, None] + tl.store(state_ptrs, + state, + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate)) + out = tl.sum(state * C[None, :], axis=1) + if HAS_D: + out += x * D + if HAS_Z: + out *= z * tl.sigmoid(z) + tl.store(out_ptrs, out, mask=offs_m < dim) + + +def selective_state_update(state, + x, + dt, + A, + B, + C, + D=None, + z=None, + dt_bias=None, + dt_softplus=False, + state_batch_indices=None): + """ + Argument: + state: (batch, dim, dstate) or (batch, nheads, dim, dstate) + x: (batch, dim) or (batch, nheads, dim) + dt: (batch, dim) or (batch, nheads, dim) + A: (dim, dstate) or (nheads, dim, dstate) + B: (batch, dstate) or (batch, ngroups, dstate) + C: (batch, dstate) or (batch, ngroups, dstate) + D: (dim,) or (nheads, dim) + z: (batch, dim) or (batch, nheads, dim) + dt_bias: (dim,) or (nheads, dim) + Return: + out: (batch, dim) or (batch, nheads, dim) + """ + has_heads = state.dim() > 3 + if state.dim() == 3: + state = state.unsqueeze(1) + if x.dim() == 2: + x = x.unsqueeze(1) + if dt.dim() == 2: + dt = dt.unsqueeze(1) + if A.dim() == 2: + A = A.unsqueeze(0) + if B.dim() == 2: + B = B.unsqueeze(1) + if C.dim() == 2: + C = C.unsqueeze(1) + if D is not None and D.dim() == 1: + D = D.unsqueeze(0) + if z is not None and z.dim() == 2: + z = z.unsqueeze(1) + if dt_bias is not None and dt_bias.dim() == 1: + dt_bias = dt_bias.unsqueeze(0) + + _, nheads, dim, dstate = state.shape + batch = x.shape[0] + + assert x.shape == (batch, nheads, dim) + assert dt.shape == x.shape + assert A.shape == (nheads, dim, dstate) + ngroups = B.shape[1] + assert nheads % ngroups == 0, "nheads must be divisible by ngroups" + assert B.shape == (batch, ngroups, dstate) + assert C.shape == B.shape + if D is not None: + assert D.shape == (nheads, dim) + if z is not None: + assert z.shape == x.shape + if dt_bias is not None: + assert dt_bias.shape == (nheads, dim) + if state_batch_indices is not None: + assert state_batch_indices.shape == (batch, ) + out = torch.empty_like(x) + grid = lambda META: (triton.cdiv(dim, META['BLOCK_SIZE_M']), batch, nheads) + z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else + (0, 0, 0)) + # We don't want autotune since it will overwrite the state + # We instead tune by hand. + BLOCK_SIZE_M, num_warps = ((32, 4) if dstate <= 16 else + ((16, 4) if dstate <= 32 else + ((8, 4) if dstate <= 64 else + ((4, 4) if dstate <= 128 else ((4, 8)))))) + tie_hdim = A.stride(-1) == 0 and A.stride(-2) == 0 and dt.stride( + -1) == 0 and dt_bias.stride(-1) == 0 + with torch.cuda.device(x.device.index): + _selective_scan_update_kernel[grid]( + state, + x, + dt, + dt_bias, + A, + B, + C, + D, + z, + out, + state_batch_indices, + batch, + nheads, + dim, + dstate, + nheads // ngroups, + state.stride(0), + state.stride(1), + state.stride(2), + state.stride(3), + x.stride(0), + x.stride(1), + x.stride(2), + dt.stride(0), + dt.stride(1), + dt.stride(2), + *(dt_bias.stride(0), + dt_bias.stride(1)) if dt_bias is not None else 0, + A.stride(0), + A.stride(1), + A.stride(2), + B.stride(0), + B.stride(1), + B.stride(2), + C.stride(0), + C.stride(1), + C.stride(2), + *(D.stride(0), D.stride(1)) if D is not None else 0, + z_strides[0], + z_strides[1], + z_strides[2], + out.stride(0), + out.stride(1), + out.stride(2), + dt_softplus, + tie_hdim, + BLOCK_SIZE_M, + num_warps=num_warps, + ) + if not has_heads: + out = out.squeeze(1) + return out + + +def selective_scan_fn( + u, + ssm_states, + delta, + A, + B, + C, + D=None, + z=None, + delta_bias=None, + delta_softplus=False, + query_start_loc=None, + cache_indices=None, + has_initial_state=None) -> Tuple[torch.Tensor, torch.Tensor]: + """ + u: (dim, total_length) for varlen or (batch, dim, seqlen) + delta: (dim, total_length) for varlen or (batch, dim, seqlen) + A: (dim, dstate) + B: (ngroups, dstate, total_length) for varlen or + (batch,ngroups,dstate,seqlen) + C: (ngroups, dstate, total_length) for varlen or + (batch,ngroups,dstate,seqlen) + D: (dim,) + z: (dim, total_length) for varlen or (batch, dim, seqlen) + dt_bias: (dim,) or (dim) + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended with 0. + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + A tensor with each cell is a correspondent + input and output ssm_state index + has_initial_state: (batch) bool + A tensor populated with ones and zeros, + indicate if the ssm_state at the corresponding index should be + used as initial state. Not providing argument assumes + there's no initial state + + returns + output: (dim, total_length) for varlen or (batch, dim, seqlen) + supports inplace replacement + last_state has shape (batch, dim, dstate). + supports inplace replacement if ssm_state was provided + """ + if u.stride(-1) != 1: + u = u.contiguous() + if delta.stride(-1) != 1: + delta = delta.contiguous() + if D is not None: + D = D.contiguous() + if B.stride(-1) != 1: + B = B.contiguous() + if C.stride(-1) != 1: + C = C.contiguous() + if z is not None and z.stride(-1) != 1: + z = z.contiguous() + if B.dim() == 3 and query_start_loc is None: + B = B.unsqueeze(1) + if B.dim() == 2 and query_start_loc is not None: + B = B.unsqueeze(0) + if C.dim() == 3 and query_start_loc is None: + C = C.unsqueeze(1) + if C.dim() == 2 and query_start_loc is not None: + C = C.unsqueeze(0) + + ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias, delta_softplus, + query_start_loc, cache_indices, has_initial_state, + ssm_states) + + if z is None: + return delta # output written inplace to delta + else: + return z # output written inplace to z diff --git a/vllm/model_executor/layers/pooler.py b/vllm/model_executor/layers/pooler.py new file mode 100644 index 00000000..76ccb3df --- /dev/null +++ b/vllm/model_executor/layers/pooler.py @@ -0,0 +1,63 @@ +from enum import IntEnum + +import torch +import torch.nn as nn + +from vllm.model_executor.pooling_metadata import (PoolingMetadata, + PoolingTensors) +from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput + + +class PoolingType(IntEnum): + """Enumeration for different types of pooling methods.""" + LAST = 0 + ALL = 1 + + +class Pooler(nn.Module): + """A layer that pools specific information from hidden states. + + This layer does the following: + 1. Extracts specific tokens or aggregates data based on pooling method. + 2. Normalizes output if specified. + 3. Returns structured results as `PoolerOutput`. + + Attributes: + pooling_type: The type of pooling to use (LAST, AVERAGE, MAX). + normalize: Whether to normalize the pooled data. + """ + + def __init__(self, pooling_type: PoolingType, normalize: bool): + super().__init__() + self.pooling_type = pooling_type + self.normalize = normalize + + def forward( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> PoolerOutput: + """Pools specific information from hidden states based on metadata.""" + prompt_lens = PoolingTensors.from_pooling_metadata( + pooling_metadata, hidden_states.device).prompt_lens + + if self.pooling_type == PoolingType.LAST: + last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1 + pooled_data = hidden_states[last_token_flat_indices] + elif self.pooling_type == PoolingType.ALL: + offset = 0 + pooled_data = [] + for prompt_len in prompt_lens: + pooled_data.append(hidden_states[offset:offset + prompt_len]) + offset += prompt_len + else: + raise ValueError(f"Invalid pooling type: {self.pooling_type}") + + if self.normalize: + pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1) + + pooled_outputs = [ + EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data + ] + + return PoolerOutput(outputs=pooled_outputs) diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py new file mode 100644 index 00000000..fa51d427 --- /dev/null +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -0,0 +1,69 @@ +from typing import Dict, Type + +from vllm.model_executor.layers.quantization.aqlm import AQLMConfig +from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.quantization.bitsandbytes import ( + BitsAndBytesConfig) +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 + CompressedTensorsConfig) +from vllm.model_executor.layers.quantization.deepspeedfp import ( + DeepSpeedFPConfig) +from vllm.model_executor.layers.quantization.experts_int8 import ( + ExpertsInt8Config) +from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.gguf import GGUFConfig +from vllm.model_executor.layers.quantization.gptq import GPTQConfig +from vllm.model_executor.layers.quantization.gptq_marlin import ( + GPTQMarlinConfig) +from vllm.model_executor.layers.quantization.gptq_marlin_24 import ( + GPTQMarlin24Config) +from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig +from vllm.model_executor.layers.quantization.marlin import MarlinConfig +from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config +from vllm.model_executor.layers.quantization.neuron_quant import ( + NeuronQuantConfig) +from vllm.model_executor.layers.quantization.qqq import QQQConfig +from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig +from vllm.model_executor.layers.quantization.w8a16 import W8a16Config + +QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = { + "aqlm": AQLMConfig, + "awq": AWQConfig, + "deepspeedfp": DeepSpeedFPConfig, + "tpu_int8": Int8TpuConfig, + "fp8": Fp8Config, + "fbgemm_fp8": FBGEMMFp8Config, + "modelopt": ModelOptFp8Config, + # The order of gptq methods is important for config.py iteration over + # override_quantization_method(..) + "marlin": MarlinConfig, + "gguf": GGUFConfig, + "gptq_marlin_24": GPTQMarlin24Config, + # "gptq_marlin": GPTQMarlinConfig, + "awq_marlin": AWQMarlinConfig, + "gptq": GPTQConfig, + "compressed-tensors": CompressedTensorsConfig, + "bitsandbytes": BitsAndBytesConfig, + "qqq": QQQConfig, + "experts_int8": ExpertsInt8Config, + "neuron_quant": NeuronQuantConfig, + "ipex": IPEXConfig, + "w8a16": W8a16Config, +} + + +def get_quantization_config(quantization: str) -> Type[QuantizationConfig]: + if quantization not in QUANTIZATION_METHODS: + raise ValueError(f"Invalid quantization method: {quantization}") + return QUANTIZATION_METHODS[quantization] + + +__all__ = [ + "QuantizationConfig", + "get_quantization_config", + "QUANTIZATION_METHODS", +] diff --git a/vllm/model_executor/layers/quantization/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2312d92c096a557e7e9b20f41d000c1db6f8ffe GIT binary patch literal 2808 zcmb7GOK%%D5ZLj*nAaY_vX$oEzYcH)VK(1EY zl^;P)ZT~@zZ7&6S=%M$X`a^W>sYTNl+5%0VLrRgh>1Dmb&VDoGM{+pBp&G-(Ne!O0 zFTQ$wH>GJ`OC|gafburH{+GI@J=AQ?)|tk2qH{!$E_K9W922Z?j1BNOi7SlRAWx8l z!UJrGCrMJ_I7@MZ7zz)vVV)*wg%d2pvm~qV5X>BfLNg3a8j8A0uN58*H3U zkO_r{*(9GLQwpcq1wKut70$33K1*g5&a#WVNQw&Q*d;zk<`mAe%Y2^9D?Gxk@CCA< zaDiRri)2yZQC8whWJ%#Mc8xESWnFVLdz|8Dx;;VTASY=S3d4Z0BoTdekGjty0 zES&>+kxqat(piv~XaeLModS88ra{hA1LPID2y%huL0+W`&vaU(%l6{O=rf(HP@R@3 zqRC0^2u*6VVwY}e_7W}I*Sc10FWW00VzOG-?DD5NS+gq-w9@J~GNvWe#&?{g|Smzjtc zN^)oC;dZZga_4@nzy7S$UUi^vTi@vOcJFV!*VBwwIJs)WrtF>)Z@a1kCc_a~dEzmFOov5Vosi<}0J8}=K;5)zXzOgs6m#aeVrh=wTXVhT*3YhngIFCJ9_?o~X; zTL~;tr@_kWy!Du`@skR34l3RmjN?|08RHeTza~AUhpnI?D$F{g!mos#=#L&e5ANs~-oE(9?QC7Z(ARVqy?J3abV0mDbzw;LLN}xnQ(s9p%nC%HegCW%dXxDc2%XPMSiNac5}S^f0GzGmm{H*DdZ&?-I4H75^$@YW83>jUKo-XIl@pR#IR&UP9s@A zMEZObC@3N$DUs75RG&mvGOtICX^07xP2?mwa{9`8)j6LcFNx)-vvx|U7?Gsa$fRy@ z76nO*obcfyD@G+Xa=4*ni?MU9@#Wnj#wBefs%xs`iwWR#YHiF9L_1@emg_bG^;f_* zO;y_3gJ5M0G+BB>LYDe=;wDtK7Nc*;Tflz`BCi?9(A5tC>ijyouC$kJ@I?sg=pT&G z-#CW;!WjLD2hbllj$YtF^gB+X-*5{3iVgG&PNSc32K|Jy=trDGKj1w29*>~sxPZPx Nqe!3rc5KRs{|geEc3}Vj literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/aqlm.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/aqlm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97c6107c026605d390bb7393907871da52025489 GIT binary patch literal 8995 zcmcgxOKcoRdhV`%PS1lBN%5hV#}Dj!7BU&Fz4A(HWi7AmWD}cSuN-gEws+E;>KP8z z^Kf;KBofU%m|h?k1G&crNF)~@l1olGBtQ;1B*!3!ps}|F0`MFJSj(1}@Be#dI5U!U z$RRVR>Z-r~djIe0mCK%n-<7}pr|qBrK+}FpmC0WPl~SqSZ&)ut&uS^U)W*FyX7+>s_rfSw<>pKsc-EGwUy2fBZu zpv$M_9NI6+dCt#-7Jb(=9_wP$l?!t5Q_H_3m$;p9&yR7b-`SZSCAlnDF#A*A+G8~| z!xhXh+p{9q=L{vUdtmlx3U=|9lqMS1B{Od#93mR5V_ znC@@0W@g6C?rvtj-fkqB`BpnlGWVT+((ZMGsHSIyp9L!Dgh{ASDb^c{UZ1*HVfWGkVAm}FTy?{phNw3>%w*Z}w!=!#UY`3p58=#=^kIE*}Pmtn2M3#cKZtKcPv>ogMHGx_MHNiClwX>)hTr*L- zfSP$rODt)n+FxnX{=zuWKhbN>=COQuZpcy_L;gRVKUruo~P6+EK6-h1F(H zRre|{V3E~EPlj8)-tD+%XI9cvjcwm-catCb7V;}s`SarUGpb#EDf8YBqCxnEQazlW zDq;LDHFXxB`?ofdPJg4{?!S-(sud^js7l9vAc06iaHw|%ox?f zdtqaM`8J|pH&pS)#}g;J5qunV*896z83WZZbDcM^fO%rdT;0|u)4Z-$0M1MqBtb)< zIQmi9K;qM0LLt=-wM5?$N4nHgeNkK1KsjRbm=J45Ca@x!K`l~Us8uS{HhGhIwv1{v zH|X|*#_fD@nDr8XF%7aL)OkX%=wRclFh1YRjvGM~#tmA}=tok(Oqcky@#}q@_0c8`AvD+%G2P z4(Q|1mWH&BL|Rr^KRfT=QGufE=HHbWJH<%n$8d4%hI;@=$OdNm2VF^IX>tb$C!R;V5`&gE*n z9f1D1J1yda@5y#2zFh4N!1#lt-cr3me^_sKr>f)Y`)GD@W}3+b+zA_?9_$9|34_M= zqbs>Qu>_iXJox1?8`k6YUU<1WUe&NZS`}BPmXZ_I_2F4o@3y0;x)oMCgD7eDqjo5( zTf5a{yNz`Pw7$z|iW5Q^AIFyJ1+o#u35kZg?PUA3X>?NK^*nk?R(vtci0tesFSfvZ+X}LKfRHywh%k#}5HKCfG3wCz%ls zI%qXX_S8z5#WOLd`!>j_ON0aDOn=iahn=lZ%67L^-wIlp(~Ka~!)|8YX@_?+Ga`Lr z;COvEz8my|dtvN5D&zrc6?FllsEd@5MDQ!qB<7c?KHf{l)iD+L$9tB2WcSeRoRt<;4 zJd2&iKSZ`PXA1Ac6|QdT9+LggG%EUnE_C~W>C6KrPRtY6Faa@7Oo8vo1thhB&ZJ`c z18D56b`Uo#Kt?|s1*)@(dK7kB$+r3~0qltNKQMm})tSpwR{--DPPZ?I`=33b zxl@#SE9`Xs82w^WBEnt9&aw9g6-&YB)UsvVAvRryOiS8iRPfct4chYPH-|zIiMMZp zrddv7%Os~$1Ns8rroN)>+oTI}9RM6t8j$Tyf@3()p*KX1nY!mldkp)LZlV6SMZEq; z+9!Hq6ZWVT&}#&hV&6k5rsnI~N3ZUe5(metAG%VcURpXbQo-e7T0q&DfLl@J>p3JVjs}9b_e&T~Nm4}^f%%%} zh#<&jX=|kwc0<)})MseyL(CczZ=5SE!8WQ`tNJoH&_r426}^N%xbmc=o8MGy;zXA@ zRK0>OQ`AOuO4P^H_ID^G+95L=G1ES7k#F=`A`T7E0P27`sk38(!kj}3H|oH;(2w*Z zK~%;N1GESl1H?NA9 z#*azIR0^)B2phtc{&bnE0wNc}x1y_?KxiOM{m9*Y6)FA=vQYDNsritfz9B^jQGsM4 zSxB~YnjV>kaJ?*Phjvp8T~8-;hxL*yv#VKPh!+-jN)SRqR^%MR%M3RGnU`l6QW;{} zj7RP#xvF!3E`@MX!!u2~YPisFbkE9VZe5To7h5%C&1RZBP%!SHxo30=swheyGMK9J9T80j;WxY;#J1k@6Bq%u}Q& zp}e%Z26d&?$x%+2>fYMv*w0VdHaGUp!a64(dg_pG?9Jy!`^dbB zZ|t3!HI{GO*K0NNaXbG~S8vdnk-wT}?8@N)RM=Pb`riW{SCEnI!uAR57x%%;5#+d! z90A#ybOirR&OOH?#HMeOoZ+dy)H9ezAY3pvWpLOMGW^Mgxz#Wi516K7`J8CDp|4bb=k?mcaS->>j zw@%HM(IUoq=ZSMK(cExN5YMSM+prLDBk{kCXMbvD_F2r#&LLBw2U4PAtMyG(DD;pb z?9d{$2j1YC;2JoDYX;ZACtNeRMwTq>S_la10AZh+wfQ*OuIWApOURY3-wt;X&QWjC z2HvNPq!!DI+#wT^vvRLPGwkTwZtIG_XA0xb3D@(`Ty+Ho7+W6*_JM3FA# zGdySNKPAAQQ8u%idAeKnSn7!SA(r>=K#J*hJa$y}&QEM*7WZ2zOT9-^ol+G=@@8jd zs$yNA^A%Iof70yC==$C)UCrXwRx=1E>Q=93UcC?6uf&zVbff9f@4Um(TA>#ha|&k{mB6oz;pOz{nBUkls2Uzry$LWKc$$e`~M(godf z;T)aVx_7c{^GFR0GW8=__bO6+2idF*16u>3FSc1I*zN3)tx}K!e9H)&^A~+6cwR=Z{ES>cuzXII!GB9kc6%S>R9($amV-V- zM?<^frfS94#{rg{`f`T4UVU#ntP*N<=tgre837=YK6!{MD^2#8k}7mFl$d46q{~f- zQ#R363s8hE-@u2<6BosIL!5@?RZI%@kvQ8b;i)S)Eh#*3Fv%U972mZ zlnA1BW^?cPce>GTm9&2qA;88!Q8X2hvBszHAX_P+e@lJ)NXYt zVn@Nj1Smt&Jq_ZqL%a^!rdfh6#TOgFARdMvd%bEzuL7f3A5lmYg%MA(zBiy*bMN>6 zaauTrBmmze)u|BEY9ow>*l!@!GXU32zmQWlV#S#^bl15|hBmoo?T&gKm}(8#do_U& zTudq^pjQay4cK9~2_d44Q7=$w93HIA=W_UG1f|!BoL^a6KSx#eCrDK(@*46XE>mfR zGLiwRNZB=H{(=fy2u{K&VFu(EuF`~Z+$Kz7S=FYDcvbBn^B2NH*$x&!9{x3`r|0Obrew`F9#6A9SRIO1-SfnOAI2^FMeG3eSY)K^Qjm{aQFS@X&5hEDh}ii~wg?oaDWrWSrqh<$NTJ1H4kfLmY%s zWvG0o#NPDEH#7Yn2YuL~Ymn9){m8>3ImSZx-%y|^+P;Zg=hqs6s00i)n0gw*kxpvm z@eW8X6D4DfUvGvmk6|WYh=fS%woo5Jj+p!{N*mu7Q%EWJgg=6$a2W39?TK}0BdSMn z5|`p6c+)D}*6B4z-AWK$gB(Qc4{;JFHI8h&7~je6!Oj-*ib}6d5G2Q1j%-0_Ek@+8))_m9D1R!D2FLH z097kbkZBiK`MEQnj0fgrAzsIJtM4K{&#yLq0hF2-DRhibw+pdD!IC?(q-h2Hezf}r zHW6=P65Gz*T%tzL@IV)yK5tg_RYMf)uZa3afAhX@MrHoq7hj1plMwTMZV^w>6HmuB zA{hObw5>X2e@R3K0A|caCSH#?K%blc1?v3b6Obnjto1EC;SYU0MTV2V_PCydqzx~fg(_bHo5&<> z{Ytw#j1`d`9{Po!c9Y?i(1WkKOW6ixKcMUt%5GA|w@ml)xC_hNS8}WV_d%9g{1?bv zV^w#>0>x~_Jm*V>UBWww?LCB?Lx>NzCcKB@d8(QDo$7Rh;%nCy_f5wGbw1i=qs?{1 SSElRzPJHLd?epiCe*C|RO{jYS literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/awq.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/awq.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33b34e6a65b323c2827064be7f1a0f5db9ad200a GIT binary patch literal 5548 zcmcgwNsk-H6=tt&HkaXyMx&7|*^HeaW+IW2<#b9nio;VmOyU@r15 zFCxFf^gFqF@hN7ItDC8Dskf7wABHWNn%Bcvrq0cQ4Ewzx+Nf)3(Qgf7+3))OL0s2U z_j=glf%qVZd0NfVH@MvDGb(}l%BRDiC&N)dJzVbh+F=K+);|{g;Xqa0X$29#6^NkA zB^T+DTS4m!&c3{U{l>JgMH8EU9AqxyjsF6{bDqX>p3XGSU^+LMf!E{~vzXm>)WS{A zW){!4b+v5MbH=$MD?Kv30&+0KcIkKy8f8{NS(#T*Y9hbHs*lWvnzzKON-wXpmf4Cb zU54HfzQXhTNLxRi!RlCa8(C^XhHFk{W(fSx$M{AoMXD2Ym`doi6u#+e` z&TMSNskCtE7oTRf8hw)e*IH2!$F+7})KJ%O-5Q?Tyv+sgwfKeFmXw3|LbDl&2jQ2E zzUVZA&A9p6`Lk~{&Ypey&9{_(TD;H0&X)8yLm8)ThxWja!x2y25f^=b&<}euwFQ?$ z(c5T|Yfc_Y(|Z|j{2oN2B{@Hr=#W}sKq9pb_x-{U$0YWFa=(YWer-I|eyuT+@+Wf8 zGg%I6v^JhnccZJ9hXPBG(az~wzZdP)UO$7$)v|fk?#=m5EsSgNa4_f#$=SpZr)!%- zS?ly=ZDiMIRgI@LbX&JX8GoWeVhJL3KM$fI|44|wNQ(ownBNXsvM=E7X3V4Z0sU`s zVhzo{z1@`EL30oe&d5M?xIFV_cevF(*S+72!p-Jj2aeurei=pGX1C9HzUPOXnN0iOU_8Qq9|2qE7uk*^{qqS+5ug*EbyQA{y+RPVJcIFQ6eyi2% z#zFIF;wVN`Hp;{?WNBrB4}|Z(f?Q0v=VCqPzNtIkEv_e1JG35J6Rqd(|mm9Y+CYV%`Fst=7ts>*gHBqS_`SMLA_ztJ}8MrvUfTU(y9 z*^c^wJokoYZUu0Y=S^Xq#+bUM55Mx6+!^rrr;{m_w9(5`+c*B3`k^UK%pp#qBbegp zC-_$e0I(XZC_ZS673#Vh_WV{b2sXnglsk>d986<$a&ZoeiY+KEJW~S^+A@tP-6JG|#NP-wq>A`xl>wPHaQyhOV2>MlUYT z_Mm!dXC*C~=j5@FpjU0|1tgN(9-u0l{S$o`p`r2yGdnu+2oaUHIvR2SDFB+0m=6uO zp1t8&_X#RflaR$L-ScNrGJ}@7=9OtVZGkiK3%(Nrd|t-L;zuMFVQ9`G$__={YG$2+K6KS4l8t$aUqeILj_j3{05$qLc9Chz;K-$JoLhpM^R z??+-C1EfYQg?Ireaf$>%op^~VEQK1PM(LtD?=y3XSE!_feUV{nT=YW}3EEjflNLWF z4ZYV(?Hg(~!lKgCB+ihS?U+iD^9jN|a?kj62nVpKTe_})hv<8zoBF?X%l%iSTCr9Q z-Kc81xvJ^;lbUPeklM##wZCSo9$IL;@KCoG@y5S}ILOVwrx1EzI$&qq-p+43%;p7_ z2mdO7Jvm4h@Vclgwo5GIXQinwxEa`$Vq{*qP+>=>HOpvGVJm!z{?SvJ{=xo^u+>MF zS7j9h^qO+#(J!-Tsb!X&Fo`jxx+FUB1x1#KV9f*Fh1%UMPULHmm1M!&$oN%hzQF=8 z(Js-e0+CL}j6cnSPZI=Jbv=tUO3!VY+AngRJ9w3wy4q;{K;Tblf}luE*Tddmh;s?_ z&L42$4+0@WBH`&;e<-PHTB4Ld2xl$Q;#fzq!xA)uZk(NJEdqwgL;gdUMF&&-HwYqx z+c-?u5V@d&C8;x=2%t0)5W!t1aa6rIu0M%-MG%wxp2f^3+HOJG5^jNh#+oN`$b+Ik zHh|(bv$OUIoaV8;>rS=Rs34gCORJ)E$79S|;ewEjfS#Sl=58s`_KK{KxKDHqn}xKS z6sPH8Qlhk?vZW0{2lYsje<%~y$ZEt_dmyGk6n1)GtoK8?Rg+LrJ^}?4DK=hC%p3uKTMGIwHOH+9F}gtX@p`v&ws4Vc>Zh?~Z#tzbE>msH$j$1xFV!0WIph*hTs4jWNKY7DQuJCHk`1vozcnxGi; zRs`?hf}2H0b^)!I=T191lf^qE-X%fz9`Qbex56Jt5%^;x1X7CFJ3bdOixH=mAU>kj zS4ezJVg|{I3RGHx2a=Dwj6a7;{40p6rEUfLjs+0L&)j!W0EUv_xs5)Y@0;09wY4= z(^6}$)S8ysbEUQlB4gtU5h=m%4MV|WFs*>q^J?S7TNoxS+?sqM$PB5RWNOU`ZYqv) zz_OKuDBrcdro2|R$}9D#qWu|Ta<-8Hz^Qe?G}4wpj{C;rIX~xz4J;yfYNg-iNAo+}F95aV>6u2(YqD<4+&S7((9I7=a! zm&XPYq@@Nv5qT9{-+YFBRiA-)7K?T_klkVAIaJx<-LC4}GXutagtWR`s=eI6km4o@ z6+W|3X+W=lkn^)a-SMEzf8f1z<<_}3{2Q0<-uUdgfAyo!ZeP1|_u6Iu^39D8uid?N za|06_%7X!D1%pAf^A);{YY?_ccXCI2QCrbW)6`dqlY`FM`;JaArr7yH?5Oq`lU^2d z?GjBAXGxqRL9U=E;LNSX+f?=%3B>~mMTNTM-9XB-?<&mC*xYASH}h?kQpA8r`gOc9 z^+QCiV!H<7ecvWJH>Px^;XYe?OD~$wfEBor>)$ybdZtl%mXZ7;sf{};0wN=n1`VdB zq?jb}Q_rLi9m1xX+m!zS1k=k@oPx}}8L-4izP<_tHYRBsHT!8D0RF^z>akuy4;@)?8-h0t2 zm+c&WXaD0rcZ1h+x&LBj_NOEB3a;q1lFNBHPYH6N;wYi&sG;U)q3-CR;TQ_*)WCF1 z+_fMdT8<^l^q>&hPEn?fpcIy!vP_#nC0ufr!m3jZmz`x%_!735d^Rpiehf5clwe$}rcf8JBy%GK8%a31SMqP^O^ znOL811VQ6!;3xVQ+RZr8-)Kj1Vtu(Ew|m`2uw7S@!kZ1z2>sX>$Q0e?Ac}jT+v`Wn zw!7`v4SG8}sMQkd(jfA^%e{-q$~bk|k9T|C=NgfpoXE=jcl`FwZv0lW5%_Nn`u(1W zeGlCB8||*&5VCDGP0w&wBR_VpOHQ{D$0Dk$$;wv-jc(i?HaObP_qwh2&ZNwr@?xON z!tV!-rtkJAFJmVgjebA4={Byv>xK;xw7V`Yj37D6qPRCM9r~i@_IvGa93|_`UB9_E zu8GFOPd2h1!p5HO`e8r5>F&5~??$pVTXDT7_M$$ z8KUW<2exs|7wy)~gCjd}Xhq~kyN$k&si|;%gTN2oxcs``@Gz~LuZiBEFXvV+(nJ7~ zBX2gEd%kz=jW;e&3eiQiIS|6{#xCYKZuLZnvJ&5>A>8<8-;bJib@HC(Tb_=~@C%-aEALq?+bekXZQZfGykBgoUePNdSMtkQ zzU)(GyiELrlao#mrBw4<)Dd_|U9D518nodvHtM7c}vXstLz=y}vd#xEXqNs-1uAxOOZBkbs&@w)- zK$2pnEmzx+TYHCV>PGFMpP0Li$Q|~R|(4r7e5&ecEhvb^)o^H z>Y4sc7{~6JYe5j6345L&xc&{lIl$n~1dW@%h|XjyBMsb{siE5H!*si@w7xE1FggKZ ztEQ6vOKM566zgF{GgV#D75jtX=7Jn;joF*fz{!v40xr4K>=;X(>tMO$5-QmesBw*M(vEbZsht7lwt?wNwT#3|CLr4v67}UnupuF3#Zji}R|8 zmsrZlP02Z=I2SPR!gW7|TvP$E@z(l7UDZF(HS3p$y?x4X;O#jBeET0!kvs+Orfw(p zbn4|(?<Y)q6bgeNmpsO zI<{kOQ(}RwPSoqBQ*+p`RynM+rQ2!;J{K}% zl&i@uN4dMXk;0+gQ}+0z{if4G{9c>@>5mr`%bv1735Xby> zqSNe2%}DGiIC1isB3**}&3k^UF$m(i>XiAOEdkI*Gv-w1vTje*P4O~@C^*=|Y}ul4 z6=h%PTW^C$>RUyvC>6z4RKb^eG#PcIvJnw0DVVv%&_s9c^nKfeVF6QZ;qGV-n<hX-P+Wkz!dhb`*>9CW^Yi7`hK(B zYJ+>$^j|>pnt!7ow43cXxOuu3_2~JVd_Jl;6#oE21_dQVXf+}L_*%Q$@&VMEK77|^ zJDP>Q&0g4tn_(zm62diN@vE4W#7xnBqJy`WiHcL&ZKa^WZMJqC8>nBTd8dfCq*8I} zoFjO|wMNkP+-ZwM3!|Mykvh>sd(rm%F%*#w&qdUM;Ncw%BRU3RYgPC+^`WJyN)>lo zwUm+yORo=4FRJR}I!p}dT@$k}{8syhSVrG-LP-JTMU99k8ZneNR&J?LS*Fxms#vD~ z6U#9a_7?1qEH_YoEH={;ikP?n=6XJ{+Ypr~ZeZp`8AakG31LUFHW~SB$Rljll~~ui zm_{;nEp>fgChOGTN>3=Apj7PC>82C6hnCR_2q#K?sB`KtT^3y9kf3oER!ZYP%lv4g zOUJk~ez|ldKh7~N_+)8bWk};m2PDmF%puWIVhD$t*F}b;ln(0uGP1YP=SaB=Z{Vr- z)Vs>P9Ly~9nlvR$=}76a3?=}kLcOcq%S9(W6GlSy@_U+iTBbFco#@O+&+Z2A;%r)n zRo4M(KgC=uq3cKkc!kmNtUDUab76Ztt^OJds2GVghQTvPX?`Wd7t!uIuBZl5)ohq)NCAK$ zuT2f`5O-TuX}mu;2n!x%C+bfUcX3%=H?dt8mU9p=U2>_$K~|G@U9Z;!Kzj;LcZ>nT z=#QnhPs#U6G?c)l^?HFgk0yy0#X@|7siKT=(s`8DNEj#42_EcZ@G*3sk z2@HTR(Lms6cJdv|vk(LqeA_P|AO-b`xJsxi%Jvmc2ER^e(mw4)hQ3a@pnGeRnq_cR zuvN6gJ9^W2Z&`B7$Z)D2vBtXiDuy=PNljBNU5SO5bIZ1C_`NE2qL*cgxml%sxkkSV zdy@sJbB>B`7TY#kl2b^ruH%v0b8w0a%8T~+N3@JG$dsX>w6OOT$$4K*Eu`#aK?f*V z-z7Wyv6bNy6ZQlNyE*fxXrz$WW6_tSv4tE{-ZP4b$wBTXD;y5@62p@D@UcuX@uc|ZcIs(X#NkrdK}@!DZx zXT2fq*xV5CJ(uZnmd>^ZxY2wXeodA#6)AXAmdcp%~^SrRGejIVsK0l;&fI zY^N{+{sX5S7dsH%4*M_5@?59VS?W}&i2GW+%rFj$tU)PPP@kV`cf>Q$N|&w7kuF=U zqJ@cT4Hs2@+M6NUE9!fqAEpz!caddTO_9EUuy%kQgVe)Lfe&U#Z+iESSH1^Fx}$T za7N%PPXirp(Lmc$nU>l?1mqOt=ww8@!fuX|$ib{KCB<_csl-S()K}!qjc{y$4KNRb zi-3w=Vp|$*@%tc7C5`LcOi`V3mQVH6DP-AUe^K5~_C*gpMq`0@8!c)i?Gb{l|3=>t z;Tr-E2($GNQaz!nj?K4+1Pc6Hiur!YP=T)aN6kzwt+Ta?GCY5{AA0O466FTB02)z~ zaA*8unz4lx`*TRZi$F}do6zAMJF`;w4czGkfF_VC;Ic=Wr`##Zt*&~JHVLjRF;kUy zN|{NPQ|A=29A@s1@C@RMs;SDkPG_8R&J>K#k7u?c=8TBsPh+%Y@eMxVH%Wer#3eaJLg+9f z|1D;}Ng~0h$JFnV{0T^6Uh%sS%@le58H=X|Ri?Ny5>@><64A>bgqJI(imh7p165Oh zY3l0xFrDw=eSV?m)t{>c`={$)RA1EJXA2!IEVR&#%6qb(1F({D!Yl0jUqPm@(${dh zpn4kMq>eKcqXU?La}^8cDi%N?p{3C&%$%-_+u(c!Kr{uG#lljdT7ZErl)U4UUY5aG z0+b{@f}YZold`w&S9}Z5awEHM;=Y7@ku6)R9!^Vc6Ka+qdMCtN7_Ri1bJm=AjTgmQ z?2s^~7W6@)O){NdIkjA^ry&x8n@K|%U5_K^1$0-WN5!TJb}FKzxZCzTzndPAEW<0~ zOwRY(bbYi>(xo;{bUFSlAwg64C0x;MJaA+(`i!A$1Zv+`xd|2PG3FU+sNv{CU&W>r z)eevhGY#M*;r$(TAD-bZPP0b&djL&wag-mKx0o7p!;3tE6DkZ&Yh-1d0A%FM+Lr+B z3Ots%*KOKPs4bOnNbvFUm0}6D2fZCfSH(U58sDp+oMwU?9xK}!eieV9uwTf z6ddAdo#TLm~pXB)$WR z)3LMX$L=Aqub-PqqayVaNde_q(w6O0xX0oY`PIvbIp_mn`huJGj`7-8zx?)_iGf|O zfuFic={ZMyx-@hjWon<~TO<@C>Fwq~-CwZuFG(00O5qc~!_?o9{4I!6nc_hgLvzZr z`E%mCZ1{I1e@{Z(wx}sITF*gnqd!CfW(D@fdf&r6QGwN~(y75FbkzXnlw_h1l*uNhIK-?*ypy3~c_SDU)ZIol>VP z)x3n*I)BFiz*_jt@zgwN~Gj%w>A^8g#@B^gm5K125eTHY%vjHYe*_LWc zkjSc<=$5Ldh>;Y3O>&E5BKL>}m1q&Q9wO|(ujEM#`I&h?ic>8wG|4;BmRi?cQZZ24tI25!H`o~4dI%hU=9^;K+=aG=hU2fWKdO#2+0 zO5Hui)I@>n_6#bLCV~^@tYlzz#-KUNJ%ZQ6^YIAEWMF^3dNog8@R1Ox5>?+CT-IAM?^kB<-d*$15bXewr@j60Tl=i zMhE~$2KGbp*hSIqSl>|hi~FUKE%7@8))ccdgw)`c8FZ%DJ=cLnfdwOWH!@wq=Qu6A zi8uZd2*O#nobnjo#PC&-xQmg9e*j&=R4D;*wE=hBpn9qT3M zHL##3a#39vNM{ zh1#e14zD9&<#061x4^bFox1(O`bG8P_>J_RjAIIYChAZTb(og=Gqa%n)U?%~=*7y9 zB|NWxEHUvN0i-?MWS8U6IT(XPj*WPS^NVj4wz^%3fgGLUA}qF%#j{J@pvYKz`7dp1EL>9UnFXLZdDc%F=t=?$wQws zv}T}`2Gn0=ib6VrNM#gv4rPh;>GUA!Gmzk1^c={EoQ3b1h&ET1Rkf;?)R&Z!QI#c@ NTGgs??%S)&{|izNAZq{s literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/awq_triton.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/awq_triton.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35acfdbe64312e4b70095afa6fc0744a6bbc3880 GIT binary patch literal 6237 zcmai2-E-T<5yu@s5CkbomZ?w6Hf77Un8cDDeYlNYb`657hMRKD-x*&r$C7<%*=I?Zk( zTPhh(p5gbTh{ltv1ik=>a&Lcl>zy0rJJ+3e%byOkKC5QH5dY?Zn0Vg76Z}cnv{2h% zE_3+U zInkAZohQUxQc|2q%E8W)(oQCh!_Jei?G)OkI5l9yDXdL#QWK}Jw%9ul)56%WHZoXo z+E2DH#6d_;i19}{!@fYYTT4Deh(2bHQQ4{mPpI+4BESh41cm`h*@%br$Zq?sHNVm* z*WAdg_?70G7a8?d#f|J*GxYqrSGng!Y2ORhD{U{zt@!AxR)Vk`wlF{|GQw@pZLjV% zye6pIQ7Q;qzE_%evUl%$wYBxI+z$Q74yu*97f3v_$tQyvJSN#DEY`GHri+`n6H`L5@eL$4XM z{KyQ}YpY@4IIb|zKNmo0=())?g_Yt<7p#hG#8Bfsd zjc3RzW;KPWR`#*x&th%9`)o06v=`g8_UW+VuX*9=OO4KY<80&pV!gJq*xnA;Tg}CL z^?GBm(Q>_d+1v7}9qgFJdSx3PTD;q-G{f3XC9HvuycPPju+>~>Z%1R)biFQzSKjpe zrdOxU9fSb6T#gm=0-IwNw^)%E^gOee`D~o$neoihbF9cl0O>z%j5Ges#ABt3tiW>m zq?F8&WRFYHZAS8HlpI&1P*0wX>;7>}JdieT4z!jAS~sHfo!cK?UoPLM4nZ#dy$OW& zwal%5MTjPH>k#6c;S^@Od0-qTjg7NnSPY5bY07wbk88UJ1mDm%jNMU4rf{a&s~lO( zc))iL(y8*ooI*YSpHtFq8uJy9;V7q!VFYV1zacN%Zh9Iy+fB*ucPQ8S0n(|7Q?-23z=7M)*-kFgZ`>@E1 zY_BgX`;LgT7*C$eIMWAm5af@l(G@*Wp-d*vaPSq8;g8}yE0`$aeJsql3|4W>krg?r zGrr{4v_G(4O(RcWX4o|+CX%zOED`5<OdN;Z(mTi?mdy9G8&tk8b?q-DH3h%S_pz?9_Z3lsrI3bQuZmZb z+=I0MG<{9z;uXl8mKvWHuSty;q{a(k0UEyyx5PqkcP$_a_G>n?Z)Fw}nw=461~ilP zM{ET&JBvKJ2+dxXn!PU0O3lteGfSN7&3X=g9YdXzfy6wtiSY%{7m%aRAy(k|hdC9C zh#;)M#*jFP{)31{`+OC8~pFd-yC?Q0^Dy_kxE0V|V{44kkuB|D2e*@SMGW z5w%lZAabd;#oC(ULehFkySUF=mj-z#PQ?HZMHB3P0~Y1QC2?N70UyORM^@DwKAprq zT^dR(O1V@R_*o+52+(y5bc2$HPp4mrA#-=avScg``+4G`(j+pfZtb2wPVET-M+uA& zpj(??0EkkuR`CxIVU)laQFXT!`jfy)Z#t<86&p@!MZK!@@~4S)idfmrh%H6zhJT1i za|BKhI7#3XK%}F}P&LMmlc9Qqs*a|jG}No<9H>;Qokpi#K?+l*IN1crKTN|N0dVX% zw^tgDr7|h79(AP|Q-SrUJ3T6}9yRDuf%VFt%|2S`qbrUn9T7OD^h6M`&A1fmA(e<# zV&Y1~R-A0CY_O9Ub|+R5#BMdU+W6Je{($%uuF;rW~0>eK>GQ(5~ta*(_w+N_#82k|+m$F&TpaQ_+w#AYDr&GOD zZ;Z-RK7!J~VlznP^c28b;B#zVmJKs0Dgg2*AM&zL$gy$5Mw!5$&C3G9LZ2!jM%bJz z28c4ks4&O@8`TLLXL(*gS#cht7MOp5l*gS}i_+bi*e?=^Ha2SNdaJrAOW^X>Kx;cP zeJ||z&1wpa@s9$D!ivTVMp54jZ3EvgrnKFZV8C#9-Zf*Iq@WMALK^oOwrTjtjPwqU ze4zc7#r?R;QP$=DLOcc{o(E?n4_D*&c;3bdC+ERPbhqQdHOYq=Hd*k2!b5gX3zm@` zQs`eMT8J8@rx)oAO25)XOHc&B!O!+E z0Xg*%mn|qm>OR7!2@J0<^)DljF|sOA{Z}AM7q6S|F24mb`!r&oCHAa!xn60kxRopa zrip0AJT{E+!_KLFb#1rey^MWPsRvgUV!0CUPg_dkq9yNS#kYR>QF-~}>&v&6%6C4w zef!qO%kN$*zxVO2Pj1W4I(MQJ&Zg>ml=2$waNGY0CYDZ+wQI@tFA||hKpwQ02z!IT zB7xHc=!~SzNcXeISo3SHvg-27@2qFT_zd|+Vxsk-YnN}KMe)~rZLcJ4zxC_l%C=rr6kHV zGKhZN4WiHa*BUk;1rb)>5rrAe9rHOd@FX9C>T?=vkMH4=(Ik>Hbm0Cv zTKc9QY^XlhKtQ(ToEc#Ozo&2+oL}Ig04;+rh(;my6<^i6L1t61Vr*!%ijOe}P+Tk! zVES$r;b(WlFN?5~JftCx2UDamzx7}UVVHfGme-c7$Rc7gOZ`}sA1LH;_~J06eEhO7 zENFYZQ0(XKpF+z28Ng#*?b`(++#oPZfOe|?7Qif4JfMSg6gNcU8Ry2+o!MSWc}e`y z;UOcaWIV5CC|Um7#6dSX{~7@qQave|2_ik{G)g)DI`#L1?*nQ|T|Xr3CV?daG7M#v z@*XvRK|q>wi!d2|{dA~w&yjss(QR9KMB9vvyE!%^Q&)kF@SG{5?+42qjJkfy{Fj|K zxc?Cb!MQ_mB0)*-mYhGWO;4%2;W!1kz&bGCm)4>fRMtss$SWmS({`iyJy;C;+}OGP(a4 zzX6WyZE*gBeDvH?-r>ytKBjV=KjjAh-Z1R%`2{lH{xhf?K`DWVB>a z*7tww2WJf9pY$;ObMWvkuIy_J!VqR;bm_kpnX%QiV!Lab^lV2?T}T1zt0wR<{KjCu+ru-9^)QV2GNiKQ%;MTHn~+wrI!&*}hH@f~iw3W?D*EwCQ!GH9Ma<(ANZtZ9N#;QE7RgksemwM(JRHdnSd4pq zCP&w4bc4vxGPjp1SA@NuR5HokCnd3Pwba>qQH%TosR9ekS3k0xiR(jL|E7oB_}c84 zqc1m=^mFO%$#AEi>$82A_pi5FZuxuIpHS?DQRHq*H;D%>9CGIP_On}bkq z{?07UY^C|H<3S{2@F~T6N~Nmf6iz0io>~MC-9`Oc&sRHHcUA7gcwRMybF=RZwF0{T zbT{&NurUY+H*=86{O0X=*pF|;dmB->y)ig|wUdqAD2g}YRLIDa`!X2jsoG#(vW-$l zKAC=F8=CZjH0gyq>w|-0IbaQx@6FnzAkv&uAH8zi5bLwlMP12!sFD+Pt>`@HdH?QU z5FN07DpiMxR+XElJRc^5VeaNK$x=0?`}g(-GJs`fUrd|&{;q^2Wa0*X6iI=5xQ2Ao zWXcX|2@HJas8jgIN-rGJc2R%4NB>#BdN>V=qUIByXK|~QP#dkyQ34aaQ06~9@Zk^2 zmGQO|B20EN7c!4s#3e%r;x*cawsH9eNxAikstO$SlCY}u!f`+igCH`Q}o1wShrZTjx{m)QGu@vG-s91d(4(svh^C|&`Nk?juM`` z70IjVd5IrO&nsG*8Cqq9fR~nut(!leSjaL;l1ZvP%3v^%qfOn`>Jap?1 zITq9!F?5jzw$H?Rd`E7psBF;z8F_Bga7@cOYg&zqW~X48?u5IDc`eGi z`vuBExdT45ec;CaV>p+a4Uq~c*m}bxpv1?g^^vl5%IN#a4mp}iV^-} zj$JfDxFB*-v5XWuo!U5^*;siIxyqTP_qM++AW%o7BmV&0OA zywb*8Lo{g)vrBThq^~hGgA!gj#q^7EMb_l$o(0%q2?Hg)Etbl?tCEWS8L4+Ji&Olb zvvOHX?^=-!q5$RTUR|6Pt4ww71gbOgG*dRkS#gdj&x617kmjXFYvLtY<2POauXV8| zE-=N#83`_mOFVl?T!!yn8G{Zs{3ch&(bapipaUUHxP(cNd#HkD>&L^0LaIuLRsD0f zkQ6F}Fmq1n$GT3LI%!QpoQqNu{>EA57dTnhC3|$KOioh_uUB+4Q9X~Hs{{s#ooqN5 zqzdT71#Ap=wP+#-G=yYSI@OP9IKt2xy=c7E!&Xo4xktpWV;EgOfxUUVUdZl0J_wUA z9>(s2k3K`082H;^6y|^vX(a%M@I1XvK-_!lJL@;x+i!4f`82L z)sODSYQKbsj^!_6V(cRDhWj#Syc<$C0Y{tTHH?(XKyv-E9&!;Jq+z1VX6aWzNwJ?( zSxk4OQX%0kjuoIRKu>CP0bf7nSJEW;1l#{T>0Q1a8#Z>9&Yz{e09mmrL=a zUM`cgr`i2IG_BS#j5g*N>v=m1=dnDaVT=ueF&Rt~^)X#N?6?JxE>CVEbLbn5}Nj_$4>A%~3d$MalsbY{~}65y&HS_Rin zV9lO8x>_|1&)bTW()ygH?q zQQ^#{q9J6@A4d5x*Kgt^RvTfLQA6X&L-Skf3#iI5a_eB#gxX9fux|Vob9>Y}-qGqx zD%wHK*?_ECdh z)CWrT!u?~K7$*VU9{az-1c^y_mHi!GKT2dsNf>_58F+eGR9`8o3Nj9%yXb50jGt(k zKLgxxA68!{W)G{N^vo8`xZcwn0?t!Kr~nL~S9u?SV2OGVOcKv0{88Wt6ZG~YvaP#Y zX#&)(Kb&Y>ug?~#%#v&QxV-bm83s$-G@jE1)Sl}4?wn*_KCH|Qe<1tv_m1|7cAchw zAy(Nfg=bq%fvTzvVMS6?jI>;)QDjiGw)xEr-!Y_^YKz}TkDH`>=zysb;Exk7H^@k{ zcfa^}f>BNeYg~o((Tvo%wEvNGgHN2si8c7>j;+pRb%l1lOv4XoaA}}?SlIaPs5$$n`**tgum|)J>Xh>jp~Cs}0&p{imY*s(wO4gNC2dz%l%Dx}zpX zy-EY6CiONA@6a%bWxk`L&GAo;kiCyVcg8EG<($W5nX9I?V43>S;P${HDc4QLuG=4G>KwapdVdX?Hi3tj%%Q~w16P6x~Y literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/bitsandbytes.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/bitsandbytes.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e8362e361f6430f4521eb7edaf210fdfadb030e GIT binary patch literal 8809 zcmb_iOLH98b?(0Xn4X>o1`mP&C{d%A<*_1w6lqGKX<8x_SguI093oLXj?!$*?HTlN zUcB8y5|bWB5$KR!aFNQw8!yO8He6(%RTf#N@&kI6s#H0ps_c~5iA)Ol&h4JT3<$_l zRR-0k@8jI}J>PjWx`l$R;CJ>P{%Pl-;;}u0=kVl_F5oHRnej|7v#juvXYpBX)))93 zpTA>NvwYSw>WnY&6L<8g?d8(^B0qUY-&3o3SyJ$d&6#Bdd`o;;mX~DtEJ{}RDOob- z&7*C>GbGU_k7Dll9p}w$KRn2tr`+OpXWOBaQ|q>S{m^rGxZCsM`EmITx(SXf zTpkzp12A^`+bz$L^>Mk?YGa7-O~>oFR9x?!d)=w^`cAh_)8g^cm}AEcP<{47dg5cZ3)0bXz<&g%|ckrzQn4T_B|yo^?FIMUY616i0~|s2XXY zEYd->$N<$N6V!+@pk`!&W}+;p#g(hdBX#ppRus{Pm?2UkGD~ER$UKn+B6Rn~B1pxK z^}uV@zgDA1ia3R~pZ{bnZ1>iBe($x=6%8+Z?TvPSr+v14YpvyPul05zqMfyy*oHOi ziPv(x+g`07cEwuD-StGUCbvEn(lwl6;CA>n-MZD@ZfrR?51gw*^ABkLX2Db^|7>+| z>dI4hb#>IP2C*LhC`}J-@@E8eT*YN)n}T*%rXjqZaGY-<_Zgj& ze%!DL-?77r(`bw#tf8U2Rdf$R zrK#d!c(jIgRS`=l1oRp>klK#OA(u+v*$c6?EoJfS1##5OmT1JF-|KZn=yAubg^AP! z&?-tmdsJeBSC2TFcmo{xpx%5J3>omC5T5_QpSJJ-G4Ah!R+ZS}oB z5TYw8x>&^ku`VkL+^f6&R_NR$hS-oV@hvbUrz~NiD6g_x_Y$tRP#e%uFg33(;IUM; zKe%wj!O97wH|J8TTXS2%<<;rVBo=Q}RB@KZBp}~PketprfpZ$_SWFiMs$C{RGA?Ib z$qRxJxlSP^SYLICZt?n$fdPQztDL;!=g`9uwIRT6YhE zPTDjccfcf}#KvYoiM%*D z*YyNT4!1cL7$tf%Mw3j&J_@QO`~Ckka2RRmc4!5XeH@2jT5Oni4Ha0fp zC99kw{YZT}PHa1=>4S9Mp{2Ldnv4TutA=8IyW4HWS`Z3h&>RTJVneDL@hlaTsURn< z%w*HVi&RWlAYP_ALz;JjlyR)VUK77ZsXR_6^lNF8*NB_WSDdFx*6GCN`_d>AzfYxP zV@;ga7^|f%Kr$9=gIG}X3>SL(PyaPz>0ja0xW7 z!&o=C;hD`$(}I7MOs6M8o@>B8^E1N1S!GI zi{-5yZ`uKwv?R+Zy2_?;ezN71=A=tS)}KUy_mCLOKx-;3g|uc9`t!OXiXk?9PyIyw zQ0dG9e+?T(b^szedzXDhJrtg-o!6^UIzOI$S zZW&(C)IiBD`vXs`LM6U{>Bl+8N#_dj&HbQFCZE)q;$2jOOWMqoPWUxXhzZ#uZ6PIl z7m0wh5rb;9@#y%v!RfYx#CWY`(EI2|C_wi^q5%On*;5n_`8iArDT2z+ z)w{}Fc2~P=z~`n=hc&eBsVfLQUWKB)H6j^)9(C%namfn&20MUfVe#lfM zG++Z|_K=eI#!l`HVi@)h;EsMEXZVyFpt)VellY5D6RuvUK7j6es5UjE^iXXYlwJWF z<&UmOG@MzT_2JQy^X+#UPOnS2N@!a(>VBu*1#IK)Rt;Zfj6S)^@Rmq!4jpxT0-C1Q zZ@KWp6S)%`GIDBY0x=DB)`_CAfsjj;U}HDp&OSD zx&1+05Z;Zx57*vl2$xH7|CFY&RWagha_6|;j_vfyQ99RiYuBARJf^Owo}ehiq@X7O z4!hxXJh^Y3N+#8MUb`3WLUH!nRh_n9N?+p2)1q-(GB=VYHk&HQ( zgB`c$t%@IGy6Kfs#h)Uxg`=oa;ZKPC8ORYQPNA!cB^+WQ?^&L=Aa6I$O4;||X^KCm z8j^P*Pg|?zCZwxshuzR^Nw5>wz=U!Cn0ODazz38a+d1)3SAZt+_h1Xi%D2=!%gaO8 z_e*+m8JK30%`?6V`eR&#%>U0tpuv-=o7EYl4+24*PI8^!r4^9NQk@fC0}WX)O&H_{OW&l1KO{n%P@VH`hr*o-7Ymna=^n}{$bG7q$*@Q5qm(BpO2uio`fGF{ z166s$Vsqx)thETQhvivOoi?q52gQfVJ8$mY%@bEjCZ4xh0ufPID_flcr83Z-nrz z>pDi1JvLW@U%+@sdVeO$Jb0F?*p+z%Rp!5JCfvTSMKkzrgV{wT{LP(2s?IQfq~B16 zCAmUf{3@AE@T+vZ1sQ1>l5%4`n!(EH)MnC%&3{EmVYHdZ(v@b8 zn{Occ?c|~vDBWCY{IIz2z8kjtt@pyK@a(+UY|HGF9mH^HNF)dsKclPKz}5dKOA3PH z2%5M^%gW!d8vyx7EP-dz&X3gCkdXa|#ccJ=2ujI=^bX%{`%()~Lthpy{~L`$L#`-; z?_HUQI!|5ir$y_)hB@d9)1liQo7}tUBd9>CV?#RBThQc&oJ!?6X;6zM>cn*-vI&FX zK&F{^g~&CKc$U6uU`2w{w+eg=3*s3nN>qgGteQ{u&q-GK5w-0SA-xkDe(1G>M9F-E zO8c@@xSfUL68i^V5AquNG*pYx&#O*tcszv5a4-S7d z+wTN7`kpuNs@nSPE0u)_q&`5F@k--DSNIJdAODiiHu`qmxt**vGg@Kl*2(n*)xzZR z0zvDGvoO0H*j&z;gXGy*Lup*7^##67h4{e0z1y+sO9fFiI^Hb|G>vegfzJK~PtXA= zWq{|v^wDPs8?mkud>IpOSWK$|=z75Fjc-*G|XKf5%0e0#yq5k3sc6kvRZW38(;80IC+k7KUh= z08|sOnvt-IpaNi(HM0^#?a)p^)OwH`gJ=#d^Ax+K9>80WAPRs4=oUkWb_$}e4vPSW zBBU|tZ3eMliu`7&jOYPeJ}5mTAVv9}1|UjcssNy#8N(FP0FJ_q??id1IGTAr1(h&D z3U&D?4;aaeeW+Y|F-Wf+3m&3{4H9i4bjG4b<3Ap#_RHvmlN(W3xBgfs++ zePuQg^Se|)5-x5LnHF)eOJ%o-jHOl#C@T#^5?ApTL?Vz_4J)N11x$|dQ{wweBG-ug z6_JmLd_rW8$Q>d-BSIEUVmuw%jvZ9YV><$hu;nL&;y8g|#tZ7wzO5yf`>R~-i7iZQ zm^twsn$9I66m%BkPWf;t!CoD zeIu8GK}BTNAa=o@gCmvmngWw&DiLhzS(C!>s!9n zw|(2Bb{3cXlKOS}WxqUabNgjxm`1Zs+Zzsf5l%h zjqe)V;pMLkUKY#uEq@i?F0bIbB9`!d4Bu5=!*@-TduwPp&e<)aReu5wjh3BT?b&2sk#!z>ykVZ6~|dG+Hc5uv;mrXp|Z`VEn74>+~7%zX8eQJ7@WIHV~b z43ciNh1O;&vS3FW|>Sov{>;eaa-8Dgcn=u zI%-2CFh@9DrnY1IW#LZ0>wM`e`@ZQ{(9#f9;fPw7ohGpuU7IiSrs}Qp*Z3+w_Lc1~ zVO~Qxrw!0rZf@b0za3pd?>{5)=&t3{iYVx_}}!e*BwZ2-s@ez>3y!nDrDRH zDD%?Ma5#`z>UD>gyfESTz1*&PcvSJe+)CfkRcf0GelyC@nr#a&3+0x`W;E4NdDMH} zyX0ZvDJhz|B?qHnkVa$ij;?!DSL{tW3Zrb0ocFlshNC$1vVnK;tto*gF3$3a!3-m> z6?n+2`&4sBifkm4jgEz}vxn+rIUnFn-$5}kCq`gQ7)mo^JqxeBMPH?f(SudpHRVg0 z!_m?+eqnIySJr*=OQ;aUY(s0GIG`bGD8}0#jHDDv7Vn<-21&f@UAlmst6i&#=5tXr zp%gQT3yAs!M>7?~<|y;F2AMZ@lt|mHvRtCo8z}NpHjtg|+}-TPgD|^z$uG|;wYy;) z@gUnBiZrkOIE+W)Z>1c_mX+J7h`akl9vkvFX#LS&&-%mlVKls;kMqKJ`lIds#s1EE z9Br--cVU#t`rSD0ulEOB#DVxibVgWzJq~w;OxI`UM~1r2p@}JJVt3dc?&fX~L`jqd z0j-p-p>Rr$*)Xe&nas4A^UQv3KOdi&tLRLD+p~!s6O(2S)#v9))Hu#lofMsz`3vJ$ zHcrag7{79e;F|@-xiW6F&KFej%41d#iz$h+l8r7-;FESD#ZU&p>-b7Z52~hZIxpZ| z4?V;?@lU$=pR@^Xf>*ZT*9yF<1s|Ka)rr$X9Kh6qBygAir*J$Iexhy3+=($h0*!RLU%D{91=8b zSs6WfjtrZ;4`DwxH(I7VN6ek3f^V#KlFR_YBL0^TUw=h5n+BL`DC@ekY?9li_ zY|PF(C6RBzi^y-x*F1`aH6odI!dUPi?C8@@-=1+{o8vc*=HlR#LyXMjQ3tUMrh=Dk z;aC+zElaMT-M6`cV>e!eO`>#Zk6P6SQeq@KNlJKll0}^y_U$mfWa&lR0RdP;=fBjRHeMV!l- zAY5s28wjyfDe+^3W_#tH3m{+NE^urG7`B2sY5?y}uhy+|!n!5DzAzqGwztHW3DfqL zFsDhsU6-%$Rg7KcC4_}Fbzh`7y$E@hVU5 zm{oJU538@=EXJZaa@mk%Jo%|88IIu90e6F;kijsNSwvVVKQS0()HQEWgMixhXXLf% z7@D5fF*59@fzr|rIjGq~7DUQO&@|;J?v?J@Fnhq)y-EhC`_P(HCalXInG*+4$>!z* zYpCRMI<<(KR}Ga;pi4oBM~)PHJpO?3fu_iB^c#O3b0292M%G*Ut03Lvrk8?2_p z_=McIq@uZ2p|P>}0%ab>8}wZU|r+9kR` zxXma$D!5ltHpC{SiS@{u z1y5UVg~MUIdk-U2^mJ?&y08F*7q+8D zYwy*RZKAtQ#kZ-rNCib56%5re%kR)90W?WyRtLPdskVV4cYY|6bRcygyG6}M1+sJK zpnas5P*5OiIIL#>mxBOB!OT&?jD5{Y?7wZN`bYLn`ziI==yTC$TaBkGN*oGeB=Yx& z@IRn97{q!dZV9{RAZ0_T8YF2D&?t!6y$S*x<)UoA%9{vbx${yN(yV>*560MK+*W_%^#)bRobN!Cf=q%@?++WIL2H+iseEVsfC?w zk$4F%)gZRHt1>U@Dy}m%c~$wMbUQ?NnS0{})g}Wcy-=$CTN1-@KwBdw4{P@Ta_=Co zkRcGLY=_D17#YRAl4*QqBrCR&vE4UJW6v>-G0cFM?jwkN$v9f>IXmVK!a~o1*Db?S zTRr!TDp9MfAysw;`NNY)h+#~XyARl28MWHOI=p##QtmB1vi4jAXE$g}ER@S+V1O!> z2h`T2o?a7uO#QA*T%2TeLq(af^X>MgjO#~y&XA7w%Xp!SRkQ@ zGXRo{V@P3eoJTldFhw?ZGUeYEAP3<> zay-fy2#}DUhY^@}n5?7DGOK6bvC_u(jV~#^RC-8=3HTX(lqRe|k=X+vDX5~gAwL8K z=#cYyK-rO2EqA!M8+Al()7@AdulB@$hk;5GeyKAU?gsMj(U7|{i`Su;PH2WhZ_|WD zfk|TeU+|`+FJ~bLhr%H)j>w|DcBCkZ5NAV2J(9req53@PQJf$+)q@n2JR0iD%Ise0 z4ojO;iQqOt?S!d&Zm$AqAR@M`@n_1b5u*Q8C1btN)AA?So`uK+CR@sT75$usMX5LF zq6>CAcf#7c5j7>_$1HcCg}@Xt1e>j@)~Zd^WrvElsi13|RFWp?B0S7J>S6hIY3u>{ zUPIfj@uq~XJ=+1`K#s3?m)%#g1uS{!2EM3b{~w`JE>nwDWi!Y3UMN-s2Bef#`kB)4 zGf^UH-Hs9tB}{LtK(sFfKBx62TvQs~72fx*-S944^=^Io{ta&v{{z^Gc(&a-Ablwy zkN=y@*N|owoh-U53Q)TseVtjz%To^J5yz09glvx@^BX!EQhg6z02gZf!34e@pMR0+ zA4No?r==s&lhU6XSU{_j0#gBflD|~eCrL%*HWmB2qiW>Q6v+4~DrpnN6f2q4=grfr z?g>}*wobq;?G9lJIvZipGztb9g%Jn7tMdEpBvJUyKQ{YpcSi|COaDaktNV5N2~B#9 zlJxDxBz;>a>DwBa>iiDn#9BO%G mZYyfzN69O9-__Zjt8{Tx{~t>(28m;VRS(m3@1 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/experts_int8.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/experts_int8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8957b279dde5ea5e5a99e50702fb337f6c902f48 GIT binary patch literal 5901 zcmb7IUvnJA5#OEtb9;Mxr;{v8mN6i}aX7#dvJ)VQLkz?w5G)`zB(;!QRyQMQ<=wku zc4gVB(+hG6Uibv5q^w6m+uZr<)D?E<2&ueJcM1b}b4}`lm&Fe&$ZQkHxx9#u*XIG6@^Dg8yT8^?_>D*G@ zYjK*!OQ}%yFOznm?6;DGlzg1B3W52qPQqz-liRnJ{2@BqU3)A5k_coBV^nR zEq;RA+<|Uv;czD0WyW1zxovNoVMX_NqPphK80aT`Q-4kO2k0S1w`!*L80L zy$vxYT+v)+Gq3?};3UR)Q@@o8$1vLEll(ZOIgTDrl=DsTY4l8S z3uiZ@#xLFM30dTCbc&ZsrEET1{?E44IL~L7yKSa*~WaD zEY0_B;iNnBH_|klFO5aq6z%n*E9cYrmXP`Us66unA)^Em&-GxA5voZMMdMiSD+Xel znZ^MMn*kTxrE~9sWeOc`^-`|#;!dl8BQ|`o>nA}^Q9ku+B?Y`wn zVy(phmBLsvQP>LHPEMesWhn=mXs?Fe(sJ63i-i|Ldo_l7PvY%iVNTF0`FgL{m4)C@ z+%6(s-0F$^1ekMLDPWt=+^?Gl4ZBa+;gQQ3l8r6ieP>UKqvV;OiL;~=wc}pAl%z>< z3%F>=M?p4EA1%j0_6JI!PiqB6r`1#{;wTX&Nrbk>f@nEO1szpByC=k%qlGv;K)Gd4 zn{+sQs6JFk?xN8*whsu@V)7Am%hN>m+@Z9a+Oq6s1HtIMJ{^~BK6<}EBb>_7E}`6> z6wAp?*}%xQ1u}f7PDNYP8B_|RZx-e@lU~6{I^DL=9xrUj3v(yWEp5$Fev&7hyoft( zA!`^d$tYzNL`{we2BaQ_NaXGLS~ZF8F>IlpE0XxGHVKj z{wGmRnZCtLwu>X#o$~aUHrNk5bkG-3@;`&@M?_&Et}$*=MC6ExYwlWwJHq2GN(IG3 zUv=5%RqlhwuhyzTwN5|&$R0l;-&&22Y1+h?aSwdeZg0ngoYkiD>@T}0J7>schqFso{-3Fo2{DLBhYKO*DhihRLA8T2IwE{iP>j!$=a0PJptwWr3%~|c&iIY&PXfj#M+|%HLQxKS=vjb>rNA72Vq1m4{wkYt>4>v?m)Tmq-49Fg^kTj7KhE69z7)akE&CC9)TI-5dv^RS{L`uTIM z#t4eO2nP8Qk)IIxDe+H;n}v*ri|T|m^hvPNHssHU>G+0>dp$th0gcF?(=0C&`2~@S zMD}rlELk<$Qjpb^19Mt8Uj%XOfDu6Bx9hRMa!p`_Y2RbE`(5DpWZnlg)7bD9D%`|% z3|F-+A3ng=EV9kM3;TEaUcb_>!kWDg*;Z{UfTys&xn`p{{aQbOjZ@u40VsZ6h0VH) zl9Gm&sp`vg>lwm9%M0s6Kqzn;xqKaL@=YSYBJu_i5=WBT%PSz)TCNK89tTIKJl#q- zj4SE%(zq?c1}RD#%f?GF(noJ&p)_%A3oTEbrqp}xVCp?rrrvXZ1bVeZe@w z6`D}INC-2Olcxz%WDm^B<>Ce!iWOZoygHM^McVzMxWgmLW_47_H3%!4$3+}E?QZW@ zq`gr2@5kx7xFls)wtRUN^U7;P-Xropk>7%Zm7=G=poD&DMU*;(j;_lP49YE+`GS=x zIR%E&ku=ahK8RpKONE3uJs2!L1A9)f!nIv<3duPVba+_<_tZ$U*|VEx|1aTN4no%AVrxP^N7J7r zLfD~mXr23y$oyk^L1O9zMP8!X5{R;wy4|$Q<_Ue|Z>TfCMXX5L<)uLjK7xsQTY5*7 z=~4oc6Q+7bgEe$24-nk7ng2a|eh&`6cCl02V)tyDea9U3ALg=eS%v-E_1HIVmHo^0 z+1GZ>|0jF0VU{_MTS8YEB+0#o4~a>rMaPg6UKhq4{pn)IT(h>QhV<UV;FA%mBkI(e{1AO# zqT~Tc!+Z=^C6COVNqU1%X)ZpdAa^J)WBQ=<-Hh~59v)rg!lsU=9GRa?fGmP*P|;u^ z-z8SMPfLRN@QH_n-c*0{+`op0=O4m6@SO6K-4ec&4n8eNN@nzDf^g=j;mC#XO^cG! s(qPG$v`OkMk0@x*-jsKtyz#zF=Ahl^l;N=^Ynu46fHke=nUfp;1>|(5O#lD@ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/fbgemm_fp8.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/fbgemm_fp8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..54b49c92cbc879247371dc318f49a51bc8ffd248 GIT binary patch literal 5594 zcmb7INt4^g6($-BL2wxkXV+#)UKv|6k>y3HavXV+6UQ1mu^q=hRviw3igg~1iH^xpuYB#fvO8{CK)XR*o6*y5H!cox~*#@mdXxXjBMXGLyY z;T7GtBQLJ_w~b8eh}>YIGt#$xp_o_$fWEMW^F4{7n1=e?pJz(b@PMKd1YR=zRPnf6^$)s(k9U z!JmG&B<4i(PDwQ7{B4szgYmp*Vce1pjGqCx9$*LzFlZ1~&zhmbBMko6_l8!V>l! zi_PCMXUm|BGzDgC-S+g zZ@&A^wQE<0mkXuuo+{pFJEY^YkjdGC;w(tkVQxLBK|-^z&qkOGN130Jto_ke$An<^ zAL>(k3oU&HU~DLBY$QgnG%oq2F#|L*cFz>XeWOM=OO(J7+x+S?P`7P`Xi-up2rL8e zrR^Z~g}f1Vq~8gK!B!ZB*-q|_Qn_zQW^Sc2>K?3oP*P{G_S?@kvUs>L42Ku8K=ovH z;njGw9bb%ZZbaeM#&8D?l5E_FqIe@FJMrZ$2_qk<4Q-X_#>8o~Z)|k8dNPj5me+?n zx$FDd1ARIubpRTMW3;T@)gyvk-zUr+C9{!AItGaMAL>&haeYlOQ5aen+&efPgTaXJoJ1U;XE&3j>`(je( zu@l+Lhf^MoI!dYOP@M+n;p1R+u(KjpspIk{&l&Nwk`D(&Q@Be2R=*g zFk_!pO<#P(E0`)gg*A+tv3v5U1ZHS+Tf1QHAcb_pTdIz=kB>7XgbX6cjN44LGDcCR zZ=&DGtlQ93W{*p^;aC)DwAR{A?uBWXq*;)3q^c0p^91yVRME?qrraIMn5H^SbU!C> zk-(7;Zjz96-0+dAk$_(VQA)Lt$7+ZWu3;Iwlh8U1zUxO9J=XZt3v}kx@Y+Vf&JiZG z>FMw+v>R&Edz%;`I|2j88sE_co9$H!bF|p^bIPTLC5zM8D(y! znaXV)C)8O&RCPuvgrS}WMv=O5TgL{~roI^_nR=e4E)XyY5T|MbAZJOUD8MS4z)J*P zCUEf4x<~O)tpc1x)2YYBr!F(*HVtMjqtSa2?Udo!%wUfk!+W&s>NOp#IrLCKyn>eA z0XWF|(nKa@!bB!T#+P>A>6eAA^EUE4vNPp>7p;OhuUiwIsDic%+OnvLIx=@d%mGuY z)Wza-2W0ntUChe{y|GJ;-d$I;#KIk$&xtymcS&6WD|xLju4Cc?NsO9~3oUfU2Blm$ z1*KV(E&ImL)D_IC1^`@K+p|U-2vp1REau-sOG&Q|dF+{EePWv7v{&S-lj?v9wjoMB zRDfAX)n&ds7-dvAOfh~6q7<%J)Dsxw)d>yFB4MW zzK7g--`I1%u)eS}V{G0w_9|m@Z1@xo4Bs4A0IjhDXlG7V?z_M+yeop;MT8XA*cJ9& zYtI`S_dMZ@z5C24WmTdq_7hM(Mm5kgOH`ni+IBps;2fT>7q*JlDUaj4!T zKt7--;;DB4cuU^ORNzlOKFBhaYI&#+h=hz>ktG#n=V3R<8yzKsOimW3o1jZy2k<-# z(cEG9A+9?nA_`)=^^jTAbH1&W9kXFrj~K(~;j&dTJk!%-=R41w3Bj}ZW&QAB>KqQ* zJe2(@2vXBt{v|q;`1(d4R}!*ISQ3gjV9Xf|yr6IHF-nH&Wh6sT^$`zH6_|ele$U*q z#wf!_hFZs3Yr2zz5eoI0ec8S8efZe~gX*I1_ABbW3_f$)FmUdl;Q>XWMYVBNy$=Rq zg;NRBy>)tbXxFq-3(;#s)pRRZqZSB!1dtn9q3u(IX%f(Sw`cS|-CnPlNesG1K0Q=} zj>N^aP=D$NT{1lKuQ2KDV#ky$)rFqiM=jDPR7Sc)d+0uL?S|1XTFiyEU1mMpy>xsO z{J~Rl<5q5Mg+bb(V7mWM-$T>7BmqF4D8kjEE@7ZE=n0pnzKw>s)i0Aa?v_!|xLSvL zUXZpa(v|uirKY>CuJqdF+}5(55q{<7KwVFVK}T|j0<51*-q+QqIQ`?ctNw`h!HPnj z`Z7iYI%V)xx_VJ$&j#0J;)h*d?4T3~JMHR>rsfWBM3=k5+q9EXdMdn4XCZ4wnSY=u zecBO6wDn&Df5UX4FUsNMag!gs>>E}oFca;OAO2^i&OGF4dbgq)Y$~cv)vE+96QD9e zS1&q-74b)xAmsSE&e2g>{ho%O0C4ByA~5M1g%kRKd)?D1l`=T#Ii)JbEj1XkU~Jzp zlRb1S_CIE`|HAsdwaVT%tryry>!E`(#R7#33d?Lf)HOi+x!gfUgz*+tl1ayrEfp0K z88dhF)op!JI`P*2a9yWV`V*-Ls8zcDt{49!DVJ8B)0o2-;`QP}yk7hzQ`F03xQa?; ze(48R%wD0_xA6DWH6efDLtV3yVABvp@je9 zr|neKUV0;aU*MO2+WMQ9gUf!gTzddTIez-MJr2}FX;J4>-q3{me_5r)|0AA87JIG` zDVcKaOf;$unXK+ObIje>ijw|y0~G0bfK`OS8cMwyTQr)i#+v9g@v8yrvZlFs;r~t* BYb5{x literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/fp8.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/fp8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69d150f8ac359a52060ffbe04b77de2470ad3b8b GIT binary patch literal 12469 zcmeHN$#WZLc1JI0G#U*Mq_|453@>9Fo)IZ(GwtzsEX$I|wnTd@*_rfs(gx8D5@0s~ zecjN)fS!e(EmNABL$=ApDe9DRrE*JEDwm`x`5$uVQ>t?EC6`ntvsh++?|qGhl%-kD z0okv={`T#?_j~XAI>n-u!EgER|9RsFPA2o8^fCU^@$n*_sH zIa;VYdYE;x3Z666!_PaP=RBq03*m9+ zc(~{+^7llr6rONSaJU>ShbNtr9G(nLg{Pg<;Th))zNfrH>#B3spY)&EQJj@mGTyW| z^FhX&@sI3i&a?QQ_2%$B=g;H&IegE13;15}r}6zod>{6X;QNSw4ByXt>Xl6O=*P4c z)oh}jYu`%D^R*zTtp$Ffzuc_HiGHaW#fkYwH*R*?wP3ZXB*x{A*9&}v@^98eE%akw z;G^Kydr{m8-A*^6&t|(ByFq7t9jRJkUhGA_ce!&RnHq#H`|(D{d#M)r$i+po zyU|=9#`&Wv(BDa_{PlOv*XkQ-0*cSoy4~QG+vu*iVNC?hwu=WHNse}f->nIslY-hU zU$~8qNHd|fax-#ebhO#*b}@)tVq<3AdeDh{ciBtIqpH(7XX~Bzbzj78+;Qt0wRYPN zZZt7DGxc5^)S`&niUzZi9Bp?*SPL*{*FPPeYP6}G4E45>nJ&5%`|Sw5^8CT{Bnx5~ zc5gJ}4fOb?8_{%Qw9{BvKafi>T-jvqOWU^(Zq!7(*_TQCYWx8NQAKzC*l!@Wd(N4%qmne%k;;jtutvAZJo z=+=yUSL(Fz6-;C0;+rdr7scl=2dl|Z%qO;KeY4x4`K0X``J&02!AmA<^|*PRcRZGM z!w>zW)Lh4I@jbaGqr?z?+!O6ON#4Pa!Wy3Qc%m0T`bt0Jf(k2Eb}{336eZJFW9_;k zW@6n_`|8g!AF4abca*-0oY}tCJ+hub2=W=@(N?#<{xW1k9V}cy7b~M4FIL_Re6U-k z-HH85yitomd>UqmU^A-tH?e)2A+}L1sMOkCCF-rUP*WxDpqg5H-Cx`~_I7*743*K4 zD{{z{YEC?fK_rG0K}jy%b;;DZ^H*Pe=iJp--&l38od3#&%NMRB);qPJ=U)(_Bg9iE zC8{7*J<%gSXmCBF=#Mhu1X6$X)>0gHm%7dF@wg_|{rLDZVQ(W`4sR?4&9$ZOEeN;v z()Az+mqKDI*T3n5kvn3EL-r6Afp4KQUZyOtdIMY(_sOwNJA1SMl-dT*gzTuc_ji6JDm(M!-Mh(+?2AL0b zVV>F;E~|@Wb-nmj*N?u4%n`L?t6PtbCg6VA@9E-TLc}T5fywyz&0*`G)sHEvIE1{f z*SfW}X3&gpeVGcZfNUMz-~Rq=_idS%VjY|zo_%1`)6_In0aTUSXa+tlNJO(M8$Y(c z@q-x?L&nr;t_D8?fh-zluJo-8W?j3DA z+t2j1zTVGz+1ncSld+oEyoFr=0O)aX8ugC2q^ccYo2u#*X=Dunh_LD3ik$MkPq!ng zhB!-=yh!965n_W;od-8df$NgveFZtAGDt}^m6B3cbVbFV@#)sgXu`(wCs}60#DF4i zG;a#xwYml}`0W#iUqh^?0DJ zBL0y$KZ@y7si$L0#-xH$(C&op%%6*)vI(kG3(hpjjzy?d8MDa!6&H{*v6@k{4ZR8d zDN2YCvqVT{37Udr0z)1LLZU#iHevz5B;kfgxcI%Nlvq2f7;&awm(B*795`dWO6l31M3CKJ?OL0p-r?Y79#*K0&xCo&o} zhiK^H36RI}M9+hm7^|w&Ue`ZSRrOO-rv~n#I&?^t<1xQ6e2;n>)3u%Rh zEdiHfBd+(6sYD{MhljN=sR3Kp39@Py=+^uJl{AE|tZ$O}X1m)1@*``OELYgp zF}8Q7oy>N6F(nOSI8y{2I4qGA(mYf=v5~VDV!X7?>LgAFzaj!8vPrN;W$4`Z_4l(e zV9k!QZT54!07Q3`?L1(P;sO$ZYJC$_?;D_)_!b17I2N0&d|T-o+g5BLTtFVRZ+Yr% zjYMNV4}hd^7y8Ps?HT>T9TirSEcv~-NcrOuwTRFJ%BaYnyRB~9vAJ8OxI0Q~vTu9v zb$wTT9$Nvmn0BSiFafi6QH#s)Ce@jI+~vS=tIxU*saSvfKLnJOvOg%2^6))RdjIipn&-6Du(sQ3fqGIVo~ zbictk07&4jL|sZ{lR(9Tew4@hQRaS3d0Ftz zd&3-)V>u?dH*M{0h+S#2Q8@uf7zs7TYWk-R0QQn&z^02vj z@gv14lB2IF{K#DwH4hkCU1C70*1;B;=m{hnSVLYVS}FX&Dey}bM$q6(#;}`Up9thy*bJ>sic8eaWg;Yf z#T!JZayI-t>-CY_EPCX7E_E8QK|foeuSVx|}IM7tmlHtb;3 z3kLIK@STa3jt}UkmiwmqDaHTLDJaH=1oo3!IQ+2RDq>1yA7RD2aJIN6bIbp zkQ#B66gHZk2g$+MeiH5zUxdEbBv3?AjJE0qDjoa?M4DFn%tUMpa`ANJCJw|$T;U^`?)(n#xd-IzOh50v2rLuj?hsVU>lS%`es_j zJWwW2ZJ~ytm4J5gTqf7gbD2Ty$eHG2M+@>7QB3Hw~DP&Q=Z7k(QTur z`LtE!l_4Jagl7Tu{EM80=tptcE6|zEa6I?6NVZGPv!(5VdJFfC=0G29E>V=$1xnbm zgv2C-%NU(av5iq${h0QEsDWG*F=%4lSU#0%YG|C3gJ5|SExirN#4u4tNH6i@sAKE3 zi=7@F6!B@|XPgJo8QSOsN*+4hs6})HRT&+g(LvQBnn8cYjFY5wql+R*|q8j8E*WrPvH? zq#cpQK#4+O!n@YX!(B<6P2)wvf4_ybDu=!2ftW8SgM6 zvd?GIvef>8JZV`HCij+|rn1-0%Dw`{v3LSA<Pfw?hgadsF#w=aH5NX@Sc8=pA=vvdd>xa5krbljUnU_+7fS;6#zTfy2(s~s z)ness99FIK0Y5CdX4nhSj_BZ8L#pDrjE{Rag92=(;y6qrhY}=JMtlHLt&9oXM=6#Z zJkG>E2TyN{MKLaTkc$VmbvAt|9qUwpMdV0I|A^vPsFG+FKPEx~G|39OVjy}Hah(WD z)Tbykl+1z!^&G`LM&uR|vR<8s?*S?M-BpRf;yw~bksG$f$hgg8=2AD-P^3F#Is*ww z(az0tZ%8{hRz%B1dQdwaD66|VYQqTjrl?)0H~kl;MZLrQhNt%_q6eDef=te&DBy?$?j#>2 z@F^}gbluIQfIClv`&dbtjNWzR!^JNczTM5lA}XVj@(A9hF>~q7IjR}F;x>A1T;aj> zC<57oA0Y;MSOt=aNggIXkjBkN;4uy6zXne=lnF2^&>#$Z6W*r6whcT;B1gIeAcQ;} zerj?JNTE%hIr3T;AXJRi!C`c z$K(62*2pvRXi@e&?D*nm)RzK?Q^Hjq+#ni^=fRk1AM9dB1CMqzP#*XG(aN^|ni~9& z$jG#31BlH1e1 zL4x**bWxC=7`Y=PUi}uuy-9?OC;@1d5%+aXd%P&I&cB{?ahjUl@7ye+^6hxn3y!4_o*onfLwl^ozYW_qr**pJ^zh2|uu% z?DsP|P&Bq}R=kQ*Td(rz;XQ}Io7a!m>0T!v^^T5Gaj_1UQ4Ssv%k#R=wti`L=!wc8 zab}0F$k4qwQpnNU>C17?FJ9%bII$9K8Lp6?0-y%;nl0tO~oGx$W2@-JP zZ-@|+3c7kAO`}I1z}^Ah!Id{(Ca46hhI8rxCB^~I;u>xqrWdRP*%f^E$2sxa7T>&% zGgNFQafn)d;kPXY$sRGC3tqAKVR&|5V=|VdEp7C~d|uw*c>$X}6P*S*nla(Q#~;&aWm+w(wpzyb c<7yektPR{?s-{}j%3scWv%He|=5+o40JH;qb^rhX literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/gguf.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/gguf.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72e41daef76b3b8f8619d4694f92ae164d3bfb17 GIT binary patch literal 6381 zcmcIoNpl;=6`r0Ag8>LaBm|LKVmeN2BZZ@-~O?+w5)0Wro!~+pl}5#`4qY*C zNVoc&YS6y$;V_8Pa4#S+Z};QPursL>Q=BYs{<7Z=Hb_(yiRj(lhTuGmI}lkEl~A~Xl>8G!XddI5r*q~R{46)Qg^e@0F04(R+q{I5Jt{f8j8X|D zN0htnvIY*9S5Q|GRn*R*T;+40ng`6QsTxOEimS%wRqZ@#>!L0!acWavhQe@d(^N-9 zeHQp4#$D_-a6A^qJIzm{x6PaU3}iHw++w^XzKohP+`w_2%iU|&ZeJ~SYp+@Sue75e zNme%da)laKb6bjZDC6}u!RsyHTZ_VtwZSfqF<#q_qTX7s&qd^m9nl`9eYq9|yFw;wcc+ea ztq%@i)Ur(77Zp5aTyN_MV%OKFgWgMNONw zZ2VDvybXzsLZ%&@m`O{QE2x&=CNi<5LJKRK5{|JL^(C~c!?5+e=G084SaM4lQf?2V z*bH|h=|ok*TH!bI=TG{8sY}WF_V8ypag22^ueX-ScTi{F z&t2b#ryE9;ullsMXw*{jeco@w$&i1^jU<&)d4O{yko=V- zoXY2k?^{IZP$j9Fd=Vtq<2bi(D9O^I+V2s$NaR?Bqh>S= z-KaC&Xds$nx8;G|1 zcFMYF&)^I*`*Ze@_6Kdhgpo^KyRH>uLR#tR(pu)aBx8N32yIEr$*p877>L{eKJgYi zonGYkhDbr`Z;Q4cT$X3SM}~0Yy8q$rx7Kg@H{bK#y_wG|UM@yaP2}vBSJaLB2bWup zyo}!RheTedaoBF|Zbbd|9X|n5%FDE$Dl30X&33WhQ^%;n)ttXMOvHF%uYofm#i|)i z-E^F;Hr!ku$`UZ{WcsZ$v^l+dT;&GR|L=Z5?agztS^ zIFyw_*N-8J>#pE?OLmgiRgOx9zbIVf>npdmghDe#^#~=d6s@bO4>Xlf#>lBY;i}1o zl#-I$Atmt@;wk(elh;rSOILo3@-ye4ta%;v%2B&)Z+X@|PSA>SgZWYz4~7W7IBFku z;|~Iv0wwq3yxAY7)HJD4ObI;jq(@#E@hDBHf-~qPMd)smv5h}ESz697kdi-vz{eX0 zY~RV8Ooz#oQ^pzB@3VauQNX2>&dS{CXdNxoxcyn_KxZ1YWGa`j2L>-??jhBhsnxa9 z5{-wP!=3wp$rwxDub@YTTinDL7B7Ehjz+0WjpAk%8igWEtFnHz#k@HQdkLjT{EiH` z{1IlWEb>;X~xepx5{wt z!(M)t6yfQ2AX$SMLI6qhDRaA6b&L~oGp<_0iHZVlo4FSn{w(e!7;_#Tm; z6QP5a*Fn4mv6IT+#9hZ*p3<6=N+)kn$B8eRQSQ9fmLf>8K6P_SJ_K{}CWvD>y2}t9 zW`5>Z=8@}GSyeZgd8F(1W1V8i=x6>Ht))6EVAV8)SUo9cUAf5I+z5lDh%6Kz$6p1R zNo4su3It%fq|n-dv2SLk{0Y}Ha~t8mgX{L^EM}RBdhk5Zxp~O;q3HW|X1=5Sb{UZZ z^<5(?Eo(3j)-@4JC|}O(2c+hDW@QfI4X(n6jA?f-q)r!YP;eV6@5Xb~U&SZ0>%y># z9^l@0GlOeGCU0l%gG#1)XD-yfv@YMrtaEl3_{dYaJ2~4{O0U!(Mqvg!VbZGRB>*0} zuc{;R>|wkS#$0}YPDn==XrwC036lYc^EJ%xXQ}j(%Kx(F`H}ooin!|ALY>CyG4e(Y)8R3T& z$o0fws{soCT9>Odi8UhMCvu4hp%dj;)oSFcRHO(ee?UZGB?Z%djbd)!QqG`2Jrv`n zCVUKdD9VugPDly?o|-*^H@f~vNWs{-|&pcE(y)Ht_gn9Q3nKyaybLm;09WWABuIT!Hv7PB*68j zRk&*-GYXnS%Elqlrw7J+0aepPJL6D<5f*cBQK%>;f%Z>$tG56+*f4{&O#S-qI zqx=;}ZnY!4RZ7}wuLc-#2T*;uq3&t<9G*A)wC`_-cE2a+QpoX4ii@BXrlObRB|Jis zGjCbS9eb6LB*H7XrNEs0m?V8dM4jB^`0@o@KZGn^Q~jdjU%{r-iu|m}?8mNA1ZDg2 zf>9&id-2&AWcA7M)?4nZnkhUI({z;ARSHLI&*yzVh~qxi1L&IIR-2rh^JL;m5&l9m zbwFJ5ylg>NwXJXDHjotoCuQxVaD^OogQZKBIvQ_5y=<+<@v7P+dR{J&62Sn^r1wQh zqbsZSo~QEa$@j+9;=OS-Qcun5Awv>C_FnlK=Fv4yEpc@NkI|zmge2doo?5=P`!+pB z6Pj1A(f-7m)q9Dz{4~MCG{g%T8E6p$X{(E%r(!z=F=eZTbscNSWVdtYjpF&?O)@xj fyI;~Ac%3>hSzS^58?~2!$zb~KQN3Qj+Bow+xY6PJ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/gptq.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/gptq.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9d75f122173fec9671dabbf7f16291d5ff065ea GIT binary patch literal 7056 zcmcgxOOV?}dIlOG2!f9xIit~NB-!*jk7ZUSEos-gQ7p%{Bzu*W#t+Gx1jz=4VK+G( z0U*@?WetnWVMnRt5T`2V^;Tx8a`7RTRL(i)n93msRBkz`a!gffW3N2<{suY2dDzE* zg&*C2|NZ>m%|W^BXn3yv!#{6Le?!y$lPZgkiOSn}<9`L<8rK8Or7sJ#P6yk$VtLnaFQH!M z71S$2NBuJDOT3DDRn$;_iR%xvX6-q4rEN6z)VvdoLVWG_q~A&UgQ#hy#yinYYToX5 zlGMD{kCW8?U`S1V0K&qDzVt(p2#HG3>x|-L5PE}Q++?Y9uOA6tzU#*#t!Cx>BH0>n z!hpQ~(a4XI{@AC{ZVjStzXxoEMhpTG+`Io{;qzuLUB9*EN0A5=`Gbxhh{H+KwfE&< zG(4;2+=qVWk>Ixm!6=N5IxJ`CC-?5%KSEYN9(;NX)1d=dd`LXr#vA_!fY4l>YcAuu zYj7q^Zs0XVj_2@NJl}P=%?r;=*W#AQcNuqh5j9&BX7v&;KQniA*HM_FC~cQ7XrQa` zC55jjya`N|*A!-HMwwT1b-t{yRbW>{P2@$r%Pwe`TkD$q9A8zf%lte>UfDj!*F+vO zuks7%ZSi$}5vw^5Oi^UxUE-I4S>t&K!%M02{)Z1g%J?z9mwg+Zz>nibcOV%6!>7OBle-0>iCX)Or7E%4oYoC$~EsU<}+lF>#dhqIqQR3@#wg*Sc=V4_bn zPn$46ePRHzi3w;VCP&M*wo5HJDslEim#-ui2i3Or3yqtx)0h;Hz4KV)T)<|N*&gmd8>9AC!U7o_Uu zxO}B?Bv?1c#Vd`uY~38MG_RZzubWNt%oEgQ1Itmzn1S!0N_&@LC_V2AYHCak7odyp3k=OBu{$@Yu zCp$OhOE8V`xmdXKy!+ZeCRjgZ$-?y0pM52EZy&RWw7ttq4Qy8S>9 zuj1dIugYTMmrtzfByDPCm<};){?L-h+9?M90SXgs57v%WxzDCh6IElX24zt-L)A!S zbWoZ-ox?GGW=vuKU?B}y#@w!nJ?1x3lMK+2;c@db4S5f~IL}4b9|cL1xn zr_>rs(d|D?t3F7gtdVVzwFC#Y3)Yh(kiP>` zm{_MD_oF!R;pHR+0kTX$S!WXYrjeP6?-TY11WxQz9h4Fy4`C95pL-z63L2E^L+#cd^em%PSd>-h8>)++;;hu0ly>OraB3I;$k}0j)%t5&e{8o}_JFAk#

%RkjGU3FHXSM&)&Yltoc$-BT@#GHP z#vA`Nz!(ysBJWNO{W#N{x6n#G^* z$X$BkU$S&yeC%`sY zruVTIDg|@&@r9E{>M$OXWINKeel#4xwIUk!hC+HnUnYG@jMCM?D50i9 zj3Pq(eAFW?&S+FHT>_;a#+hU8kc!M7vdbhDMZEF9>6-TExk-Kx4sTzd+7tFya2MNb z6#?=iL)H_6tFRbW5#MQHQc&%=ZEHsHDkzY}{9Moa@f>=!vhvi45Pnac1X43C!reIQ z+O)*AX&Gf2{iB6GS-Bv80%~Vkj4ZgQjGj%)NijioG+SG4U7Hk8uaGn2jOTf_=G?^Q z_Gi|#k}RRM`j=W)=a9^2c2b+LUHvw!`m7h@475eKLZU2BO7mTtwpJDaCPG;U@<6x}VXfCwy)TbkG}(&Oc{9MhN@oxb-}*Zml~mNy7;_G+ zgYorKhNB||{DznDCvLqN4B*(~w@{dmf~e#n#*m*7pdDZt3PEN0Q$iT(Bgs+7KguWw zueu;3Qau&okRr`L&ZbOrJ+IH7rq<)Jkb^k2RIrexE6ww%f#~IE)AD7E3im$>K#jon0MeWYhslncLm+YenydrmTA7fgIq63|At_>%RRT-Y zT9=}S1ITP=b|;mVMdoQeL~w~{u4ML_46dX&Ox`E3LEr-d9|E{_@idYC?07vtNo1_f z(7mu=WRKhEEPN(&?=Gvu@cf8-bUMmbyK5(qhf*m2l&1X|bu4D5=|P)fw>Hz@SwEnp zN0fg?K*csngp~ADj?2RqNaBA4V2-0BqQTERu+5VCBJx44lta?WEZs)xO2Gkz2)2R9 z2yrMS@JMK1ATC15qW6ESlA~Lc;F{)FEDy}D%pwD&qO(vd5iRKw4jRsRbL6x5R& z<2q-bF_=XpyTBL=7~>mg%mo@#(c~5|IgDbFLwm&JCK86iwj*yR#k~^n=DxlMSG8T) zhZ)#5Jr+@Ii5r9?%yyNVa3ZP=W7M!NE2-}-1EYGV@mBT?y4}qZ5Zpu|1#`<%JRbF> zh`pY~ZK$rSIn^@ka=4*N z8MN{(BBel4_6ShEnHspybxU+T;l*2{ZZ{Ar!<&acN}ZHYDxie&@=y*sB97-)%JaLV zndFiBQSaX|YD{!>R@bv%$+Y!#k}}Ag^R;c1@U;)jUyWO*?BBNzo$601rS;Yp*;p_I0>#YY(B&es-fEKgQh6N}5w%RWZ!XgSq5m zWgl&BK^=2Xgkh%aO6?UVkC;eI$t>&&buoR;J=`ER2(6g$h)Myq6=1W#Gz{-Lvw64& z)18^J6G3a5^M}J==fBV~rjak1b-e=H0PTlv+opq3mXsgUQPnBSmun|d8F`adLatH; z#LBWuVuHL)Kna^l|45u=m%uiFYduuqM3&zT2z?}9heD=O1d?=h`9!H*#bSmvOYW@0 zQYN8-G*5(^f|Ek3TMhT}S#ut3O8HxIwVBgZ yuA2OE#@{607Bb;leS3ZjIo%(ErWCq$&p;=T1d*)o$V>W>Trj$9zdDf5&{ymoOjWiY@CP^Lwuc#~!2weumJ>Bp9-s|b+a#jMr)BpVMoA14oNc<;dW`8;|ui^@hD2ardaFuq#S8T;sZPnLo z&DU++PufY}unj+DrxdEAwoTi_U2CU(%g!)QZ)g3So#S+}o%iSLIlo{R{GwfCp3yG( z^Y*-7w#)v4y}-Owd(mIAmpE;p;(WS&*gs-pQ0T{MAN8NGpYV^_$NY+2 z@t?GxWPYaol>fB-G^exeikw1le$(=`j-YX#gSyz2KQ7zx6d8?)(?X}KU zWS+0J+qLz!7wNAz>tUo{Y6f9se!UkqyPaA)(%$yMDEUqYxwWbiWt@6H2)n-1?FE!= zcABBn?rv;&qN+va#eU$qm%A6D(l~Y53pcy&S89P59f^6~H@)V@X83l!*7n}+_j+9s zdM>1`OU;f~6I{0-(=*bAzzdz5Ea}w3P*C@!H~Y0t*c{You;;s-MsuU8M)Q}iNa;(p zEl)^xv{c{hb_37xdrq6XajtoybNnvqd&2LBo)gfM1ksam5t{RAVa#>99S1Y)h+5}b5G~#CM6PBj{mxWiUb1go zx^#KUEWFjdxr>8wWa(*Y!U?x}UQnl{GW*LS^D3_3DQq(@VJmLJR$ax`JiV2~Ww`2H z)lRvZXSzDBq?dLLTq#_pn{H%X%gy{yw=LK7G7Z(ux;f;sUM|kla@Mi5^URs^3a#Qw z0`+KF6TifK6C7ID#F>w4X1HG2U0`k*+(mD}%Xo_obtQqm6Aj&6a+kSuiDi~ODYN1p zV(yB282ue;9d?g+8H{JyJK`Q)NuXWBeZoD4aUKPCE|#jePlEFVT6${Q561Phdz@Q4 z=BBaPPDF(_u3UZdGA|>!6bDD-y;5)2f}qmqiV9V!$U0TEs@^AMb`^J%?olRQ_3O zSFjE;)Py<2i@c@^(^W^Bs|}SyiDAM?3{_BNsDY}(Bq&OTdIQ{+enSz@hDlc+D(@xS zomE>>SipriX>?H1N1xA&#RBae~ORAdz|X+gC0)mtVUQS>LF&``!g1x+1c;U0%sm zBhmw}-4IWc%vAcBaTE6v;zumLZu_VFuIsfO@0M5ZV??Lgq-=syvyXCWqZeL}_xfrNy4G=6Uph4A;1EdG)HO>r z6#C~?T~QVNp{ke<%i3UN@48tXOV_dWCO=B2aq*j^r*Uca6L=r=hUmTE&9zkAb=)<2 zPd}lBX{-&7?Cs&X>3p3V#F`h*UXO(oB=7ZZw6=_ns1I^>(mtAb8DA2a5FqwV;CN>H(dPYwhctBzXEsIfj^YTY%#!wXHr%YTro5 z(s%BRBtQE9iFQc^Qb^F? z_=Va?3=`Xlp+aNbQAgNQoKrc6oy9qgbJ$;;(>VuC*-{{v+)!M@P2JH(#*lQWHq@a- zjoV4AQFATQNy|<5OQb>B*{OJQU4%JKxn85zZ--UY&e3Bx1i;rdZ!541yRuGKR8!)M z7_gwBKT^-_U2++2P)-DJbs{K()@vOSuH0;fo0S=bbF#A8z3HK| z%2u~u33^_=*=Rz1);^r0`W5e1uidOS!}bqr;%Mbq`hkc5rA0~8Cl;CmPb+d)e! zzzV7lO-)q_xLc~JWL4M$eelA*z5KWqqa@qm$mj{L(Yyt3U~wux>r+JYQ$8FP>1ga) z#k8X7;#Ev)D1;semD_5NU#N*I2z!N=FLt6Dg>LQIb;zQU+8;>Y65= za!pbpq=x+;L~IpTy2rl7>Z2ej4Q^c`}kt+ny+OxxW3 zHUt5McX0(zfD|+fDxCf%u;sEwOM*~l9tz!fICy^F;&`Ous8$HB(W`Zw$Z{Mg)P9@N zS%>y@J1$8%j@zvR*BwLecB9b_ zD#JdJoJ3Yqg6xPmM`cNdLW0nuc#X(ci0DLUrHMC)km(U?Ad%YXM8;(vryxH{TqQy# zjwfkm8Bg|=sGyW4E})4w6F{T7t{4y0q^hb9R89R@)zy!X`%pF157dwIW~JUWazovMTt! zCN_eXIYqhIz5a5AOf;><*ahBY@M3pxl|`e&an#0Hryxhi4V-lk#*&PXD|oj&hY4KU zr9NnJ&?;f|L2nnHDg4@y{iGIoNzmc0az~{Y8+#7o+o%O!h8tW)db(q=d)b%aqCI<$ z*7D!t3TU|*im4fj^>A=(y31zm&%~P(q~)XKW~bK&_J>Mhg-wcTMT^~jNJSG4 zS3)3QDiLMkIvf?qqGrtxq~}m4JUjj^B0;|5*Kh^@1!n;EO1rI$Okh&=nl?%gQ`AEkQ-=t)kd=c+51WR2l08cep&so{YLp36!wlNU4zn#2iOyAbTx7@@ikCEgKB}Q`{!!?joWiHHz zb1mRVmMJ1uVIp55q>Wu+y0Vqx(1MCKMul)*){g6_D7Rb-sJzJhxgm{RYoVQSD=5$0 zRY#>sIivWPJ3q{`nfTWb`vE~Lv#AW{f2#aQ8I{8Ya7;ID&&}P{h9xdp=q!heTxPiW zyXvHV9y}WHSYqz3K9+##T5=1+0?#6)i^Jk1T^g1r>G|OTr?bNhLS+bzxka~phoUvh zyrvh|qFfwBg8S)u;$k&r7aM&9CS!K6S?hWB;&hem;*Ud6KSz zw{ElN060YVCQkr`zWHt0OM7GVv9E7uY^{2~t z-^^r@r>It5%>s?-Dp>`%(+~6{c{e8Tf=-fD=VKs?vFtC59P=IkaVW-zdRj#-u45SQ z>shLY@{d*I%#3u?b)Y)x6D=VY$~hJU)HCx>_@X8eRhAY z?tb2Vc2&c(vt2D90S%gf_YV6ZEvfW-RPoy=i;nh0x9$bOc$SJW4*)eW9r6#+i1-~MdlpQRWuBi5-u?(5&6_8cPL1A$KPLb=z; zri(i9<{T{jPB&zyIsovlP!RzU-uI$GsSP4EBJ`}RX2rXdqM411t6nESM4gfI?-G0Z z2%I91lY!n}BN315l0w9*cc0QREWWDd`P+aIFw3Am6swNbxV{wjUOmdvWcRZH^eW8 z-uLInOUzPcm)Iny39+0Y%4{}W*TX3%9V!)J9&oPdxlK}ZtT;)v?>asz zgF%asf#i=cY})!QKu3VOB|0ji93Wc?&~2PHLzO_}DBYn0rW7vXB#ARLwvAz$Uh`d* zlt+ly+e(94v=6z3%ppEtDpOh3HSXwc$~Et(BMT4-ZqLx#0niNdLkpo*O3k()$AA`w z+8*f)OIyQCES)((Izzgd>d470Ih_5ewf`??f}{G{Als)7FcFZ0#M9u;Q$#up3|{Rk!1=J zs6wWAfXv>0N-R?xmg2FM4v^W~&pgYNhV!w^yp&;0zifL%q+#fj_bv(6arSvSwjq)`w9zcl!9(&GAz10!Nf zc!4BpOSEW$D^rUTrUSJb0<*YH7{+K}Tw@ozT9DhGabFIjwUckW_4RkIL`g(n`<|UT z|JB#l)-GI%jD8Q0*_*2UD1Bx8wSXAWvl93dzeR+ibD}`xOCZ(qj3N7d;{E}VKO{ms zU;F^X&c%R=6vFAVHh(36>7wiz+e82doo{T3KO%u26ZvBze?sI>iToLnKPU1RME;V< zUlIB2gs?7q7uhe8F8gmJvRMKu1e5wHqCG) z6l3V56ZoN;#N&E^O0&E?arf}J#VHScPGpnD2U$~&pQUlKo(}5TOVgYO> zhKfVHPpV^liaLc)QKuM|wxOO{Wn569tuo#Rumr2+g)+?$ajclL*jn02j?(<}8H##f zCZzl_YC@g1miNgMXSck?^4X!q@+*)!gzGTfXX+ktb9Z#aY-pzQh}%$KRQD)d`+6bH zxECpzWy!f=mLyr#7S`B(KWJ-1@p@RHk>PqbpyDoyA0a15CyP5oCb|M)a3!z}(k0Hd z_}-DO?nH}JpOA|11WA$}iIN<$V1;K}?BC+cE_jHfU!o)w{RY${d3eyWk%~ij)_nwB z#9bncjo+gbtH+5(iZp!NW-rd3pwY)(`0@Zz2RUd~rK{9Wng`8+dDeQ1)E*7mKDG}> z?RI|U^O1tA=JqP!$yn?HSqpjsgge-N*mGodW*Cv;@$=XvdF<)1Es{R1T^#c!CTz-@ zSTcOwD8F)}mibe*y-JxJg?}TZOb)N9WmV}xvKI-(i~}c}Clp9Yj~$ zLwEtGk%hmD?_5a9!ZS5LHjx9kM$F;UKHfZMLB$BCH>P4`sxbbaP*vOQ3gk>bb#6>pj<%E8E^q#W8vMP{906) zf@)b1S;UXiOYxVJ)H21a8~sk5BD{4%`{Q38i8)+Sts_C_T=XgH$U-1hZKW#CHTB2}J-N(S~s3aGgS70?2TZBfuQ z-$E$W)$ioCi@?J4QASd3YkYuEXi#uHK61vvk5@JA+}L4&UB2O@t*BI9} z@Wb;o&i^uxtN%0mNz)W!aM+=myn1nMZLL~5DDdzJF$JDxLTwyQ5Wh#Mbs`24 zjR-x$j7nj|$6)U*6r3f!auo?WZ$nH$8^;vJXKlJ+eUSM6Vp9Fq_-XZDjH3$rOx!_5 z+yQXveIu>@(y-KD=$ZWg%H#CYk%4eE_MAK@p=(!|`M ze1_Md{9Z!Pezl+fR4bF0syx`wQe%wZpL~pHo4-w#p9hd0V3=;7eT>Ae=)yX_vyP*s zqDaH!PZb}rfp$sC&}ty#%@kX%(cBQqF(1I=gI za$kInYE_85NrYDKLCZQaUy;0^>> zl|yasOixd5PtSBuccWjc*#-PA|LebP{rXa&@O$b^{tR@k;fekOfET#dErgn@h0JB4 z?&_i88XDnQ*K|$1^{y2b-J-%7-BMU~ZPhlrm9XkoRom*;!n#`zXWSXpFLoQ@tUIgP zrEW8vbLYZ&cRoDjo(dP-1%)qnPlt={qE--9aps}sp5?at60f-D@SNvW_hqrT{R$qZ z!`xSSjaTr#aHGKMeCG24pAoYUb@w9r4L*zhte8RnHO}r9mYPqovci&)>Z`ro)c7c9 z#i_9tL~&ZWHHd?L&+o1;X=(YkFa1!&LZVagTEi&rhhBdWQ8(xXvDfW)Iv8cCy%zL@ zFF)`jkv6jSO%ZSPIblHF{AB3&;$Y+x_YeEMcF@7+XKD2Q+S<*752a^6-12)p(N*+! zTYgs@jxFAh{o&xak-6J`Ye(=YR`_Z)0s~)q@$NuGPa9bC(|;szjle#BEi|TnMf9%W ziT(f}3a-WrF5{Z3b0!R~<1x6|wz$QMpBvbHLs)IbOT3I;QIzt&%`2Z9*s-$0*rKvs zJyQT(mDd!$rtk(Zv{yt~&nYcMM>|H?8DN{DAuKW5W@ifccA;(XIX;}Cn z?_@hPdL?^1t*#$MPP;FiF~PF!;FnfD6G3My_BMk!O6?A9j~4|aky=v3L)lwz>Cof& zM}}qL`4CU^8-Td5t>J-!zhIB`M0=v`=?O~;o{{LhFf2r;554)V*E13P3mFHii;BzI=Gd15o4w8^4S7}*4ZvAm z25j;I)M2kLv8N>N(-OewjcnQ8m3JK87m?HJ$4)dH4Ei#5KBa@LPMGt)Gdi{G-0F3A zoe8}VddBgk7*7#=`Gp)0)i}xV5E}vg)PY=b79c1<&ZRj?r#meUgI@g3=DTT`8t-mi zzVdFWcSI~t)97oo#oo=;J8K`WV=dm@d$(`jy0dZpBkz-YtLq!y#!qiwznj`W^1DNE zUCO?E!HgSsZr!`>-Tm058HYM>%e4L3d8iCvlwR9!#o$zqgRaQV#!E-UxCv=C{_4*4 z)sMXOv=#EQusVdUqJiAjCSW_+SQ+Uw^#41uB}~PORJ&(z#G7yb-SXQmIgwG zURU(sgp5dZ+j4>QNm`N@0KP29xA6MY->t;qU}X>t-imz*J-+qbaJUs-4nJGz2AeB` zU3j?O%KdIPTnYPJbUpDvw1!yjO4r{NGFnlJ$-Ul6XApnth04O9{pG=KTJk*QMm_pw z^bUYw8Jh8prL(Usv!s_arqN^V8~e7wG&^rwW?i?kKf~5XN=akO%uIH9%(-a}) zI@QVm>A-O9F-&5c42eCVn}My5-kj$2#^L6^|FgX}ox^j8?$VN;TJQ^@A1l2n4SuP~ zH^EzJFayiz(qg3;NYA^CUPOMtz~YSk6U5JGpA(;!K;r+A*1CwGGUV7=Xc!m0(QwoCGvSDX@yh*n@STyf3wNsKdnex3=8UX1m+>k^2$BJs66|)m8Nq5G&cLZ1kPy{qpij^PZD+tS$LAIAU|3lI}Ax zjuWU!jL867{=nZ1x?Q3$o?TZ}fPAH96cq9(|vu2(WC6OTHabc7OEH^-&Q*$$(Nm9Er56K{QoskZ?K%tqC;mp7Lkzm@qO+cQl@$sb&i_mx&D12 zWxz$oEqCD_f?>q8I}j&Y1gIKpzoA#w;W|X1M{PJAuNYe%YY&+$$2hVNVQ92n@?8wUGuZ)j9L0XGC1e>Wc|+kd zXMLRzKOk`AbT;TjXDZHCpMXdmQMFcWba7JF4>MCfI_G4cKt|UY3z>cU@-b2<>!)Zw z_mj%{e@>Dp7p|RX{YU34ouP|}rh8s$dmc{a5P^x8Ry|r}Hy+t|i=^OlORJ1Nb#F>}U05X5awo3wRog>GOD>)0m0iL96cD z4Ti&MY4-ktr8u>~Wa>wk*)=@TKLZ?1e2}Q<+)xRRDXi_{c8Ob}%!^1R%1BE}XqWLI zvDmJ(tGvRipsn&TZH?EFw#@Jbu(fiXpE?+s#f&;qoEdt9Q;pv3l1gz*x54XhHVf*U zj#`-+IGNgyBqJ&tS(TmWJ=LPzB=-vHJih0U6B@@g*-&oMPUd&>e5N`%YwCia&7()z z8r2UHI7@HBzBG9c*d>;k6SXI{QSBg|qDOetk%e!6b<%Jqjr<^ZlVp|YV$d555hze# zjkuMf*f^j(I$h`wV;VZZC?aHHhcnVjPNUMYDn-Cyl!X~BN_NM;CK}`veuyVp!%F|b zOp07PQ`jpd7DYKbx}1#}vM}qnh2JsCzG5Ahpte^|kT-1`+vWnYGf-Uum64Q@gQ<*c zoANa)&(|_aE2sR+14_!zjBRi}poISdrkGQ1fU=Y@jAtAu!yEG?x)QhXWv? zp+tak3$jVVD`+PrKKrP+SBY~@yqR&yIPvOyHXDV2@oaFY=ChEa=Bs1!-+7~Fj#mhI zYf7?OQs(u_SYI`$1J8l5ifRv}cU}Nlf z|0p}g*JkGndyTjW?sEw&5bt?#*NOWhwl|BQ0;Pte@`NEMp!VscdeB}>W)9kCl38l2 zfZ-)1c6mySsF({mxpj--eXm^l5O6GMj)t}>@H7^X&R$o(be zT3p-@aK+yNqo<_0rDAGuKlamdwhkVIshz74ZP%t%<6fn9EpKO8yW7Y^x`X6h{tXz) zv&5Cjzo)iBY-R|0Ns7r+;{&9)sr3OxOn$0GY5B9h+=;+lD7Of&PS>Sd%y(NlG)s?$ zp>m^=LU4JJz#9ZqV*VX!y+&XLAT>oeh<9BR7xdd#)a&w9S)<`P0aSJil5REE90}Qv zzLy_jKaORjmhPT8@j$xG z-Z1n?by0R~u$^h~(~L!GkY?O!wk7HaMDA?PUIi^4Dh^RA`Uff ze`A?8T2+?KV1}Vlu0mljV<=yI#fmgy5k|BAXf^7pg^_10=c~s5yHO`9%-S~&@SrJL z|5K_^siU-yLI}RZAS1Jg(n4{h1X-y_!gTBFs#(>l2)w7mzp1-)S|5_a_Vwp(c8cHB z&Ax^PMbX1#0PN*@>06DzWvrn3NLanu(R<#adiTmBfBh-m4~6WTC_h+3YXCBdKQd z+S=&?O2#!+EY{>&QhRJCy2@5-P`$CPyso-Fqz7r2Dw)caqLs=?+o^Fs5TB)HwB-*( zs^i}q?%Z@e$?J$P_e&Qr!8Lr&y`}nu+vou;`vd|4+XN^}RNLtq54S z-E+&i1G&OeD{CILY11DJy1V#CQX!)6TQCLsYciYJFg!9u>Kk7>W(gSNpHpRnDRWS) zvW#$6Qm`lK$jh4q)(KFap%NbDpygfaQmst+&n(fQn?-gBgaEF!A$n0?W?4;4=)*Is zTI85i5dTkTL{t%>{H1Qeg7XZ5yjMSGP)2)!>$Uy=4_)ke62$=NeS2l@#|%3E)E4>S%jmvy z8)^z&2l;=rRLMWv!&M@fBU7I!?{9p(c3r+soW4z9iNHAm=Lx(*;8g-@$Eg-@+`gry r?`0|S_i_9R(YF8=3nd1(i>CqSyU40Wi5cYh@mJDuz32^ne(V1L2g;Bn literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/ipex_quant.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/ipex_quant.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b11c2f2c624d6d2f0a93d8b550abcd93695bc18 GIT binary patch literal 5668 zcmb7I-FF+s72lctkhGR%*>RjWaX@?(1Vko~0xbjxNebynL;``d%jwz3nvuN9>cg|E zI9BDJ9vlu&`~{R#yKi~uztBHqU;C7|p3~Ep7UKTytQ^bAN6Vfwvom+^%-nnDer|Vq z+BNW;`s?3Tf4^WDf2Ya#V`1_ET67zoFa!&YHuG4Ud%SIWX4~>CMsOb5o{hH|I_->? zX=lBxhFf8-?Ru^r+u>9@@8$K_38&iyufU9t43QDpJBG;0{Fdp>V4f2$=B{)xFADyd zQJ%Vo(~Yu~m}fg1iFH1x$BA_*h~gysNiPn%omyBe^TfRrbYx9^Sc_y*OvjgHyxJ86 zDYIne?DebTV1VYX_G_Iu7}RK&bKOoOXaZWS_mz^J*zbk4xY1SZI-NiMWHI>wEpnJ) zNW){o@VH=}DY!6&^(XFG!j`shq$4tDS&?gGg)64+SRVM5c7uz&n8qxNnS)tD%wU$2 z?&xb#%-*rKm^Y;%c{$xGEE>R@6NfZ>M#E|4yf~~OMLA1Sl(xoO5Jxn04tPi9A!*C` z2494nh0(CYqBy3%9@e`ojMf|%OB#AaoWOZUTZ`f;@Q{@Y;%V&Zh>~~)G#(o}nf{UtFwSd%LtYCf-tA%ePbyYf)6{M?t4qq6H;LxD>C-(zz?2 zmsV=^o3bNbbxU}HPCe`k!o$r~H0U1WZQSGNaN%Zcea*9)y?8Bg+WMxGsdX7NS7U!A zh@-@9s&2pMN5McQj*@X-b*gnz>Ai=>)Xt$rH_#2)(D02RN6&^PdR*$zXt84k#+bL5 z;J27M5nDpfzB7dNy}8A{A}mALH@SK-wt?#i<1+(T4#D@a%9-lD63!{JdvxyLv=+bt zDMkHWudCvylyXucPD)>%{@tskm3~}mcH>gQDyl%MUw*>sFCm+;HRX(&!4g%ZZWdiK z)q|-0My(!qRpNe83;XhdQWzhiT_;Q>$!i+@RLZbKE0SSjV#~XRI*y(H{oCcZ-COPj zy_0cGHD!GAt#*I4eX6~_90n`Py$zUTXZcnbwwK#oAwyr@mi0cUSPp9&Qbo(-;#dxP z^0u!bmEJ~@_5Gj|#J&$_Wkko&IgY~|mS--vn8n;(*L0bMvAa9ajHK#V*;`j9{n>kH z?4arEU&26NHeDAKpvCZ}OXlAh-#gH6t2&q?(WE=Rl#1faAU~#YX9m-+miCqIOxc_i z5mQ98mQ&i%vBZ{=J>~naU>1=}1{2KMwYYPCf{hgKAvVY&G#hygVh~S+FG2_JvMQ*U z0yH^H)caRIKU=-#U%qhd<4?}}m(N~FDSQeDlT;nW-TC|SA6>p5h}VD|(TRCBcy3=L zmB-;fL@f!PsQo7yfnX3WuO=29rD?d!60;sglk{Y1|4ywf-=L**zQqqnXYc!C&x^=! zlXRLmQz1FL(1Yr5?Ova)fzodWTu#YBM2p3Tl79~A~HB` za%SBhJUv0~B!1$gkU9~%_t+Rv)7t(X218>Tep}#9clb8jF*aem8iG*Vp7tda&ppjVQZ@v7Qzi-CcUzWKyO*}PJx9#7Q8&P71-F2zT zw)z>Ct7YmAy4W%4XqimuwJA%#0=m}5IbLK%_^-jjm@vi=>n1Swo&)iUtS~l+<48xF zA=^f@65x2-gcQu7Da>1jS^)Rf4m<%xBsek6ZGs4-V+hD6cjGxhwF zbY)93=jB%$sSoz+ji%@JBdOB4-fWW3rS2`MR3K!EGs><;?&C7HjmSx5O^X$ImS^Fv z2QQ8>|D@FrI>m^@=_%O=ZmSG7Iv^pE7B#IMM=^+@8!_AB%7q-D89T6il0<6#X5t1& zM54IXsY^wwsX2nj)NcAs0+5AIvREWlNR1?o10ImHYP+0V_u%*#LzB1>e*w8ox;V3%-WM|NrVRUDj(UXeDE8nNG3>~7Z&r^GVacQUU}_cDtd;Un z{$1N>RIi02ho)`x#sfx}zL`lIoy0FMQS_pvzMr_hkC5IEX`J^-K;h_1#`i_HjxhHO z_D{?xR*4yOVs!#j^)z+Q(uXOX9Ht`G8mLI+RT&-X3v#&XWdfWe z6^T>5P96CP^>cIy?{pI9vZhV_f}p>oZr^F?5m~7Ym8751$LO+##kkFxb00nD_u;sH z(Yw~M;ejVNnnkM^9IPUw zh0Gq61hQ)zm4wsEw6X%VM8o>VXfUcL?&Q5(E`z#Z@2d+eR2XuK-pKS(d$e3s9aLx7 zUj7?oy52N2X-1J<4BkYFR|<8-R7yqoKXURk0pH66AK{rv?hjO%yh@r)?^_#F7XU5u z)VOFob^}`Yqz*UHbVPO8V2R9VA{Tv-%PDQ1b3m^ih!7-JN_a>aqJN-^jV;QE5F@^2 z3{zWuplsXZgBcWUMre`8Y}vxRX&}FM1d6Y(*ma{bYZz&_ksXdR*Qi!jiRp&cJ#b6~U9{4h&33hhPx_RSh#S zFtaek`>~7PJEIS%48H!!Yf86*TImLAg`aTFjZ)l&u1Z}!o_t7+D=1I)l5*`bk{lv+ zk35r13Y58x95?1Vv@0@qEXu5`h(ou3Z8T zaA4B2sIvA70A1I$?s;;H8hYBB0rYG=zDkQVa_-87PyG+iUOV@(_X5SCQL)nPgc~C< zjF|S@wO$WQXcN`EdzrNq8^s zQTPyi3*gk+gfQ3{G*_xgyI;Z&xXQ^u7{k1n<@!*K6%^$mYci*!h# zVAn*8Xh!k8U>-C1Y3oM^7i=M>XA#pabM8kS7R$4V1A8Jv&}jt4kTeI;xdc(=JwyL4 zD&$V~|6Nqle;1WhL(@!>l2h-MCm@g8cm@A!6xCtg8~^W$6}ab7jio5&c?G?4@26#? oGy7DeB!sbhdtd}f_TAKS-=jbg7113vvb@OiC`t0XXwHB6UkQGiT>t<8 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/kv_cache.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/kv_cache.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44e5b4106e4e3c76336208752489270b4e7240ab GIT binary patch literal 2593 zcmai0OK%%D5MFX0l4V&<+@L9f6d-aaDioHICT)VC=*wseG>H4iqv#$=(@V-)YhNX~ zb}T5DIM@7wqG)o=q5q(0|Hoc?>MiG9(jj*(%XS}YEIk~OvophQzEQri(sAIq^2=}I zUsoLG4>MVOJeYh0uet#PcQ{I%(EP^jXthczb;v(<0;c|o~`HEDBhmy#BHI+Isb$y(MQ#4A>&NbJtng_+opLI ziO5)4eCFPL4_e`bh&v(XD0Dd%-UxG-dyl=)2Va7n)=nEV0dIk3=}^Plu+kZMe2I5J zzkH0Bx@l6gwNuV_-_kTx|S*52K*Ad_M9QL+~&3E2{)RDy%)n|6!wevSN4UrW6^G7%DR zM8@FWmKfy{zQ}g#&x^G2nAm*_M!5KYGb5p+@!XJg8-j04Sz9*?CRtj0L?a84n8r*k<6H+#FPwte;BZ^zTZ^Z{GIe0Xx(0%FcF+#i z&Qq78XNW@;Ji#+Wy?@dB+=B%8(>wEvT$jS6_|l$;x5xStd5cXLX>e>|+~BEC02rVk z>^N!%b4%Oc4m>Y@|E!O~6$$W&rBHNRGQQCdDj#wbjVqjG)zVky(uEc>44?#5E~{4a z$*R?S?>Uw&uqKzyU@oQ}mO!d{k#(BZKC9P2r@AooS{Suo_Ao}>ms4^&v&WBBffTxw z*{KA(V9F+V6$I_fY>IybI{XPAphu{WH>MW{fX`%~sBvB4)#xc)*I57A8 z1x#jW2YjBmQi24~n0S(agy#|h(74agQ}3Dc5J0no7v^sEc5}*+!{!u7+2K{c> zJLDV6^*#jn3cx^~fuX`WuL7MXvC@^BvIkWshG>*#&}l1w1pP)=o%<{)YsiM}!(Ikb zQ<2gTPv2iC%enb*)%HEAbIQvii6fwtlh}R(_&gF&1WnaaHiAvG>2z;`ZFL!jj(ZNB z!K>&T^4tLR{2nrYUF<<6v~l1Cu+l*o{=Ao--$v8n>FW5;FZEZXA&ja+DG+#>KrL2X zN`bbMx-zR4r96+IL=8tv(>$bM7YRR^oGQ3&4@tufVQ|;7mppHli5ZTV%1VZ=q^V3Z zvmDiSJk;x~y03RHcy^`Vvev_^rl$;!_mn~1QwEDYWv~UHX|QPL#bU cYx|$vsoL+?aen|mXB=f5YddFf7oT(g2Hj`L!2kdN literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/marlin.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/marlin.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f1f5fc028584de84ada7010e96faf6d0281f15b GIT binary patch literal 6937 zcmb7JUy~chamUX70kHUU0Pc7^Qlz~iqgYTxoJduagt2T%)E~P-o@JA^lTG5SJFMd78#fwu3BT+A`m3FPex+3UcWNyE3^d-s6aNx`m$(*|BF)tz=CVk4 z^~i7yjp!^iT@!CTw4$x?5Mc6*i)#+l*G+71b_>t*Gs`6+X7V-aQ{}xEoqY)WwCzntSo161REnrzKt!?Z>+N654g%K)WGU z(7wdk-BP#t3|g1EMy9_z+Ru#lgMN}3w}UvzD&LzV!Fc3{TU{-ye(Xy>5{Zy#)V%&Q zPR5Zpp2XA)MnU3*`wHiq8p$z`HG^f6r5$n*WztOZGqYmZBZ7T0lQGb*rkEN zSNWRyx~lrDiM-D_zOJa}_<4-CzI&c;h%)q9;};;);urZ#&~XElTG8VYzYNL+Zesnv zmeue0G7O59nEfPQ5vL#eaqJAn(#b#fY}>)_2Vx+k81={#+`<$8IY6qVC9jk+Kw!Jrb9=hnP#7aDVO*!yKb1aVkF|f$xB*U`n}7zl z08LI3EJZ2vGVls7!5WpVXMYd*&yMh$^>0lj=9+~2uRG&WxbG;{9a7!7e&dfFf5e?@ z*PU1Uxl=w2;NDIUJMnZf8OucQCG)-RY)=zsIL5P>zcU^<&nh%ri@-YoK?%{MTa_K) zaup!6f8>W#@qv_Mnbjt6WN+a26Evzx5Q^N6jVKs-BQM#JFlL;o=H8-NkNij8q}W|q zWg=wcg<{mzGb0w^;CQT0OY#ba{OpIDNi^A<1e0rtFNY$z_D9ilC%PUz+zf;5&B;FY z@@Vrx7)G1Xn2XR8k3@e8l{Q0vU&wf~SWUIR3qR;h_OpuTsX*af#31n{0L!wpHI3oV zV8$255!$-?Gi-f!ZrLq+^S*rz{pLTyO+3mk)`8Hf(yDWK#;%69{t5ZGu{GORmiFrL z8oWJQd)+zXhHrnSb-QfIGMaoD;+1)Fa1~9OkrWf@d4GylYysGsp;eB~=$><((Vg@s z-G56*2-lz~+sdqb=cr*m)iS*w#%IRGqIe^O<8Py1%m55_YRoyunK6l5*{cpDftB#E zAx?r%wJo@WiFNGnxRvceIQEn4Z@I>f43pfMJ{?!hZPp6pLVti;nebvAot z+3vkB>v2ZQxn<<55Cf|``*@+ii&W{5VA=$If8uWkVUX;9gbulB^x4G_$pNobvkW*y`=Z<56yw7iqOdTIHPH0GpLoEWSP0%VM*a&**iUJF{|1pUgS` zk+&?|o(4`!hX{$J=Tml|Tpukq)VR)#C;9UvJboWYg7r?c$1H9rj6G(uji|26c7oIVF}5VFm%Ij67}5@4 z+&J-(naCy4v1_HFiObn*cO(c4Uw(n4!%s zpRfS!kL6xp9!Y4^4(AHzqth%;>!D&sOqnfL5mm*E`=kp+0E~Y&W}N2kUZjJGI(lAa zdmbEficG=F>K^GD7GKJq$H#q;b@+K^Y>&sG{5_13>2V_EHwde#Brx|5N!}@G7P3d* zOcm?pCK0X^xIutcP?Dzd4++rrmEQ!&*l3hlca$dbU8256;CK-fMk^uP0Bd++>Q}*( znaLRYg6Zsc%&?y~UuFgzQeVf@VoXQ=pkLCMh0vnakL(Vzi6cw1kJigdQXfK2{Kzlf z!4v-@!09|6c{w5qa$}R5!rCqGR=9;+yDX5;BMYZIzlz7kx7wi2YrGEL`hvH?o5<2v zcnj1P!iHv@Q-jLLw_n@LOded$3CL+(e^3<7^ZxJUq`%(&7WZ>`0VSH2N z(v*+q&XtoxWJW3* zB7A8H)#{N4rFfLP+S=@;v)g`=%Sqae*+wv$Op!<+V0y?+DN{`XDh;#s@id`N3yNZj zvDX(pvRc8T>WMmKxlx>_A$_Xo=09>O3JZURC$=!hzcte`*DjO}DycI-l}@`95N zX=ArsaNYxFC1v=Yk5yjcB)$r_G4hz|9cmT5#*1FHoPMfTF3aZi)zZb%fer2KqUU4u zY>>uD4Rfy-a_x(yVg~ja){r#5SmI4^H2BIB)WE#OTRB}pJFW2clk!0^I&@_6-*{_e z&h^BobtS)^R(UhENDBINe!&hJX`Q#g|C^-AH?Rv@ftEJ@MfY#x3TQ2ixAqG* zr_NUaI+A9swbH>#(uVX-x}tPkg>;>yKVb(g@iyBzoBY{l*1zrl5 zf0+~LC0&3_doR=Ctom>)_hLvBDs*+vspHHo7pp5N0m2bDjZ~|x!_X=GXg2;RQ)S4`jL%t_ zzF0)jtlzd;gyY+hQAkCv=d5h~ccn(vNae^vehWF!gyAxRQU~LU3N(T&q#V}o>Si7J z?>y^T&R%;bXDcaZ**bHaQo^K;)0;pjk=@mHajnSN6P9Woif>T#C5o;n#u5cre@G7W zEmUf5LyG&;0Q<}vO5Bl}l4>ORR|J&f|21K>H1i``<-4br>GDL5`y!6#>*VPR!+;nas7!ynlK$$H8l&&PRv6qEvG2rQY)>bIN+YrVN^RrIS3w< zia$f&w3hP^wDYB>+Ce?lQ7UTkgS38FPIXoO)nQ*_OGQEz(j{Fnb}PAE7beY&2Z4B) znemQ45t)vwZ?{%Np`nN(*YG*_x{al+EFeY>2>1lH3DDxW#v|_rcP*9jM8$irc#qs_ zv7c|?HZ(_1Z#2`NOv3&Df~dtcnx%D^jSAb=nzRp&9KC`TDz@U{>xAd3?s?_Zmb4=B zeF7g4_>cglOR5-9!BO6(<{bhQ!tx@3qPXl6-~g_5PmJQR%rpKyA|HF5!mbjHO0g-P zn95el?VA=hS&`ja278WN|8wN>pBZKQf7XZ0HlEX01Gzso<#WSqex8?emop1jO>oz| z*w9Hhxm!Md5189gH(9;WsHaYI3bbmwxCUKaO3DspK#VnN_wuq^(siu+vZ9>4-0bC- zo4ru|b0e=YDMfbQ{0e53X%|U&ZyW#4D6WHK?f>feVBGh&7q_;ah_(f%?WyWo_wtLz zJf0?PwxxrJPN{sE|Aa8LxU>>4TB5A-&AeXx7M;-X2EazC!s=QbPfMfshE_3K`kMd0 DQKSDk literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/modelopt.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/modelopt.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87d5593ab49b052bb9d50540931ab20844528266 GIT binary patch literal 6188 zcma)AO_LMH8CK7XM&DZPvb!u__CON5WCyJ=wnGt@V~oKjhJ_Twj;CBT9@H)Eh#5)g z83C)(`VdGt7b*uId|i$l@)vTPw-Qy|s;cM0g{bD%qD6O6)$`#}bih5J%7t(_T5(sR zRd+Qy=pIyU#c(Y;qIS=8UqZdg7f@di z2T*?*^%`G9eNj|VcersqQ(yWHXUo*B)I8VSO05rrW|CT$gE&d^SNloO>-ynFou&4b z9v_4PrNT8|`jJS4M5PpTgTxDa?KY}LTD%-|g)cwwW0BT$`HDz3dz@O(zx0QJ-%Wy{ zPYa*#bz4Du+9qa3vUHJVeDdjezqvW?sGDic@Atzkuhl=}g=(X8r8!7KKaN#fJm~j( zG7&sIB*mZgPQtqrB%5C3-}U0Aj{_L#GL9m`&qUDPOr~c^4_*`UhUmsUd7Z{hJ6tuj_6k}^m^89F9+^|)NTk%dfYzF*iJVw$L3^=V2D>ftCDpsc z4`Ag3on^i(a@cu=uVA*#SNXyFmb;3Ug3#-&@k3}i$g@y}Bk7`I4kUlEe?|*v_>%s0 zn#5?Q)sxO-jC1kYna0KRXb^i4Q}b587jzRX3LyhO42F<}Ek!br-HoOJK21IrHeago0wP(5VylKynI^mKe{R4--`+Q2pe)KHS!coxmnttEE8H7Qy^%_lZkPUyi zKOOVKpG!e(V-3#QT2jO1&6cNB#A_-S+33W*u4d1Ege?$u@?s z>`&a@DXFbn<_tB9bx4bpOy5Ldl-Yrq60r}BZDV9`_?~T(npBJL@-u~)N8YDv8tu{0)IsTz5fXBJZ4 z8XlYQW7pVc=A>EWM^d{lMJu=~X?oK{Hu*?kR98ZK3kA-AQGi{xO_oWF5lhT_hFnOj zQHBIWEhB1KOR~C!Ja66HkYpkh^3Np+go2n|x7}rOYz=?X(onR>lNgy6gE;6yQr)JI z1!_1rbB2F|iFk$m0_N>P*u{&RJ8We5SHL1unSqQp)~5B7&6vWY4p|y;JJ!VttK5 z04?BoY0>i{1-ei!dvx4z+>`S>-fN=Sy4mZ6@(A`yODa5TZlq?MNcl36NvDt}Xh2qZ zPx%Wfn~2l$SJXw$RFbYpB36<|OO0+WRyiR!5o`)0)nbUa1$<9$6RPt&So+4615-l2>{p>?4$-lr+TaI%*Y+3#qG zk{hw>6C@RSNN$HM#lmN#LC)oU8u&M4&q_l{!ouwb0JoFtEaS0~|#aY@l3YkZS(_yCZ%hkt;phQ8== za34DZUTLz(jLyk+d{*x>j%{b18!)1Ye$JDi!`X|BhxGeAY0?rhjrS9F4}ikM)oJW! z>xQxwxqu;(ocfH@QF=?h?n|VYyoAllY&fL{v2r%XOwC2bl5{QT_6Goz&nk4g^>2KK3$gQQI3lM3yHQWbLyjL$tVw$Zb*Ky43Mr#32cT&n(RybBRv zmHenotGr-#Wo~w5Zgyc*q1n@mT5%(-Xgk51fY0fPK%NB~z9S%^(S(&+q-t(KbC(B^ zTO2bNWl3=}we=cMhD$V0wUx)~DubbS2PI8)rC&t68$_vz*q&B#e+$rxYd=pmWBCEC zcW6d>^TOBKMmIkdB;CrKQOHG_ofT0(*>abpXyaK#tB!sEk)+9~3CQi{)Wp+oy4hCH zZS_(U(7#@t!7@aMB=wgcQg(r|i^$w1aW|2^H$EP4m5e3D19fb7a%>&Ee^2eCTnI9B z^gY&*R-01z3En)EuZlTF#vdcIiiC-5{b%H#Qh>Hi1Y-PIPYjEG?Z@R@-mrn6EcDwy zt(s*vQ?qK!`hE_|%o*GK#&QLl&foaSUMVRLuz>>1f^`r+J1~QedC&X;*NJTtnAF_K zK2(Sm3d}r)7Q6>2b%xQP#ya*+Zf1l%MynU=%&pMFPas9?wWZJNc}amNB9zEzw)i9qmN*%sqJTSYP&<0&h-ZE_;59v|Pvh-381G{J2Tnn|xHJ=K;k{y1#;(#K$Q=;bJl!i?auWj*N}E zoo1EE%2@%q+ZK3p5)PMWvf3Z7*ao;+6bKhmKsEx zw4owP0q7H`x_KR&$H>}Unp7Jqme6aeQcJg_ZddvdN|J=#X zcP&|@!KYs^cn<&E#8cTgs>>45OE}McDMNLS?YGb_hsYf(pR3h`?j7#Zx$`$JeR}T3rK=m>_46NJxN_mTM#6+}m3a1( qeVTt49L0n7dlZ>sy34O+@`mfA$7t zMfn>C`A37nHE8kAAeh34ubB9zJ_)Ev3HDWA4Kz~=3T7eDO+6@@MLDnerJ!tb zo_~rkxCSl$3j|k8!W5G-Vyb0YR@n)rag7zAfzPfk{e?3!oog;-6;|EWOdUo=t}QDt zQ)6|QDe*GQRbbp;GuzrTVpe%Ar!~vwMT3L=h#u) zn>YCkd|zjaYze+UpZnG_TY;Gcc8ne0R?H);2pE2p&UN`fM4?0@gJJaLtuVQ49R2MuMh_LIZfav7GegWDlwD>BBM0rUVVe}Qh{&_vxNVlmN}ilY|_(-QTC z@;kuy8`&bDRoFsZ011v*=$g&$&4>5zbgg?2uXoq18*4w_y(y5%qZ8v}j2)9)Fqoa# z=&v#Q&hTO*j1pth_C0ns>2YJaX55PgJ~IM4ae77+awBpL)L!R)^w_|U#jRpWqd3(! zc*4WYw3-KI<-x|JAJQzF0lpdb?&QR&rXaMb+fUR!ynO;>*&&cl z!~*WMf&wAUZMcMAiU<9EBofXn+Zi)E{si@ogBYDTeERnp9dZXxAkip6CjmT=&n?;= zU0Uk_5#2%PV6rmCY(rwMbHlNHpEKjJm-G%qCFHyLZj{^#`aTbM2rf$+!L;i5@xjA2 z(a*pOEywQL>z?l=!&Bf#tb?GG67AjS)bzvcH*!FK-!r-7*LcYrfU-CNo!s^ZP+mYs z9GQ_V*Yi2vaNGiuSY9z5ou2N-L6cO^W*jt|Jm>>9iSrlG*-~CATaaHi$Se93vW6T{ zIfCr6rEXECKBtgJw5|+28FDSU7kMpW*3n~EM1l2)595|54EV4>^!7}0=auBGO8v~k zbD$YxLTbpeM?KzFHxq|B(qlmQVHI3Rfi zcaTsUrhTK`QU!XF zE+Rnr)H1$b@iDA|l8o-t&=;ShI6#QmUF&h-xClxTuNo;q>XF<$Ih_=8)3Mp1fSWK= zoWx!CSUD~BB`aeHwsGJ~R$f6n8TaquKr$E>sksp}^lafyqo@E)i!?;2CpllL%mR ziYf?nJ3b4d1a*aKz_K^MwYOBIZ>U!JuhD3#RGo#D1wz&1WJPb5YO1tsHN+g)r428@ zQUR5>JxDx1iv=Bi2fT12^Gy0E4~L$aC8)$suN{WcTk`@I>h}IZ-OdViJ0DH$bvx#h kkdk*POBfzv+Ue5Ll??u?NTwJGR8neWiPUI~med9He}g3fvj6}9 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/qqq.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/qqq.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32780cf879b3abc7085de3d960b23364a00b2be9 GIT binary patch literal 6996 zcmb7J&669)b;tA!2EgERu^-%}NNGgHiC{}uiBcq)krh%BEwj8_lFOAHkDW|r(GBb@ zF#y;zAh`&TN=)Sz<{yY%MOL|7(INjrPC4Y5^Hgq04m{=Hs#MvcxcR-F!N)FFuF7on zL-*^~uRq`G-|Ov{%T@;8+yDG8`~UTanaqDuVevCixQrD38wk&Et(yrnM+=z40^QLA z!!b1Cvu@VO;#cpQLC(o3p3%(*1;S)e+UgpIPb6(~ZlwSEP z!>hdZMTXbJ(nH;`QC{M8lm%MYVCe`65& z{hrs|ZfJ4grYF5XL_(rabX&tP>IZIr5K__a`H|c0cRHwLv9;y*geO1tLJ`-J{B;rS z_c?JOUcWW;dXYc!Nc$)KUfb`W`z5Nqx3zVB)=_-+lYOt(6I~^LyXAGoyl(w7*&hy0 zt69G3weARhOx4pG=J@n7k-s5ATlw?|-);-JV_yNK%ShpWfQXEv@r=W`=I8~+U*@_n zxPg=vS#I)NJJ0jH@I}@!xhZmO#w}j_!f^7Ukd{ll{Dp!2vJ|H%N(beQ4CE@js`wSf zH^8a!CB>TnV*5?m4j8jE^_F3jc=f} z$h5+O4h^hIYFzZPZydK1Fs!TS(Ds ziA(l=6b-^lo130|;NNTZWoL81+nX2Pee1pETi^ZeTkk6Q-MDyP_?`X8-SeX`wmP(1 zZs?ChZ2F-a9tN=~MKqMX?G|mu^rO6q^a)bc7lB)P7DpEy_aTJCHD_gt}~$Jtn~-6A~Xn!h{4vT`n@`56cqmAt91Qw0vKLmt{+>NrZQ3__ z-NTdYs_jWJZ6kPdbm3HG^Hp5bHRp!Je)-g<$)ZzzIUNG_0L7;^7TM2W-5X1C4S9Kn z$T~=T{`!@jt!vvZ6uP%Rzj^b<&hFJu-R;k>yB}ZMy&YRW^14HDRm#4M?NdcR+qv=i zP51US=j!cv{ZxtDKe-+k2QXN-?X@B(sxFA_Ufa4F7b3qal9O|0q3$}ywflpet5-0f zxE%OBx93LtQg}R!zd`PYiQJvj<*xjhd-KZ9m94F-TX8Ay9=HQIylz)?0n*)FFX5`6^h~9iZ814se2lqF-{@&)`5H7H{d9T|I zHiJGFT~|C1ts&;T+4T;E3^(sCxW&!8ckea_hjHF@m2-73poj215Ysd@Q?qoYSsK&K zBjd<2PS2OJrmp@BOCQy-iHV6frz%?-)J;Filkq9TBX7yI$2j~4WK8S{T@-A4^!iN6 z>kH@ldw(@~!=4{P6dMgaHsM18FPg8_mzr!quUe~wd#c71LXX%Rw};E@Hnn6R23zcHTKz(AB&yb@ zgiJ(E25Zy7+I*>PW5!wRWozHb@3p&qFM9jE*x2`QSl)*4$sw#$AE}EF+{Mv%d2~1s z;g2C4Qa>f8d_cNo-z5IR5T^?w-+~To+0&b|-cK7xofMPtw7h}0=Xd?+@Tb(^U69ea z*={#q6nmlP+2Z8UmTO!sOhX^NPguh4x9v9Um{vWsqrM#iAhh&Q6XkXaMCT#9Y zL4M${Ia*To;IjKOS-=gM6!tnx|m z*i1Mrn&0%J(>5~}Kt&h|j78&GM|;91EG;or!kvZNOMgfvq;JRB`*a~HC)?1RGL5h; zfu!z;!?0ntQ72Cgq9>a3zZ%-vLL`EF^jQIIDbu{D2sji2Ml=+8jwEl^=n` ztk;Xp>#Bd5B`$58{FKOi(N&I4k6Z;ghZK^+JZ8%Z#xus)bEdPeVG_?+*7~aQCNpXp z)7OybcSU2lbDEW98au+>bF^kEJuRp>_K{z_j1>Mo$Rc*;Oc2J^xdHT<1$s6QatC?f z(gM#3U}GR!!psFE3w6bIi5GbZvZa}9nOA^#DV{N$a-jn3JJqs;78QzV=ogyG^b2IZ z%vXTSYrF!>TT|z9^poV++sS%QVQ$k>nDvV6gqKu~VDF;$xn%FcL*Y~;+%$Bt6B}%* zCUI%8!7*<^dqqiEl!Z)!Y3Ej?N%GPSmKe?4W6yVu4$WU9sS^r2Uq4~NvpZdq$;In_ zZ!p9K1Gw!1ybzR$e2TW>wf-=osu@QK0ie%Y#Klxbq4g5Mpdd`Hr&bQyr(YczviKh& zg_kkGagJ*nnMr2QPEq%a0-J}?flYXIz!YCy^u7YL#! zeB{g&$S0C!D*4e&GN&X7u1_W5nxHe6O5TQKeoUY~(U4n^B)L4d&~uN%3M!?odb3u= zgnyz{qRZl?WdsHj3-hqjmOn?!vYKNNb1$WOErbtgn-#)R#({Y@!z+*|^Xel5=1}9c zgs-SI&hw>5xk)i1nG)&>O1etYsH^e%BYlDZB%J}1{|dQUC(*eCDScAzskYOZ(Fl|d zsx>C%aT$7+f59dQU}jqWJF4>yKQpfUO8WDx zwWyB%SI00h{H{^|r2P?_ERD6tXZY&4_=IUV?#Qo=OSAl$adnnoA1_f}1yN^#Eu);#ChAilUQYq#jce6rQY+6rswi}d^yc0F`H8hJlO?^oJDH#N2E}f3QJO6{y8z& zoPj*Cv1!qLWLvUeYy&n{*f`_bFc^ zeXkJt77-Q3zD~JUiBv)2tOy3tp_9dJ{LV#LRxG8>F(c6n`%=D7vUH8f3Xwkoak3*J z`=Pu>`MMMx;Mv5dVom8itiAwtZc(cpA{1Q6T_T@@ICb$LlAfC$Z!d~us8-3@_{O8^ z)O*7~Eozt?C~S0`yPK$xgkzmjx>hdj+`L5UY#uL{&;s~E)mzWatc6^OB`ug? z0IynzWZ4m8hGu-la#TaHE$DB|YN|^EI)PsEx1|E=JAQk4NQ9bh3k6;_P0sodlZs>ONnuhQS05WnFGA11gT8S! z(b3k&VvgDB~G~0?1?Lq1q?G-jciu74rK;J|pr2A{0=lphkITxlKiS z%2EC@31;Yyk!>OruR7*#;%im3(kJ$5fyxzB+(%N6UKFTQOc-(+sF*5HVPC;D{LaW( zzg_FGqVbGs4TLc;{AWhC@>LRxUWiRRt>Ew{L6c4$su)Z4Pl1+GS1&isUazUJ26=U5 zfsR4Xb&^o8?2=%S%FX0I9g^(Gc~xPoHEXosnztApZ*^SYWj2l literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/schema.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/schema.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1ee39ce62e7c5b9cbbf30372990eeefd85891e2 GIT binary patch literal 3238 zcmZ`*OK%*<5uW!Bhs!0!$PR5McGH0)Zy<5)*hX}43?-D6M3yMVdIT6G7!0R-cSoAX zvU`>i2g^(7tA9aYz&`pAAip3#WUe{+FZdAVtDarHBF$oIr(adoUwvKG<;qH6;Q8>M z{~q7!8ODF8asF}8xP@2kLU1EAxXG=|%&pMM?a(%>Zv5iDSwgA6-kA;ne@+`$a-BF-n7xXzvv zZshG3PgoKId4gz;HkMQ+yR&=q#^)d3|A@VxOr^x;GP08$29r{$bPr)H3-;xM?Yyr6 zoo_pd3-Yd>qi;3N?KG(x|G`{E-)WY9 zA1iU6KyQNWvKXc#J*D>q8n}k^Pe8X>)yPK&ZEqYzSL8 z-2BuKuJH8R8Cq``^rGKFzt3&%d~NccCbx5JhD)b?z6iL>jiJMrc<>dCEx%-JnRoSA zm#^SJtTyXkJlQ7H|4jUTsEOaWygHnSrZ)#BYJq9ocnGS0Z#E8Xukoa)rm{#(Y+>=x zHyz<^yoy20jHfuV&n(ruYjBHN|1h}yjeTtX-JBWlz}X2`hG2YkGZG)?9}SE8lY1@y zFNFoXtwgb4>mqneyUKpc>bIDTK_f{FF4zT?n|)I*!I?f>S0WpJZ^$(?e|$Ws^2uP5 zPHt4O9Es}2Z}RCl|1f_($kM&Rz@h$sPMVrvj{tF*7 z^Qm!U%`Ew3W^wZkR?$GM}YZbH|n^?Vi)R}e2>Vcsi>vbK> zTWSNne@BNp*l9Y|BvNTDnk&ULkH#tIqKH~F>$m4|=QwnfspKIWBMCp+Wwe6Tzh-;W ziru?+pNV3WBF`XCoFxP~Eebls`e-lKH0D-)dd`v7<4h=)#}l3KFiFP6GgfbqraaCl z%?ZB6K7WA(jr7jy71sK8osG(h)yu5F->{q(e5>y^9ZGECMb+4uC?wUj#!V}ctHy=m zk?8w!6`k@YBz{WbDv8%1G!SGx477y^PN~LBH7{Fng{JjdQUEoYtlmYRx(?x4j_KpK zX7)@=lWN`Wp$&?ic$W8H@1*N=&45O%dh?ZJTj;$&v<~#stBLjqAE!in|1!}KaF7g? zb1Q3xi~}NEIr3&84QOzUh&S^$t9bQ z*s7AvY81r<5YkmfMUkZB(Ada|jjOYrq@c-l5*~@4lX#oN+3xfgF-w~#Vk%k?7{0mT z`c~KUy{o3bzG3R|SC_3nV$u1sMYpuC&CsEug<{p=HWe%UJNREhS@KY{Iw(;7#RA2b zQ3ATnnyx;(ben2RkaoFmhF2%D#AO7BQKO_LaUxU`wCR|NROmlDh!QGba+$V8nYKM} z1o8!5Q(YopG_P;tkUtbL)?pSXVcqaTE&m%=A74!jmo5L z^qtGRCd=odt5e2W%iS}G?qZUn%tOpYg!StnNuUxr_2y;h=XL08{Z79t-yuVc#3l*4 zzJ%R&3X&kMLDHsv%1`naB#7NDAvG6&Ma`Gd@Ru0S0inPJMqrbJz6Ao($WDK`=|mAP z6PyVTzyLDMdu=ZCC-WP-q}Xb&TI%bQpj{v>FEiZupTd0{w$JFwq9}Jb#@fn;wPs)a F;D0l2TR;E+ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/tpu_int8.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/tpu_int8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..470be3208b864b21fc3a97c74fc6e9e3e1436271 GIT binary patch literal 5095 zcmaJ_&2t<_6`$_;*bl8%tCcKEu@fdCq}T-SmN6tDq=FL@f(u)5u#!+yz|?5FC5}=h@bNTQ8+WO7BVSG-V>BmCnD&F*uXoMkHWW>y4G52_EdS+~SR&0AVqi;NN zJO{sKREeuzRgYOwEv|cYP215--0&LltT(IsPSlL&yg5x*qER%{QNIx;GEnaXsVrJ0eN$#z13@ETZhtTek}Mnt zwBWTtvJq}#bPTS6;?7xWS=%Q{Z!DoiZ}fy8fkb;7#q#_DngqYd;MV)mi+fO)gBsrf9; z)f97jbPjwi*^-u=-{30-q!^GgFWUNRTP(;0u}IS8qBw_{j#v`svHrQKwU)&SMwWyP zbzdlEuP52-Yr|2ghH;O&?zDc~@{8M#f;4qE2FhK#^`ZMt(EmgxqN`@1fMP!AXW?C~ zPCxB$$ygSSlG#Wlt9=fJ!-r@S?=`%siza88ahIu?j0u*rKO2AL57?)ma>M7szy|s1 z__+_0bTk|eR3?RcEXXYdxgE1;zkO}(`bTfCU4MVozkTiJ@7{d(c2WN*h(_{Vr3OmP zK@wnLr83&kozD!lh_Np}?q%_?Hw=fDvOsOh?9yxTXe+)P-|I!;dT+P`%p|?LQ55%L z!iF#J%l;@Es9qH8NR{^POyQ`P4M% zd&t^nU^{$>rdMl<99A%`4X=7AMxWp_rWWu~(`9oG-J*)K_zC>s0(xJN)-85g>>FIN|X_Il7`2AoQtcOvU?L1E>a?o(j zVd$~@^r=ox=!5sfb7=4{$l#lRzj_A5lxPC&l$jT46MY@O{EaY@WbyPR@My(aeEh=G zTR6>LIOWI*KQ*xi?Los}8ANXE!B+*_Wcz%V=SI#2e?*8hS4;0=lOJ!E_Ozh}vHywO zNjs(@L#r34Avc^lj7I8=W-YhyGQOuIw#n;kymE5yQ+$Q3-LG(lN^XSrivd4PjZu+iEE!s}rgCTSKVeW_~TR-{s#M0?tcH1Hj2 zP8p*`2iF0?M;=15>48b(q0Zs>Z2a8G1D$4n3f+fjP%cS3M#5Qcrjd{W*-~g9(@6Ls z&09fp77k7`cbE?4o%0mCXu9tgb>BzaLxKg}@ab^TTwNr3M(2)FD}@;*nfewDH=yGz?3d$;+((g>6h#H~ZE7gZDw~?`Qq!U4SnZl3 z6saZ}LRb1dG*tvUZdqT!4Zq@M{YkU7XmYcKu|>wsi_AeJ<_8X|A1pa~W*?JIJQVn@ z;!XdCW;(kIuJgDlEM!ny*wPV>tO#UtSwS{ND{6S_vbJ5{Xz23B@@B*=`VG;P`j=?R z-N@T>qJ?#udYz`2#|SbwM&@esVi}{g?bfz~d``2^$$9#Pl%`&xsx!OowZ%N#a7AOZ zw6k0Kfm`bJC}ZoK-gS#ciP1B>p7c(yb`#23IuB2|22)os+To=q5ItoLZ3sfbn|O5q zf)ign!!?f*o+4i=mcwK?LXHT>(jQ9Y4+E8j6t;_{!3fDNJ08(YDg5(jMlmztq3J~f zbAvc7GfbbrbNJ9{C^Y{9Z~6wByt0Rsy3cm28O!*#nOV8{faN@Q5ZFus3+Gi~KeBdf zIoq>^lh^h+6}8ZHWwn^Sr>jICDNvx|WQu(KF*&TFrehVB2(rMdl{*z-?A0e51+9Jr zp28{D=s1OmP@o8N6VH+HFx!D=jbg8@LK7%Y%sTQlqI;X5wH2X>hnVfK^KLJZhmaN+ja~QmZSx-254RgM1RmLxzWz>^P zlww*~<3wH|Gmu#{!u|pxXWIty3EN})#vaeiZ3`wrp4l}8-?fC9!w}2H0t($_W7j_J z<N78D`&rgTOqZ@`&$N^2&B4w?y?3BRaP)7%;E7SHW9F_NwL83&x_c&vGZX_f0LY z26-6YtCRek6CiT*H|Q=>Uwur4#pZvvKL5i%{`c><9+Pt@LeJv^JZmT^O)_oe6Dfxv zs8m3P-vk4m|NNvgqj7=yVK^!{$|eeolfvoB_X{&vPdzJ$gZqVz02vPpTMb5u@GL~g z*S$)5XC&oVb|&^F!+DK~L3MET?8ILWv%-|a)T>|}DnCJ>Dk%w!+{Jy#xub)9QMpsj zK5hP1nYPFQONLd5(&f3sTgY4%Yq2J4vjz&-C4$326>k$+ZT#X{P;~T%X{2z=DtZAAOVRj6I9!efDdGH)3f+d0qRM9di5xNhO1mD``Fx5w; zK-WLIV_lj^(YMX+Ru+>ySB(<)?u{EaU6lLKXtz`KUOC!ESKdcv38|_NqAvDzgiztf zK^BjqVg_|TZkXkLQOC$z&|w*n%>n$Av^b0-|-a9*c21&?3SjN53u+L35Dr&1-( zXu-ln8aNFcI~We59SR%T>A2MZG$`8g18(vs)!(C9K_n*N=t6SkEXe3J$_jQty-YiK zg&Hb|)Fo;Ndu63Y5m;TOh5)3`r~{R{imvCZ$t1<4nbKwH4j+Ijc#@Qr*so~(82ZOW z7*)xX*-}bCRD_|7H<3OL^g&RJS-k!gpF?qW&_waoZt;foguYw&R>eUrv-zZ~wa*m} zE{QOFSqqvZQtwsQf_i5nueU)i=wk`z@*^Qd00m TqskkIzfc)#a71Nu@#X&mT{-oC literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/__pycache__/w8a16.cpython-310.pyc b/vllm/model_executor/layers/quantization/__pycache__/w8a16.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a134f3a804bebfb9aae757aa318493f79598283 GIT binary patch literal 4003 zcmaJEOLN=C8M}B81RtU(S`Wu@ls0`Zt;xD+9e0|hO5I61Q(H~Pp6Se9Fc5YrK?O*7 zv9u))bdr1NjBh>mU>|enwa5MtTzk?vx6DZH_bo_?)T7JU58t=@?dyAU)M$7HwEO@1 z&)#aoF#e0ge6xVqf}Z^Y05>>^jf60gP(~AznTf?Lg5xx{nGIhvc9IIK=rJpHlPdEx zZpXEx&gw~nH8kzS&18u!X}l7*l4Z7>v{_rzZoHCoSce#28NAB9R|faQ(y_^I0bS#D zpzERm^eU%Mjc(%vZf10?!u(=*SXjT0LRDCgqf8a)vDy^FQHn+I_3M9^{um_YoyaJRnrCeSGssfZNs{7uC0URE$!B`E(>Oj?b1E_{* z@+Mz;WgHW>BwD;BmW3_a1G)iPVUJaQ3wCw*Dqn+r z*7QlXK4+b8z{ol`K_BlGwZA+J?msByH2JXnZiaD?Wt)SM+|;u@ZxfmZg(HO;%i(rN z;MKE+IQB=-Gqgf(JT<-{+ur~vtW3lMxeTKkeD|!clC+;j={*(5U7_xMnvD07`^n3G z9PRYeL(t{0|00f)elp@B_Qin+$7&?|ad0SP)_*>y-~LOyeJ?#MT;Gp|k@Ecx-hl$J z%t>eNalIMCjW_>R-<*J0=(cRm?kn5pn0^E9kEcnIO5fiA>RWvpxqMp5aN}um6_&zD z%+_xx1$6CMAR~VW?r;Y$vJPPJq}lY9lki$vBXvNOu}`4WU2sF$f`P7C*dWz#k6C*` zJYeq5ARYzk{)1~DypE@2<1`&fCAc4i%IE4Z71;-{l@@oMOg@?y_e~3~TythPGl87^ z5w^Tp9Bte(AvubuX8W0(hPG#Y1Ue$zrZEhqOK_PGHaL0CE(ufZeUcv z)xKYNz7N4`93x)$@n!LJrsDg26hip81&azZQ&O(sVHijx1|WGGCv5Fq@;$`OXsG1- zh}A)`Dhf*xNp!uUrepPV1ob4*m?XMHL5IV_d7|$jZ5;g(f*&KekpFp5(;*L^lKl{X zYgmL@lzL}PyJ>dJrb$hYQ1cG)OiJiGhj{N+D|$@`>XhM?>8xHF-3>m8qSSVebyrSu!=JNtBGF1f{o1s( z#au`gty-)2H-Ko5^vLA)6;C@)O4>Dw^=O!mAoBo>RVg`d*l^dtw-J5n~r+r zCRC*C6v$xYfG^U^=B#UY0X)ii&nqHwGR+{ir^uFUjblkaiCSC#P>rPV_w4Fy7Lmin8|YypmhFjZXjCI&yQ98)G8x zf$5mA#?9R~X70Sjs>|L!c?a#}3;?6g@Q;84HE0s3u5%&3!p`=BRImyd1|o$C{|i_> z*8d88?cCk!oC3Gt`tl0^tn$1x53L4P1zq{&R^D_g?jOKzV|7_Yc95p=;V$fvVJ>kj zhukJ@__^={zb-TFf~@t5GAFhqW@Y&?f}bGxDFR(zVh+%TDSwXGCkXlobRs}MkiP}M zoM&Q~jbs_b|A506Mx-$vZZOlLeP%xcfbqL&O=C5sXB4|hPc73sr5{_T4#aQ>?`N(a zYo3;QdaZDvs*8rZWfsx-MBhcH56n!gt&7IqaM;rk2LGgK2V)l2>=Gl5{1O>IM7pzE^6iED+Gx*eCyodA{K9?ZCghX|W?QDzo>mTX}IcjCK+e65F_rajYUJt1uFj k8bKLeC)~pQtW4;iV+6`j-s^@->!eOw;BYMpU$fKwAK*HwJ^%m! literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/aqlm.py b/vllm/model_executor/layers/quantization/aqlm.py new file mode 100644 index 00000000..c35eac1a --- /dev/null +++ b/vllm/model_executor/layers/quantization/aqlm.py @@ -0,0 +1,374 @@ +# Supports AQLM compression, see https://github.com/Vahe1994/AQLM +# and https://arxiv.org/pdf/2401.06118.pdf + +import math +from typing import Any, Dict, List, Optional + +import torch +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.utils import set_weight_attrs + + +def get_int_dtype(nbits: int) -> torch.dtype: + if nbits <= 8: + return torch.int8 + if nbits <= 16: + return torch.int16 + if nbits <= 32: + return torch.int32 + if nbits <= 64: + return torch.int64 + raise ValueError(f"No dtype available for {nbits}-bit codebooks") + + +@torch.inference_mode() +def unpack_int_data(data: torch.IntTensor, nbits: int) -> torch.IntTensor: + return data.to(torch.int64) % (2**nbits) + + +def dequantize_weight(codes: torch.Tensor, + codebooks: torch.Tensor, + scales: Optional[torch.Tensor] = None) -> torch.Tensor: + """ + Decode float weights from quantization codes. Differentiable. + :param codes: tensor of integer quantization codes, shape + [*dims, num_out_groups, num_in_groups, num_codebooks] + :param codebooks: tensor of vectors for each quantization code, + [num_codebooks, codebook_size, out_group_size, in_group_size] + :param scales: weight will be multiplied by this factor, must be + broadcastble with + [*dims, out_groups, num_in_groups, out_group_size, in_group_size] + :return: reconstructed weight tensor of shape + [*dims, num_in_groups*group_size] + """ + num_out_groups, num_in_groups, num_codebooks = codes.shape[-3:] + num_codebooks, codebook_size, out_group_size, in_group_size = \ + codebooks.shape + out_features = num_out_groups * out_group_size + in_features = num_in_groups * in_group_size + codebook_offsets = torch.arange( + 0, num_codebooks * codebook_size, codebook_size, + device=codes.device) # shape: [num_codebooks] + reconstructed_weight_flat = F.embedding_bag( + codes.flatten(0, -2) + codebook_offsets, + codebooks.flatten(0, 1).flatten(-2, -1), + mode="sum" + ) # [prod(dims) * num_out_groups * num_in_groups, out_group_size + # * in_group_size] + + reconstructed_weight_groupwise = reconstructed_weight_flat.view( + list(codes.shape[:-3]) + + [num_out_groups, num_in_groups, out_group_size, in_group_size]) + if scales is not None: + reconstructed_weight_groupwise = reconstructed_weight_groupwise.mul( + scales) + return reconstructed_weight_groupwise.swapaxes( + -3, -2).reshape(list(codes.shape[:-3]) + [out_features, in_features]) + + +def dequantize_gemm( + input: torch.Tensor, # [..., in_features] + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks] + codebooks: torch. + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size] + scales: torch.Tensor, # [num_out_groups, 1, 1, 1] + bias: Optional[torch.Tensor], +) -> torch.Tensor: + dequantized_weight = dequantize_weight( + unpack_int_data(codes, codebooks.shape[1].bit_length() - 1), + codebooks, + scales, + ) + return F.linear(input, dequantized_weight, bias) + + +# Generic dequantization, slow but flexible. +def generic_dequantize_gemm( + input: torch.Tensor, # [..., in_features] + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks] + codebooks: torch. + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size] + scales: torch.Tensor, # [num_out_groups, 1, 1, 1] + output_partition_sizes: List[int], + bias: Optional[torch.Tensor], +) -> torch.Tensor: + output_shape = input.shape[:-1] + (scales.shape[0], ) + output = torch.empty(output_shape, dtype=input.dtype, device=input.device) + num_outputs = len(output_partition_sizes) + + # break the inputs and codebooks apart then combine the outputs. + # Surprisingly (to me) this is faster than doing 3 de-quants and 1 big + # multiply at the end. + num_codebooks = codebooks.shape[0] // num_outputs + assert (scales.shape[0] == codes.shape[0]) + assert (sum(output_partition_sizes) == scales.shape[0]) + output_offset = 0 + codebooks_offset = 0 + for output_size in output_partition_sizes: + shard_output = dequantize_gemm( + input, codes.narrow(0, output_offset, output_size), + codebooks.narrow(0, codebooks_offset, num_codebooks), + scales.narrow(0, output_offset, output_size), None + if bias is None else bias.narrow(0, output_offset, output_size)) + + output_slice = output.narrow(-1, output_offset, output_size) + assert (output_slice.shape == shard_output.shape) + output_slice.copy_(shard_output) + output_offset += output_size + codebooks_offset += num_codebooks + return output + + +# Optimized dequnantize/decompression kernels, supports 1x16 and 2x8 +# at 6 and 9 times faster than the generic version above, respectively. +def optimized_dequantize_gemm( + input: torch.Tensor, # [..., in_features] + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks] + codebooks: torch. + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size] + scales: torch.Tensor, # [num_out_groups, 1, 1, 1] + output_partition_sizes: List[int], + bias: Optional[torch.Tensor], +) -> torch.Tensor: + weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes) + + if bias is None: + # scaling the output is fastest, so we do that when possible. + output = F.linear(input, weights, bias) + orig_shape = output.shape + flattened_output = output.view(-1, output.size(-1)) + f_scales = scales.view(-1, scales.shape[0]) + b_scales = f_scales.expand(flattened_output.shape[0], -1) + flattened_output *= b_scales + return output.view(orig_shape) + else: + b_scales = scales.view(scales.shape[:-3] + (-1, )).expand( + -1, weights.shape[1]) + weights *= b_scales + return F.linear(input, weights, bias) + + +class AQLMConfig(QuantizationConfig): + """Config class for AQLM. + + Reference: https://github.com/Vahe1994/AQLM + """ + + def __init__( + self, + in_group_size: int, + nbits_per_codebook: int, + num_codebooks: int, + out_group_size: int, + ) -> None: + self.in_group_size = in_group_size + self.nbits_per_codebook = nbits_per_codebook + self.num_codebooks = num_codebooks + self.out_group_size = out_group_size + + # out_group_size > 1 is untested, and probably won't work as-is. + assert (self.out_group_size == 1) + self.pack_factor = (self.in_group_size * self.out_group_size) + + def __repr__(self) -> str: + return (f"AQLMConfig(in_group_size={self.in_group_size}, " + f"nbits_per_codebook={self.nbits_per_codebook}, " + f"num_codebooks={self.num_codebooks}, " + f"out_group_size={self.out_group_size})") + + @classmethod + def get_name(cls) -> str: + return "aqlm" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 60 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] # no extra configs. + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "AQLMConfig": + in_group_size = cls.get_from_keys(config, ["in_group_size"]) + nbits_per_codebook = cls.get_from_keys(config, ["nbits_per_codebook"]) + num_code_books = cls.get_from_keys(config, ["num_codebooks"]) + out_group_size = cls.get_from_keys(config, ["out_group_size"]) + return cls(in_group_size, nbits_per_codebook, num_code_books, + out_group_size) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["AQLMLinearMethod"]: + if isinstance(layer, LinearBase): + return AQLMLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class AQLMLinearMethod(LinearMethodBase): + """Linear method for AQLM. + + Args: + quant_config: The AQLM quantization config. + """ + + def __init__(self, quant_config: AQLMConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + del output_size # Unused. + del input_size # Unused. + + if params_dtype != torch.half: + raise ValueError("Only half is currently supported by aqlm") + if input_size_per_partition % self.quant_config.in_group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.out_group_size != 0: + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + + codes = Parameter( + torch.empty( + # There could actually be two pack factors, one along input and + # one along output, but we don't currently support + # out_group_size, and only the one along output needs to be + # marked with "packed_dim" in order for QKVLinear to work. + output_size_per_partition, + input_size_per_partition // self.quant_config.pack_factor, + self.quant_config.num_codebooks, + dtype=get_int_dtype(self.quant_config.nbits_per_codebook), + ), + requires_grad=False, + ) + + set_weight_attrs( + codes, + { + "input_dim": 1, + "output_dim": 0, + "packed_dim": 1, + "pack_factor": self.quant_config.pack_factor, + }, + ) + + codebooks = Parameter( + torch.empty( + self.quant_config.num_codebooks * len(output_partition_sizes), + 2**self.quant_config.nbits_per_codebook, + self.quant_config.out_group_size, + self.quant_config.in_group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs( + codebooks, + { + # metadata indicates fixed size concatenated along dim 0 + "is_metadata": True, + "output_partition_sizes": output_partition_sizes + }, + ) + + scales = Parameter( + torch.empty( + ( + output_size_per_partition // + self.quant_config.out_group_size, + 1, + 1, + 1, + ), + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs( + scales, + { + "output_dim": 0, + "packed_dim": 0, + "pack_factor": self.quant_config.out_group_size + }, + ) + + layer.register_parameter("codes", codes) + set_weight_attrs(codes, extra_weight_attrs) + layer.register_parameter("codebooks", codebooks) + set_weight_attrs(codebooks, extra_weight_attrs) + layer.register_parameter("scales", scales) + set_weight_attrs(scales, extra_weight_attrs) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + codebooks = layer.codebooks + codes = layer.codes + scales = layer.scales + output_partition_sizes = getattr(codebooks, "output_partition_sizes", + []) + + nbooks = codes.shape[2] + ingroups = codebooks.shape[3] + outgroups = codebooks.shape[2] + bits = codebooks.shape[1] + + # We support these formats with dedicated gemm and decompression + # kernels. + if ingroups == 8 and outgroups == 1 and ( + (bits == 256 and nbooks == 2) or (bits == 65536 and nbooks == 1)): + + # thresholds determined by timings on an A6000, one GPU + use_gemv = math.prod(x.shape[:-1]) <= 6 + + return ops.aqlm_gemm( + x, + codes, + codebooks, + scales, + output_partition_sizes, + bias, + ) if use_gemv else optimized_dequantize_gemm( + x, + codes, + codebooks, + scales, + output_partition_sizes, + bias, + ) + + # fall back all unoptimized formats + return generic_dequantize_gemm( + x, + codes, + codebooks, + scales, + output_partition_sizes, + bias, + ) diff --git a/vllm/model_executor/layers/quantization/awq.py b/vllm/model_executor/layers/quantization/awq.py new file mode 100644 index 00000000..8849e4b0 --- /dev/null +++ b/vllm/model_executor/layers/quantization/awq.py @@ -0,0 +1,173 @@ +from typing import Any, Dict, List, Optional + +import torch + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.parameter import (GroupQuantScaleParameter, + PackedvLLMParameter) + + +class AWQConfig(QuantizationConfig): + """Config class for AWQ. + + Reference: https://arxiv.org/abs/2306.00978 + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + zero_point: bool, + ) -> None: + self.weight_bits = weight_bits + self.group_size = group_size + self.zero_point = zero_point + + if self.weight_bits != 4: + raise ValueError( + "Currently, only 4-bit weight quantization is supported for " + f"AWQ, but got {self.weight_bits} bits.") + self.pack_factor = 32 // self.weight_bits + + def __repr__(self) -> str: + return (f"AWQConfig(weight_bits={self.weight_bits}, " + f"group_size={self.group_size}, " + f"zero_point={self.zero_point})") + + def get_name(self) -> str: + return "awq" + + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + # The AWQ kernel only supports Turing or newer GPUs. + return 75 + + @staticmethod + def get_config_filenames() -> List[str]: + return [ + "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq + # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq + "quantize_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "AWQConfig": + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"]) + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"]) + zero_point = cls.get_from_keys(config, ["zero_point"]) + return cls(weight_bits, group_size, zero_point) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["AWQLinearMethod"]: + if isinstance(layer, LinearBase): + return AWQLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"] + + +class AWQLinearMethod(LinearMethodBase): + """Linear method for AWQ. + + Args: + quant_config: The AWQ quantization config. + """ + + def __init__(self, quant_config: AWQConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + if input_size_per_partition % self.quant_config.group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + + weight_loader = extra_weight_attrs.get("weight_loader") + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + qzeros = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + scales = GroupQuantScaleParameter(data=torch.empty( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.qweight = torch.nn.Parameter(layer.qweight.data, + requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, + requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, + requires_grad=False) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + qweight = layer.qweight + scales = layer.scales + qzeros = layer.qzeros + pack_factor = self.quant_config.pack_factor + out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, )) + reshaped_x = x.reshape(-1, x.shape[-1]) + + # num_tokens >= threshold + # FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256 + FP16_MATMUL_HEURISTIC_CONDITION = False + + if FP16_MATMUL_HEURISTIC_CONDITION: + out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) + out = torch.matmul(reshaped_x, out) + else: + out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, + pack_factor, group_size=self.quant_config.group_size) + if bias is not None: + out.add_(bias) + return out.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/awq_marlin.py b/vllm/model_executor/layers/quantization/awq_marlin.py new file mode 100644 index 00000000..b3d93b28 --- /dev/null +++ b/vllm/model_executor/layers/quantization/awq_marlin.py @@ -0,0 +1,464 @@ +from typing import Any, Callable, Dict, List, Optional + +import torch +from torch.nn import Parameter + +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.layer import ( + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported) +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, + set_weight_attrs) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.layers.quantization.utils import replace_parameter +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported, + marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales, + marlin_permute_scales, moe_awq_to_marlin_zero_points, + verify_marlin_supported, verify_marlin_supports_shape) +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.parameter import (GroupQuantScaleParameter, + PackedvLLMParameter) +from vllm.platforms import current_platform +from vllm.scalar_type import scalar_types + +logger = init_logger(__name__) + + +class AWQMarlinConfig(QuantizationConfig): + """Config class for AWQ Marlin""" + + # num_bits -> type + TYPE_MAP = { + 4: scalar_types.uint4, + 8: scalar_types.uint8, + } + + def __init__(self, weight_bits: int, group_size: int, has_zp: bool, + lm_head_quantized: bool) -> None: + self.pack_factor = 32 // weight_bits # packed into int32 + self.group_size = group_size + self.has_zp = has_zp + self.lm_head_quantized = lm_head_quantized + self.weight_bits = weight_bits + + if self.weight_bits not in self.TYPE_MAP: + raise ValueError(f"Unsupported num_bits = {self.weight_bits}. " + f"Supported num_bits = {self.TYPE_MAP.keys()}") + + self.quant_type = self.TYPE_MAP[self.weight_bits] + + verify_marlin_supported(self.quant_type, + group_size=self.group_size, + has_zp=self.has_zp) + + def __repr__(self) -> str: + return (f"AWQMarlinConfig(quant_type={self.quant_type}, " + f"group_size={self.group_size}, " + f"has_zp={self.has_zp}, " + f"lm_head_quantized={self.lm_head_quantized})") + + @classmethod + def get_name(cls) -> str: + return "awq_marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "AWQMarlinConfig": + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + has_zp = cls.get_from_keys(config, ["zero_point"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], + default=False) + return cls(weight_bits, group_size, has_zp, lm_head_quantized) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg) + is_valid_user_quant = (user_quant is None or user_quant == "marlin" + or user_quant == "awq_marlin") + + if can_convert and is_valid_user_quant: + msg = ("The model is convertible to {} during runtime." + " Using {} kernel.".format(cls.get_name(), cls.get_name())) + logger.info(msg) + return cls.get_name() + + if can_convert and user_quant == "awq": + logger.info("Detected that the model can run with awq_marlin" + ", however you specified quantization=awq explicitly," + " so forcing awq. Use quantization=awq_marlin for" + " faster inference") + return None + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + if (isinstance(layer, LinearBase) or + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)): + return AWQMarlinLinearMethod(self) + elif isinstance(layer, FusedMoE): + return AWQMoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + @classmethod + def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]): + # Extract data from quant config. + quant_method = quant_config.get("quant_method", "").lower() + num_bits = quant_config.get("bits") + group_size = quant_config.get("group_size") + has_zp = quant_config.get("zero_point") + + if not current_platform.is_cuda(): + return False + + if quant_method != "awq": + return False + + # If we cannot find the info needed in the config, cannot convert. + if (num_bits is None or group_size is None or has_zp is None): + return False + + if num_bits not in cls.TYPE_MAP: + return False + + return check_marlin_supported(quant_type=cls.TYPE_MAP[num_bits], + group_size=group_size, + has_zp=has_zp) + + +class AWQMarlinLinearMethod(LinearMethodBase): + """Linear method for AWQ Marlin. + + Args: + quant_config: The AWQ Marlin quantization config. + """ + + def __init__(self, quant_config: AWQMarlinConfig) -> None: + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del output_size + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + # Normalize group_size + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + verify_marlin_supports_shape( + output_size_per_partition=output_size_per_partition, + input_size_per_partition=input_size_per_partition, + input_size=input_size, + group_size=group_size) + + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + num_groups = input_size_per_partition // group_size + + qzeros = PackedvLLMParameter( + data=torch.empty( + num_groups, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + scales = GroupQuantScaleParameter(data=torch.empty( + num_groups, + output_size_per_partition, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.num_groups = num_groups + + # TODO: Update this docs + # Checkpoints are serialized in AutoAWQ format, which is different from the + # marlin format. This function is called after the weights are loaded. + # Here, we handle the repacking + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + device = layer.qweight.device + layer.qweight = torch.nn.Parameter(layer.qweight.data, + requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, + requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, + requires_grad=False) + + # Allocate marlin workspace + layer.workspace = marlin_make_workspace( + layer.output_size_per_partition, device) + + # Repack weights from AWQ format to marlin format. + marlin_qweight = ops.awq_marlin_repack( + layer.qweight, + size_k=layer.input_size_per_partition, + size_n=layer.output_size_per_partition, + num_bits=self.quant_config.quant_type.size_bits) + replace_parameter(layer, "qweight", marlin_qweight) + + # Permute scales from AWQ format to marlin format. + marlin_scales = marlin_permute_scales( + layer.scales, + size_k=layer.input_size_per_partition, + size_n=layer.output_size_per_partition, + group_size=self.quant_config.group_size) + replace_parameter(layer, "scales", marlin_scales) + + # Permute zero-points from AWQ format to marlin format. + marlin_zp = awq_to_marlin_zero_points( + layer.qzeros, + size_k=layer.num_groups, + size_n=layer.output_size_per_partition, + num_bits=self.quant_config.quant_type.size_bits) + replace_parameter(layer, "qzeros", marlin_zp) + + # Not-used + layer.g_idx = marlin_make_empty_g_idx(device) + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_awq_marlin_linear( + input=x, + weight=layer.qweight, + weight_scale=layer.scales, + weight_zp=layer.qzeros, + g_idx=layer.g_idx, + g_idx_sort_indices=layer.g_idx_sort_indices, + workspace=layer.workspace, + quant_type=self.quant_config.quant_type, + output_size_per_partition=layer.output_size_per_partition, + input_size_per_partition=layer.input_size_per_partition, + bias=bias) + + +class AWQMoEMethod(FusedMoEMethodBase): + + def __init__(self, quant_config: AWQMarlinConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + extra_weight_attrs.update({ + "is_transposed": + True, + "quant_method": + FusedMoeWeightScaleSupported.GROUP.value, + }) + + w13_qweight = Parameter(torch.empty(num_experts, + hidden_size, + 2 * intermediate_size // + self.quant_config.pack_factor, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w13_qweight", w13_qweight) + set_weight_attrs(w13_qweight, extra_weight_attrs) + + w2_qweight = Parameter(torch.empty(num_experts, + intermediate_size, + hidden_size // + self.quant_config.pack_factor, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w2_qweight", w2_qweight) + set_weight_attrs(w2_qweight, extra_weight_attrs) + + num_groups_w13 = hidden_size // self.quant_config.group_size + num_groups_w2 = intermediate_size // self.quant_config.group_size + + # WEIGHT_SCALES + # Allocate 2 scales for w1 and w3 respectively. + w13_scales = Parameter(torch.empty(num_experts, + num_groups_w13, + intermediate_size * 2, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_scales", w13_scales) + set_weight_attrs(w13_scales, extra_weight_attrs) + + w2_scales = Parameter(torch.empty(num_experts, + num_groups_w2, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_scales", w2_scales) + set_weight_attrs(w2_scales, extra_weight_attrs) + + # WEIGHT_ZERO_POINT + # Allocate 2 zero points for w1 and w3 respectively. + w13_qzeros = Parameter(torch.empty(num_experts, + num_groups_w13, + 2 * intermediate_size // + self.quant_config.pack_factor, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w13_qzeros", w13_qzeros) + set_weight_attrs(w13_qzeros, extra_weight_attrs) + + w2_qzeros = Parameter(torch.empty(num_experts, + num_groups_w2, + hidden_size // + self.quant_config.pack_factor, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w2_qzeros", w2_qzeros) + set_weight_attrs(w2_qzeros, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + num_experts = layer.w13_qweight.shape[0] + device = layer.w13_qweight.device + + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + + marlin_w13_qweight = ops.awq_marlin_moe_repack( + layer.w13_qweight, + layer.w13_g_idx_sort_indices, + size_k=layer.w13_qweight.shape[1], + size_n=layer.w13_qweight.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w13_qweight", marlin_w13_qweight) + + marlin_w2_qweight = ops.awq_marlin_moe_repack( + layer.w2_qweight, + layer.w2_g_idx_sort_indices, + size_k=layer.w2_qweight.shape[1], + size_n=layer.w2_qweight.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w2_qweight", marlin_w2_qweight) + + # Why does this take the intermediate size for size_k? + marlin_w13_scales = marlin_moe_permute_scales( + s=layer.w13_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w13_scales.shape[2], + group_size=self.quant_config.group_size, + ) + + replace_parameter(layer, "w13_scales", marlin_w13_scales) + + marlin_w2_scales = marlin_moe_permute_scales( + s=layer.w2_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w2_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w2_scales", marlin_w2_scales) + + marlin_w13_zp = moe_awq_to_marlin_zero_points( + layer.w13_qzeros, + size_k=layer.w13_qzeros.shape[1], + size_n=layer.w13_qzeros.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits) + replace_parameter(layer, "w13_qzeros", marlin_w13_zp) + + marlin_w2_zp = moe_awq_to_marlin_zero_points( + layer.w2_qzeros, + size_k=layer.w2_qzeros.shape[1], + size_n=layer.w2_qzeros.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits) + replace_parameter(layer, "w2_qzeros", marlin_w2_zp) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + fused_marlin_moe) + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_marlin_moe( + x, + layer.w13_qweight, + layer.w2_qweight, + layer.w13_scales, + layer.w2_scales, + router_logits, + topk_weights, + topk_ids, + w1_zeros=layer.w13_qzeros, + w2_zeros=layer.w2_qzeros, + num_bits=self.quant_config.weight_bits, + ) diff --git a/vllm/model_executor/layers/quantization/awq_triton.py b/vllm/model_executor/layers/quantization/awq_triton.py new file mode 100644 index 00000000..bbb7fc8a --- /dev/null +++ b/vllm/model_executor/layers/quantization/awq_triton.py @@ -0,0 +1,317 @@ +import torch +import triton +import triton.language as tl + +AWQ_TRITON_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + + +@triton.jit +def awq_dequantize_kernel( + qweight_ptr, # quantized matrix + scales_ptr, # scales, per group + zeros_ptr, # zeros, per group + group_size, # Should always be one of the supported group sizes + result_ptr, # Output matrix + num_cols, # input num cols in qweight + num_rows, # input num rows in qweight + BLOCK_SIZE_X: tl.constexpr, + BLOCK_SIZE_Y: tl.constexpr): + # Setup the pids. + pid_x = tl.program_id(axis=0) + pid_y = tl.program_id(axis=1) + + # Compute offsets and masks for qweight_ptr. + offsets_y = pid_y * BLOCK_SIZE_Y + tl.arange(0, BLOCK_SIZE_Y) + offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X) + offsets = num_cols * offsets_y[:, None] + offsets_x[None, :] + + masks_y = offsets_y < num_rows + masks_x = offsets_x < num_cols + + masks = masks_y[:, None] & masks_x[None, :] + + # Compute offsets and masks for result output ptr. + result_offsets_y = pid_y * BLOCK_SIZE_Y + tl.arange(0, BLOCK_SIZE_Y) + result_offsets_x = pid_x * BLOCK_SIZE_X * 8 + tl.arange( + 0, BLOCK_SIZE_X * 8) + result_offsets = (8 * num_cols * result_offsets_y[:, None] + + result_offsets_x[None, :]) + + result_masks_y = result_offsets_y < num_rows + result_masks_x = result_offsets_x < num_cols * 8 + result_masks = result_masks_y[:, None] & result_masks_x[None, :] + + # Load the weights. + iweights = tl.load(qweight_ptr + offsets, masks) + iweights = tl.interleave(iweights, iweights) + iweights = tl.interleave(iweights, iweights) + iweights = tl.interleave(iweights, iweights) + + # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7] + # that will map given indices to the correct order. + reverse_awq_order_tensor = ((tl.arange(0, 2) * 4)[None, :] + + tl.arange(0, 4)[:, None]).reshape(8) + + # Use this to compute a set of shifts that can be used to unpack and + # reorder the values in iweights and zeros. + shifts = reverse_awq_order_tensor * 4 + shifts = tl.broadcast_to(shifts[None, :], (BLOCK_SIZE_Y * BLOCK_SIZE_X, 8)) + shifts = tl.reshape(shifts, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Unpack and reorder: shift out the correct 4-bit value and mask. + iweights = (iweights >> shifts) & 0xF + + # Compute zero offsets and masks. + zero_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1) + zero_offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X) + zero_offsets = num_cols * zero_offsets_y[:, None] + zero_offsets_x[None, :] + + zero_masks_y = zero_offsets_y < num_rows // group_size + zero_masks_x = zero_offsets_x < num_cols + zero_masks = zero_masks_y[:, None] & zero_masks_x[None, :] + + # Load the zeros. + zeros = tl.load(zeros_ptr + zero_offsets, zero_masks) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Unpack and reorder: shift out the correct 4-bit value and mask. + zeros = (zeros >> shifts) & 0xF + + # Compute scale offsets and masks. + scale_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1) + scale_offsets_x = (pid_x * BLOCK_SIZE_X * 8 + + tl.arange(0, BLOCK_SIZE_X * 8)) + scale_offsets = (num_cols * 8 * scale_offsets_y[:, None] + + scale_offsets_x[None, :]) + scale_masks_y = scale_offsets_y < num_rows // group_size + scale_masks_x = scale_offsets_x < num_cols * 8 + scale_masks = scale_masks_y[:, None] & scale_masks_x[None, :] + + # Load the scales. + scales = tl.load(scales_ptr + scale_offsets, scale_masks) + scales = tl.broadcast_to(scales, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Dequantize. + iweights = (iweights - zeros) * scales + iweights = iweights.to(result_ptr.type.element_ty) + + # Finally, store. + tl.store(result_ptr + result_offsets, iweights, result_masks) + + +@triton.jit +def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K, + group_size, BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, + SPLIT_K: tl.constexpr): + pid = tl.program_id(axis=0) + pid_z = tl.program_id(1) + + # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. + # num_pid_n = (N + BLOCK_SIZE_N - 1) // BLOCK_SIZE_N + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + + pid_m = pid // num_pid_n + pid_n = pid % num_pid_n + + accumulator_dtype = c_ptr.type.element_ty + + # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. + # accumulator = tl.arange(0, BLOCK_SIZE_N) + # accumulator = tl.broadcast_to(accumulator[None, :], + # (BLOCK_SIZE_M, BLOCK_SIZE_N)) + # accumulator = accumulator & 0x0 + # accumulator = accumulator.to(accumulator_dtype) + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), + dtype=accumulator_dtype) + + # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7] + # that will map given indices to the correct order. + reverse_awq_order_tensor = ((tl.arange(0, 2) * 4)[None, :] + + tl.arange(0, 4)[:, None]).reshape(8) + + # Create the necessary shifts to use to unpack. + shifts = reverse_awq_order_tensor * 4 + shifts = tl.broadcast_to(shifts[None, :], + (BLOCK_SIZE_K * (BLOCK_SIZE_N // 8), 8)) + shifts = tl.reshape(shifts, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + # Offsets and masks. + offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + masks_am = offsets_am < M + + offsets_bn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) + masks_bn = offsets_bn < N // 8 + + offsets_zn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) + masks_zn = offsets_zn < N // 8 + + offsets_sn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + masks_sn = offsets_sn < N + + offsets_k = pid_z * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + offsets_a = K * offsets_am[:, None] + offsets_k[None, :] + offsets_b = (N // 8) * offsets_k[:, None] + offsets_bn[None, :] + + a_ptrs = a_ptr + offsets_a + b_ptrs = b_ptr + offsets_b + + # NOTE: Use this in TRITON_INTERPRET=1 mode instead of tl.cdiv + # block_offset = BLOCK_SIZE_K * SPLIT_K + # for k in range(0, (K + block_offset - 1) // (block_offset)): + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)): + masks_k = offsets_k < K + masks_a = masks_am[:, None] & masks_k[None, :] + a = tl.load(a_ptrs, mask=masks_a) + + masks_b = masks_k[:, None] & masks_bn[None, :] + b = tl.load(b_ptrs, mask=masks_b) + b = tl.interleave(b, b) + b = tl.interleave(b, b) + b = tl.interleave(b, b) + + # Dequantize b. + offsets_szk = ( + (BLOCK_SIZE_K * SPLIT_K * k + pid_z * BLOCK_SIZE_K) // group_size + + tl.arange(0, 1)) + offsets_z = (N // 8) * offsets_szk[:, None] + offsets_zn[None, :] + masks_zk = offsets_szk < K // group_size + masks_z = masks_zk[:, None] & masks_zn[None, :] + zeros_ptrs = zeros_ptr + offsets_z + zeros = tl.load(zeros_ptrs, mask=masks_z) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + offsets_s = N * offsets_szk[:, None] + offsets_sn[None, :] + masks_sk = offsets_szk < K // group_size + masks_s = masks_sk[:, None] & masks_sn[None, :] + scales_ptrs = scales_ptr + offsets_s + scales = tl.load(scales_ptrs, mask=masks_s) + scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + b = (b >> shifts) & 0xF + zeros = (zeros >> shifts) & 0xF + b = (b - zeros) * scales + b = b.to(c_ptr.type.element_ty) + + # Accumulate results. + accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) + + offsets_k += BLOCK_SIZE_K * SPLIT_K + a_ptrs += BLOCK_SIZE_K * SPLIT_K + b_ptrs += BLOCK_SIZE_K * SPLIT_K * (N // 8) + + c = accumulator.to(c_ptr.type.element_ty) + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + pid_z * N * M + N * offs_cm[:, None] + offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +# qweights - [K , M // 8], int32 +# scales - [K // G, M ], float16 +# zeros - [K // G, M // 8], int32 +def awq_dequantize_triton(qweight: torch.Tensor, + scales: torch.Tensor, + zeros: torch.Tensor, + block_size_x: int = 32, + block_size_y: int = 32) -> torch.Tensor: + K = qweight.shape[0] + M = scales.shape[1] + group_size = qweight.shape[0] // scales.shape[0] + + assert K > 0 and M > 0 + assert scales.shape[0] == K // group_size and scales.shape[1] == M + assert zeros.shape[0] == K // group_size and zeros.shape[1] == M // 8 + assert group_size <= K + assert group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES or group_size == K + + # Result tensor: + # number of rows = same as input tensor + # number of cols = 8 x input tensor num cols + result = torch.empty(qweight.shape[0], + qweight.shape[1] * 8, + device=qweight.device, + dtype=scales.dtype) + + Y = qweight.shape[0] # num rows + X = qweight.shape[1] # num cols + + grid = lambda META: ( + triton.cdiv(X, META['BLOCK_SIZE_X']), + triton.cdiv(Y, META['BLOCK_SIZE_Y']), + ) + awq_dequantize_kernel[grid](qweight, + scales, + zeros, + group_size, + result, + X, + Y, + BLOCK_SIZE_X=block_size_x, + BLOCK_SIZE_Y=block_size_y) + + return result + + +# input - [M, K] +# qweight - [K, N // 8] +# qzeros - [K // G, N // 8] +# scales - [K // G, N] +# split_k_iters - parallelism along K-dimension, int, power of 2. +def awq_gemm_triton(input: torch.Tensor, + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + split_k_iters: int, + block_size_m: int = 32, + block_size_n: int = 32, + block_size_k: int = 32) -> torch.Tensor: + M, K = input.shape + N = qweight.shape[1] * 8 + group_size = qweight.shape[0] // qzeros.shape[0] + + assert N > 0 and K > 0 and M > 0 + assert qweight.shape[0] == K and qweight.shape[1] == N // 8 + assert qzeros.shape[0] == K // group_size and qzeros.shape[1] == N // 8 + assert scales.shape[0] == K // group_size and scales.shape[1] == N + assert split_k_iters & (split_k_iters - 1) == 0 and split_k_iters != 0 + assert split_k_iters <= 32 + assert group_size <= K + assert group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES or group_size == K + + grid = lambda META: ( + triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv( + N, META['BLOCK_SIZE_N']), + split_k_iters, + ) + + result = torch.zeros((split_k_iters, M, N), + dtype=scales.dtype, + device=input.device) + + # A = input, B = qweight, C = result + # A = M x K, B = K x N, C = M x N + awq_gemm_kernel[grid](input, + qweight, + result, + qzeros, + scales, + M, + N, + K, + group_size, + BLOCK_SIZE_M=block_size_m, + BLOCK_SIZE_N=block_size_n, + BLOCK_SIZE_K=block_size_k, + SPLIT_K=split_k_iters) + + result = result.sum(0) + + return result diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py new file mode 100644 index 00000000..75fa8249 --- /dev/null +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -0,0 +1,143 @@ +import inspect +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Type + +import torch +from torch import nn + + +class QuantizeMethodBase(ABC): + """Base class for different quantized methods.""" + + @abstractmethod + def create_weights(self, layer: torch.nn.Module, *weight_args, + **extra_weight_attrs): + """Create weights for a layer. + + The weights will be set as attributes of the layer.""" + raise NotImplementedError + + @abstractmethod + def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + # Not required functions + def embedding(self, layer: torch.nn.Module, *args, + **kwargs) -> torch.Tensor: + """Gather embeddings in the layer based on indices in the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Process the weight after loading. + + This can be used for example, to transpose weights for computation. + """ + return + + +def method_has_implemented_embedding( + method_class: Type[QuantizeMethodBase]) -> bool: + """ + Not all quant methods have embedding implemented, so we need to check that + it exists for our given method. We check this by making sure the function + has been changed from the base implementation. + """ + base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding", + None) + class_embedding = inspect.getattr_static(method_class, "embedding", None) + + return (class_embedding is not None + and class_embedding is not base_embedding) + + +class QuantizationConfig(ABC): + """Base class for quantization configs.""" + + @abstractmethod + def get_name(self) -> str: + """Name of the quantization method.""" + raise NotImplementedError + + @abstractmethod + def get_supported_act_dtypes(self) -> List[torch.dtype]: + """List of supported activation dtypes.""" + raise NotImplementedError + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + """Minimum GPU capability to support the quantization method. + + E.g., 70 for Volta, 75 for Turing, 80 for Ampere. + This requirement is due to the custom CUDA kernels used by the + quantization method. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_config_filenames() -> List[str]: + """List of filenames to search for in the model directory.""" + raise NotImplementedError + + @classmethod + @abstractmethod + def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig": + """Create a config class from the model's quantization config.""" + raise NotImplementedError + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + """ + Detects if this quantization method can support a given checkpoint + format by overriding the user specified quantization method -- + this method should only be overwritten by subclasses in exceptional + circumstances + """ + return None + + @staticmethod + def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any: + """Get a value from the model's quantization config.""" + for key in keys: + if key in config: + return config[key] + raise ValueError(f"Cannot find any of {keys} in the model's " + "quantization config.") + + @staticmethod + def get_from_keys_or(config: Dict[str, Any], keys: List[str], + default: Any) -> Any: + """Get a optional value from the model's quantization config.""" + try: + return QuantizationConfig.get_from_keys(config, keys) + except ValueError: + return default + + @abstractmethod + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional[QuantizeMethodBase]: + """Get the quantize method to use for the quantized layer. + + Args: + layer: The layer for the quant method. + prefix: The full name of the layer in the state dict + Returns: + The quantize method. None if the given layer doesn't support quant + method. + """ + raise NotImplementedError + + @abstractmethod + def get_scaled_act_names(self) -> List[str]: + """Returns the activation function names that should be post-scaled. + + For now, this is only used by AWQ. + """ + raise NotImplementedError diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py new file mode 100644 index 00000000..faa8d92e --- /dev/null +++ b/vllm/model_executor/layers/quantization/bitsandbytes.py @@ -0,0 +1,316 @@ +from typing import Any, Dict, List, Optional + +import torch + +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, + set_weight_attrs) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) + + +class BitsAndBytesConfig(QuantizationConfig): + """Config class for BitsAndBytes Quantization. + + Reference: https://arxiv.org/abs/2305.14314 + """ + + def __init__( + self, + load_in_8bit: bool = False, + load_in_4bit: bool = True, + bnb_4bit_compute_dtype: str = "float32", + bnb_4bit_quant_type: str = "fp4", + bnb_4bit_use_double_quant: bool = False, + llm_int8_enable_fp32_cpu_offload: bool = False, + llm_int8_has_fp16_weight: bool = False, + llm_int8_skip_modules: Optional[Any] = None, + llm_int8_threshold: float = 0.0, + ) -> None: + + self.load_in_8bit = load_in_8bit + self.load_in_4bit = load_in_4bit + self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype + self.bnb_4bit_quant_type = bnb_4bit_quant_type + self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant + self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload + self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight + self.llm_int8_skip_modules = llm_int8_skip_modules + self.llm_int8_threshold = llm_int8_threshold + + def __repr__(self) -> str: + return "BitsAndBytesConfig" + + @classmethod + def get_name(self) -> str: + return "bitsandbytes" + + @classmethod + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.float32, torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + @staticmethod + def get_config_filenames() -> List[str]: + return [ + "adapter_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig": + + def get_safe_value(config, keys, default_value=None): + try: + value = cls.get_from_keys(config, keys) + return value if value is not None else default_value + except ValueError: + return default_value + + load_in_8bit = get_safe_value(config, ["load_in_8bit"], + default_value=False) + load_in_4bit = get_safe_value(config, ["load_in_4bit"], + default_value=True) + bnb_4bit_compute_dtype = get_safe_value(config, + ["bnb_4bit_compute_dtype"], + default_value="float32") + bnb_4bit_quant_type = get_safe_value(config, ["bnb_4bit_quant_type"], + default_value="fp4") + bnb_4bit_use_double_quant = get_safe_value( + config, ["bnb_4bit_use_double_quant"], default_value=False) + llm_int8_enable_fp32_cpu_offload = get_safe_value( + config, ["llm_int8_enable_fp32_cpu_offload"], default_value=False) + llm_int8_has_fp16_weight = get_safe_value(config, + ["llm_int8_has_fp16_weight"], + default_value=False) + llm_int8_skip_modules = get_safe_value(config, + ["llm_int8_skip_modules"], + default_value=[]) + llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"], + default_value=0.0) + + return cls( + load_in_8bit=load_in_8bit, + load_in_4bit=load_in_4bit, + bnb_4bit_compute_dtype=bnb_4bit_compute_dtype, + bnb_4bit_quant_type=bnb_4bit_quant_type, + bnb_4bit_use_double_quant=bnb_4bit_use_double_quant, + llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload, + llm_int8_has_fp16_weight=llm_int8_has_fp16_weight, + llm_int8_skip_modules=llm_int8_skip_modules, + llm_int8_threshold=llm_int8_threshold) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["BitsAndBytesLinearMethod"]: + if isinstance(layer, LinearBase): + return BitsAndBytesLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class BitsAndBytesLinearMethod(LinearMethodBase): + """Linear method for BitsAndBytes. + + Args: + quant_config: The BitsAndBytes quantization config. + """ + + def __init__(self, quant_config: BitsAndBytesConfig): + try: + import bitsandbytes + if bitsandbytes.__version__ < "0.44.0": + raise ImportError("bitsandbytes version is wrong. Please " + "install bitsandbytes>=0.44.0.") + except ImportError as err: + raise ImportError("Please install bitsandbytes>=0.44.0 via " + "`pip install bitsandbytes>=0.44.0` to use " + "bitsandbytes quantizer.") from err + + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + from bitsandbytes.nn import Int8Params + + def calculate_quant_ratio(dtype): + if dtype.is_floating_point: + return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits + else: + return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits + + def create_qweight_for_8bit(): + qweight = Int8Params( + data=torch.empty(sum(output_partition_sizes), + input_size_per_partition, + dtype=torch.int8), + has_fp16_weights=self.quant_config.llm_int8_has_fp16_weight, + requires_grad=False) + set_weight_attrs( + qweight, { + "input_dim": 0, + "output_dim": 0, + "pack_factor": 1, + "use_bitsandbytes_8bit": True, + "generation": 0 + }) + return qweight + + def create_qweight_for_4bit(): + quant_ratio = calculate_quant_ratio(params_dtype) + + total_size = input_size_per_partition * sum(output_partition_sizes) + if total_size % quant_ratio != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape.") + + qweight = torch.nn.Parameter(torch.empty(total_size // quant_ratio, + 1, + dtype=torch.uint8), + requires_grad=False) + set_weight_attrs( + qweight, { + "input_dim": 0, + "output_dim": 0, + "pack_factor": quant_ratio, + "use_bitsandbytes_4bit": True + }) + return qweight + + if self.quant_config.load_in_8bit: + qweight = create_qweight_for_8bit() + else: + qweight = create_qweight_for_4bit() + + layer.register_parameter("qweight", qweight) + set_weight_attrs(qweight, extra_weight_attrs) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + if self.quant_config.load_in_8bit: + return self._apply_8bit_weight(layer, x, bias) + else: + return self._apply_4bit_weight(layer, x, bias) + + def _apply_8bit_weight( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + # only load the bitsandbytes module when needed + from bitsandbytes import MatmulLtState, matmul + + original_type = x.dtype + bf_x = x.to(torch.bfloat16) + + qweight = layer.qweight + offsets = qweight.bnb_shard_offsets + quant_states = qweight.bnb_quant_state + matmul_states = qweight.matmul_state + generation = qweight.generation + + out_dim_0 = x.shape[0] + out_dim_1 = sum( + [quant_state[1].shape[0] for quant_state in quant_states.items()]) + out = torch.empty(out_dim_0, + out_dim_1, + dtype=torch.float16, + device=x.device) + + current_index = 0 + for i in range(len(quant_states)): + output_size = quant_states[i].shape[0] + + # in profile_run or the first generation of inference, + # create new matmul_states + if generation == 0 or generation == 1: + matmul_states[i] = MatmulLtState() + matmul_states[i].CB = qweight[offsets[i]:offsets[i + 1]] + matmul_states[i].SCB = quant_states[i].to(x.device) + matmul_states[i].threshold = ( + self.quant_config.llm_int8_threshold) + matmul_states[i].has_fp16_weights = ( + self.quant_config.llm_int8_has_fp16_weight) + matmul_states[i].is_training = False + if matmul_states[i].threshold > 0.0 and not matmul_states[ + i].has_fp16_weights: + matmul_states[i].use_pool = True + + new_x = bf_x.unsqueeze(0) + + out[:, current_index:current_index + output_size] = matmul( + new_x, + qweight[offsets[i]:offsets[i + 1]], + state=matmul_states[i]) + + current_index += output_size + + # only update the matmul_states if it is not profile_run + if (generation > 0 + and not self.quant_config.llm_int8_has_fp16_weight + and matmul_states[i].CB is not None + and matmul_states[i].CxB is not None): + del matmul_states[i].CB + qweight[offsets[i]:offsets[i + 1]] = matmul_states[i].CxB + + out = out.to(original_type) + + if bias is not None: + out += bias + + qweight.generation += 1 + + return out + + def _apply_4bit_weight( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + # only load the bitsandbytes module when needed + from bitsandbytes import matmul_4bit + + original_type = x.dtype + bf_x = x.to(torch.bfloat16) + + qweight = layer.qweight + quant_states = qweight.bnb_quant_state + offsets = qweight.bnb_shard_offsets + + out_dim_0 = x.shape[0] + out_dim_1 = sum( + [quant_state[1].shape[0] for quant_state in quant_states.items()]) + out = torch.empty(out_dim_0, + out_dim_1, + dtype=torch.bfloat16, + device=x.device) + + current_index = 0 + for i in range(len(quant_states)): + output_size = quant_states[i].shape[0] + # It is more efficient to use out kwarg like + # matmul_4bit(..., out = ...). Infeasible now due to the bug + # https://github.com/TimDettmers/bitsandbytes/issues/1235. + # Need to change after the bug is fixed. + out[:, current_index:current_index + output_size] = matmul_4bit( + bf_x, qweight[offsets[i]:offsets[i + 1]].t(), quant_states[i]) + + current_index += output_size + + out = out.to(original_type) + + if bias is not None: + out += bias + + return out diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/__init__.py b/vllm/model_executor/layers/quantization/compressed_tensors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5510838be166de8d784807ca728a2da036e2035 GIT binary patch literal 199 zcmYk0F%H5o3`JAa0U`Au48`iif)FQQ;Q&!+EgFfc zW!Y8LOt6~Av(NRN@vnlL4Lg>MPVB_I*xp6R;~zdpAnUfHcmV=P$Lp^#PqLBX*K-h>rJC>Oqr~3qQW5E54Bo LuJaFZ!YIWTuR}JA literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db6f166eabf1cb1d0e99f12352b4bf585ec18f17 GIT binary patch literal 12415 zcmbtaOLH98b?*1{3gZ?%c4Mu0Ij5Br!|;v00U0X zjBYmoVlpT@rX1&`aw^%R%H=AuDjO}b%raT!CnTF*q*9fY*IA^J62*My-0q$S9(Kw= z^_}~EopZnQxVJq%UQ+OP`5*py{ly(c`Ac)$MPuZf&9SiTmcwy(Jgm4C&a;|F!U=aGoOCBS zU1&~)N8O_wx0}=9G51(_+&v!7xHI7i_e6NoJsF;IPlc!5)8VW;%k_%QGvQ0_OX17z z%c0{s;aT@=c+Nc+zT&g$NkC|ieCvP zp6KrDNFVXbNSA|gq~Ab#!kBoUbh@)u~_6E>+D$zt!4FjQ1M#I5F-uqBybd zcj897Rcj_jy%xobRW&KTTZ@8^+J4YP!oIs11^!3vJ5@a?-D|XhnxNujLdHJ|;`O%A zIWrGipKsP$aibgfgN0QsnfgQ))M%*N?bd2zEzb-3U0}FVx7%Sy1QEtu4q8!LL>OAu zsM^Wwr&n)XzP9x7tp`hYu3Yt&9)A4s{Rhi;-t*r5V0kH-{&W$gqq#GCdR(fn2Vs!B zw5LG!-l>TwxNiR*xH7OaU*Y zlT5725syae?aij|HP%{f5qQnoP9Q*#NuuqC-@jc$cPSmzWTL()M9_-8PO}zM`{Ycv zgq&`JEKn!z8~iIHaT8zk_c*74;;O#lYDLxmwyyb`mEuumS_WMqv16Q0SMO zR90=@@(W*>uI*p*Oa9mwhFkP)S$|dY%l7~7cIIO-q6sCH2BkE6Ba&-f=$I*ryR`}6Q^Us*N$Q~qi0=@|FT zb4Gu&{u$0aUVO=a%YQjI;yb|%dYnOTXZ>@S`w9P*;G};ZpUi#Lzko4LjpV-OUqtTd zk=&~PI&x=6a^LXhkb7n%_mV%4+?V>f{sQLkrhgf^FY~TvYrcYArX9Dr#<%>dn5E;Z zK#=Rn-h(7h@?tV6xy+0BV1f5Ki6z-6DKo2iDUTg_wB^-_lhUenbATx*uHm~)d@+ll zr}Y$1>FEg7o{CWG83=V>SyH}I7psPN1+~O^3eHjRDgt1f5e3avnbWU8WnU@cB7Xii zUWmiaLZ{KWBqx69t#ETayc}*XG#e`mogF}0YhkO|3>QK|K`(e5)Hma{Sl~kuEesx? zg?bKHUMvx7pRD=LPGWmrqt%E#?tAl?)$;hY%OadCbCS$r6GLXNH?1;Lv)o%5m zN$qL1XF@b7BIAFNJYEC2crtS2Ib@AaWSY8+>a`{i3ozxiYGDvvq7G;-hT5GwY%Ze> z_e>|>AA+4&*--ImYmkd_kMzU@aqH`DVYS(=#h0(S_DUN2N=@{7Ga8-m37+p}r_&a3 zI%Piw_(pGH0N%f{+I{oTIq%bW&%B4Wkx!wv_^Z1ZLeA$(9wV+6OwuWuS=eadn00C^ zjb`4$T0K5E_951Z3)vjb-%zKss6+M+vE-^2O`lh~$-5nbP+xzauz%=KB-#N-2) zSkMSo8;`{}TGe#~*&jg$u^91M)Is2@Q0t)b?dqx$YXl%0y03q!iHX>F0zQn*o*G+E zH1P^b3IG+&H@?(%4FHSk*>O=G9jevSQEM(P^>gAelu49Z%EQz&zy7$p!h6J`CSP&HiZiU&S9i+#-YD!$7d5Z5)z!SMiG3Kd0V@*&? z3FZ2qQrxbBnCxdhL~s zqs8f_8kQ_eAwFQ7$Je?oj8JobD*EjID}x=*f!rQ7ahGL0YPxJ@$;>}Joa7+E1 zq?pfnl-FiF0Al@7rhDe~Vsy&zWa~C~jsA(B7@D3NhM{#A4&tfcj@XW~;g3J5{_ub?3i+{0M5TbrS`(g23ozyLNE#Yq$; z*7BXjrTY&ObF0?e43fg_@7`KmymK!p_&Z?yMjgAo69VK#qh2jYI{iM{i9evmMOc#8 zWw%Gt{m6QEsZ8p?Tn(}-Zq!E!cHBcPS(H}xwy)Q&{}i1^QwV0YidI&&Z*1LCjTaNz zl+m3(kW%}0bucwH&_DL-x21-~^dV{#K#ey*jRMo+Drj*Pxi&)3Lbw@{SWUfiW+6CAlS3zqE1CV)wOAZ+mC{lTS6=)*S;H^mdV<@!GDVm z_SxXduifCiT@7t8$?2Tkzd!=gSJ_k|8@d&KMTrzw+Voi6(7^u3=qSiIv%d~TS2ql> zGi|6DTN?#@_NunU3@+w-20jy?6}`!QX&Z)sqX50);6R>hvK5Yu-7B6 zYZJPmmh}>eR{)v@-2|te4*vj5d`3xv3Do?A{FR-u7Y*?o+OmHThT6WzeuSOU-b(tPwLEB0u-c=GAgGw4=q7D5{n593L+1DLfR0BDYup>2H$ zMvaNMl>5WMT=aGexm+F zg%MHJC8v;RNlKDr5ghN(#ro1FqCy++(H4Fn?!qi3O$%lzAzyNw*hC_hWFe^}E@xeD zqLOX*p{}?tYLqadtEk{q!agS8r)ceUc3CycF z`VWi~k;(xJ8H%R{ET(_!dBy0bkp8-4ms6ZJaJ0A%YH4N`E1UgX8K<=Md@!z2ofn6w zO8ZVxyo2ctok=>6CO+0)?;-+!BJ2%1FJzI>c>~xX)q{x*Scik9XZY~eBSa3L8x!Zi z=$T}IKT}nO?2@Wk`x>}ea?-zjcqrXovQb*jW~)IKvI9$m%#+o2v)SH;40g6_J2y&s zH`2>|!=bAzr@iX*T`iFV+osi;$3502xRNb8CvH0{fg>CF^La0+(M=ts)|@oQZ#c{A zfq-#TYf-z!;ZkRNy-{Cx0{FA@o@jOJwXMLZIVsoj9ZPyB=AApJJk$$nN2uIP=bz4G zs6RPOI)|FGy4h?FtaA>nMX(k;W@8RL(Y*~jo93_sLmgoi!y7W+-^fp~#|?o^WNQjb zgF847G&AatEXR}*a`PPCAg9*y9b6CQqwJBV16P91#m0Ov@2qw%Ut3sN#l2UC6l8=F zjd`S@86mgj_(2@hN7Krugz0hb`;Y^Kz%5qPX}5ghEHDR6 zBfU9G`^x)4DXtZ0CRAz9LYLSrN1Nd@AsVgKK(vB-(C0D$E1kB)!leROl07JbPMCNf z0bn^-b)fs4ru=hM-FOu2L}HuLjB*5UnQ}ya74Fon3}5)UV<{T3H|Qo+8=ASgA;|wP7AUxkATim)F0N91tlq3ek@Q0*I+St= z`@chFTC0^<^3;oWsWs`T;u9(|R@&{R+(pvk1j_}I22zNytYA^ayG=VZuxq)hDbAvR zL{Id02y8{OjIX7(t$m~E8ln5wrSg)RS;U5EU{3c9h+4(XIcfie?w}NMrB&* z8*7qNYUzbl*(Wxe=gnj>jT~fod+iZ)lo#z&VqF)M1K@nI#hIkp@I9iOYmeU*7a?$dq z^ZFAg8^>FO3jOj;D&4c1?h*D>9b=A3jt-vA)NwfE3?}}d?>5byyuu`rJa)lL{!_&h zD~sfKdow0KbDraoIvFLUev4!*twSYA88vGmoDgJT*XcNAe?>&- z`w$-+q`ksZ=nZhalNJsq0BC2hbKLA%JsTQ%0XqgQzG`&;F~{47DQ@psZi=)x|LuAM zUM=6*Zp2ayr@{(QT^9k0^3Djd35=ttCUzX2Ii!jp=`kRC=AES=fGR<6Tt3R*wQwD7 zl6MD@RBJS&Je`cEo-m>(UQHAv)BFq@Abv{0pHlE=6#O{_e}RBh`0r8TCW53QC!DcI z=7(U0==Z=K=H0kc!DIOK(l#Ehltjc`k8=KB(s0B=qym>p7CS35AMSJ z@Z=KpF_5%Ti|S;*Wq*X*^znJPfC!xj@?rWaoIqnSyw+s9^bDpyu10VzCu#m~`95Ye zmwd+R7}GJ^7S=Zo5x;pyqs&2Vq#^^}sC7C`s0zpe^|x!n2SY@WJR-{X)1aTk-l(qbv!D<1D~HMtGE~RjW5cs8Cd+F z$PS7@iJltq10i})L=S?-eg;>)TLvK6;ugB<{(z~vB(F^(Cq-@N(blCpJ!S!!YfaK1 zSJ~DEs^Izy%4PPDEwhrj7uK)Ykr+wUP%L7IeMJk27x@rIA-Bi|iqCCa5!LXPy zk`r*|H++DeH}b}YuL~0s))lUyh<=&0p>grPq3C(6O?jRq4-eGMUcQPJLzi)c3iL5D5VQ0k4S)&KX^5ryxVGJeJV#7N4D!J@9_hI6(tMMCfgSLX z;GPSfpI<1y1c`tnRNF>vx-l!F))mw$^>ma=oT2hDlmmgHngDlA8!J{o0eUh~FR&d?{G{NxVa+rv*fIZ4K!vJ37Z3dN@6au0-08TA-YOp3( z0&1B%6UcVix_9rRd^R^&*nqo+o=V+d#M;YLs5dm#yqiU_fxTvTP-3zRTl^t1x?XyZ z2Mp}bA@5{p?sQ+6-XXnn9V>vOe}{XTbJpA2!4?cd0jb*90;c5^yM?qS`*Cpn#~{#+ z?zj3QI^f0hRE!*+bcaWej`QL7(2_V#pfq5!(GuW$mk?49n#~MGc&UFwm3gUzqY*8&gouqNT~nrJoL3oNtC#SQ zV5fsqA?-jOf@O(G?FrV|1a$4rj!zGa>w@%6_dGwPfo(bPL{Gk_iRM?y5{KOx@Z%Tv z>>*WgkfHGL6B|LZq(*FUKQt2uH literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors_moe.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/__pycache__/compressed_tensors_moe.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2bceb07e56fb1a361490a58011b9a30763545ef GIT binary patch literal 11478 zcmcIq+mjsCdG9`bnVz1W+1Z=!NJ9d)$AJ|qVI#1>0$qqPEJ9i!?m(K>bkFXL=C-GM zB<;8dJFF-m6;(NL5?6&d+09F$s-*HC?wg~8pY8tKXGGa(%r@ujx#qk*-`r>K6SQn&fAfHSK;UNMVDpfDsCn2vEKAQzUe2>_N}lD* z8;X5oMe_20E87o%Uht+spYkoxkD^@krcs{ukD`1GUH?u-0z54YbKQm2MEWn@+nEP}2+D!1rEipO0q- z*h_x6+V-Av13x~J()`!`+RAEpx#Bkb%iT_=%|qYAU~{i8J-hgA=klu;FTQ+f`TRNO zx$iAsF0*+0Y`fXveh^^R3vJ$X!+7qMuGT&kZ=S5SoUol@-CjK3@wpTFt)R^v&tFRh^s$a&;D^q2u|~%YLmpradM$8P zYn^ze!)qt3WmDafY%TrO#bL8}>iLVy zue{{)Mh)A*E|$uf*l?Vd+w>hLHXWzg_PPziEyuwc8)=Q|<8orv?S}1GzjXfMvuD3= zYnRSHd+rCfC4LC=3h?QT<6*ONyi@Bu8oGSN4Vbnb^;-m1vxGx$@0Xn4-z z3ElvRq&gJHL1|O=*p}Rv$I1%IERr@^Bu8wE@k5~!NgEJF(A1RnP^cxeQ1?LlQ1Y~h zeJK4LYko&A>r1izVy)%7JkBI+8^qJKR;L>}ZY8W;6So)i9(c7C45Y0z5XDkZS@oO# zQ>9PYBPH+S}KOO`v*auM5 zO_|B+j-?Ps88Tx9lzQbow=-mig~^eY%BolkFJUxJ|=)+)*T z{C9{xl&b`zg^JF!x1a!ENQ^P8kL@s4yd&!Na3jrd2G2)-4=~_H;zjR>r+Dg3)mA*s zSL<3`_jEtwW$+kyvY;7N(=)vs+GK}qEH7VGg`!t&Gh56~XY=&$JuqxBgPu7kd4h8o zY2LTIeR%f!c|UVd@(!fG2k~1#*`k)!Iq#5n_@-u0d3nghQE{uIldF)D8~hPIWOt4N z*j91`)7NFe_KesbG_ZDE8q&b}psAoKbuHBEnX8OHj1o4Vl!C?30K=nRusmu8OfOnx zEzIGVo`ePJOPs`;|2Bb#0D3NLMd`xDQwgt^z!0TGt36U`w;JmusQ*V3^$#I{DGI7+ zpUO|kZF!|J;VHS?FAb$

DDXWP9bX?2BM>EFm_nBQTyCzEh(&~3kj2c!)oj#1HT z`eD7I8@jhwH+)?_3vwIxzRM~~>Bo?Xv_OXyFogTIwgycvted646rQk!b5jqjw>3)( zY%sQzm(J5duj09@|MlM+KY!=%{`Ws#*|+K>0krD$pT3AB({`~kcQjqY3YwX9+aBt7 z@EdBOGc=I8nFl#=gr1c;`mO$ z=Km$!X$6U|H*|-8-rja&%!ifHY(BsS;xP;VVPrB0EMAjtW7pi#Q8LzyOWIw1wQ2Gi zO{NUOr_e<{g(N7+DD8+rD^AtK#x0kOV$*To(+EE;j7{v4VEHz zENy#Fs~HVf60MDU-$nIwv2cteucSlTjK~ybWJUIpwg#DlBwE70Sq&WF2=}HIKsq6Z zo{~dlqb{7kxSBg{wp4iqbMI!MDt%xz=(#MEzlqF5C}~wfW2kRJF@+}np@u>$gVG_& zrnRmMs2gDnNl8(ilet8v`YEcHm3E=dLZ5}#T9r@YDX-r(0_$?8l}Zw#W?8;~KgCv{ zGA%EUvp`(=?n(`~?CBI6^8%UZK3x2L&58UFlbG_J*lh+`T zjB&Hf*D(1SCcQm7nQS5R>KP^Zc^dvzN?xO+LJR-a`^=+@?##7ZF;{;xKrY` zrx~{Oz%Y#W9kceJz9^}-jeFn4_#2dT4ar2xA?NjJITse>-GRJ2EbpXmtAQ(A;X&R# zQHn~NzVJCMb6SbYqDr|nU)03RO)IGIuBc*H%;qzS2V;=7Dp}4dZ31Y;#$d^4)BBGu zQrDzDOx)?;0yIo^M`rA5FwC?^#(M*M1iK@I8sBvAPrJb#TFWH;q{wOKxLhisg%4^O zl=9HJV#}9Q?hyQmD@OX_SGDiz-O`ZuQn{eLO0=wkoyC2YwOuk1Rg`G#(mXw-h8$5; zX@+vboTHk0HajG(q3xRe4bFq3yR@?#n9Cgg=a}kJ3zX)$1U_?VAv=VzX#OSK^j>9A z8k)ktMen(o9WJQ++)cZvBb~66Ef>A0X9G8$wK*?JqRf4dWJgivH^6@#cNM136cw&L zhVp{^m+bhc%^K;W6QX*H-u)!g`F;1%Pm!i!ZBtv7g!JS|BB)7pV(5-SjLV?-!2ATXa(dTHyNgWRy%w(8c98#4cl*ZZIBMMKOyIzTIgjZYf zASzlmCS6DK%uT$Xq)neYTkMzQF5!04)GncSVv-9KM*v)Yjb;VzfPvWtXv_)ubxPAe(dPve5xrHISY&-MX6wCump{Wq$%N?~aO539x#PTR5BIn$L8Xm;rU~8 zp-ji9*1P9;qdW}Ddk&rk!?I@?9@M7FRo=JmZy$b0V@(VVX`|6a+KRLksoM0fkzjv7 z)R6+W3jKS=lJ-;B=^rw>`l0>}?FSl+j=^n=N3_BIx(&THHcZ-Wiwss_zNVjHOrCMT z=nvOo|0;CXG*{&lnD4vq!yjqdxplP@T?39K#Z<#ypj<-YF(UPap z#~!*v?~VaZgO0KTjuFKiQ(P(!p?`K$HY77F-=gH(l>7#gW<~8DQkl#Nxk0%OlEB1_FL%`cL#xDq`Jd39!9vYFi5*mj=Ro@kC}r{C#VN4LU1N2%S^IFPLVTaXEBrd- z?c|Pf_GOtG5Hk937;^!&Dcw&YAsB=Gd|v?>qP7q2 z)bj$Lelcnz>Lr!W!rNjorom&<*E1`e#16;&!oJDQuE^0fV3~h3N z;hExH510wunGDa6Bwmu_w&{K7hLvR3g2jg5AiIdV06==T5EzqEB_pAz1;#MF=2yjy7U&$v^HX<+kO*t`!g z#XYC+M2wk-x31?H_iAqA7n1sZ`1buSYNxkvNiP|q0Z$xpAmxnR#^r;~~6U`7csukP0E@rWv>+HSELJZ~*N9Gvz z1qxxW<_ZH&c5+^hd$LtHF}c&1(OT#FKJj$%B4n$0d;b)f$@YFndHI%X3@unDYt!o2 z$*=FiI_)^`kViv;39p9p?B!Dqx5pkxDf5vpzpcHkziq4`PJtMiH?)7O3HvV%;oLIc zB6U&04Wu5{vKi`$yDnT-6th_a5suQjF&=S_RfoUIu$=*Xn^64dqCDxVIouq6%CvX% zceHm9JQ{h=Ixp;9UEV;OyQX3lZA+z#Si%`NcrVATLuhgSphd|`%06`9LXob_OC!{l zGGfW+_l1wtAxru}6;A4g_6e)-A0b0~WK)-rTot0(afG(xc56dhXahZMkUx&RzN8Vl z6EMX)E{vp#fZ4mo7qxF6A_Rw6fQPyg5SuY{fZB+LaQ+g4`a^fu2Cj2<+^tjmJU2au zpszD@h4~X>7vUg0y+ma;A`Z}=1C4+v8X*}jrbWQqfuR`-hh4rsZJ z?0O`Bm-_u4C8XVg3Vc!ocZ-H6h`x&i5jl>tevb&mRr!5N2m#8sDH%UL*E_k*0(B5F zXVtw-dyA13ydp=Zh4z5@RA|j$kIK;f_dHkG74QAJ1w1%6z1X8zt+l$kDz9Mz&BiZ- z2&-tBn<@NI5H(GMr1rO1V?@gjnV0Em%~{|f%u+K7==qR3b^s6^Gx$dYq z2HSQR=e$eNa>OlYRMZ)K-fIO%+3p>Kls}dik;~V6@?2|>w&L>_5H*3TPXPmR_0_)$ zAmk;Ei!#YAg2LDvFKGq^4arQZ6CVm7BD4rEKw)h^ zE01xb!J+m_R^^~N1^gkk>BU@q%KbgC?-0$uWV&@`AE zbo(uw>WOKqvx6FCHVH`wz#2F&R>!n?oeMZ89i1Mtc;_|;*yJMG=WeQwisem|4i3vG zz8(;fUnRxy1p4`|qx&Wu;?$u)$V$L2kTb?W)O+Arjsg2nz|bji8JlkTVVdj%IB9GH z4&lKJ;S0zPAWwcg^sfq^>+(t#F(94nE;Fs_^i2rOaDUCNh{X*~Y zAo(n_k`ALsfrvKA+~^`Xr28E1^i?D{Sc3uEGz88LakgN}dAtr8DE8FgcbEuZTWn$q zDoNdhPA6k@ScvG1yU|MFphz ztAY&a4xZlsAfuH2jzMcjXT!TV6&f8i08nfIERYZqxL6s7jsUl>>hi?a!qCS0dIT2W zecHDz0pC)dg>Px1{_NHQe0F%HyM}xLZOtK$69FjvO=qV(EDr%Kh6wlTIFIhG2-IH1 z(bLww9(jY^eM&$l<|dy13+{11UkhYTDN^{CSBLI zl-9CWn>Kt}R?oM<1Syb=2tOb~G)4GBA|#6N70Uk+C0GUO!h|0Z;Uto6d*w0?8MkY( z3VpArHISdA#^u}+opu(~=}=cpKJZ9^81M;@E2DngYDc*TnJ-7KStw;b>egdr=smE{T^H={*)xBz-8^GFh>bKW@1JMHToiF z1y(^FMDY;~W9p#HHR@{v$xA`25Y(A0@VTArT!zjU?5xNcs(YO47&nBY&wC2jdIe|r zovZvYnj{S#oO(oCoIc^}H~v0=519C~fZ!94>cNI*3c`^237&jJ5B`{<_&knRR)^g%rpn_2tFLM4X0?5xd{ zayv@B4_uWGpb$yBC0IJ>Nov&NRA1%#G};O1)8@L4K37kJOBFfmzA4kdHghcPH`gsGj$dz5x&?!By(%PV9t(wC7~ t3nvk6pu@h3UN`E%m=%3qub5n>yiw$hdEK(`u~WHGZ=7tj8qMWb{tr%#vgrT- literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/dbrx.cpython-310.pyc b/vllm/model_executor/models/__pycache__/dbrx.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24267367896a3b72e4b787755079f1378d46aa12 GIT binary patch literal 11986 zcmbVS+ix7#d7t~t&R$3^DN>YdX)P(1H?}Ctm)J>URk9^Hc4S786(^WFoh)~TAQPh1aTA+<3OZ|P{nO!a^ zN=mznIdg7vuHWVNedlz_Wmm!P#9#g0@}1L)@*7Hw{!AoJ<4OKnRg^#p)JTa{PmMKC zi*-+r4bO;8&s3>Qi!9ILw;em4!+CmCh+WT(i(Zk_MpTN+UYX-&REfvDv3T4Yk0-nd z&aWN=J>H&p+MAANycy1OqP_8~H_P!tv@hQ8?dP}~JrO_YJ<0K6bRa(H z9gJ&Ujnk#*srZn0C_d~Rj*oapIIkRiA%5C>IzH+h<#Z)_Ca!yRj*mrOjGy(M<@k8? zTzt$shWJEKjppLx-f@mkM$gA5yc6*Y-V5=I-ixa8rV>mAdp=ZxJ)yJm(z@=wjGXCU z201g~B+_3(dT%g`^lV<^73Ay-_9JJ1p7Sblo(P^q&Xal0myvTIIEb8sdCpgmQwyF# z&Qp2LSCMllIES>a5YhHWa%CWg^>!n6@I(nej)if=TU%i$ZH_F`+Pu4kpCF!j4# ze@S$D-RuCR@Ojb^e%uMd$nQ2pBZ?3ajrNu7$ivI8cSIEUNozGkC(ij^5(e*d&Y_2i zZ+`u~i+L*-TJ5kQvhhn{H)=H-X&B@=6PG&Iw^u?dV;4G0tu(nPI?XV_kdRgVX0L(C ztu|Gr)m}pTdxCJW(Th_5de~Z8PW`CU z2+-SX+_A^%R5?I z?13|DEP;V`~(G5lF*6>55mUga$?Kw`)XI7I*J1ccT zb8+Ga>CJA~)JNt>aT^c4lhX)NWnI}&Hx*S$wRNSh_rWgBil8HA#iL{e8qCr+>2a-F*7&T+^R?+~HFj^ExkjJCAt!4$^ zZ$^zI@%`J%XXlTnarbz))jgIr1UTl{D{*f*J`rC(9<>&ZcW|pBOeV(zQ083srl_L>nq@MLz%?D!RUK7R4g59!m+q`i5*f=ao*PU$D=)W#AZ+^y zHako*OKxvdLo<2%J z7xB@(lI>QrIgF2o72)o7g54Nm22zy^08fUXw(30|;B5p)f zgJ<_ejK5wK&)~)TDW0X+a|pZwaVvKwj!}AyIC-R3W^}nw)QTXwzMYfp5y}`YpWy8u zk}G%g8fqkm5LlL_S|C>|=P=?n>%Xm{1I}|0cki!Gjcm;p%b6?%vodR`AlF%0G1py0 z#M`!wXoxfb4Ko4K;%!@_ty0%$`L#7G)%!-@no{~kpw1{1Ti4d?zP@1u+NLUAOU(@n z0#c{icx=?fwk_$KU|<%U%8#_Z4FQVxqCTtiO&Jf+>W@gUa{ft$TQt$)^8dHRz7<$B zz6V;h(dthg-s(mH=){;Aa2ME0%1%9PzgXlVdjMGAi3U3m&GYm@N7`was#@8sf9_ZkOF!o-A)qL9Vze5 zp{6)b!JGUI)Ci$7&*jbbsSnRZGeY7q&*b=sq+xkj-UeX4g_e?&2rP|AZCZ7866aOb zxc8jlg2ZIpfw(vBJNL&C4r-8C(mp?A7NCJie`da?vnJhFigj& z+^y`zWbO;9_q$0^iGP5AN?QmBlGFqevR51+At_0%zy=Z`3}i!+7AOP~l2WwDQnbWU z)TUJeI9~myGc^gekI~K%%&sB(VVG#nWr%Zb3jrNXYI2d2T0?|2-sUiGv)ky%?LOimqeGgm^MbglSWKI*>pOY}_if2;;wi~%6J#|E(7e>v1Z!Bk0-P|MU?R!v9E*vw zRKAO#{zK#^QwUs3ieyWzXof~&x^!=KJm2r3SZ2f`UPk?G;D;846emmHmk_~nD3F@! z2B}BLQ^+K|H=sZ_RV<3K0mf0Mk`ntpp^vIyRW>08M=p z<@NGT&L`FOX!d7*XUOz$R%fY2gV>9Jfs(tE*$-3&)p~%@Y1s7p&p$x04K7t4giAvf zm&T3_W5;1*hr$BK-4KjUu|*eRq72})#qwiWnO0?Jj!Wp2m6~LV=fIzlK$EtA@FQ{{ zh0J{Nom?49J7vfxD$tbT8$=v<3`RwSs$ifY9~^>Mfp~*sE@CcXMZ8OCIsb|jJGJ0| z$aLT@;48n9j^(9Vo{Jj(cshYxs!vZTts->S)j$snwuUCtsda^j5q0;V46$j5$tzm& zy|Jb|q99D30pyka9w2S%djqSe7S)va#^ePvEFYIXgqSFX`K zx?NAagcYi7;M}XqKRf zR~U{l)6uN(slIhp+Osw}KmeaTo7^C9QwW=i4VpsTl#++h?{H1LGC<6h3u!kQ`wKLY z5VJ}d3xUIbn4Ul?@=xZc^O@!hOi|~qX2}vVP@OxLV*$~Sn3~e7MioMc{nTc?f|E-n@x&#;@`oK&>yV zArCn=rTYbFPGi$t1LtozeV6+Kh$lO`ZjifBk6UdKVn9-9sF@C8s~1Frvc8AFv*1C9 zyFRoDe0Hf=qynOH5m2y9!2$vpNNkL^D1%lP!IE*s&#%^@}U-+KUivcil<f@oLTiTU$>AT00Zh?mOdemuN$(=#NoqssBK7)q0s5J$;9a5>DGM;7i4zIv~DyF~UW>nPrh=!($KInj{|#kxPPNzWe-{{Kj4L}NobBQFpri#%3C z%7P4*6vdESQk0}KOz8$b0NCi$NY^NCJ(PIFO{%oDt1nWLaQ8G#r;3PSOOLCjx_616 zARZ@%douUCQ3%;CaJfB#LW;Xm2z~beg%F^iUqaIyMb$8#Cty4SDduiSliZLnqOH(g zbl4GT2M!-36hd-Dx*Qh6BE3+h4qg$f4f2HZ}ID<%4t&Pi?&a0Tq%sB9`{mG4finSj{{Qhjq{GJuzC6YBr!iF7J(mek=0_T-?oiCr81C{8M25>GzKtjvy|*kW6SqW0`XjDt{qpoO+e$LPJ(bSv^98#Y;Nmm${SR z{YI3WLPUPBYR@f@JV?Y3@YT|PEdS=?zxkhkIfYX!J?thDV!wov`t%4>@1;K7Yjonq zu#s)mOur6y{vi~3PTQBKRap^72Lfkqq=vErTxWUI$~(P;>LR7)q<&h=Z0Wd9GJCm^ zKxBx_ZQZNZ<7X}lFD_jhPZLP(YNN~mrcb52L1EWE=M3x*JxxgxOA@K_q-g6NgIT|i zD$mfMX>Jm;g3#^^{Gb-&)XE)g|3J7qIdZiu?X!|y{jH!D9uV+!;z+;b9}tWR_+a+& zZWEkE?c3oxami%@k?w{?xKE6*#G<}Txj2K8;=LSHJ~X{bFvgCS@o)^vz>weJlnNI+ zApFOJ$w6K)s4+oq8@|?Ic-3G!oD8Qzy53Qsc2TAUGr?YbJI#t?286diWB1khj(DTd zOB&IIcODCDh-+ABtRk|dG=K-lj|s&g#!}$Tbrt>(QVi^!!F%AplmNYul3#{Sn~U^L zOYDgOdiBd`1sr5kE~}rhbevDc$P-gofd|H~vO3V`S6@zBAsNvDZUN*M77}vswmWGJ z2henf01jXfWsO964ZRXbuyo;eS{^KgF7S01E9K8nfJCr4y9i92s{mku-c zy++gv&k4~HQm_1otM=N|U>Nu#a-JwGEAsgUMioQk&LBs+DyON;j-(?*Oi*fEp6!x@ zLS6_FQ(P&LH$KFl(#H>jfD6P6gw|X>OC#1b?pt8w3dQ)(?t?p)4PkuW(NK1ObuX>d z2ssQ@pCDP16$Uq?U!dB+u3)kC zYh>)~gjjnwVYNncCA-!oSIB)sTb~z{27AL(|Pd;GBQi<-;NPym1ZY~4h!?6`yUSnPX{+NcxeG` z@#Nh055%9N@dpM>gXRH`BQjWPxXMQctBPNtY%or0g~y4bNH=J)Tw8VSc7$PN)I!;w=ab3|EC} zeG4Xq0b`cb>|w+$s9L*kTvJ8u>I{7UI8lZAL>)ezg@aT)hkO{iQWdWla1s@ulFh!o zuB*7{(mt4)aI)$+yM;@Z4_nS%#c>NcUh(zUuTBiU#-WQ7SU8H;2W_kqCY@lRW6y8H zpoA2OPzX^aPa{~uWWGCwLkoDft|`I|)HdKU4u{oWDW7PI+E2Bs?1_Rw^fgo$ZzJ&B z>nC2!PpZAb^%wH!su4%sIo=Mqt+cu4`S5e+_P8e_pg5wBQ@^Gyl8deBm3ugsxwrcK zPBx%(@w|g%6wMAAfvrCrn4q(&^L5o5$1ShUwa^c6^^R|`@quLB!SQaZnR?@NtR8IP zNMKKa`)&>3%^FSeA_e;pWHwHq2$PCgq;N{bAY~JAZ;w22l8q!jE|qn`o>`H>DK0gk zpUj|s_#8OPPkuS$O=|ue1>6y%dJY)cmqus&OIo5Yp{k{6y7l?EQGz>^T%`nM;4p=2 zv;@iM>Xhf=IE!+KDG~p``1jQ(9<`i;4t}IK+xxq~o;p=OMZswbNC54Mc=l7lUr?}z zf(;6Y@B~qwxJH&a|GX`na?A#t2gf{#t=NE}fCvo5#g7Az2|u$k>hM={bB;Xs+EBmnd1GyH|6d2YG#y zedfFZUXe@WZ8CObdgGiEfJ{Ye0kSh-q|BEL*h%jl;R>wcacxXq{` z%GB>ukEjagJuYnWIRRfb<6{Ev*++fB_pe5DG^uXy=p(-1yK?!JPWI3~vi3^6RJvX12zYkzDJm9ORhL4F?Td_3XCOX q$`de?V8-H^QcD^houtb+QpiIt{>ZHwIZF0Ev_sodC^IBwGgtQCwt7;OV1u8lY@Y_(3@K1^c}qajy2(%eLH z){!s^7?puM2|2B*hsCN#=x3=7ugdb581o9`tVNXRWYh=e)F?b;3^8 z4Z9Zi(_YpO`*z$)2iZ!vV#n=tH5-OQ7pK!tRm~Rz%dKBE|adPi|kQI!lL0n{Iq>|l~2agjq4L+{+_L6+>qbP<6p6pGP z9Df7%KqE)D?X^9DadZ z6t00a*7RDV{xjE!b_qh=0KU5!=rZHEvYH7dg$SlfmGXDn+tbD_6R^uTF18I33+~->uHEqv7P2VG1C~q{z7Zr6_D5>z z9+l)W_-iB*WANaPD5?^OKB%}j3T`R|LW1!ceui3HfCymKAg{7rE`nkb+<9;_nDU6p z02moj7>~i{gyB@oxDW|rJa61C3T9DSaVuO2Thi!=FNOV!tfDkMy1FED)y1Tw_Xu!_UzPo`dRdqeqj^$lRTKjMVN7iSvhj2I`+7 zd6Btx<}&Ojwf7v)y_&%O@n_E5pSR}ix&`Z4j1RBW?lT84|2a^CvCCk$xKX>c1M4>q zuP6`3=!A&duotLrqIsw8?ECX>-K{&dSGP|bj4;aIIX(Soar)6=|MYZx*k7Jjt-8Z} z1?QGDj0=J6#$j4JBhM6{!-E+uMX?|Hhql-FaO8-0KpEOUSj-%p&NO>9p25-nUZgm! z%I0{-Vuq5ib;J?B1R7y$u`Z0RkI#t$9N>uZYcOj} zs-s<~%-P>0yddL;EKWqi-1&_!|6Ey2qaa=C8voJkYDqHn1G3V?2b;~7sD9l%CC zv1wK=0SJEvIW|66(v~!f$`X=-?%CQWsbQ~O!(I<;{bWmqUAu#trLeaU+ahf_jRd0! z)1KX8MG$WrmKI!uk?tE}3sUk9%*YP_v~j=Pw|+e&UA#h!hxhS1A-D^pHueE?@w)R8 zC)j<_CFF_w_3@?GW8f?kVN%O0G0=hulYDFpJb^)t_R&7XK+{`DE+B{lGw*=!;(eFl zO;kHDYkb!G1)BS{4<6{B-~%Gx`y={0ZZMd~&wTJ*rzX#xIaaOxHY6|KJl(zo99BEV zhc^(c6NB4S)_4#JjX8Owq;^=6jk-3|=-$*8I4U8Zhc;z_QJcEilpq1GM+E6=8kJl- zNe-r)CH%4WpisxtaIGN*#EH$W_4e<&hfji8}cXK4mp2k zXS8lIvT->|b(d1epH*tceM+GMrHhporL2f46~@%U6d7R>pJ*FkT)-w)VUnXT4{hzh z%ST*?QRqM9xh#aZX4cF-^k9+NV<@~p@}*N>Fk6ggivfW5p*8nKE+!l1e(raKkRN@A z{NTIfd*3795r5^|w^uHW?1|Ba_&prbzB)4T*N9_-n s!u+MtUUS4|BBPiy8he=w+8;EgyJ51Y+yLOdwL#jXkB89gk8C*q1)SNff&c&j literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/deepseek.cpython-310.pyc b/vllm/model_executor/models/__pycache__/deepseek.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4533427f040ad4dd00b85733b0926f1f01d4d360 GIT binary patch literal 12235 zcmbtaS&-budBzP4=GfVLNlMgGqD+q@E=|d@Bul2Lh@uXYtAn!Qz)~T@Y0U1-0yDD> z43EuhB75Z$bI^&CRGh?iLfW~QIF-uroV?~Ck9opNQkA^uF)y)2kz&682j*CkqFt4l zLZkcdMx)XF|KERfcgtm0!SCqb{`2~gQ;PCmRG9pkC>+NX|AVS1Lh%%#3N2J4&C?>? z(<8$(BGWS?%d;Zevm?iIRBEG#1+Rd+5f&oXbD3s_#i-8^%|>(HTr}^^M+@Ep(+lBZwB#*u-VOIed%eA!FNXV~`@H)&UkdM!_IvxI z2fPQOWp6n;;2mIkIeajB$a{$MmGEHnu=g z_jL4(_l&B%q=+T4=Y2)&3EcJ{dkA)*9N?ik*2p`TG|2eq zj=CZUS3acGuGy*neCJ|noM<(Y)OjTdWc_Ryq{gXMjPh%}q}A=z!_<7M-vinFY6s=j znwn~z4$|uDGDu{-)d|E4-A=Q$R@2hL^GOnP=(%)ylsz3Jbx}|1c&7M5y|EtT&5g9Q z79@VJ=da0bznAW(5+0Ac(&u6Ny}GQ2VG#PV-q}bGetP}+t_+1Ax3+?G`R+ZSA72Kd z-v}@a_e?A7Ki?6j`yon&WWi7!TH10h<|T_h3|Qb*ChTBV4$GAd#2y(btS&x z^_re8wqjbss1?WPf|o1`@N9Z0s&52-5cQIaeruw)e2HVfS#Km=c`+?=!?1oaka2?; zbMjZt3-Qay1d68uj_Fav9kBJ98xj!>JW50s`V3c`KA>^Y(+ zO7AOP;RD4`JQo5|NsB{?IDP8%)wj~(dP|6)ld;hBuHiCQxp_T?oR&6ex%>t*eOk%5 z3d|K_a^uv3RP<%1VNHAmw+rmTS|AED$?N@(~zQ(g}ZCl&c zQNog6Rku~-u*ghTL08Wx*VI+4)#veye2S7Mk<<+NG;XPS!L#$JO;umgc(dd=sM8jSGfypvmyaEXOKy7M*KEz4N zRKKI1P)grD$Oa|dNR*#b)*ZIRJ3;( zSr!R77(EKBTk{#*FZXhd2ME{-mbVTO3arThvMraB^?C2FbcS4am5FbtYMbkt!%4_f>EfrG+a7H0}Jx9`5kRj!*XaRubH2U1P}<~ zCazECp|^MI7aA{P2h^js^0`!Ri3`^>`3gP4Xu^`iLgPLc!q#bYqu#L%8Xp0YrD&G* zu>r{;u{Um5x3)ezlA!L3BavTuE`+KL<73DSJ2@#oM{oHGlG>7dobm)M*}JEZ^9(X& zWEI)ePI{TXsdk{}`>-4Qd=kA`%-3C6>)K6aB6Q zsh#(b7JK!^My8Epxk!%?uBHY7NQxK6>vbru;X5SBQ#Pptx@l#1b&fz2Q^@$t!u=XO z$r4Uq`gL6KFCcMEOLg*p3G?V;KCN5Et*YTz?sdnpv}}Hzo32T7ZQRnBl2J<^8#gC_ zc4F3d@fN`x$s+^peI zq8u)fd^(_L+eYF-`>Hz{da?oC=C(C32bN#le2m((OOQ!*M_p5)wbeaJyYfTzP37CH ziSvE|^5}PJ)bA8vU)#3PuRYN2((l*M?-ci|aliI${Z6rrYO9hhv8wzkJ{kJ6*9$s= zl}2i1#vSVoVi?(2_bk7kw8Ge1@S9~kNMP89dp4bdbli9mQfzD;e(w^9Es zT&#JC5lCrO`I-UyEUOi*s#5NtwzW7Zh7K=qF14Q02V!nzU-cr(;yBPD4+%|0Smwc4B!oITG5KU zaxUK_?zoY=V3t9pBBJM8kuv(fkY^#(-1-432Oa-NOSyVZl=5!{g zh!W+Ru9y{buvwmpBsrOVv|WH4C_ZGXZ43$v(EJ-(y!fi}O_D88_7aN1-$6>Yk~Ju7nDR)L%UiEd z>rP?7_Eb)LnrMm5wiKS(tL)HQfacUI_erf;YUk(;E=sV;+668ZDJS$d z+x<#1#XZooG-68_14o#bsjixkd-gEmBt67mjk z!hzjJ{Vn7^93#AZ2y?iG3%~#Tm&~@P!Dy9Tcsv(ZGhl4ABCE@@a~6^j zy`f9V0+Q6(M2oRIXmzu~KB^mj4){zKiC{oUb!u*=R(Pm1sO)>VU;4nT^U>$YH;Bek z&O$#jNd6vlcDdc61@y6?OG-#xbxxl#C||*!C_pBS5Gq3|_D4kV`gu?8;p zScF|eE|9(|n|K!@JUSM)X3IWG(DaCEB{D`{CX}3~7=3 zPsggW?nu?mRNY9`b6$}@I@2<#mE9*`nwRZ3p8X$OF-=3oUdEEmV3FX-?zwZKhyJW7 zTWGy1+mx=>rf*jrCqPflCW2l`ra=gwveM-g%e;f4Q1?x4;UFLk+4ltciooq~UEXU~0vAmzUZsuo<>zo^wWvI?A za!gCNw(cK7%_O6bVEt}zmpTXmw*&%bcf@`vY{ghku!4B5(g&^#;86~7eOv1gMg!vq z=2eY76S5Fga47r*@-HtNM`CEC%&AX*w@a@>`VrI+B|8 zso*UyQ=_o~JwoOhP4a`NbPAX#)xdN)w*X*l(8IXDVK_u_HAmgL@2-=WL-sWdI9<6N zrwJM)fpvz`{~uW1K`)4~EAba_2P|7)hL4AUY>gt0mFwD}+W{EiiR*?v^O2-J8r09^ zuaXzy_mCNauuwhr1q!;D0R#i4au}wa34&qx0@#_*awyJrCJOKc+%|S6I4uW!N83tJ zhA&_S737^@ifV%jc@hBSs+a~SR~eM20Xws4c{t682L<8D)iie)N3xSzJsC7x7w$x3 zLT0*jT>Lp>G`>Wrhn)!A$(}_#lXAE$wyN#KK^|KwAE+N_O}0SZcLoMWFcGu7tiMCn zJM0V}cC`<5r6+M!U^!F2Q%Mz?$zIb&Jx=Gg@2(IJ!6Atcd0-qPK-f@CVE$au1l)6h zIoUxF@e;Y;24=%pm9L^HTQb+w%=FcH88fchMkVOqp_`te3xmyL^w7w3Or7V#dURIQ zkA0mOh?vq)s?6fn&B<^eZKe+Q044*1*~44*3PW2ge~-HV79{~CBwV$bF@$@i9!zTX z%b`C#1n+p)j&y12o}(Q-wy)&_^e}0Kw8U;%?#fH}e2O>CR(!UPmAy7C#_$w!y)yxbvc-}@l##5SE;J;73GGIJ`+^a*taDmG3 z`x%X`Pc*}TQRf&l*Jo@^gQb1TxwW;hi<3t)n-($+(h@O)TUr=e(J@<^>D;azJE#?G ztj4=wm^_P-N&RgUh|$|BU^w4{*+%eu+Xf7q6m+9dAMq9p4KlIG*f2R@w?M`;3=Ri% ztsA>zyHEzCdLLdJCqwmGSME zGqedVaV5cxSH?DLrkOc_{9VxGTa-|^Mg9>be~iQ{p&ho!uyagtX2NOtJtVbawq`#d z0_%rAp&Xg%@=qxly*yQ0smbiYru*GgKtL;c5KlRd3mxkK>Sy3lS)>kB_Y+6Yl!5a} z6>GdTzk5xG2D`qxDt`xkKn$Lz0gR$N-o7Sn)}N*Lbpl5*@kL+K)-r8aPlSNRs^LX? zo>-0JK$+QYQju+Uo&&Nv<#9@0qGYVs*ocQOY%pQdVVj+{ zugac{mU}0+uwcErw3UTtim=^Fc&>!J8I*|{lyYT3fhE&}9nYIz)nJM?%d_X2mY5bZ zq)No>kcQAx)?yl}#u9U49`!R~A(##3f_Zu_3uj?hYEdkqZh_U%A{cmII@)JE*_AKU z`*A%ybvknmCaky>e1F5BsmOnnr^R9oL})VBKooWH$^Oaq!`T+vp^`~aSj1pq<6}=; zy-YNUv>*;E50R+0i*1Um6iDw-%N^Kmq85Kv!m#!dRtn1n?^h*sAhJ^cD8cgEeCjs$+i$=cTg2Yx2)gFaHe5a2*07 z{3Q}7%q$YVd(>%#whpm#mj}v!H2MHQ2Yb8J#f z3|N(afx6*XsTUrrSL_5AaFB|F{1Zq=)0|ovyZx93O0Otk^KBfznw;gVVQOa^O7X7! z6jG)mpTtC zBEA|e4$&pYtw z(F=T~Ofk19#^pjmn*(P@SK$YA9vk)hZvOPUo29(pAn&)G*R(fFqkelBb>1&h>UW$^H)(A(Rfp)Eq`s2mWM^(%V?j+%`5OB4}Kg| zOOG?z9-gRq2*j<~JO~WHS!s37`Q(&unJZ^;;%BP_w|phv2wvHmrPA*G+?BMv`w*Ce z9X=ca^s02`LyQk`#i^YK08$6mV|>1qtjmY-e)(fcKA>b7$B3~dXrb?LuevDl9syMf{&zl#e9U^&Y|%On8Q-gEBWQOiDHSm z$~%hxiTL{jnj#KRAUM>>)h>BX_fzuMl#tX)LgXxX`Hz%)iIVYrjBT19qT(Ag z7Zlo|%`SY^jcmJXXwOtW*7h3L?{3YwA8943p(8qu;oi^+PF}K5I_OC1zxI4;`@ZNl zaB|2>F49tGAytC3Y`7-n2r#mNvoZD#6$?nb5>AnJVxW421AshVUWLwWt&9Rhyf47k zoLA(U{BRDLfL@g;0!J`p>nxq3V1rqb#l?FZyb0<(%&l0BlY4_(Ym_ z@E&bPXGMAc^`1@Hh-SBn6pwe{9=&4Q4W~-^_#AJTWG6P3=&?ud^%yIq36p{>4e!uB zp5UX4`AJg_ci*%BF~(%4Naa52|G|6oKeT4ZG-0Oop14Qr%(ERHG#VREnW06_u6Ouu zk7vus*dbSPoZvGK3`Tcf>pScONz&z&x&JuW?9*Xsc9=Yuy-6HT#aV)bXq1{+=L~%0 zBD24pS_rq`z+4speVNGXl`Y1+5ZEcS(d^8 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/deepseek_v2.cpython-310.pyc b/vllm/model_executor/models/__pycache__/deepseek_v2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb2b8a7d900c9f44d3861383e517cc4118f661ca GIT binary patch literal 14350 zcmbt*TbCTybzW6=iPCgZh`EI%Y(maPQH9ZnoSRiA5?R`%7%Vp2N=L^642iNvnhVkEdGx;gtW(`;LpG?D0 zMqnsYSsk-$1y(l~jX}>5EQy@;F`>n>lA|`e(g@N>jj>~ zI^}L9sC1`-DY-9ns@>^eTEcE;raK$VcISe*?tCyWamCI;cQII$u-94YE(gmJE_F_H zPX;F?>~|jPJ{~;YJr$hlo(@iTSArFZFL%y#&jx2DTwKa6Lhu5@^J<~< z#qO7aFG+Z@^J4d<;HB=%!OPt*2Vd@fCHRVI{Jf!-)bjg=S`NLy+rgdt#%>gI}J}Slr>;86# zo-4f5=%SZ(QqBV3+>N()W296@FSzmxVM8Sg-|jaXSBH_8x>rM`+Pw|DWnP7?#%?FB z-w4|q*W!Ao-%uf6UzIYEX<{g5-|S()yP;~MJ?~1dM;`$Q4CMFT+1=jmYqa4lbX4W7 zx309K2#;TG+(Hc(+Ytw?o5sl?bwiYuJl$<Ifu;@(qctMY-- zGF3_W@0SBtl~v_^BPf1g*hb)CX;hQ)cr9GH{MP!riIJ49wUr8c>GYaDQgNAaUY3r^ zC;koxvfh*lpH$Krhlv-V+oQzU)?urCGjXsoc6F~wgdKhuc5&r!MVk<@abVmr?;55N zTLW`oZJ5fOGn6&32hML9D)&80*|+lpqn?ZH0dh!6UefYNb8b6G%MWs@u$9y2V@J7z z9G)wFFLz-6iWwIU45YitK(oE|k1V|LjlJ^f21dKSyItSb{mpy1#J<{YMEW@->hlnL z-m2=~=t*kL)?Z}2B(4r~BzA$5sL}q)-g}oalNkRQ9j^cqH=nXQN>Yp67SoM&Te=~>%P|0Ompzw zdaDN*v!k%n($BJvpQHFZ#nF1-Gm~Py-t07@s9wKke0=46+}%FE-QIpKZeYR3&%NB; zz1ICg_s027`|A1aTk*Ai@BH;nr+YqyqVQ(e+{NTgZ=>@ngVg%<7go1#=`Z7rKZh$i z3*i+l(=#o6UDN%<&NGzV-FKa;>05gXN2cNGupgQnrs2m%fV1Tyu855r7czJ;5J^R@~V!y`Fe=BKU5HVZUC- zqUmHQIjpZF7sXmTct|fpU=sHG9qB@SiTCAfpqe2-eX@qH@ex*2KLf$$MGUxxRbcfJrT6W?6Ras=(Up&2P5;umbW`G_*fCynO^);MlSiP8 zOHksE5ST;>S2jD0(vMW_Uv!`o4IL6s%<9I;C9sR%{Q`=t z18qh$)W-unpnl(%l zl2SCBK`59V*A^-~)M!K6>R`Ux50C0>_IFTW2S`h)60Qb?Ue>Z-lo}3Sv%ou>pdw2w zsT{2snoZ=ayGP>lAx#h zPx0Z``M_+e4P&8xv=7&qEJ8+?=>C^*Mdu(q)5n-XxyblnBp=w8{lLw+h!ZqF^m9j) z4#sMzbgUFiYrcL1CF!51_yr1&Vx(C#AzhYKf+en-*Agd%711;LGE>(^Vys=RAnrX} z5hpgZ3pBX>33ZDjx~1~TspmEGfX84>hKbXa-%n2&!uV*)P=&*6Haf;WhFw`(ruJyC zDNKzyV_;!;jk~0PxkZw;{JJ*rgbd~e0{HykYa8GFO@4m=YcLG#ZX>>yx;c5Jq@H-Q_YF1gTQ=($$ z)I3zIuNK1cW<@QEToSn)PN`XjTAtX~c2E_SJB==TJeeJnl#%`UQT1uVSpUdaLBEix zq&URUNQ#{0*I-peNp)wd-h(YBw{=W)iJ^5ij`6OK!=)XpHr=S}MsF+ex2{jzdGZt# zQ`|^=31`rqEOr|=>)ZXPO*;^lQf9kFiQlAQoY@r<82LPE*&JF#2phAw2t+~8w4OPDxLHIDFDhD7%Z&c zY&^$WY@#fZfzdOC(KGYC8RSXJd^ndo%C#_hFIuw+$Q+ea9OYS*x0AXI-@YUz^9h!W z9@umijfq-#qi2`n6Im|6GeXv=eA1h=btjcS@KpI#BYq71ok!0Uai3EaG0h$yc=~?~ zygR30wNf!ukyc$c_NU|1gXwr>P(3gYto<33?GI+~4qoWx_zYr7iv8v@H1G#yH6taS z6>>D2;z9-q|MmNC>=jpc>J72kgk`|g$-&^P49X)5huD$W@h$i%@&V>H45nLLAb~$Qo5A86QK@M=YEMisBCb~5rr3l?N?5GS&f@eI zqdOE`ifa_Ypr%-G2i_8>C9d(HkgY@AW1ejap)T&I|G2!cX1l0AnGd=#DKk-g6^znz z%z+2e{z{y*W0RRr_iIv-xY46Yan#MJaqFaAtxvGXVaEa#k{pm)5IM=W;KYe_i8<^> z9QSIo6I@DqMcO;+numKsTFOC5n>*@V;ijqXzmC+Pl66hSE>l%<3_1H9WRCtGgg5P& z{CXnOa20oc8f!TjH6s;sK*iicsCUMJkkK&q?mLCad-uMD`;pmv-*Na1$y-(E0Lr=V zBS&UPf8wG{hSRTm-!AwlwSwG^OInzw`uF!9J4PjpNnde@#!t6Z5_>HuL_=6Ntch5*WU!4j9j1!0wu0|I{T z=JxYF!0DX=ZTaI;2YFaYDPVaUmI4snw!k_aa>P5O;ZytWz?FOijmdn)eQ)62DJvVb zKsk3RgW{d3yC53-fWSM|K{0*nerZses2AaJdeK1b5I=hV^u+u9{qg|bC1v~tuw6<# z;q$kZL(ADBSbu=}%H14lTR=F@HG8H8tZBSv%(cGK! zn+uzZD8*Fd(9V*9`k1P4J2$Mk0$2lT^YLt!cj{;6of_pWXL+X|S#BAvS_05dLDA&~ z_T4-{y*hsPUAQKt4byA`#(8-(&eKw_DxNr} zjmcl9->Y^qg}6V9NWD+-=O})Og0_ghMe)lJAf@n9$78hA-(eE5F(~f#qMh9^ghBh~ znbm_cB5rT&_ID%wE6gQW`aVOn!U8XK=62d!p)N{9Yqtk)pX>|dwi`-+z}NyDpfok~ z0q$xuDMEdSu&gQl7a96Bh{W6p0GOm%feqjc1HS`Xiw;%NpTH(yiaC*3Th{|8Wn_tY z9Vy72m|HXqRXv8=g zyK=_5rY2v{44Ro^4VpKRL;os;m^S2osc8e~F?5+?nL?Zg95`L4@G00CkG6C|q!P`e zs}Qhs{2`#Cu|J&jrojf9@TOTm2L1ovmS_E_QnLORp86xFZ2eECV*Q~tW&O~qR(@c$ zY}iXS>?Iq%I6iGVGaqZduZio(hxOzqqDG!DGfY)g7Cdk{m<#a1iLXD8-vZ192flvz z+TiPlA5MJz#jv>PDNkfcq#u@OelW}@_}lVTMNPp^=Zn!$#sZv9CY-UahfQpdY1!yl zPYs3hsMMI1q&8lj)y+O!7gbywm1qvrZtSuH^CaiBVz>bSF4h6SUcxzLAnZukR(XUA zbn=61!;N;^RPM3?li!Wuj04cog+~reCQBEc7sKJEiU*ed3hw+KY)br=0Ze500H%Qj z7EUf(x(y%(TZiLL)Kd%G-`uQj<#igG2J82?EZvF;*6DL)JXg6*>tME51z%M~%J5MiVYM zdg_Ez)>}HevYH5(OtpLP)r>i8vINo-vuCp^jwouaIKeI-n)pYV zWKc}&&{j9~F*NZa;g?9{6wjkj5?rsKlG-#<&Y6O)BC9f zD#)+wrwDCy4;{ky;oj*nz)mjR5x>*NX(1Qts;#gm(IlJ>KMeD_h%2J^2Smfz1&R(p zD4@9Ep#f>KdL&T@gSe1}N!ljd>U7P)iD%xn>D>YCJCXzBPo8FMiw-<%9s4oIR1UGf zg?H=Ur1))$??Tj^N9t7lJ!V=Pi3M4!cH94#8u}|J-~#LW66!w;vFIV= z?0HoAh}n@x1TWDve+z-Jrui_{G&%gr=1`nJ072F#;sRrExd+rl#hy)%4?}SYvCx7a zm>+<$Q#-xy4s7U+bmKYq9yI_6FMU1rDJie2FUHd>3)c*MAS`iqR3bb()Lt9b=xy_Y z{O)=aBRl#q%=toiil{eZ<3lL94~-8kE}nF0SYw4-Gi7A&nPRct%cUt}>|R3=vUiZU zuXY;UtE%zZ?{OxPR9MD%w(mXrP}h$zV)a$=5k#*ckQM-ciJ)7g@33^jf68RJ)@lo5 zMG*Mlz^R+O-c~^gcRY)M@Xf@#&T|^Fg`+tQ6M98bOsCIKpJJ(^NPKYwX5Myvj`2@J z1k>W3Ob;id^+`$*ye}D)C&jR*QdCczEZxUDu*Z#KC49wL1=!-dc#73H{WDH!9NDdT zXi5sw-(>|aP>^qamz~XK5b5y+z!0n52X3Cmd&P8}nIF5}98)Zq_lNG`-qNFnA}OY- zs;wdcM{${mZ4jjf#N3hN8PH!iUlSdHak@$hxCF+Dom?cFCSW|XOF`1E?@^qg_#KE^DOD2xfDxivewU$Nqxgpu+}M%w6SJMnM(H(39NY#icYYUoHmxV*hYahYOmBxL4Z;O$?B_^WK)D+ss- z^obnMNc9ajc&Kkua|in7$)UbMdnML^zVWGVSk9;P%@iW}iT)5`sBdt1$yBF?ri>gW z;GqH@UBbL;xs!(qkn{B9!2%dG9uF8GZwclMc2q*2&xK{ihJL1Lc+e>yz_gJg303To zyDw>uY3M0Oaks?|X0Y2bi@g?ib51Qm&Z7*!<*H?M0;vl) z$G8|Sh0828-N$KJ*#CJPX(z;(I*A#*l1v;Ed84l{Hg=;%=kk@zmKwu}C^uXBDisV< zFNks@a~vx-8bR3DphDtVD4|+ZShuO_@-&h%^9ZJ*7%te>VB7(pE5*JjJgBnGJkBsc zp@~|u*OR;)?GcQJmU#~EtWAxveZoHAQ0sq8@i@ibq#)!ag-*JCq?dSCO}Ag^KTl8o z>64NvN{{yGzmL1@6wbp6U#908(#OF1-#ABC5al`at`14Ei}@*3_<@hDQU?TjlD$oI z`AEO%M*+eS+$3g>aFbqfp}85Y1=;1kH84wm9ZBdd_C~Gne*-vc$eAugS${O$SKRoGW>7?5e+Wlis+@61_1gwVc^Kj4Y1dm zR^QV96zTe(Kn%x2u<@TG0;)9KL|V-FPYF0Mt)7q{vO{(mq6R*Uo z@)bUTdcqw(+0$v(#Jk#vo7d8FCwav^pTtq&^>#CCvV`G>;n8FvXOE3o`~+BJxlOx8 z>j#|jcJBCCWu;0HhsSV&;a}r<{Rb4k2LbwVhG9ZLkcXqPAEk?h3vSr{f5jZ*=hk>D zg6)=dHp@nH7Fap@`!GfiDyi}L!2Njd#pB27XE39rkRGg!jvNaiw=3e30est(J$`Eg zmSK#!huuJ!7w8ph6K7Uo8xp{bJNdix?*Pg`F8(rz0~QyKTIH`BdSz#EK+5s*d+-VE zyPz8`;!XYeowGQE#T+lf&toIz#htT*{2h=Ik_uR;`8zlTd)L&Lvh+(kXF);mZup7* z81iB-81%&(c$-B!^#`CJgW^HX#4cR#@*szIKD+M^{P%2mZ+Z0Q_Rf2g?-`Z3i}>Tp z%!446aXc(9C7&ME>({e-{rb-0sEpr0YqK%{**HNLm*5M_YIk6n#?GREwLd)0m=xvc zBKI=obVPQ_F)3t6TE%s){|$yi&OGXWi%`uDW}92!7o$hzm|_+=I$)YcPD{?LG>Ou0 zp^UxRY~K@9eYM@YF20vNA8wc7nX=Wr{OVO4rp3VwmuF-Yi~ulnIgiF*`RJi+U(cY} z;6ys*(p)_2hMmW5V|yDYm(F^+f)ZQdgg(xi;c%rIpSO+@_}#@Rl(?`2BJStvCy`(O z2a4aII1K^I{i_UrA0k+Q0nywV7BhNTFrU@*XdMz*G*Ty4AhB72pe)DARd&cc@gzsu zd8Mq4%%}eyUYF!0RVtJeh7J4otj*d;aLpz@OU=rR&T#Q|5RJ2(ru|bJ8+!ag4SwP~ z+~%`rS72FJaqiOxErmlLmy1nDl2g}8^T55I#SWDx%D{Qt8I(FqA$TMF9s57-ojo2D zCM%}@8f*P^h}vVvxmxNc}Isqevm}@F}L0#E2KTL4sIZRjeXJr+$ORRBP=fRoJC>5tY@s|`iaUcie<*?D0=`C}wZ|9K+k7lN~&QT~tslMc%a55D*r zxl_LhfkXcLBhC?eRZ{YOij8^hxX06B6m-(784~Tr?M@^pc>Gv@ic;zomS2|gqY!(g z_!E^7lE4$!1CFlXZ0Hlnl;qRG&mo+`F3n}}+$0X1dpP}`y5z4g`d2Bq5M_@*Tv}p5 z%9@z~!!>3RM08(8gHXt8Si(m1DulgoPO!~4t3bFp)6dbfzGV6~f2;WVh{xAjGH2|W TCzdYF6pUYhA9$Sl5H5xmX;#46?At1U*Vg?J{ z4bEt$IH^>mlut@kPC2Af(YSK)A(z~8$tkD&1YC2{Eyvigt$eRR&gKQIZZx`Izkd6B z&7@whYIv6a`j7sfzo%*crjNsqiI2C?5d*5ju91{LS#CoPV+`sbc*V2 zMJ1=C<_cjssyG!jHp6N(@3U*odZ`>1f88QQxI-Hkxzh)VLhZM+?q^ z8dt&-(W0}c#?|nd=%jN}jiR_CSlXCD-t)6BT5H5*Srx7IYX z^6iuhuM_gjTn%KJmERkrK@xjmR=hVHgnX^3XR|A5%43?o!&8rWsfWqw)ub1s@_mu` zT*^dXs(#mtutmH#Ob5fXX=JCa-?+K*lhu3fhwr_!^0xc_%Ju8F*KWCYZ#y@##`~V| z!jOlnci!b5!}b%5cfDblx?4Qx^;0)YJjNB?I|<|A2fPOY#Wo;k)ImYqJ4Da??HHUy zoCRLW@9`KUNq|n>Sh=-&JHmx*xjn^?#ps#L&O|zMY zMW?}L)#@Cd9@z`*lZ=D*B-EL`~1ao%dO?t&RxD-weftwyIkVS z)AEz3eb?*qmD{Ucr>#y`2vm1H=#J6IoDA`8XchQ!%f21k`!Hwh)zFK)>)Zz)V%6K` z;utkHOSp_*PVK(8$!&!Lvh z&cN|l?^}In4X81(FUI}DkMa)#>IV0*Imh!gQRgIFP$Vevda<31BMA0v@Ys#KfxXoa z{5}q2Lj;Sl-EAhm*CD)705X7%JTg__csPf5U1cdyOB$ABE6hUrQy^QJr1)*)RuS?xsV_5 zd-#yHheL>u2}0Ne;mkwBA&I;)lqAUwzI`BqX3JiIEuJ~NLrG$SEQWHF?v!bKBlEAk zvC&j?#P+_pTA7tgG_wRxhaz6{jYDC(jByE#qPP4Px>S3hJ=XU$T^s2m%{9`(NLx47 zO{Vv>2ZdcTEk3qJ=AQm@W4E|#jf(kdPahed=pSf-HY)r@{~hf#)6#l*q_3NZ-Npj2 z(J048+2BEe6(0e&TgA#0-Zi}Ic%!LzBb~;mf!PYpq%+KVWMX|5ICRP#pr(}0DHz&O zdO+CbnTCFWX^7vIwXE(5zaK#L;GD}R#Cc|R0#Dv->f!{xnuarnfb7P@$mLOo!yDqB zbZP_gi@2wZDw_%^J-gEPLf*qBJpXyKc_J&yA>w;hcHJPxx+`X}l9ga0B>t?Z0+Xm= z(wW8aNw~>fHRt-F%#1j8tkodqo^Yo60b}rQvf<3=`dRhBjI-%P1ieE>o-KTn_MQ1* zOjIB72SJsWs3@|+$R)oOr0#4+u_WEWq~ypd`w^yD&P>U}ZqAMgo0plju|;n&Ps~&f zn9yCZh^bHYtm1+zq(|5NRQvo^JBUMym%pf$ zEH$#es#bRzDm%6&`tfxjnLIS6_70jPH-IH+>-t^rveq=jDa>Rg74@iY5-;FeH21q7 z(&!Aj+h~&Ps%-4k^W(;wvPzy7117V|AdxD}Vz{yC`W~WzgmtDVV*0oa%oYcNZwARw zx)W=F9RX|sQFOl=bMp`*eNyA(AuSUNikCGE6ff}Ok-n$zmPXoR`0=_qDvgRg)MU(f zWMIC`3ZwGKV1*6i+IR=N2Q#C>$Q)VBd}QrbMiqEcaqY=dJSYJREQkb&t!as~cx83P zq|BX1iWcTXn>tdMIFGJbRKj#>%D>z^35nBK$%;_9-_HsN}B$^J~vAf^^Nz4D};Y`lK1x2 zw~| znCqqABYY!ppnXCZfwquaz=*m6XsbRDFoWIBU^ zk}1j4fIJ`p@hT?alH`6(Gbj5kj1}3=RLt+E6DvFFu$h&+<|cq89Z@Tq`cCbz*Jh>s z!X%GmN%A#nilkq{`~z**80puwU))C?gkqs2eZvsXDZe9SLSY-4IIXXggAGJTh z06~@UxWO8T9u}y=Z5*ckhmo~D&8C;g@Qe9q!=zdIUeb0gMf1!tPVqgmh*Ln#KAr>K z`8{n^|u-^z8A)sI|mcS=~H+`e<`W->%+Mq{eW zsRmQ`Uxu3i-G-%2w*s10iEL7G9=X~mL=Cmx8fVVdPA&h=`yDHPKcR4_Tym$p580TE z@)Prrxu7zt_&&iuN8PjNvING11l7V=WAZ+SX4yzIC{6&w2yW(;DSdM>c> zthi2q6P_lMN1aQo0_Im}@?CUh3F#T3)}o~R)cAGPsA6P3ox!_?z>ts2MneUQMPmVT zSi^5or=Hdu#)7dkmwV@7CUXBjNp%;|>bhCgrMhYu(zxcj9}T^5yi#&qmiQ>msK8a# zL{>)4gYcJbs{o>+qhsZNa)<&!Q9u_I9z=t>tJKx0TcM8ftazI`%KCybKvuwiDT@EB zKCXg%9)>veKtwMSu&Np;q|Kp85^7m9^~E{kMM$J-er6iRS7yQZvTPb(7)9fMre%C? zm5k5Ka`nIEB_yk-2&zww8GY&ShDp+g2Pa z!*gsZz*?w5^8ce+9nZ%dlgXCX0Y9FfDoHr6DzMy0CUsg%-RNcOconINOXvCT*fB1S zlo*{0-vO8xz=QE0TL;Cts%BMP>zw_j?Kz>!e8o6WxWhCEr6AXFPJQbhnOEvBeG|af zssLpWPF=xDb<6g-G8ZS`5BbFsl|;ma%#X+aHuS%lY^gsjls#tF9c8sjfro5GSq<4r dR=$?o^^YjVNb={CC^Tz&jT+o5f75ue@P7ja>p}nk literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/exaone.cpython-310.pyc b/vllm/model_executor/models/__pycache__/exaone.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d88bd0bd99c01e55a2775dab213ac9c913bc367 GIT binary patch literal 13503 zcmb_jX>c6Jb>2C5cJ{zx0RjXsK@vr3B@q-Q%Q{R;qIgKMNQf3CIhjbAEVl>1f-}3z zo&gVJkvN88DT!Im;ly?iJX@cn$SZn z(mg#gJR>qaGqOA@vOPP>d-=%m9F58tVZkfluNf91*K?U?g~h1km7=m&=DZzNqA_nQ zs(Mw<=fm-+=GCGJZz7uXCYk4i+oCCNiqnN~I@<1S=d>HnL_544(N1q?w9DJYykfXJ z+T-ovbSb>^;os$?#**BivpF#%cD#`jI|6t4>25^BozXyB((jxfqC}op2>;pU&Do*=h$3nT~(@g;zJTUg})j$U<{t zCp(L+Bt9)W%^=1|!K!_FrGfdcVzE2zY;Ib57mb#_fke4LWD|@1Cas1aHya_1)uB?d z(+#jt`7@0O3oO%emUwX`>8>O+@MFhM9C_vB%l@k`9zF7mfBML=W6#e&>z{evJDyes zLp=3d&=Bd=tDR=!Y@c~NIvWUq&yH5MiD03z5+=U-`hM7H2=q2}?!xGx;y!;SYFs`W z_%etZT_0t5$bP#K(YUKTM6^5j&{{h2d>dOQ3Ph`s1TXV{jeAwYu*@ zB;_#c(wUWRwzKAh8PoOU{|!EmYpjEoh=BN^g`sN0de7jht6_GiT;J7l*Z)R@6E?i^UPA zPM)4Wk>=9kQcH-S%{w<8-!O98%&D8b$LQ12dD>>!Qhboiv#my)+SpwStxKsbgJeav zoB5IHP?{bCPrL+@LJ+o(jEhqG^>%zEV^>V(Em^~`d ze`Df}#c!-0=6}wRbBQhTJp;9zuN&*y=d>ihoFgmCNV{j>G zu3$r}YKjsldozeeRX={zMShXL6g2= z|7RjW8|f&X_&$(c?wwpuyQ;mWujy+Ba@bH;wKWYXY#run;A>|v*ZI0352KEJipVEH z>ZW`ee^Tv|m(SKX)qF{OZw#Ec)QA%)2`R~GkkmZe>4dzD@;ST4nRPuInG_hW>rs)IYE-{e9imZ{Xc^-Ko4+-OaU{sIcKT zljXki6(lx_Ac!Fok0o|M1Q`%PCd7}a#7|&}VqgdPzzGTq`b-X@sf2fB@LZ%TK~an$ zT|&AVIAUDr3)@*-)x-qERas01mE|$9jp-EA>7XjcDYf8=?PBH)(;H`DR>J|`nU1Ls zFwpn!R`=&}^Y7+nb2;?|a8OuK@i|Bf?Ul%1f_jS65+$KdRL&*c=O;^m3u%edq-G%S z)9I*j$?taJ7HL(8_Dqk(!;_SjlrGIWqH`r50cn;%`o9Vi1w00xco{lpC$$a34_nwbd%0z^SDFF@KBvdipUQm>+nG2!NVbtQ%=tDrcCU2Kk_S~PD?LZ0+l5|< zxEb#;;w5>3eyGll+&cB>B!wOzHh9E0!KJ=kq#=8kT}b4dUqs3ot=xm;UeV;2 zd(JzPS98KWr^_#k;#xV`mbLGqN2_P|%A#}yYv4~M(|E%p#@{j?r{|)TGQZu#Hw#B=_HNNA8wr5kkQ{hn^eJS zVGeqN8HN$d7}h23qHeX&!>SkwCbR{7UWfO3UX7N z6Yhv+s(Kr$B%UIjpERFzd35q8s2GblmUt}ZXyO7-9D=wNTcdw9tpw_7lSa*~X}0ca z_RWf3VXmz~20~ubzf9{~*Ijo+2bs<8xYP~ZQFTo)57TcKN1xLl4z1K&5iXMG#T`owPyFMWfB2&$mqZt ziw0THSM=A6wfvg1RzPVJKq=pIt{SKT9yk`|anOvOdDU9O;e5yLx!htAEfyg^jrn(P z;sl98FCZf#Z4l4Eo{`TYm1^fx z?Odu|@QOTXg_}}a{T{6R^EG4MmG9{DCe*eGwQWLeQw`Iu-H_A^eV7fuzOt65!G=lo1n+P0{C}N=-CLdo>ZOTi!;f$l{o)?T2ylh<47h0ku$qAxSOB0F z{*vtsxGM6nn_UK?jwrC*>;?s3og&i`(=xzo1$Dqjt$e^-$`R0AD8VwX0-Tl^oK_%Q zYiUg_;jy3zN0?;Sff7_|w+|FL+nVZJjp_t~i-U9ZU&sv4RZk;bK%fy24w_)yShF~d z6NR+hvl&M1Wu8eqxfgX{U<>nbj=7ewQ8Lfzyl~cy?jxKoPy@cJE?GgzF@|Nuq=XVwvMj1sXeO0g zIAbgu#^)6*Az`n;(QHSy;6DT;8--7!z?>lr>4mg15C%TEDNZm>^ClSi2bcW|WI8JY zVAYyK*|wxz<2g~-9lE;* zq|`@EYkHdy5E&ZQRh@kUWFlyza&!&*4nBenCEhXEd%)ZU{Y*~&6gq&c{XG7tPxU6H z-XN~6)_MyuEZ?F+L%}$Zw0CG&+#Pu76ih5_6>zh^Z1x9LhZbvf#}>;pbSo(NQ6Jk9 z=+Lq#3^PqBMmY3MR~r(EI^t=BH8Q-|@tcfdI`|G*0t^m;4%2B-eV$WvwXvlzPm5tkHZu3!mw!4$$CT&C7ui1E?@7dF2i03)Y=KMNL@c01oMfpThK(2(pWD--E3YB|{ zT&v_haN1@vrd*{-6>s2Ec~g5+Ux0cP)*DXGf@HTLuk%-o*Jva#-u($YlUcjXWLwte z6wG&$;M0Q^r<3ghpJtI}QDIlZ49oK3JX_#wi^Jhw>&Qhmt2g;nnm8nbAEjxfCh){n zkR#G!cmL`V+QcZ{e7>fS4L;ebdw=iqN| z39pD8U5p@oDc#0-%X3PD?BNo|Q0 zlKfe!M;d5B-cQLpe!v9?Y|N)oZZiP-dE$Kx#B+HCc#=vzU7=S4^>mm%(Fw3z$0moe zZr^lB9YG0|bO$ygOns=Mf%^Ht0j4#%(o|qDfKX zuC|)povHzCRhV&qTIf3!Vm7?f$qjcl>hcnv2<|CXyCB|hD?{6y#1PNr_mLnGw5-XA z3_2Ej_FBG83YFdHYk84dbJ`VFwpu*HK1oYxGr2YR;{*^eAS1aZ$p2@p>&J4hPmx=K ze1Ul9!Y8LQY$Tq!9gbm`gwDF9oE7Kq<=*r0rvM>gBnsngeI2Wc-V2mx6`o%!_VU2h z=KRRv(nr!Rp9T5Ln2dZIB(-oD76bWf#P~9iuMqhXk-Ld}6{K$^^2cIb{&jlQCh{9Z zev`=HG^biCwfHo!n`6`GWu+R@*L)DQoPt9hDc#2H22hYzgHKAyl@kAT-*+-6f%9S2 z!fnFDsHExZd~<$2(=)uqwsvulh#b-UWHoH5{%KjHMTCI>uYyC_=ZJiyZ{z3i7EuT4 z&rlG<$I+3*|7a`8U&*2e~m z5cS`H+7rg2F3h)e&j82zhUS?9w|~L`#}b7_&9ia;iF;4F^AuxmTVB3UrGMf9XnF@) zzQC6d6ZlJrU=n}nB4V4UX0{)KEa(yfH4cdFeEBfLbO%rYE+9~LXE2UHib7DMdcg#7 zgIZ?)QBZ~DO^Q3X&TOy^B?_wEg3aL+rZ*+--tcxBmZL59ihF=qwu^niOt2%^NhQ?f z$$~ESi+k~Im#7Cjd8DSePuvgAZniV`;5^Kw<7(fW=*XjumADa}JcV0!_<6~ro!aNJ zzskc+7l#W?|H16R^JaDtXxeD3mTZs*+n3tb{sS1O_F94A3@8K>TCa$^cJ_@AZ z7ZSQ9A(NMkTBm2m4<+R`+FY*SmRj9pkKs*K9wphhJibgfA!B|mnSk*QxW&f%WYz|t z7vIBdOQsl}5vmYqg zjKJ0KmPAeD4~Tb!$WbB#9PwA!WijR4ybS?O=4}6;^S%Sq*VL+H@$f(0w7(BetmeFD zySPw5d9vH=4_(92Z38GGv)M)t-~da=k5Vifj9S{A^w1zdY;Ir0;KM4VaGe+7nnETw>6f>i+cMP z595@FpJAub&#m1wwwDmnml#>XwQ5nk6Zx@hgRQ# zoqko5uS2pLc>ntOqquE`lwH?l6J;?r>Z_MucF_yB-Yc#f8e)Rl^+D}kR=bCh7Ad{X zZL_=Q^WgPc`~pUmwYXvQBco;mv@l<{ARu4Ef+Dg<=|?G%8#0x$l<@|p(ws6;%tuvq zrAC#uGUO=p5SzKH6WLWSaQf) zacS-%bvez^FB}Ci6Al`hWO4M-HY;3|}QWp$b$tjUBhzHcDnmU|o5X~gO zZ#s;+VE`gm0tVNC$Wf9|xn}A$h;Dh@{nkWOU5& zVet?wHaKZGf&IWj_623ZWU)6s@FWaaGecG%bY-o1cb`SraHB2GUPf>NB22bA)au@x zcI{@P@)N`xx|2!UZ)r->YRPXB*-7NjiI5VMbR3jv`qz~DJ0dR?KZ zBK=9eYbS>b2ph})p?$+C>j){dKZG9|2Uz{HZ{Y46>5X1$WtGB9{fK#^@24KMM@xD4 zx?Z9;6e8!C?gt2O%W^hyyPO?Ic^vg;)4cDCP7~34Hg+^*p9&u^Xrw$hUJYI~o+TUB zyjHx+D-<_Lw@m7Knqy^n?Yttt$->7-g5TA76AFa$X`^%J-NOac8S6)4 z&hZA1=Z5!492V;By-iuJLU&5+)AR1UT~Urw=crVU65`k(@BZ7>;5hXW6i@Kh+kcxH zqf(WFGrXt&kEQ2sjdPz{Nm^knN%h@!m_xm+;?y~|$-C<|9rx`JvPyWDdk^2Hv@(-O z%H)8k2svj&YMSI6{0SEZ9eDGW@tj0ky+e2729MX`qK`i&v!0rV}EmeZe zHO3eB)2j#Qm3@ju9EYHCXbWGIvfrNlLwxW?Bm>I`k__qr&iz*DPo)Md#8C#we+n%k;I*#X{juAO=$tww-8M$%UD~q%hRpP2w6KOjdi^sk3 zc*2_yc_*sJlis9Am!hfoh<7BO_NL<*Zzi7gW(D7kj>gBlVjjw6Y)v!WPHjy zC3uzSbUf$HiF7r3D1O*`Sfp#wnfR=CR;0(GbMblae0;&XAoAnU$Kr<9h(GRqT;wOB zN8(R-pAhMK^k_Wq&5QJ8bTNL+drYLKqQ~P)-X)}uu<7WD_(|_ck)DY@8Gp+ARQ#0p zRQzf0)2b5MVJ*D0s2x?jDw_>I!;Xf|%4Zi1c8nc=%kVz;Wrdw!C*M-o$*{C?d0Y2B z56&re8l2Mw=V@@}*hAnvRB)aF=V5jRoHGUIS#Zv>bKsmKPWa@u;XTK+H1>uOV6cg*rxXR#31=fm2>o{ltc5R!+kMGzZ>u%io(d}L3=en`{45R4v(0hv^K+h?(iPa zkIzHaYlawxOFae6)L(2j^nB|1PSlIruN4ow(rSkR&nI60%GVCEu6EYwN5j*-e-O(z8*5xYA@j_M_9NR^rF;X4_iyisULL$7SiL#MW3iNGVzlywXw`` z$XY=fUK4YipsI8A>vDYWVV^5txNr2j-44h5yoQ;XX(hBgaVtsComh@IL_hh3I9Lt+ zFz%)seru%XV)+t(5wqLj8+ln&j7VgXd@SvxLFBi4vA>LWOLB{cX^*#?^zi=ALBhuM zJg($lK$w~;J9KH8va9Z@lggHw>MI6dMsmSp+N@GmCdGv^lj*m$Ee$D)8O+Gk83kz@ zTzyMV^&KbEGJRLuLP@5q>by1DA5$}R3Qsz;Pcto1n@pWmwzVy*{{*zLrj!{Pt9*{J zWh3QghIpEp*}K}3#!O~mTt;SKW7-SvX?VN$bdZMXnJnGthI|e=PSbHm&2`p_8|B=H zgLFAJ7o$#x=SI|7x|Exk+vc)oNm2J~lm+P%mplt1++GSj6KnFsrCjfJ)^j~>wR620 zqz#9kL&sjd8Fi8{!N%*XhurTh`j>L!dXR*kB{+#^B@x6l@!arc$^&1NP136IFb-O6 z5(Tk-UPV@r&F(1t3G{#W#fxd&z1VGaA58-enSb=DxVIcX9IiyeH0m(eblzKunRBLZqtEi4<$lQm9 zZmGterMap`r5_l^IO-TW|0J3;aqasF6$sA%eiI~ApxhKsE2$-w)tDCQOb?CFJgP7Q zck8IKXuYj0sJ7ZT-A&)EA9X=#}TOp}lF_1Ge zWyQ>lX-MU&mK=Rq`7R`vIL8!}T1cbKiYan7QgoS9C|l7`|8Y@YU2#y?L60TeajiJ~ zIpkdA%Iz}pCH^9#XP4V}CVEE-_i|>en*19wRgA!+9^0ji>Xi!~BVJme9;nSR>?-u& zrd8yLC&nbo)Rh|3Go2Z?!Slz!qc^Z=y#6>+7*Ua%U}jq1nWX%#dYsCqFk+|sW6&cN z%1@^=D5d^2@I=2zPh)qYPk$D?qqrZ#{W$I?Pg zkc|y%G{zSM3JE;Ac1H(5w1OnBCabNkPf0&ujDHq_xP%M8|9Pfi@XJW>&l91Nye9Q6 zP$PkwN;W@s74XkD=7Khx3usf%1w5RKdub2Kd1EfT*^GKgYb~75T~d>Nx(vgiHGTnI z&BVb?;8oHhjS9U}Xu+hV%mF=h4-KemABgEyM(grR7zxyBh2U#-P2!dqUPI^4(I_io zC3=qN->1$#H_Xa8@-Tk$f4GU`m_HkfcM}g zpQItu?B`>my1yE3_-jGb3v=s+G>dGsAp3|vLe-4q22Z`QR(s8F!;&l1bzbfE4KBPu zJ3roEz|Q)JDJEBwPf)kSTG z>g|nVEd4T+zHiy1xy{)FdS!kfo2CAS!N%>2DHb41atFd7lullyN%vvY2@9Q{$J;l_ z`Ro5k=M1c|ZzGc_J4&YQs&8sr=9ZONOzEk7EHg9fDP_yd%oV~VLROuwB^_3pc3XQ> z-?F!yEz%zbe*}G)u%hhD+11exO2DxwpOrE_Gj>g&nzCbOu6T}%=eU`*NbZcj@ZKJN zyn&&>pm_y1Y|3ubO8ElP{8b{?i2NFnUnlYgNTbZZL1|L;+$ZuaA_0h}-&kFfyGOl| ztE;)X=9R^?bc9hZrG!iyAu74O*x~B|XMY1uQU|Ho5DRq<;!($+mThz9;Gz_5?q~}O zcdCRR?H~?;AFfZXPA{T##m8n3B3Z&o({ob)yh3D}2uUd?^C!2Y4bN$PIAL&JEyTJR z!qX{4TMYV5sv?B^DWqtEl8=Kpie^F_?`fL$p`~jdSVrxh78=MI$Qg!q59RM0R{bug zcK2CN7MS!)z6dh3o|rpoJ1rW*W-^!wo5^BUXs!aZlkA=Tk0yjX1YrZUk=Snok#!GEre-?q0$4$?D>PXBq+!hzz5sHfY#Y+) z+o<7m&w!SMRp6%OqNQH6B@GR$UFgLMY3PQv@E#4}8vh)qXDo$LZ=d3O8f>X7s+=Z9 zF4+r|B2kfIRu!n}FZ*f7e{z{mpdi=bF!7hD3c(OZP)3muTU35h)LzEJgd7Vnq}k!3 zZ?VP$5fu;pCln?h195>ShQJVsGWT^`^uGa#i|2ntQsQ z$Cc0v3rv6)hP|wA(Tqz>P%{mf06f@%N42Yp6zr`*J(Sa)zN0Qc7vDfXoD)evz!0yQ z5<{33;^l^KNFx95xDwhNHQ=*TAm--8@GXi>$9t#)09e6|uM&xf3^&;Ry~SfHr3Y{l zu#xRQ+({i2{3EVp9E5g~@qye&wGU3saw=`&+V>+JM?d~45cmVdY=2Rzei>9bpyT#g z|AfjUdNlLN--8^Ge6spUs*RMe751EvaVv$CBTZH{tawoPuPZKANZ}WdD)S4FdDVB3 zt|Xs>-A(<#|G7;q#t!7i+owcp_!Io~17#Bj^7XcJfHO|d>wc4-B=*~+*irfdzks(P ze>dzVjj4m%=F2GLFA^ab%4=i`_<&~_W)gD^BPi;MxyxtB#){f7&xRI@uq%dJi{K-# zi;#pge(=~GFi+|ofBkiJM!88ax6D zryvsv8PJAZfdyP4A=OEr(BHjRv&b?gAtw8H96=dFt?6V6S(;-IIv0P4@aC~Y#I+CA z%QP!3B16C*!tNRh{tMTG4Vd$X8ZgxEK0vO4Nx$SK$k2d607zpK2>u|{Py~QX82}>l zWzi7^jm;bwG%#QgZFiX~XgMs?9RZ_Ci0}{ERFSSB{ts?gr89uV3L9hNFnY#>(K8O| zs^{fGVNqCqAwLKoxz*+2V(aEUtDJNJT|yW9J`zJ+aEmZryH)4{+{yN_aL0D>owS4$ zZ3;tZgmPLzzWTQEw)(cVC~Wn&oXq%%!t|eNh_vW$l18cFLcGNvPba7)TGU|^k$#v= zr*f+ID|5rOFT!oGkmBu6N&N!*3HPL*5gE#;QBt-qm2-K=t2p*t#ni*CB~@`fhP zK<($w^x?i$7BrGGt!BM3GUnc!!smkHtW%Md{k1TPD> zRPNZk1i!RMs>_@-6VIq;18QZFpP}UGUlMA6fYzazQRMa?IeA;%x&$9%5O{x1p%)SWe!w}CX^2; z4P#b0s$|Btv1Q#jB6_rwCsG_sW>!nxRw>Q~6=MmtWnl|y3sMkVPHNDr;?QHU#bF9i zEP?+Hc)2O!>il<+=D$aTHl0^NO+*rkil>Nw8>CT^fc^)>5IW-rlnRLakO+xguC{Vh zh{OoeiB(!cX`5DwI&>V0Z`WbmN~o>iZryV%05H~Ub8>Wb3Jishyui27wlVQwledC$ z;Zx>%f{4tQC_7I?7{7!o(zO&XMN9Te4C3#i$`u+ky@GTtu`k~rI+x^5*523V`uOY= z4yI*kcPn=N7o2)%;@bD41)*Q^uONMlZoy?=QkeRV>cQ4v`jY0EC8K2EB-IRslNmZd zUbvZIX%Mz2XS3?LsC5!}U0$gmR9{}H0;_9uNBwFz2DG-raiqtH2eck%6K|Q`1gnd> zlVKetwyeKsu_<;0SU$<7_mxi3GuaHAMcEOY_fCg1;jHKn8USTlarS#u=zwDoz*BkQ zw7k&a&j-CEh_1ZSFg7RV4+Qh(@7nXib)4TElZn0$Ie)j$DE!2E{>PXx{zo8%D?JEf z?(-_hqZnusT7sb_5#9p)2;jV)!Ydf~6fpE~u0}o{MaZo-OhnY7xVFX5)BcpYrtQ*M z8MFtZ+t`aDmT#_%BQ;)l1lJm>bfPB?VzY30X`Fc(M^o(t zG9M!#JDIjZ{~!X3ooh$(u#q1_BYum>3q+1kALBSp!`ZFh?H}IqSz_CXJe}p=M{b|T zJ(xrZtpmj*Hqeg-&E7-OGRPwW(4=SdxiuU?;(8!_>G#7s$b`W$o zM=_M}bAX}K$N=3C`=Hf=FJ~f_hA7=o(f$d&lKuYxsKa3Cujvg*OSul-29C+*Q3*h! z?|>zt@R&ro2{Dy)V%U!{-WSdx@e>MQsy}$B0D6I49M)z78$siLigNxFkp7a0INqOu z0XIx6!SE^aMZ+(?i5MOJ{wDm-(fxfxrd=b3O!j+!zHy#B zGF+Aa6lML9QZJlF>Xq9#GezKM6@l}E!?mQ`{R0h?9#Js?@kM-hbcnKAZp(xJq^Y59 z|2KxjKpg1n9c)OEKAcw5a3oi+na5<1>X0x@K@Z zkCE6r)q;zI_Fa9YwuE~9_V1{y^h0%jyLZ&9%J&ucii4K$R@GmjWo3U$I`ID$T36FC zz+5@2WYr}dBQN7)09(vQC7r;0)Up~38GBn(l^bUT6jm4b92WEid>}B6!}rcw=reqj z)539nxLGTpZC=X3ra4buoQ#RP(v=hT>jm_{uM>edC?a&8SNDH3QkM?wm#O?EBF_~6|Q?$JK(&t?%8l; zi7kzc$EorOkj9uqqIZaKj>uO)yyIjxOXVj%H1N@|e|ndf>5wW;@M_I8H)vd5Rebxx ziZ5YuQ`F#pL0wJmEAqu4JZ=33u2o!X{4eRL0>}qwo>!<*(OdG*v>izkkPGYLqHwCC zj*G(~O{*hHL)ICUR>%Pc%u_AcYkS&WjpFHh23%m|KO}R=MRZ-mRWqD>22{a-k1{H+ z=?td)Jp z>Bkx4?qTMsVY|Q3DpW%2<{0iht<=w1$endKJ)!Z1-1dFeY2x(d5#(`zxj_>t$Et%P zk8{MRg-I<8ZUSr15g+u-x7)%8^eQ8^O(Ley8+-8uUoN=(V9Tq~Cp7bexQ;kO6*jw9 z7A5j?5^;FsjSCLLX)14BhdyM&(~)Pbq%cGr$~z}&?f)oeUV5$bIGt#%iSOCOafSEL zN7RkP2XykhW0HC&#qK@wQ7sOdVPu_o7e1ne_(-Ps!cO=xAK8ELb->`e0dWHDJ^T^< z_bouO2Zh*qPkcme83z~~o)4cRj82Lbc#j^|T+X#XC|;nr-%DFjBEa$R1-Z|l68poe zivE)jsM&6YocuQLwBU`(lI(q6%0|LZQ!Q8c>iA$UO!z}!e&Z-rC b3YW+e$`)l6jkq{^pQ&%u&(^;+d-?wWT_Fy# literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/fuyu.cpython-310.pyc b/vllm/model_executor/models/__pycache__/fuyu.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03c9d3710ef7312be97aea638730672bd58cb2a1 GIT binary patch literal 10363 zcma)C%X1q?dY=~tzz_sq5Jif#1k02y$TDU5C0o|omL*E2P13SN+i~wYvmYm51P-2g~Jva=xP zL-*J3ufO-8F*xWM_?-T?e_xX?8pi)nW$%Yt-@+6B)G`ciFoT=i3QcC>+YYVBVwOob zC$uAnIZ=jXBA2;QmSv+H%SCyXk38n7K2|uu2GsaMG{^=!xT3;k!l7uG4J(`*jzptu zG#X=Ls-6u?(E)Zqm2=^EbdVj4CfJ0k=flb95IYo2v8ia9O)H!iJ`o*ehgEqXJQ9^z zS(OXnlhILjRFwzAr=l4)qsqnbSoAb|T9t>wXQJcmxGE2amFNUJp~@rSv(ZU*Qk6%; z=b~9QtIA{Hspxt3d~}+fj$U9d09WD%!WW~L*h|sN>}6FS4_}GSurtxC?A7Qs_8M>p z*;zipUgwkS96!WTr-ip|vp4t?;!S=S&k>OqZ;CfptVv_VHfHGUSDcg690?}>Ns zoA_Q3Z{z#Wy+v_JeJ_c3?wdH}>--!}c}@(jzb{@AFK=hIP5sBNaO;{;`O#*nd}U*~ zA**ZUORbGoIcjhb&VEfCxl}RJOdw^jQOTyc`9#RzdMHw7p%y1;cA-XyFwHIo&1S8> zn&#ebCbdQ#sLXP!iH`Z@ji%riYt@7%nva52QL$2wmbuWVuhdr4Q7Ws3E3#1)aomvT zI&wZqM4jeZ6iL8?B*3tN3qf^FbjA%Yw!);g*x*5^F%>&4&M#g0aM`~)cX|HW^3_k$ z!hF5iO3<8$+ew;VSXfku7z>mxG{RO?zY<{op$HdhbrI0QqaU;ath*V|+MS8bbnsdb z;W+aBR)Ps>_!S}JS`;v_k{Qi4$OaA;t|NPaf=Rfh6 zKe;lOj?LrfG7`KNBw|_ALEJbkToa$QM7=65f~;SgxMk3G^jfRgY{(?u-Bs#!aaXQj zm67wUr17z+t*#~ULL=Zprrt^z&^`Ir>*-V^B=#H8t$=}o&==8l!8vFV*Q(GXwHpLp zbqu5SdDbwh#9n(pHY#u7iDv+W0cB?}>qA4>+?+Io!>x0M$gI04+2;)IfN?WvX_t9w zw?(Mwm=>B;@oxrUOT?*@fCW>BcxZ{3*!Sx}Bz!;hd>By}|%9~kl!x+(C@rBg}NJk_i>aHo$rw!nph_mb(=~FnxuuP=jpTiq$BP zIo=(whtnp`mX+f9ajl-5#7fDh<>$)B70FJPH3chlss!6qgVbc2mV4$?8#qhjG5mXI zfn(e*^P0p_p(=VuQCs}~hi>!@{rWI24RYv!E^Db%6}7P9$kRZ|7XVU+wgdlo-o5$QYe#YD8+#KSVi%xTESfn}zJ}r>lGo#z0$LLzF%yfMcNK3eRjl1jRBSaq zePZ$aANli(=P%Fsm*&nde|UAyzc$b2{CDQ&FTb;#PJd7H$MYAL-`TTyB|ho|M@ZjA z!EK-9ptJuK5rDykPnUX?PC+7JS(a&;?n8MFmDH{#x2vSTx*t*@BoImf#!*O&ZR01_ z7MP#J;GogAN(Qm~l01UeimB$ov3`V_JVW4B0zJXj1fG1%_B44JddEaQ&n%eo4HVLB zEI#u?QBS?zUBp$|SocHcAO)i~$x-NR(%y0svu$jfTbaa4?8I3$w;`h?L)9`=!vUO< zvE>q`?V=UO*~+%FROhDZZ?-dpoiIwq8MGyCJG*V_BP&gxRYz7^l|(8z4FFxIud{1I z(h+}Y?&F2|rMaG2k%~ccnpDJA@+|^AqNSr+2>0F_8`>k+evQX0ujod_ej*>_tti^? ziF%}}eW(b{4p=3oO?u$pqV1X1j%7Lzi?$2ZXzh5WyOXozMRZ69^lDl`7Ia0+e2+`$ z$aG)53n2bWjCF8F+iY9#>@05IwYO~Uv<;rQVad_9T>}Bz_9`x?WKg-{E~WO>%Wq3X zQTR0w&4x@fYDLXimLwlOwV}Zv#&4l4=LlRPFi)V5$4dJ6dehyrhv=x-tK?3b@;%hj zd=&ZBMysCeXDSl$-kDCI(uN368e4W^v<;lh-q!48D^{^^!c5y~XOsc3m(-3R>Afvt z&qPtRq{|dV*S#}DACbDe{=<#w!|l7*n;04w0diJ>Zc@p0&X$Wn zr5(QMIY06^u={LX9te7r6#Xd*ZDSn=r9kJd1zTs{!$orE4iO?VX8vx@d9RF2~<}8**l>XWMo=ON2{uNuKAqcQ?D0Yv;fc z152sZglEW#+L-uoi^$vRaUQm)bQ5vkH9Ws90jy%%pSoLQq>V}AFRag?bRU?CuM`bo z5UknfnO%!k`)TT(3*m!TVO8JQ#!NABcFEX0u~&ySrRb1w_FUM2m5AR!p*xJZirZqv zvFW&Gy;i*$aM*ZX3r?IK=))`NikIO@b;PYkK6W~&yJ5u`emfiP8@2Wb<`ye;hJ-;? ztTjS7l+X*^eqGZKOz~zQYZP`UTfi$`>g^s(K8JIsnK%imH(0(^kKx*iO~G7E|I~&m zW=^9nVs?0Ulz*$1tZBD`d}zN`hsd4BO!94jN?tBenV60h^)BHrH^`S|h3i35UDHM? zmdn)dL+a;gJee!`F%ap9cJ=gdMWp^K9Dn>>eU8!bNr0j?Zo1HMjs;!Ufd#wRkrt8| z{pbG9$;+S6njTzC90nH>?~|4hH1^a3Vs1#vJ$QoT6@uH1mJttflx~`Gs7=Za3ULb_ zB1Z_k<N8Yl#@&Teong{RM_D@Y_Xl^!v~G6p?0A?2y5zf`=eY0emEI!54DG} z=aIWMq2;KYI*$nQohQ;v0%2j6@{B%7uA zI)bDc?>hMil+Kn@SNrPeu>RI!LMFSbmi5lahLZu@w9mzFT!Q8sh2rpIxc)UzO8ZAj zwR~ebaIvnVtE>|EV}Qy?YQw8a3tC93Iv3#8;#wV|jM!A!Pk9Jk(~Rn)7+g!cT@4aB zOHE@-jbxr8^9V{u@Hr`AhgHr&73DLu-W2t8lLjfi)K!8k3h2r>XMS8?hz@6@yOEqe(S!^#c8b?&FYj z;`?~vudzt{4uEG4LoK*g4zK_(*@43F;4A_2&~pmFk6NCcLpj&VLgArvRH3O{55^C}{Mq3-vf@(YN+A z20_!)mCH2(;dOb(^7;qz4 z`DtEKa~x1`T#V8;);)+E<^<*_p^W)f3Vf0ux|3lC_>}57EymRvsyFhbPw>N-@gOp; zw4-k8#GbtCBtI$+i77EnJt!lKHj6*SXE5%GU%-`ThrvhBq(hWyxU`#!TT0!IP8r#g zk?NodRm!r#jaCWB0un#9h+rxQavp7`>rWl9p@%tC}if@P}Fu0?KvO>b1o8WlW6; zjVr*v>#1}$K;9nQ8mRZ%Y=F|F9YQegCLG4=%H>j?QQC2Yx&a)VFy#o5!;=;$)~&7~ zUnas>UZBOsXt9B{8l1_xnk5~(3H9En>%3ssKv0yP7Q>*v+M;x)QuXXe6xA6pBkrAYaj;*p3q#@U#-yWDtu>TaCAS<8~$VfT- z4E26Zb_clsrQvkNIcxJ^|H7T&kH$e+44{xgvFAr9=*1nVXn1h65h!CsEXWB^F5#E( z`ryYoE0$6ZV81&Bgo5Tp3PW7v4&axW#%I&;!!j5RR$hlU2KIB7Qj_mHHmyj4K^GlU zc-S^NJ|#@UpBXo7IhEiWX2C)}31af4!EKldgG>d=&R6guzd+!lnR^GH&AEN)!+0%d z3d+{0d~R9ibIV}tZcY&;<#m@8Qt=}>I-5hY+whk%j#7 z7HX+`OJ$B(hEpcIlGPkYHY_#QQuCG`_$wMnXV=HMOC=JVbbNQsy?gpS+En~b&PBm2 zSve0*h_(6jA9keFy>W|jxR0<@4)ao`Bc{+whbi2+kC0vkDyBRR&jSp#js)2XypMP) zL4bik1mQzBh)CdhfEf{@V+@i2cdh#-&rm{BF`K(2Ne65`JCB%7QvS9~gxO7YD@GFW zZ22-gW@PoiSEA3VM!s4|yb|?r8FK zWUu~L?}bvcqoI}aq`0lrjd(g~VC6rW{n zSv8~W{@VPNNdd6xaRTHr+d8m?N?x~-a-#4%3y;}EN-0mucH6-W2**{b2^nj8MMKFZ z1it27cgt&gP?h!)(r|x^wdGd?{*J)k6SxO}I8X)u+Qw?l1c9e8L4HYqjFED0SwX#? z>zZM@U%3fT8PGTOeJe=CwfCtzU8?*WfON2H=JB#jJd)YX!WW0EbG+X-;#u_1=|tx`&Y4XzT8i zW>NcO19kH5I&pblwBl<2;zQabEm6!s%EtRQs5_-uZBO=Rm~4!qM{SNQ9wna1_O~s1 z2YDZskd&{cvL%nAM_LH+8rkp6*42vtLM@;vy+W>QOx_^)RrnHup z^nyoZ1w$nz#^tjFNRs6c0dk;ZkwA$6xru2G0Xf`^WJBH|Tu-qp9iF<4>+3KY>b2}1 z(IUFZK>|+@_(uW<3H&pG0|b6UfT$#i6XY)m(3#S~j>lRRVc7Lr)fIhCU6lVy>gK=j z#7%&lVdWt=-$G6xJJv%4_20RU^{wR;zqJl|=D2Mgpf=a^cB~O9!8GhxgU&aQuZIZs zzsphYjP-l?yWdz@?|0S=wZMp=#rxLEcWW+cGdW4CS3XFyzRw#~?C@pO)%znPRyOFT zANp8|1|)8h)TXAbI%=M(L`RlWZ_#J#_0+1@*|@H?l3EzgLMc~oG#gOE%)TE$zi{GvjcXMUiciXc0`8BuJ+j{ZRJt}jn4!Db%UQSBz41qwz z7{LPacLn%zC6xOAO6IU#G94Q^J>>O#kmm4eQ#xvHS!a;m32&@NJ+}m+I T68h1J(b?Qc4r}DhqvrnsBX)c) literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/gemma.cpython-310.pyc b/vllm/model_executor/models/__pycache__/gemma.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31ad8d30894c4386fe1417e85f1dbc658987d394 GIT binary patch literal 11293 zcma)CS$7=Ab?&`;HY@`WBq%N;iWE7vInZh?)@o`oWfGDFTZuceTZ5?rFhKXru)2pN zkb#{TOr(>9+3c*A&msNb96sbRCvVB0NM8CW=j2H*dC19wM9E^ld%I^a01-Mfr*Bu? zTDz+5cW>RQZlzMx@O%6p|7GpJE@;|+Qe^z+BJv8J_}_F*6Pm9HT^OMr8NQ)Y*bL3c z@~z1B?a1++$o1XG^S!9x7dYJti+&OL?64S>{1Wq=upCwV3WwcrBAWCkqbYxi<6bx& zRsCu-4KgaQ6csM%ZAK`E*oR5zBM>$*$ABv9o$2eRGk4Go`6VXZk zWK{EO%$o=wj!yZfI6N6X5}o!>b9gE|6Furb%HiqoY;?{)7uEeb$E)F2qOba2wHunKQOb_# zzXZ<1;uJWia?Z=(JR(kmb2{g|0?rxnC^(PioL9j)E6#y)jyUp3`P7c>e^XdXTD`vY z^2N@IRI<~O=ewP7v-YNpqGm1X3K`Z~-Kf`0+RLG=U27+6wQu!TSKFP{H<~RlWP5cj zi5s8Lf~o6iAyoaK)oiWFx|4br6RDagms*$Fagus(_mXzE(+pE{NhYcN-F^>g_V+qq zE~2o}>428F?F-$`N_(|#q=i?LM0TiLIz0+rmPu1IlP1zj7pS!?ds@8IefQN-(ekQH zf?h9JRo#9sJxwuG6?av@9~ShQsu_kd46b!mD1x}XC8_;(r=0|0cXd@NM2m0Au>WdD zT<(X6mAP~8US3-4s;F+JM=wjYD#e9v*pE8z)@cVttg)8F~_d;3}McCYPaq0Y%m z-PLvyzoWV>8KZY#Rln13;+wZH?A=cOJ$mZV7pQJe;$aN9!&Y98mZcDwE~Fe1a;4c1lOUTld}CAaH%@UK$TS*PJ)ODO!R$p+w3~_iE)QG$ z2>?Jtu@8Q_sJPMAJ zH%}L^+>H9*E-##7H7*{3T|!;8{xd%N+M9R;HTwP z2vpGT1(2?FHAk$1=w9n&aT@H@$)*NM(TZibqMig(Jw;@+GPsR9P@zH43Y&3^+G(Yz z$A~-!vWSQ0wdfg`M+1NF*Oo%|{^B^G=U?p1P1w z2!?1PYwUqcpzjk{ms?Y#-sV~En2U>x^&$;#ry0p0NQ(if>V8P!au8rCgn5dIHKPdH zQ!`GKqUFwmr0C<)Nir)3ImT+L%1F$2+#(!)@-L9ZJ_yL3BH)w$^qr$w?l z0-$};Ov`NE4F!?}Ba_NPJ`(hz4F%0Lt+x0abmFd5LMFAqNX#8=y8t=V0?6dLH7Lws zE3sk3hu_kEf*iy-iuD=TkW4Lbz;OnJ4O=Z_p*(kka(4=Y0&#QRv&2hW$Tyccs@qTs4!fbs)4NH!S6r*^$qn5 z-rucTJ|+VCVv}aeFOSbwYAwSK@GG#sIMnP*R!p>+)%gyxq|Oz#no}1U2)BJVZZ$*Ffuuy#A2>C}r1ELjJsE4wYK?@*L0AAOYXCOvLlv2>51hrez$DGy2~4u)XCnNuG^hE`s!cdiD?)n+#2SsnyHPsedC=g4`?HT zb9EU6T4X}KPO&$Lyh-F+AgR5AJx``L6xlnwdLy;j1Wui7tD~q{)+OpK3USAn>}rs2 zBVi3sOpQQ;xS1xlKE?aqP>&eUBL?&c^vhkxH9m7Z<5R<%_|$m8f~K*cX`sP5hvii7 zDys879WwzY{o<=2V>(6`let}%*<@s!ku7G!7K7adTMWpMt$Af~vvc8F>8x#_PYnUbLB|PP%LMEKhc1+|j8D1xn zNu*GYDPi5Dny2sMR2gLL#k%+ZMHjZ(>vop?V2jO^y|_NJA6aQC{TaekEceWVkty7a zxgj@$tF%SPn2PXp7v0hVnTb@g!*<86t(dy7_rX`G8HGJHgJNk}MSy*5jso-~oosjD z2?%%tP0aKNfG8R3#V94W;M4}Sy*q~o!010CM@$A#(ecPm8v8R`ergmm9)V^Ck={MW zu=Mi1t)nck;|phmk?H`%NIfOvwt%U3prOxj$Xj`vxANry;BW|y+eU|APQ$vfZyRjZ zk_y&oBS4^swPbB$Wx%}MljD}jhA(qh48WkVh_N}KbKoi(tPYWwNDrj${Ga%z63Vm- zACFO#zTw~T#HT>K0!B%oeByVgS?*lnS zaHZNr7~e>}{05LC{rOLLc=7>qHYWcI0yif&k_~&}bK}H*F!KO+6K?f0Mdj}Uv}rKu z7vBOI2eg7AX7Af(5~MlOVl^h$a)+Ws}$vFpdFPC2Us@iPVCb6=RWWnJ_NUmio@Ub;K0xfYFOxTPG1c~oOX~ELTxnm= zpl@7*)4r4*hQ6O6unT=3$Y%aG!}_T;U-hJrXdRYZMf=_D(%fzjZu3u42H$U z7FnFbAB73C30PS9CfY~(N7_fmj=7|%SHP|7Jgawfbq%pQW`-P$A!K9j?pjmtg|Hbd zi{?ufYK>_}UDmeF><;D#2O2Mga9W1(O9*5ow%kVM0aVwi&Jbj14!@ih3D#g|!cmZ! z8zf})*)as>-W&rrEnX#eXl5TiNsKb@Yl5V^0jH)q1C~F{mT%@=%vw!LG3@LdMAL%o zhzw8R>|wto4<6pZ7F{x734@(44avpz*xXQ@?ONj&LEG?0SVwXGlwTo0a=5OmBLj!Mz@>q+ z?RH3su-9YTg_R78`Z;YI-k@y{Dg_qRJYsmmI@5U8FcGEqA*#Z~;8l;@Ijo z#=<2viLcBZ33njG?$LYn7i1A}0zX#%%u{VTcDbIb9_`eS_G~Rgz zS>^o!C6V#2{*cIrMA)h)F-!G!YO@fqRlje}S)1UPPK#-YdT=^ouZ893U^#lQ;D>vv zGOM}b=U#3!dtXjr(QZ}87Q|4)!6~H3@XfHNy5FupNY`DY>?9sz3XeyJfGtyf{Q2af z_zn86k1YRJa(|^!_pYPRxWZqBQojVC5=(C|F^m4PF97y=e&&po)Q$@A4@a*}~ zT+%Pb(d9@RuEHJiOR{jyRY3p8hHrvn|5AtB8rl$V8~V^&HGD@D$khr55={1wZQpIf z>ADJ+YhjZ9;v6(y5!$XK%g}XjA;}8fGkB90oE4KRnNB?<=CD@b2+k|1-eKxI zHX4qgb4$xVD$dA<owGnI+Rw5h6Q&%8X#<;iHrc7^Zj~CNf{nYS#>kV~*eW#g zdKz0^BWsmHS-TXuU8oMpH$P*8_3Oi)N6qC|U<}d>Qu2;aRkWrDz>cl>U?9U`^Nr$as%(tk1ie9jD zilrm9`Mg6NChhZ9e^s@Q(}L{T4F4W z(9M zxqv4g^&QHR_SM)UnK%v@*@33|D`evjP;3ii2Lf4V{{~5XG={m`rzFw|x>C@`nMpRa ze>D7vGVLztBQnEa{3G_3P=AZsTl+gso#&2EAut^D+jO!q>awo>8fn9QFV&)`qkgF) zuj6(=W)tom=hVqo+E1yM)I`~arpmm2<2}rB;S?~t60ygaB3~2j1h{uNnu0A$@V;#% z)K8pN?ckgxG2j!23j`L$4`D6PG9l}tG4kla4!NqUlUEK6Y;0KJJI1%!2wMVPL3qaD zD@Xdml|u=*1Ngnw)5vQMa9&4t$+`o(q6GT{p7b4@cI#T_v_R@rOXkTpiFxZ z0teo8IE!@~?rBC~#%!uUpRb*XafoS& zb}R9x=om`u-foC-423H#u9CeF$JMEa9YajFWz-|ov5$##i5v$>&885k1rJcHzDCUR zAoWTH0Y69BpTq8>wJ~fU4kDaoLuS)b4_8d!>)}3B`2I%Ti5UA3bjcqw^RF&)0X|Oi zi}4!H;j*xoh1(*vIS2R9pX_(`)VW`7%aoem9*KWW^6&*;0B0eoW10jsX8SK-89 zHxR5oI5u=xLVI{5<3*#)X}P!kOY7d&@%w;lyfyV24b&?j_1ZoY(4XsNgOwc>jz$Zk zI6Sup&_F5X>HZ_bdoature;ShZ^9{$%MWx8Kgx8n#dmtfE(v9FJ<9ONp3(9WU@3MMW2Moo0#u=-FY*}*t?zd$Lg$^`hPf)_&5qXHn-xD#3kUmmu7XJf<-XgLK zbF3mCBif;PE4O(0+56PEE-iHXgF> z9%Kxc-Qs6PnNsMS)w7CU7=>ZXLF}~0y{KPE-5?O%7LI-ke1E3f4P!;y($q;dNgT3I zu{MxI$dec~bYOPrEBwh6cuO@TkfY=J$# zYA-`~3EqP#QZ;#z>>7w7=azqlbF(QE46knF4vog{@R1%Rl>o5_}gUzm~>lCl9PYKdT(AcQsG-pL#(3Lz|8) zJYK^7lMl$9d3FfD_!i%8JIAP<^9Pm8W`sdb<(eVrC%8Anzo-*5^fwX+sj6PzaQ86vL}VfZJxOr6UN0(>_obNPRpGOQD! zZ6gcCn9Q>DlFY)(mI2#Kqz_{fZELcu*RbOzjGZ#3^eMdQKdYBb=20w9oz*?lgIL3T X1cyBxNtJ(Hov$9NUar1)^y2>luwL*n literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/gemma2.cpython-310.pyc b/vllm/model_executor/models/__pycache__/gemma2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eba3d7d0093ec6167052b358b2549610e61bfaaf GIT binary patch literal 11383 zcma)C+jAS&dEYx0zyc(}J7r0bC0T}JL9rdjiR1WEUz13*Y&lM|ak_?Z79_}B06M#n zL=Kp?rf00CRiD~UXX><)dvz!6bo$h%bo$sopfCHBnLhcoGfrb!G5h z2u;@vE!R>hT?=w<4*z6cbGR6+gh$*X9Cm^S!lUld@R)lnJnkN6 zUMZ-CC)^VpE(Z^WC*6}Ao(fKdr`^*Wo(>)g&$ws8huw!cJ`>c!N8Cp^TnWAuKI%To z;o0D9Sa<6jo(s-}Uv|HY@VrTBif*yy}Nxyx_+T(TE$!KOgmby{?SCxGS6MUb7qa+JTRP#TOgRbw8~qDV*=V`NF79XU&hj zUe8;T-F`1QMKQDyb)^?}g&%mmhHL}@0@u1S5MI>Y_R&(Y-D$^O&|O>eC8C8_{h;nHeXhBZA|x!}t+U%c22`eEl{R^Rz{$8X4F=8f;XeUSBL_u76I>YO^? zU2DhDMcHlo5qbw!<&Ay=ld;{P3CYHzCN_;gc6&7J+@(f{d6h{i3%uTsd;J(?7vAnR z8<&Tzyc}Nkg=lxykTNg)R-+%p-Zj6ywoYSg2p;1!*MUqE6F&Q`4%RjFMY|FEZ}GH6 zpWxUkXxzD!E=EMtofL+#ix+DKR<9p-zsC(AnHoqI+Yv28*p4E!!P6f4s606nHa30F z4}0;J*PduDTc*fsHJVuOt)$2qgT|IGqbBXU$)}V>qPGz66;~CCtL0Tu5}L0I9k1aV z!o+L&rf)4PA}7?A^MTS*g)Q>$mw+(^?C-qD5p5ZwD4h2VxA1{tC~gsZy`1FJ?Rw$- z#nq&^-WI~|@P0{V_Vt^#P*TX!(f4F#9ByCb)l8f#w6@s6wDS{7`f*=&ngor*hr;yg zc%yZYSlLx>skaqXiM4?`(AHF;&H)|=rqFL11I5#0V}O(ijycc=hJh+JHF-uDn_Bc} zYzcE$eNRO#St!m4OXO~bVnH64BxQ;`k7+Uj=@>2~VXi-F&W{A%RK)py6!grXfW z$;b-6TaCMiwNy_G>QPN~i;Z4ScCWWXY}i^pF@YG^W44E_g#+!$IgBN7){yG;dmgfG z$OU5MMfX~V<4NJ|M$q?Pma;2lh0-ka+-r3Jb4KI`E%`8*vPR?)B3~jxbM&#A%A@cIBNe2-iVqm*qT~UHT>*f83z4RsKOy*Puruou{UjOS4R7_J{*6d zUT@Ok4j%&n+F>u^jc7Cj<(4u~Z>#TUJKBzp79K<15C>( zNl$bGWDHDs72vdxfs+An(g997z)1%H>B7dF_f0^O<>vraMf}_NckplGzXb4FV(==9 zDS%gAO#6k6qL^V?VLIzOqC}xqLClGHfLV#btc)|hm`tS;Jrc94NuHtAYm4iNl>!s* z^d#Tuhu*s15K-b#aPw*wvx(`6c9<-Njq6^o8@1_B1IsgU6D0+bR31PeZX`~Vgmi{^ zd|;BfMjUrY%tG{eQMUy|Z1m(=%v95%Nz0xx{_}Vl3*UtN8Dk-kQ1LY29`Fxnr+7we z?kYRZfDq9dC>z$mS%8uW>^u5x3i~Fw-U_3BepQlSmsOB8ym)^8@A4e}-l{OMD z5pt(uv>JG${U^Ue+{&0c+KM{*ls0z(SrY3?DwuP~K;qb)ztZ>7{hsta-f~g15j3Qm zqkpI)@^xx>o@y^6J=H-y5`Kw&1$B(&gw;(WbgXH2^Q>UQyX7colZxeCYj+?^BpE@8 zL5KB#S{3<*sHCKZ||t6m1oMRAAwyHe-h_frGSh;Lg!&EzKPPO+NT~UKexm@HO0>w-@r#g= zCWWbl?@)-|mbR?496{1`Qf|&9nNszV--P)n&yzK>Issne!^=`^az9g;rk$nOzaTIn z#WdQ-o)$y1B3Xu*wgGK^!^CUh&Ed6a*Wy1f)SHkSkUPUtBt@u%gO_E^z`#i$nJ^Vk zb1rRT237*KQMrdRdk<&s9?m?=2xGNYoER$u^=}AcHS>OIv+nnz+QNi1;HqgSxU+=* zV@zf}BxP>;Ti#Vt)l+pu67r8>)Lr01#v+GR3uqQS;B%5E9iF8CD{Kc8`9u$co}5Kh zi6cV@HJJDwdY4SKJ20(1){%&rECJo(iJpy9mPyRCd|zY-bE41+7dkWzGsL>{SWJxf z(jhSa&*)-AC$^9SF!OH+rt5(L*4%Z9twHJgq>rY(cEM1uNwhw1R4@+bf4ma<)m(@B4xGZ-Uwub@2Ys zO`6?LsncY#0lF>hj?agnjlSNsVA5z-`HptxAUNBA@^Y^7nWQ@4WNx2NBz3t(Fwm#P7L2ua(}PJ&>g2tyoEg+lPxOxNox`pi z*huWoNzp!a=h$pt6_6vv_7UD^@e-_~GEz&x{i#u|oL^=MP-6TqL%PhoZ@N>W;!pcC zer_lkAR9C>E9M{@Dr{5DvQ1S?m-hlW7FUx@ogJOx!~!m~+SgP2W>PZH!qLl;;ZG14 zONJY4-^BV2b;H1Gvc+M=IfU&Gln>Mov=-|<@7n|8FW@Bqi3S@&e}^m~Xh4t=uuNiy zO5!aElVw1eTqv8``eHYE`1vTh)7YjYD4iMvI^ObS=Yx5_$sY9Kpo@WLs*2@k0x zIB;j!W=UOyX{$*Qx-VSU982lMwT45$HvI^#}Be+{m5UwW7Otrq(AGF5D) zCrO^BjK}{U*Xa<_xbLHV$g?d6)Evu|Z3;1@ksR(rOa51!Fcgiee)aV#n_@+>okpdnz;f@Wjr~*9-cJ{9J zlJdg^vJ9Y4<&jo^6|A8y9ToxdEGf$hG%S19Oznj$XSpAC^m?2pyFut5YP;}!p&tc` zv*^n0zz)X-H&!P$jy4f4$|;P9E#y{PLS>MR&Zhhkc)SLQi8yW~&YNn~h@=XTt9qP7#0_ZM3rhl#KiI#IOncHSlv5i4t{j%Gt# zRWdgt%sduoh0pC=D%$KXrm-WoXD3%blVeFPy;z9$`58&?C4}P@{sjVS@X34xu^rQF z!mGUtuVSpha|4?VtMq-W0NHr7V)c$Tag!L3VgALD>kU++?^8NUuFG?0G;TaJ7lJVObT3py~l1LT8G~` z4clqBEfSM+aA)1AerI2u2M$f*P-=F27ClF&hsdsxE32qT?w)B=Rbev9`-Xg5^UW3IBsPdL9JYqeEzr>W>!8c+wxq zCC>WeL&j%zX5QNqu5uvAXO3O?<*Cz>YNI{9?;)(DCwc%gJ5>Hsun?Ni*Pz~fsJS{g z=0~b)v=!G}gH%M^LY&kQho+m)Pti|2Om1>G$&0=NwXcGIQvYW0 zU&jBOn93XmxP6DifJ)TFqCXAed5P(=4@Gce26TQ1(k9|oc53S5BNtZ2i3{y=0oBP(vGpZ zc^m@gktDk-d8I2~Z1kf>aQ*^t;14i)^2bE}kjNhqxd~F!7BfTam23^@0-Y=X*ZTsAbq^lZu=@dldzWb3eRj4&<=>Z*lOcTiu9UfpDSZJbErcOq+bol z6{qbiVSn)fcmh5v85blh$y#+*|0pQLtHT@w?N$y*hGO$zSN5}y;EZgS4R_*x#GEG1h z59c}XeoDi!5h%bwH}(u|dpXrvCcB?$F03{sGdy|Z$R9bSX~zs7+Cj9FUc?)XIuB(P z4mi3h)p0qh4?C~RzeYA50V{mN>rw9Q-y(?xAnv{nbUi84r!9TlQKt+1Z^KbICdm$r zqI&Nr_7{sN$Pu*h2zfK?Cu#Gbo4m8t= zm2SBIpl(tVjtS*36K9hXo$X7K?)$Vu>gK3x4i95HxD#F z7?`O}1BChpzD_u(anMx##u~>l%KZ04ju80=B4aJ&C5ql(75NFWY|;u{N5EDrTufn| zzOe4tC4*d?x?R96Jywoa@6KVW%E;BVV%!-zyCzi?Cs+7Pb0~%G`)#9eSIZA$7GkGt zNxi7OkXZ2+xt7xr&}L5OhHxTPs|fB=Ma)LXbz<0j5nw08r%XJVZkfIVsgL~Je0|8e zntOy((r?}PcPvrQM9KL(yFLB&2 z-LFLYrNCI;k&txH+@}P8XO(?J$M;?Lt$%!6+Jl@ILhqMJ)+l-plS2_sN}l zj>y>HPYy?CSw2ARoIR{$x*{n|)iX=3AGd>ujj6+z?H&;(>Ablmu0Qf|7v5?5l8gYi zz;0T6V&_Nl1dZWAki<+|VDvmi+|xuJBJw3bIMx-ta-#Bm?G?JXrk`=w*qh6( G$o~V*NZ2BmLm4(XrMm~m z*50thDLLneB`!$GFXTh|niGEz2!vGi>>A^^rLL~(s;*a8byvM!$AWSFk3UDxT9)+} z632%J#J4czZvdFZ?A$7BY8Ql(!l6##Qn&D^XCsf~KJ{UBa=&QNmf3UjplH*!!M(gw zbZNH;X=rFa?-g@&&fu+lzUb3_u|OAq4p=*1ESBh!!8`eKu|ijD>$b(ZEIhDS$Xh#S z_{w9Ko@Hdi8uVU(chkG0CMnB^WF?%M!&mc_}L)b=VyL zp;qHsy|B?UTX%2sqDa2HS23Og+1aSa<4P!b|9*Pi9GqjC!HxVp) zXLfa_B}Ri>{7+9)QJA(y#mui0Dzq=Sszte;V%5zC0mOybg(<%VpsdH%Q~S`eEk&4< zSrhx2{i{8(Vn?~k+wqyr$S(xw*2LOzb^^cxX2Jcs>ze~Y!>RRHL0{;0S&1UaAzja{ zL0fyWhHUB}jq8{4-R$j-rCsIt!cdUa*;gXF)zky56-=7sTi6Q zdkUJ;l{_D6{{f_KTRyjSJC02ifQe<$`0B?sRgBlh+4!0Q39hbvRn(*6da=8fXIpFI zJvFMzwMThgtQlWo{xeT&Rf&d_Yulzj!G~$(u8#M_8BqKIOo={s+`iqneJ6Okf9bR; zU!8@l+gS;nA7@PJ_PCO!umOGWC{B}f#HH>fN|kXjeR1^iX(8*L(IOjoA`(RhV|TrXHJ|2?w}>k!y-OJa2Y{_;0l6|5PXaP^)%oT zV7fIe(@%hqK7g?8+V0!^i>Jmu71q5`#uzVSXaVRB-Jg}pEY{QkO|yYG7N!X!1Y{P* zAw25hDr`Ui=WubF06qm=e1_nJvp(A4;ySP-mV4;!UpUpzMozbOdA2=LGDYFpKpb5% z?(_kH>W4m=*oQX$Q8Sl_$*B(WDq-)ZQRPIc^rwnaFw zJ|OGsgB9$hGAVc*>rNa)@zy!wVH`iKlYGi)#WAZ==mjiUfl1XK6n8qJe)BB}j7(tB z2+Xv=8x?rB0-puZL-08Q%!0sgUprC>Gt<3g@BRdDP%x_X-M$=WrU zqCF!j`NJB%AKoMrvq$}H^MaTMk9Ll}8!51qw13ZBoQdvOg_t@v{TAG8uJ#*zx=mAG Tv;q>^A?e#8JUk)b=bZll4Rwyb literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/glm4.cpython-310.pyc b/vllm/model_executor/models/__pycache__/glm4.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d465ee2af875c869ae96c75452571d8bd39295d GIT binary patch literal 8411 zcma)CS#uoMb?&`;dR7cJZlWNHvNVDBRC?;3KKI=H-m~4) zGpN^n4bQ9p{4Z;ZZ)@6rP-gORka-s={U=@1gcfK*7e=fnMqngnU?x^zC3avZPT(YN z;ObOv#$MpzZN*+v2`WspV?U_|RmPpTmehlK(g+$!GiWl+ja$i7FvYkRx0C5$I++P( zIA4ipleu6nnGfbU@5jfIq;B4|j@B-&s@pqCJgBKZ}ieE}PL5J~n{N3c`;AP;`VkZ7xaxOTREC!35 zpN-EauLQ3$J_r1@;5EkQK{+8# zf^xE?ydzG@cY1p8eL00)dj?4tr+;gSXYcB$yC9xJ-RI}1t#fw!gOB?wQptW-o*VSzt%VPluAIAY?v)F#URX#5LdFZ-K{AZ8-i=r; z-0o#-3m+WSI2^{V2}O>oUvjPdA64 z+CT4um|IJ|G+XZIxzX;6ViBgjZHbZGYgeu>4^)D_r+)U+UtBFKE%o{`Qu)-i!R^B& zG-)mkR(n}`RSmi_MgJhRf3_K6b=%m|LBE`lo_jP0H5k&w>(`?Mo25S4%!Zp;?v`1! zZR`(l5uz3g+z*8l*vzdUFnw)MGbw|t{1zN(thOrICu3b`tp|I zD2c8t5$+c_F#wKa>f&ZL_+@d{OM^&Ag{fS<(o0i}ycBIAUrVB`8!{X!xzfASC3cv6 zo+&fw?}11y(4iI#u&g0WVFA+5mgc&(QW1`{g)1H5Aytrkq^bnvxF%}S6Lnb;4di{~ zn|N38ZsA?SdrH|J#%zj=E8e&d1*IQzq;W38CAv)89t@tclR?z;9 zW@*7R#Nt$5C!XBrx1OdM+VbUI>qSk=wcP14v*y*co)EIn43by+n@PBaV@dNGp&Pfx zxg@$14hLzEdD^2Yq39*KcN0}^^rAHP>Ey#~4P2eqy2Q5NV-knFc9XbeKgadbZWLqR zbEnwsyb@(uKg>u%93`_&)$cYYHopbDf@C74H^2v(wyWLK?`yg?GDccxWah56(-@h+ zt&z5FjT&=M$s0yG{}b(R^n!9iL#aJNdFX)Rj2auZI$vOA?M15HuZ$W*EopBPEptbF z(!1tPGxG$Ef%Ye~T^hYIa)wuE9_tp5;R8bot9aXl&(O#-b5tEQg(rM& zKby^g#&g1(R#_QU%08;-qsF!7g$DkqFDnwbd={D{>qRj{JB!p-xDs`trdCme=ih&< zNG`^?f8DMu5=RWxV7+4o7IB7p1r^k50GGGj#X(W#5WjGD;3D-h3Ulj5luGp~Rel2? zsQ1#aF9&zRG~0^hw!KI!(y`Qcsm6H%`yF?V(jaBGs9uIxG&fcs0XK ziq)!l%8VB*lN*YhkKEz~wQ?(!@rr^}YQ>~!dd1+hFN$YyjxZkQVCB43w6E%QfWXa$ zJTKn8FZ8?;hTS+y(=hx(dvN)DmJH7id&6_sbgc5+Tgm2H@@jJXeB8Tnez=vb4f^MA z#c^`JupaV`>_T6vBAcFHg(g`XZmD-L5}i)E0^m0soqlb-2I%WEX4|SE*U~+`fge!~ z{4D+v=IP$oM#X0;=N=j^l~Nrl2kmQTYO;Rq!S>uCHMTfD;k+`|M`>OkD={3OOmh!M z#40GSvtCI`Q`@yM>+w?q;v&6|lnQ{6b`LUjU;oV5v3H!2y-s?Aq#9af-CQ+E*WWch zGk4q_ZwHcm&l>3?cjVnS(E>D3Y|2BT%#n59-hmX}b4EV*SVfOjNR_$#fhsHT53h9}@V8z{doB0uY#xw_Fc%{bsIjJrswa2Z(ct_;-eNQu8iCZ&yNxC=PAHV)0ww`UJ;cv-hIL(s#%qBB&qGW7xgi36Xgrx|X+R$~a(`U?1- zVHkfjOygU_GQKfvLeXxdtn+YziVF`QJW3%$lrywi)j&EUb{8G8W?oOCIC=fYJ zJF^PcSB>SSupZy1EJ;ByO&0H90qoWV^OKHT2wNnDfFqU`wKwEecq@uGW#RU*Fsomp zv%Ipuh&C zk22Aw&22Ek0S$VS6(SqWgNi_AuD?f0mjHa%gMF&$Py}q93M=Hp ziWRED1J(P=XpU_WEZ0K^w#wUcjKi?Dw|)GmgzX!t$EobxQ!?!dsLZZ|nT#L8DbcLc zA;8EM+NP#zBiN&{ZQ3#VWM|0hweK6SJ6r}!rLzoKLceS6Ko0I1WXTvK>6)kut2HjM zOAO^J22_Qy4EO3WHm2jK70gGGmQ!m4RtZP~JphzH$vmlbs>Ek-a1f-+{|zZU1K?FG z9D~K13f)?6zw?dnwrt<{*O_I4>=(0{zeM$rUoWL&#kl|sQGht?aO*G;H z^%;RbCBVkTCX8mK{*1uEbeV%L6D6QgRi>W(c~P^|de&Y_VX9$OMroa!OO=;_V{uNv||V72$v02r6t; zd~ixNudouJyLE8_Uswk|d0veNDk?90?D-$HHN@Ur(4{K|J#+-b{T`Bul%4`0TXVqM zupI{Y+d;sI?kC_@=ZXCWK5G`fQFhD;9ZBc-Mai#6w8#FW6d(HmzY@#frm3Z5)p3?3 zj~crFaJ#*~5BoCTss4YMk&LQ3;K(O3ULW=T6DeH)0OPfa;hl$gLl;f5-`qn*wIAVv zDG=#NPhb&aE?C9=`1gFLT{VRDTO$A;@Xrx0_`;JGA{REhCO+Q84^_N9yy0?`d{Gw- z@P&^UMs>Z02nOJa1qY{1E=a|$*y3ziHqA9=WP{%5V^%g1&#+|+_&nYPwI!zrlPz{; z8XQ8jgSI%qtxw8nj8WEIal|R{491xerw^6R!VPl7vtj{da|j^L%VV-aHK&ago2p5e&14b!70Y3eBJ4#v=4xM4{L-ax)LI zQ&DfhH}cg>@Sid|ia|@+4Wx-MEM-b≪Wo|+uxh2cqE3`+%!Rjo5lt2q0s38=9 zt1P06QGX+!&U%ufnqty@sj(}A0O|&`rOK%332N%!8g!!@#l@II0?$*?EI{6hIj~7b zS41||aU#2E5xNYXO`;8nF^1U|f(jE}S*eiKP3r0~y5c113i3sK$FXrihHWc^7EZ)B zp5xHPxTK+E&;QnQYKEsfrngtvx(N>4*D7j*Mq~fn05*OIH<4%>e3Qc&Au=}7@9WpK zFZ5+>JMCjWL&4d?%p5pCm^`4T=BZ~8|>_zQsCDHw6W@1PUN8}uDu6H2BizBZn`58s0)fBNX; z4>wSB_v{{>Fin~#Op-O;^83c*WOa2LW#f5LFFenn+Lw3e8wHEl;qA?xLTY|ZlcXVP zHUbG8xtV0BS$GNuT=OtFMB=%`Rmo51oKA`c=ghlgXpC-GNQf0!I!({WZ!+ zJ+tkn%YtKgdw}2i_V%fx`+ul&V|lrA>WC*0%nhUNh7@J+Bn^{jI0Sz^)?Gc~AsqOu zB$|pusqkLOQ&QwR6)x;a!gdIdP^rHpK>N&57DYSEi+ZxFt5BsuOC~MqY49<~<@uxY zz-T<#_h>wvd(j}N`g-jAdJ~})522Ns=l{W|Qz->rJj?&asEl(Ca-PTcD!sj@Xa=1h z=582@K^GAw2OkD#mv|Y(C&e;I_KJn`X7C1)bO*Hzh9ZS1CsL)nw?Tt~n-oGWJ#U}d zl6%a17@Xwd;=>z%)mjt?n5YPms8rRd&uUqNK4}zp3Ob*l#bkkRfvR@{!YbHe6@llo zTy{91y2yTMTD<5=TwI%RdOKH#h0S zup%i5e758Gf+16N8U*z$KyJtM(NHZAK@fP3she3ZP74L{JW)v}<<1ojdvb)OxX#UA zTv`gtcxSkD@mH6wg`fWF>ZR+2Cqt|}p;K6EvUc}=Py|qa1WD6UB3W|_x@S;OVg|5< jAHf<^2&Z4GDA(I&JHmeu_HYk{PNJXnjkVWyPt5)w?#T8> literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/glm4_vision_encoder.cpython-310.pyc b/vllm/model_executor/models/__pycache__/glm4_vision_encoder.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2025bae472665e7b4ab44d5f0795e056ea22176f GIT binary patch literal 8418 zcmdT}&yUsUMaUKN|o9?jSZ zata%KUG<7pELMH>)qAf7)v77sy7bTg*8S~|CFxg`$zK{WZ{v;>S(04ImR#9Y0=ci) zP#~=aYG1Q8p4NhbU4X6-6#BZYbDbU(`-W}sbTKIPP21#YBPjPPcBNmnt2|!{YW)R! zp@6~C3}gNnZa`ZgndGmK9Jn9Tlq?IE1uq4d8FDWp{cqxXlkiu6`BRN z4oyAPoPuW2Z9vmVHK(Cja+jf5PBkwqfslcP>L+G+JpYE75iI(XWjSXu666f+t;pJ z_dS2R8%Jx;XrP;NQrc|wy=d5Kd&n3+8^-=1Yz3RhC}BwD8?A>P+Z?byGOuiVu@ie? zG++*Yr!#D^RuFiBbAP}B*NOaHuc;*FCw?$`Cv>lmg2bdct#<5m!e${^ULORbet09T zw&90fizTP8du-ctk1SjK_@|#9dc1zq8wP&66?<;lz~arp{rQK`+s4PERv7!c7=%H% zJ_tMhcALIBe-Xs!iY`NVk}U&f%44Y`6V$#kY}Hjgtyge0k{!)0xcXO`tv{AD$u43_ zib?H8D{gn+>u-6k>xbKIg|{aJrFGmY?&u~&EIpDY^1dWXvC>n=a!9NV#KA1=_*vZuk6dvmZ7a6UllrNYOaJHsGAt@@?DRGm<1aDM2}+TyWTS_8mYOE z6r$14V~OE7e(1-J!|0n69Rsp+*-grR=(M}75VfPE-1S}83wblP3G}qPqDgJY2EC+2 zL)`abv{r}*tldotyB-@vNwpnBjyD|wyOfQFT@6QlXE=y_nu925Op~*x(R(fOf=;5h z`4l|`<{YOTw4%syo=V@`z8Lq17l;1vLfisK;tQ|$N8SFV{{4%Azjbl=Fzyb*i#tKk zzbFvtJ@DEi^iE`>i`zl}ii4$r_dDs*SQ|cM%NT@HxFbrKvLS2yXFOlgC~Y3>o}8c9 zwdwdJy5DbYdr_h@FCMXQZpLYB3%J**=^n&bo=9ViST;T~lM^Bb}z=Ml6UzbP&BHP!CC5ojhp2QEP`ctc_dwyIghN#HQWtZ@UZa;*)78=Tf&i;|)+Gz2jJm=W zAWq_rh8~WaC0Fr4e5a(2AVbyFzN%t>Enq(?x;jaa zVbL`}h6X3Y5&&o>rFUrLX{~Q26?{qk2|NW`V{uJqEgDv4tOaqN3qdwckcqsU>MAX*xS}tfUOF96S1K8b73@UPf zRp71N3OrUu?PLKEV-KCX-a}`n6(C(DZ2_bUloBF2JKM!;Y%m;*V!PssgaZQJN@`GH zg*vJ7sVac_L#jtzW)_JIpSHVX!5u7+PlRun;ZifnwV=#)n4`Le#-$R;r3 zJ4Rb0`w_$()6wS`xYuz<2?PdI`a<1P_S8M>VuIzqQ1%MQ6?$Meu$THo!4vvkanBg* z6L1=M4U>zzu&+&YFdp_-aG|!RkBi?~UK|_WS#HF|p3x)bL`!4Cl?beE>50BqLf^{c z61Y=$vA0gD`_iOFJ#m$Nuqk?M^8PQ3O|Y~%HVN39&j_>Z9Em0gA|wWzRq#^^9Z7n> zsoEv5Pw4bp(OtWg&7NKIBd6UOMXjJ&XFs8eZ$l)7c6SuswTsLXq;!=^-lCEskklVx zubrX7?2N*;FwpEX98>_~D_-qbSjKeD6Mc7wTqN zAcv+lk>5?^jYM{A_1@i`M822EcN2LhQ319@7l_NT%*)g?5fRxl5uvLBNG$_Y0MP-U zbx;Ag4yR020mQ^l|F2hyCeh>K3~VVgK9x70n(PX_;txsu5eYu@#I7?&WUo_}dcmj% zx$aq%<$ZhuDXJe;APh+ILqiN2ycM zhY)%00=s~@un@FIWU>`vVk}}L1MDMn_9?|C&y#+tD{}@?Hrbo>>Pn7w_{yeLW}B&L z21`1rLijt}tc6w0N+@SUQ1+r90x;NLeb~mra#K6)OM+yOy3E$;S&PIxOWcJjBBYrG zo%jLMBRB!W##5E&yA_Pc_pt&E3*Gdd=v4@;HZTKUS7h@{tsk<3K;@ZU{zPIQuFpM8i~*O@I;5DpOxmQ1ONt!8KRcqU_5gK7sIY zAI8ZNFKMG>?waD{H`JrtmRUS*%Y?@usm}}{CcLYoVMewjjb*e6F@sr5TObD+ zAVr%chZ*1AC|8V)v7F^fFFb8dy#`oP-Y^oFE9WprA1L5d3&{6#LYQn09!(Z}vJV}u zB_&@-3GtoS^Z9@!YCjkzt1!EAgp{j8$T$jtkLgPr5QIQYHV$e^O{P@kVE5GF5zAmW zwLh-fX11>^{X7?OR!ZI5p{CnXr-GN*Zhq&n7kSz(N=3T66OpTbdNpQ9c2 z1&O~TaR;JV%e7zdB!Ytkdk9^kE~@NLr$JiQbo$T(ZV9j!uaW6JLs zdvYv&Eqx82{)zH8%02RtWPx{?e+OJoW$h=7Smmk0W>!>2E{XYO;72hR*?b(amh7|rc zUtw@C#ea^pY2*cKk=weDn|H}902E5?ce~3+b!l#msHj8tI@P72g7GC?lkVu;t&|_Z zMZE`dnc`n&9Xtb8<{puQv|SObAe@$?Q2S5RG);#IHU<|jP`!b0Z z#NXpSjK4B3X`^KBnsV|Kf9eMtw^Q8Fa8EBvQ@`l@qZlHOJD7l=RaYiMkoyi=T$s@Q z4Wp3LYGGpRW3(LX;M_MU#}tksBCc*D7P*XwWLCEplYc>>ntY3O9>puPOmrmka&L7# zYry<%A7MAR-owD_gH8G7ZuyZFqd0=6Xti=K_LpCNDFPo^D-=Ol<6eZ*g|F&d=~`E< zqvC1jvMS{83+HTY%43Kgoj`9pQ{?%&mXCVN9JiopB7O;!s< zn*Lw(oyLzYOh)!a`E>R*Da)U?9DLXOhn{;d3wO-T-ivkSiu3gzpj8obK`i6k=P$l? zu4dZso%0r4)2kOQ&E8La&pzZc&+1c5){5^Bj>M4#Q*&SqLeDxrb~_6_<;R7O0B_u# zPWJIl{O{oUd5Y&lc)er2YF(ZO_WuR6V%m?N_y@U}&V6To&hq2&ep9hE>W^^Ye+*vQ zY%b<@4BCUaL6(#Q9xvy^#sdY6m@g@iF?Pw;K6r2AcA~*KxWvd5;>pX5&RFb)Lqn33 zb3GIOK~>=73g?gRGzdn!A`cRGycVLJUfN3wYQfx`uW2{>SKQGbK$x!zmw`eGHH~7c z3QRfj{fy^2+zCpNI{}i;w#^szsLYNd9}70xVJ9MWAJUL2SU7VfqzR%%sUD&AOrPy( zI_siMM$uVDhCd5PkO##2Bt9c?gf;eS==p|JgAF$GH0U$_XS>=q8u*uKBk)Ld?OR=| zeOtfEsO0!vE%ivN?|u5t<@Jq^Zoqi(f+OrI+TUp6`Tq8INaXi7SNADIqyB^Z3s}Li zHdZqq6*h6|N(kpc?v=O$LD4cq(0Y}z3JtU_jze6H?~c{DJi#tEsUu`V2N)#^p5Xb# zWAqKQv4rv^Zod^%Usquvx;XXMt$ON1iV(0SX#Ic7oc|(9jM$MR^{9d5M(j*1(6`1W}{- z3(k9-4%XX%<(G^2oFUb;NMz@sUfihKq@XecHQ4pyMy zM)QgJiJv2woAmNo%NAuh4G)m|2KOPe`RoZsCC#%*(LunljgtlgXKBM0L(rKo9zA|I z1eisrjejvM1ltgHMHupZ#;(&b+nNifh51_uO}Zi?{v)o2KY#FL{=a9!4Bkt)6= zF@u`@z3-?Xb~q8!qMeI}%^BYUDO7I1_8oP(k(r+Nu5Gmjy*0uDrhNO0*nlH%Z5v&Ty)S9I|_6 zSk*&G#1KeA0~P`Z$a@e#%A0^B??Hg%59BdV{S+X`gC2u@T3xJs=hVz_hEk->5Pev6 z>eAI!=X~c<-Ez5L;CJyK|8-M*Xc+%RmC2ur${*lSo2Fq1BQS(1tk_Jfz)I}EPMp9= z+`vt|z%!}Mj&nf{|D8CO_<_$nH_j)8pulA>E+(a*l$3)q*K=_tnF(gN?8nt)HkeIn zL5=JAcrKX_=DA#mk0lGiLb4bvCQHE*^NR6uvJ$LtxfH*V91o6jxg5WloCr=NCxeq* zuf(U4w}Q8lw}ZF2J`*UZ)KMmts4Ww#kAycn$7H~j`aVGmo@ z?TnY2yU|GS{W2>L3t#usUO&Y*SH_cG`!Z^XY~fekW@CNC{8_Rd34!g#%*RC3YV_kY z)Vm(W-G+$hl~tY-ohG(w?yC-tB8f!1kw!OpSJWdYm4W_+8~t9dE3xkD*n!2iqD@TN zN?{7#v?RjV>|D~=j>0JErFX*i#B9S&R$&Xf*p+v(B6o}%cOt2pBngwh{IK$yKSLoh z0uyR&>N_U|NidqC};Z zFUq3wofDM4Hyk4&d9qzTG7F8U1o7 z_;_qQ>K{N-gN~EnE+L z`WK3JBI`796$IzP%aOCg3N1jAuw)J`v!2Tw6~!%innsM*!wU{((r(&_!^0PxdY#@s zG_!mdHsgj;VffH^{P~46>0Rixd*{=J1R$LManj#RE+)4w#O?J9y*ue?xdmPrOy*!Qe&?R{@Qw-1@TZwhK3JGwhkwg&SLP!RpA5*y$hJb=tw@6ZWNjoTuMCL#;TcwiEIddTZ z&CSfya4Km9lD0N04@IjP#c>!;Ec+sLp|#7mKxhNh2Oz#-xg-?M6U(xG@NDau=U7iI zxAMe#2jcUC1C9lbWDfTt ztYK-C6_kQG*su038|tvt@1O$5!v?eNXo{pIG z+{$SL&@~^m!t=fAaF55Du2tg`%+RzzMnBp{damG6^l1}1Z0^Gj-8V_2_f6JZt=q&m zZx|2FwYn|8#u#~>2WOB3-DM1+88)qty--t8Qu8sFx+-iu{7e zmmrx9VVTe)QX%;j5k8ThQ|T8(=zHWes|G z^&2UNJOwtKf;=t$r*XHrtrz_=n=&QBB)%lZwD0OKKqf>OwjU@6%P)++DXUcK*brHF z9$rA|z$Q~Gto%Ll5y;A$grscUZ3wh;&mjSRp=Tj20(VEE?~Z$xSk%3>M+8eSV|LV2 zka~j2hbEiDJ!dtEdpL<{!4GmBSnN(DC4Ekoze@huT304nja{{{gK~)3+A+l5i96v% z#u~BGU;n?2mI;k@!Q43j@2@-pqZfrh&@ z@oS9H;tR%zMmF(G{Rm9yr*4Bx7$b7=XGiAf|L5Y9gO5<46qQ67VL=(?N>m|kRBltP_KS(d~belwSX4F`p%y~1p8YuYO{E1 zc;?dL);x{Hm}7_rXoiJ!QFt4)iY0!>GNVFntw~<~=tpIOtvNi3ob1%NYu+{PVgg~_ zb6A$;AAw&tB^@M}AKF@lA$M zBC?D^4O?w-J6M!aS0eZg8yyk$d(DPINDo9DVo^j>@&FL=4r6Y3w6_rYoTT2f9Bk;oM@PC@Q6Y9n^{TRbgq3hZW^PJy zd*q+cy9rlY@&rxc@zf$a`I@-8?*89c%h*bhwDu~=rv3{abq0j6l`sLQU{*EC!g%{u z?diPh0Vk~I{&S74tRv%-%Yfa;MTBjkveI}LsFoiE8;ZU4Y<|iS)G?RRXiUN}`V~>2 z`V~CtZ$L<`woExUa#4V=6jWHi+8hNNz*@gkfaeF3s+M*5EX-waQ=qsGa}TV~j9=F% z96`97M_XY~K#+pagnBF)%f`Stu=o9)GyFDR{Upu9Ec{ea@3Mp+0-WUz>2%=-4>qd8xlv1Vp`>muIXz!Pj5hzVs8)QH&q?7DnI*HOsNGG*CP9?d*Ln7>4utiAa8yUOVS2+jR&Zd{53uZF%wpDGrp$2%&r_sgbqCIj5~6 z+ZZ1?CUdKBKy#JT$^m7qW_U~$r7Bw>do3%bP!a_pOWVagszFqgI? z%E^iesAZ+gaom3qm3|!<}yn%Pakl&yQR+_eJfEl2E|Tv8y5=QZ6wc+F4ogBhYKG=(JM+W+;Gm|O-hOGl;wuEBvWC%k5S{@ z{CiYVEKjH7?!jVF01tcj84`8&zC#J%U10N0K{!-WZwQw;$9DI?pX^-3I7h%Ds>>)l_uU7^zO!>NrDSwI&B2o9 zkOV}WbYPhV-b+d4!rJF`GpNca>F!3Mz=ck`iKOISWvI2Q`a|~$9G$MNBg?!utIJ74 zZNnvFR%rq-RuNlKzEEdnC{=+(wVoy`Pv3L+nIqcGG^kRpP8{A-sI0(_n4;S$`6i~w zd^lrV&CE2*A-xOVP|77V)qRZ*cc@#jhg+XS!BK3cnM2bCWxiAq!z-4|qRB6bVHoq+=?T_th|Gz_X_ z+hrHzfADZNQd@fG>0xTK^3;0MdHixKU-+$6q87^Q_)g)el^@kS)Xw-kcm3lm7lxwS zM8eif@AQCCdV%~&q=0&CPSk+;|>S2;mXC2>qMczNl(y#- zj0NXkHeBzVzR(=1f7nmkv0|lt`S!kGtt82DQ0Dnnw9}{SY<5>@5u8fpCj!v1Y zCnvDr*oWD>|yuJ zsJe%em|?P53b8J_1$)T@QVs!h$iJ}3A%8;-jUWh+OMJ;CKmbd!ncr94^B~2QGemt| z_0_xTdl$Qvilg9n=^uZ)`CqRq%70Q}@@Jv&6Qo$EiozA9aFuJJ8fi?6bf!lJGa{3j zk;N>P>h!R{3iujfA+nh*ab{SI9OlTp6_%nhD@PSpk>x^Iji%UCRAV(+w!`VD&g#(& zn~~*WI2+BeIhl9D`DlSHM2l=OT4GBQR|=P-W9*pBm&4=H33fu}E8)rL6gw5IuoYRZ zhNq)5>`e3$dr6k3!n4sic24GN;mgr^c3$SE!|z26)*p)7k65Ebv8O7IVxSz%21)V3u>t_klUaj{|c& z$NT`86Z|AFCkf+U@L%0C*_&MdOlh3jdGDi6%NKs9>0j)2!n-RUT)**duyLi!{gntH zTxoWro|gnSLx1IVkZi7e*x%R)IvekMO(6VWV>5|YzoI*Bn5q3y;tLN=Q{!q7C#n5O zFA2IGFHFrF{T|TfA9PS&Z>Xu(=>XQo(_QIyTERv`ON;L$iQl1q>GUXj%}+e;B_8Ta zSG?w?pEXaNtKCoE8Fejh_=(%=xf`O}@14> zaHyTvOx#wdp{KJ~x?w-+T+f?c4LZIj(&?Xn{P|(jr`_8JQ5b9LYIh?@;_IT@^kYm6 zRQ>0D58K(nA$B{Xr4~N(BAkdwD|zOVe$wkF*k5%t=(P`hkEipWcbneLA@ZluO`mgI z07jnUe#`5JiJRSn8+JYJ)08ziCK`>7O-*M$>fmxBp9fyz-;k#ge+8~nFlG6(ey`UR zNld4iI>XZS>o|{vAf_XYf;h&2@(3dzBc8Baht{&a^A;rX|@h z^zQm1ZW5(T{_MQ)KYxUbuP_x-Kr5<6l~=gt>sfsw5%_cmdZ_T-K#(n z6hP5MZcAA79qv%BW%Cj*-!oY0OT|!F8FXGvXU3E^BC7S&YD%h0%T1ywcUZbbH8?-UeStmr1;+e5@k<$s6J9uWuUgTf#PcT>bpi_>?tUj17o1H&3VYKEiGR9neuBj z!z@#+g*t6u$b13$0@r2;Z)1!izDVt&_#xMKO|GNtbgY4~YXXa&u0uVLoLTBqO3KJ% z1;7~tlN$q_n-8d6C8_2$T8_n<+^M7nEUkdXu#udBhCY_;GtCv8T49~Uva*4M-+%m* zIF8Sa#~L$k23{;)MyX+>X593^+T~6^a<{hKP0(yC&Y>=AP)CD)5AFZ4vWEs9hy2;Z<$Blg3VI1AKxFKjC5CSK@{XRVJB!>vD7)1vD(Lobe9 z_p$Qi+FBCz)_OtjV&Vx%nu~8m{mtl7bbBofZm#w2CY#;P+IAR5YnhDl@A%C=_$DjH zYa6}9y%}sYA&gghcf|#)2%1!h*8v>MRxQ=gjQzT%sfKFpm$kBLYrv^C@U~i232y*q zfA$ltI4s-GEQ|74ZTacW@gsO-b#!Z$;jJYQAaQDA!;)*#3LTjXp&%u6Y6(B-i_QVQ zAuFJObcI$C;rq>>5U+BBmZebTR`p_I$ zgTlZjN_}MRf zS%4}I*TEHN-)ctrX}wUzLGK(p>WK56kG1vn#}(p}ju-jh6UQaJ)ek9Oc0tTunAHf9 zL4pL6C=sCPBxR%~D5trZTA6?sbnb!43soOduEvRr?O8I%@6#BGClqf=&?+PRL&(jq`CQGY2Gq; zkvov#j+Egg+-x~5W`}p}>h<+>`hdzIO`b6;t^EM$nLv=(){*3bCPcQ@2KgmA1g6SW z2tVxsiAEza2N2T}7%M3ZGzdw3pze_%E=uvJuip^oF)FC%6@26cWF%HYF*TZQt0^Un zq%koCB%6Uw47Q~y;I{f56kz7)KsL;b1h3IZQtZ%9vf<nKMfNJ@jvC0K*@h5v40gG3{xBP1Rkn=R=j4b{EY05Q*Qb(ydSc}7y^1K?w#n5ybPT!BZX$Lq~GLtICFqb%O1 zF+Lzbl$GkBfeBU^ae?@Vn*59a?L>S`U_AAh;;*72L6ZCminlU)*K&G4xge-T6Y0Qj zNZ*1k0Jg={n5o|lyUnfjLwe%?JrXSv=h8XHA;1JZ!i3lbEw**8^i@%#Tn9pMV9YAJ zW@5C-oKV{q2}Mw0A%USlqA;V$=jxrLG$2aO>dQIJenpc*N$GwsZd49YE-Q4Pq&vP4 zB(2lpRWjPvyCO>K(tdM0$h+tngrHj^?6{Yt2#}&wmS2}gmg-U1YgpO!OKKh&1d^m# zQO+V2@)6Yi8#)`(%|zXGQepdby^362-B~tNYiCn=8q}I z0HBeMQZ=^lHq?6@P{FyOl!iIi)~-o?{v^>F5{#8epsx$kZ=taEEmm z`()&kv2SL^KDY2KpbRs=1w{bE-{wUa{!k6R(=PFngk`@>U$m?E6<$T3D)LkQ6k+{p zOM}}`<~2SIbyAb+WEwX%Gs(8s$kkm>bICiR)wC3Lg*S8yte)^&!JQ^?@bEXJDapQZ?}MUT?LK%|RLJm0jkp*)pOhY!QLYgF5lMh*GKDhs^riZx*24L5^PW90|61YJ-)g+@K>vdJO(WHjW|C5S zHXn12+fs(iCkwo|L2FywqY;;+lyKIisV}Yn$11CL#fCI`{h{+2H_0&={wPn3HiHzG z0TSiDdSAJ(?IFgH$!=8;Q9M?~pVCLqZ0fVfOD7?tJR;}RemnG{o7}tnuPDW&oXX11 zD`VP~H#Oq<)wdyUy)eFvOxA5jTiq1DL^pAR0I_6ac|yw3oY2CYyG_xBi4Kb{3pPl( zlosU{NiL;x^6vdR2`FHjT=2|aDMsEMc-q2?Q)i48ix@Z6Z^JPrl3>M=pN@zUI`q16 zK)N3Oa_+y!XbrdMG_{_V9!2I9Wqxv6is3FKt}J0y9iC^^P4PLph(DzHNC3CQ3Jqb& zp?P}hJG}Ct{E=rrB6RT!8k6pZIda=_XF`f*GH5dck$jJTMvA`=;MkTFEt+QS!`Ik{ z*#Yl^JP-ID-_~{VJF4(P>fg+o76g#CZ|_eE?}HbaLLngh;RZZT%cH}hQgP^K#L_5F zXAc;DnTr@_!hVu5$u6d{6!!O!A;xU0VrFQ%K!7U`^??RaULdamqTKF~=d3Ovmyib{ zJ?%a1J>?g5@-^To7g6U79C#r5fH3BwvZM@*J$=`{byCh?$3ISrP}z2%?!hd=9B7R> z?13%SL`IP%QNfZ}1+kS@AmYRoUS~_RfD;=4sfqH2FE$Ag5NHFiGMdA$%v&EL{1!l? zl&O@65FG+t0_4Jo9ss7HvyC&_i`mrZd?hi)a!IKgq^6{{2{kZMF#nEW;*$WjO$von zN*$`jP|Hxx*0;7%2JWDe$?i8-G7V#WeO-Kk0UGr~R@{UEM|YO_G9*M&Jx)YsRFCaB zlCzmtBezdC{2eylpPe!+{6%)UvcE zKVEIsGEnBKPiept+xoGhfQf}=k)EhxkZup{o&4*$`pK0Xlex0!XbguGd z`0PEC&G0$dXWpMh3p?xAvScLLU~_!wKUmB9WrBO48x9MbEJTE@b1vbgYgXRB`ih4w01$P;A@fU zb&H*9LNPY0U6N47*I;zOIHXz_;=sEVsPnf6P=8R%M!Wi!qEW+LMqi-GA*Nl zP{J4u63CsAYF1kpE>_u?J*1y9Y5X#(B-N!QsGv>^R*HPVW|F|C_%)w6s_bO9slx&R zp8ydno9TCYP^Xv=$Rg+F{FZ!W}_4p zq##UK1Mw;{PQijYuqXn8z?6-^6jTMmRhDXhV;hEAqtKPCL7V-ZlSdNKNYe|FZfS8S zy5d7J?s)}3YM9a$q<9`6v-HVd0vVVHA6WxyY@Xaki=l3mf^-Ln!;_FTag=yGOM51- z(6OQZ_6=?4#L>GM+T1iyIxSC`g6hNAS(7Mt_%o0pYL*+AMMiEQcTyhdyP)IY@-=Y> z^@0ch3^(>K{tSrJ%I^5ky3;B>?DP{KwhX;k;Jqo|0)>5O3ulks!ohB0fGM6^HZ5H) zyN1m0+#hL^OIF2Q)D2flqX=3pD|KLTxk+~mFBFHDY-MEmBU&m=Q8uA+CW4j|!e=#W znildWssmvTB8$<<{sx5vi)=$`!?A|erho=?{$<1_5y?m7UEc<~+;S*-j@;rcCm|wI z_B3(wmW-OiDlid7GC7=d*a8-}sAc>bFUTHOU|+!O=k}hiDtW#*k#{JMkz}hqjq*}H zl-A#GsH`UZsJrdE94}}=6S3EwYAz91v(4=|s}z@9y@{ZFS~>6*v{P@&K=*1UZ)K!@ zXC}wxEhSR?K8`{Bgupuhc*`?*&`em30*-wA>Jy7os7?zAWMfVtmZ)*=(*6b2lzMnL zi(**N%_KFbH>=2ZDW1PlrKW5leudVDFStxC^LFt&bj>7mYOpd$h#={X{8Grv*stkj z)Kt~7X8iZgnWL$7azMm}VT4@Ah>a76j1V^8^O{@OXe8sUu^V~49ypWD=GbJCjOiN`9^}sSR@4glEB9Vj&kGs zsQ4Nw{w08|kO7A%7fd(9q&P2*^XWl*sQOepXFNID8gjnY%2Y#fGTU&T;YlGcSty;g z1!mawx9?EJ16UKjd%qfKDDf7KG*6vHk$=&PrsmT zDBpzz2{`$rBm=$d!i#zwR<)#o+4&drkk63$!?CoB*_ju#9_s8lS`v{7d*cPIGjo7g zQ9g)|Xg4z;$eU#skLsSCT=vqO%OAI&1Ys;i!O_S2oSqXz&#WTHk9|DS;olF$F~T|0 zyTi+|ABz(}2y!}7Gn;`#k60many}V2$zE9i@+zSz#vn~5Nr4lLAX(x~>SVv2vG8R` q7f5ITW9B%xfka77tHCNMtFvkuqDAI&{H$6x>hIN8>aQ&S`2PSCgK{GP literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/gpt_j.cpython-310.pyc b/vllm/model_executor/models/__pycache__/gpt_j.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1375c93dcb9f92074cf3c39b29bd0f991217aabc GIT binary patch literal 8414 zcma)B-E$k)b>D9+77G#tDSk+@L|d{I*cK_pcH%a!+f<1^C0dRYH?!kpyTVQ%X1il*XeZ<_OVREuZ4 znYixNxm=EB<2i4R^G+SCPdqER|u)V&KCM%Cy>2m8HM8lVveyssmZfV!jJ1$iwa9!jNc)2nHB;jUT=^gjL^l6fuNmAJ z3BMoT2!+7X<0;2P*a`+w>gPM)N4-FV^!OT&iAEF4Hv3^0+Y*PO9i-uPUZLa>j7>p% z=h~p(?@7GFRjkHBJD~-Q+ev~Ucmd-OeY10Muo?Pc+)uat_QYt#f+v0pE83IWS%n)$ z!FDKOv2V&|Ff4fq!dBX<2ubmMTnP-KoOl zOHkXgusJPr>V%G{P_E?&S5)s9UgfT0D4q+Yp3bVITVpj^%d95rS>`rLi~Lb(i}tH1 zHI0es+Q^r1>$sEuLy{_c$`|SbMOB7sM;j`>j(-FHxaq&6A6HN^Q!AHuOp*z8s0-~r z@_s3`xdy!oDfKR=&YrT%{fEZ9BJ@oydHW;fXDam`8a!qNc~cm(%C4EZLtc|TJ^7eh zshz4Q75%Q0gj?96yr=J4oX095UrMKF6wIJHEDbF{sLQjOPHT|!6Nctc!)&W8pAiby zacb?ohgxRbXa|XW8H*rEan^C;`k(*wEZIu~xqbbShH~bRiu>sXOtxJ+2f;!d-1hsu zq)pZU{V6m!X)DW6KZ80@cVDz)K=wpiJTXW@uL_0ihP~T8dDHL9 zUPsPwu{Qw3P_c@lSKc3`1dGO_qVWfB-@&~4b6k%Uo8o}q^0`Fqv0 zKLJc$DQN}~j7~PF*{et%s#)3hn^BM?zW-49^8M8`?yvUS{qtBdJpKH)UUQ9+Y(+_xrjTtio~&O)mEoeO*2&b zY53V+X{B-gu~n+8uIgwtmHsW%8&C7r)RtOXXr1TX$2;=}%x-10Ihi|FgSJSr>Sze?{n1yp4w%j6r#%Qq>Zm2FhyTa^EEO1?wMJCytd5>JPQWvZX4 zTba6&5*9-10?${LsUi%1J1&HzO!fGm#?m6GY9+` zk%Zh*3G5E|15!b{Nh6Z)A(`+8T1VS8QYaFX42XYUC$EqKYatnxt+X`MIyTfz-J|+4 zeeiuEP zg#V+9R*3FPxLFI>Kvgy^br+uV3w4OnE<7BW1L&dBP_l+3)3N^(hA_4P`3W^4h-A%| zpHgn$Jooke2dL=dPRI#B->qERwL;rZ9h20eiTl7cB17^;h(TA%G~K_Bddg2l1r3!1N zzV?V71UICI{R9g)R!^_gg+g{iDX*h}BrML#mx=SO^<iQSi z#B-FSCCT(S>W@_Q42^V_5>i#K%;OU;I7(rEjwVSBiDNmM^Az6a>6|tNSlp?PS8-oS zk3eC`I!o3(x=3{_+^nK35Tv_Q2QD=%Ehs~CK|wnW z@B*7Qj>3K@^50_Qd<6*78}`@x30sCqTL7yiL7(Sft;q79brP(ezeJm49*I$M3NKEk zYdv->mt4in{<@6J;Ulz`A5(IaR(q)UXWUx5QxCAX36qd2I)CT2kU zb*B78{mq>E3o9(OY^MJuVT1mI&W7cVBOFdEVTJzD-VH(g(Wi?1R5(S_u-ejacyPtE zr~&n-8TD(}mU?buSLj%~mKDG_R=LdT%dpkHeSm6^hS1H{_V>u_TiZRLm#^J}o#j}M z9CV^%>=o99a`~=uSG}vX-~oho#~vEL0K5OCCXD;~=hO?7!w2v7-L%>vZf>B@v@qE| zYw3)z)>RsxN363=me!goW-+=bPc}-R#q=Om?y2{bd)gk(^*P?Dl6K^wDh2)Nxrwo{ zk)7pR&s4*HCko;lB6#;#G#J6TtL&U*45bAdo72iWz^EWf-bE(wu%oSP$dE?+oDzcR z#_T>?|&TEehqZD~hbT!I&d| zg4WWffe2n&@-$^n|B<8W(SXCtlS565MrFUUg51C3PKeStB@09uTn0bu51L`X%h+oD z$((6{I%`kur#bkrdW}~r1CYUFeQ;hzgsePbx(U<4Z0-R4$m17D4o?I}ql?ZHW6qnR zK#JH=C9zg^p}SVs7@9&|QgG;la`A!G+|$0N{CE}+r9e4LsI$R%H2@ZIU$j|NmXx8f z2RPe0#v_)JH&Pq!!)mL0aK{+EJhT9E*3e>L%Xin2#6|gZ)@sWHp^c!sDcg``CbH|H z>>w{UDT%11i*pu-;$ifXY`8V9jn@DnU43+w7pB@t-p885#9D%D{P-M%}zI03V-%A>SW_{oy zo=8{nUq~haRxFK(^4CXJ5Pvy-D7xZMR0WXggcU?sjL>DMt&%i!3wW)HsXL}OC8jwR zQwyg7#zx++WdYXf0NYJCl$l?*lg6Hvk)&Uj%Msje{jb(nmCiD7p%Y4(Of; z&^T#@q2OB9Ikw+WlF>aLP;Y8U*3A}@;@)_j!+0;=K>bGby z=KL0O<{aCY@jrGE7O*s6+9G*?iPaw?nz;~9s&HaIM%c#KHH9`bZYs&501v;Z$fYfJNG@&#fgX#)Iwi<)px!z$ zG~gJ)&GDKB@}QUxRQVR2$5H>*)`=9-p|YpR@1QM4gL5no%N+^?IS2}wLwiqG5$o-q z9{0Xp^uE63j(dBULD3s=-?A{%5@M~z7<-zkY`F{&)>^{^|G$8D^sKEsrucmn#g7q4cFGi6TDgJX?M{ueagc0= z!e@!e6q!e^h%1{F)_V~{+Bhsd8%@=Z#Xkc?tq zKSkb~2ZL+U1N->E5+{Bf^!u=~tWuQeNtr=U@~Zp|Llj>$GKX7m2+OM!Gvs!5vVnjY z2vufsll<{-)8jcIW{|lwSTR(B?{`W=O1Eia z^9d4k=3zyiDj9fAnh5?qcf#W(Otriz3R|y0LG!PF96e`V4Dm^vma`?0b-c=qCR+9D8-&ww0#=DKduyJI!N15!dT!PxgpQ^T1l$zReAg*+#1k_Xi1FPbNF@h-M|?YvFHvZT54}vVSW_n)*FHwr?_l!i^)A6@LilCOoWv2r g1rpSd*6=^4x;mGs#5Jyl0K8HEZv9mK%;KB>4}7oKU;qFB literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/gpt_neox.cpython-310.pyc b/vllm/model_executor/models/__pycache__/gpt_neox.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0a3d4aac14f780eb0961acdbe4cfc15d00d5576 GIT binary patch literal 8623 zcma)BOLyExcE%fM^n=YNDUy4EJr-&CG0sUG$75wYwj@}SsJit) zp>XeaAJr|FT?M~O|MIK#FW**_|Dnp{&qU=TTydf*icma7s6q?XNb|Hv_w>l{jL7uN z$nq?e+Vs%&Y}}2|jvUWno*5P**K@gSg~h1km7=m&=DHnLqA72R%T8F0roHK?=GC}f z2xp>MZL; z!i&+ew;Zi_E74otTdMLSMVt`xcNH-oI2)Jtb?1ISi0R#)|Wo*udTH^Yo9clU8^(0ubv4yG9ndl!M$fw3?X=o!buBIYAW4D_4N9j+#j8Qm z5RIgP0maLW=6W#dURnzhzt{8EWVhc-&r=P<9Z>NXp< zhRh#Ew*n!s`*_Nn2wIJPnE2U8@WXCH1oZeS&xuYG+cxuA2VW)%M7xm$H+YBQCr~*B z{oU*RUau?h4%e_7^X-^6G-}5&rr-^X0*p;BM2*eB529YOLGvGH#M!O#T=S!1(oFf&_}ELg}@FT2MtrXn`*DzzEDmMHskSi%QG7r?gaI3hQne zif==B3&LSqVCn|0C{n5AijpYbHN4_I#ZbHwM0+Z&=HfMyq}9}FvZ$q{CJB;1tZmUJ z%WF+zV#NiNZCpC8SizPh%D(bYeT3pb-OvV#uj6juj*IRa`bh;fGqDCLq1nEUnw>cN z${x24j1!8`H?{b}=gQAi>OU}eL?LmrnKn%3DjW7d7sdnB{9;n#Hhxmhr)QSY(-h{6 zvS%h0Dx=rbtg^4}Sp!R0`IBzalXyM0aQ5{*o67~^qUr z=0L+LO1z30pdb>lYa2&5{P>$t5fK`3<^kTm3uFWM2{e4;O&(TJf#@;v6{N7ZLjGw$~w zqK*&AY$v{-&UE^bKVIZIR4MkR63iL;)WnRmxZV~b=rAmJ1>$irwV|pQEahwTj557I zVKhossXn#U53sz=V8`EXgeXrF*Tigbkm}G6Z})amc{SN|LJ7z%x4aLTbc8 z*plbK8o!jc^j$dS*>STGLPzBrH1c&Kb&$tuTJZg5*ob4_f2@4{`AQP?R(kE;MQkWu z?Be@Te?7Vs-ChaXw^n*P$$Gc5vK@xeN(Pz1ouJu=B4^cjWv!R^ouGSXxwj)PV*X3G z;;%ql%TXP*g1@0_hc(SmHPt%QbVJSln(BO`*`xA-YgN^%T2d|aGY&QGuc7Q5SQd?? zf9YU%@mZZ+9(~o)NMPC`PRqluhi?_csk6E5vreaF+E*W_#YQ8wWRUb_r%ArX@K-{C zwE8lx_@^KP<)Jdr9;r9AJrlZPZm{kO{egB9`m*QjK^q=w19e~zoJTr(fd`IG^+90( z9e!l)x!@QBmuD$rmLjypSbaj!@#GLc;Jeja?{_xiREJ>KO$na}Qo}s?9?J6jMBV|Z zm*o3Y{(#6I68VtGAAxxK*5-Dq`l-5=s++00ofi3xyKF>KE4$0zr~XX-0HxG!b>;1b z6t_@`>HRDAtXjh#-%s6Lcy8D8?vJ(A)yHL0txh8fpjxg^wzVHpx#ar*&M<3{4n2^t zQx=KL5Fv?4O^8!-J+(5!BgqEHE7VZV1+^K3q3=(u`1hy_EnB_XmD&nc5QKHR&@Ce^e%UqkH@*$k=KmkRo6{A-N{h2V@=C zR-jJ&ArBS<^#afrJ=uo1gueJ3bv1*%qI?q*^A5`?D$zE1(?@JGIXr2ZQORHTldk{n zdWKXv?_FlD%L)y3bA^H(Pj!&Go{8W&nw(xy@@E-|6hAr!KRylO(wD&>3jI_whb0Fk z_i%S+yk#SGZjxmf{%E7KS9G5u9)lx39|7t6XVL$>Wrk;CaxpDp|d2`3lrQfJ`Bu5;jm+Q?uZ6Ub zF4|JYx3T;|p2%y_5}hTEJO&GLNId%(o|6=Sdhh`<8BB_(>HE~X#NL3m3WK8uy|`W; z%VRp%kAt!EErYl%fD@7+H!Wn#@5+cJHm$I$=y$;IF;69U}XjnKMIm4jq^#=dhwx)!n(t9Un>VaiA6z4B2=B92V&g3M|sFF?;0SL7WFB*$wQy0b8MLX!0C-00Gd$M#&i1C^@h%);w&} zyaF}Qt}6qJ%NT*a)^UBaesv&)aokE_b{bfXByp4EhD$ODQh-3tzt!ltY-9M_$j{N& zD`21DZ{b^cE)|b8JFFD{jP5Z3xeIq7GxTs34lLW%T{r|r!8vfDf)3DQclOAR50&sL z2Fq0<$Entzqu~LrZ?_Rs$Jz)Q_nguZ6w)ug12VP|h*2~#jrasEc^;#fS&SmAz-~Eg zGvHpsW;mh{6gFJpGA#y0x+A731&B~Eri^kWs1P?Ow=_gKB{3zch+U@G%2Z)xY9nR8 zN>TV~syyQ$rdCe|t@fQGB1a}?WOlYu8LRz$ASWsPeV7wmCabWO*eGM|_tbk@OQ&$< zt}`(H5dq3SX$U>^n`EBg`uK2dC z2rAk2NR<2PedRu4qw7iru&Shw@mQ7Xbkj3O_#Dca;o+}N{y^${5H_M)qVeH>pcaz} zDJi=bhPX|8HB!^%4?+abVf-Nq*_d5>nZYBLZJPQf5i$|=+8DdCeO>@Jb+;+_=9nOZ zDu)9V2)$fFyEn~2K^8+~^QOfZUUm*%X(8x{498L{e=Z@Q^ojfxNLp>RksZnUFapXm z#6I&Ku{Kng^^`Yg3~#h|8OCi;iS~=DN!i(cXk6^CvYR-h0njy&%K6%x@6 zRg&+_hJDZK7z0zNi^`srK>RqShPeK*@)t9NiyPL!MjPRx1}r2G9DNp)MP*>@14*~$ zct$&ZCvk*2u-fWAoCM5XV6;SB4E54hv+wRo@`SxIk{+$Lgq4wv&ZdmOOM#M|wLlVZ z%Px@~^(>($LaV&vDdLk+9|9+NLmB4MtMSIjK9xwyQnj6$EC&-P#vh5C;s3-HzYO9y zCDqYt0K_~d!reM>tP(gA0GWNB93boZ>Z)AB;QI8pdnzNZJk?_ax8$139}~GwL?=QX zhvc`SZ}uG?O$%Lqg)4p+By-K-0)8`g&dD9Ge4{N6!Pm(hbj#AdF4?u;NSfg}0GoldO#H^S@TTPq@W`fyZj(X6XXw$^JSU}qe2K9MJ-Kliw zV1Ck-mmB@K5nj3aoqpU1&`AKtfDZowfDZXb1}Z`9F!Ir|Xu^hH=OA;FMHrOJ(KcLp z0e5qwxKYBr;FppzdfEFlQze<&r@U2_drYImP%bIbe;yt=8lbO-v}T?kkn^a@kjM%( z6f&%YSK!Sg?SL|{Vj|I#DVdceFJ+BJR6|as`n_Q;Rn8IHj zKk!+7T+2d=p#r5D#~>AW8IEdcK(|x5shIczBaD-zzG-2YWr*c7O8gO;o1_@S9|742 z%ub*Jm_b^QKE(hahLixq&CtaCHG0(T47wR9{tk?1AHpHa*vlE=LlIV#qZQY|iya!; z?#W|bW#}+5-c0k(Q5tuc3^@{{l1V1vx{QlmU|xOZnre^qJ$*1T&zk%jTIv7~A5-$kI3g5BY+lpM=a)iO0vjX7(IInCu zNEtxQ_ci(Y78yMq<_{ThZWS<*rm%&AR4xS(#)b(aX$hsTN-;1tEEr1TA(B)Afy2)M zOi1#=KW*s&3 zrPbYWN_hDelEAxjqo`~-YQ&p?;7sxEOrF;TI4rk2+kQ(1Tf1`;%w{*v(s<$s8;21& zJIk55S+qg+@Bw1LLEzblVW~wfqF3F`RN&_{qS(X1Q50h(NShe+5UCvoGk2 zLjs&=W;sHFOwIs$-+R%BXALpgXYaxbM)3Jre&orS-xtn5*7xi{EL#he$Xl4PyBcI;CbdIRg;|esYPp7F>Je*EzezDJ_nT*;!^uolp-} z6mk&|7h~}l^&wC~iQfh>rcWb|V6XtClb2bwq-WrP8n=`*oBMCjvU{Ka(*syeOISEW-IUK@oTm6k>`8Nvs;Cz=odNcv`SIgFGpkk7{_z1 zN;K||M^(Sdakn)QP5P73E`L`vriyqKOEKk8pp?5_eJ;n z_eT%-4@5`&Bh0I|7NVp6(da?{!RR6XA?8iA9*!RIAK~z1>od`3{m*iESL<`pqyD1^ zPl@T)W6?4HSoFC6ILBvNpO2pKpNPKTe<6C(e^S?;(ZsBndqWd*p||?QP18RP&Tg>> zoIM%m1UPqzz2NN4I8TAIPt1cepK(rtvtJwl=KyiSW8veQwtq^P7qr^l>rbC)FNHE} zH^L*GcI(>wnWxUpKO^hyW)jXv9TB$X8=a_IPns87;r!KRvNHed+VXO(N;JGlVi+WN=df{Zfu@b86sdu*X!ilV?nHHDB zB((+iZt*nT~(?xtF)IUg%ui%0hKxXFJQy zBt9=YjW9+_!K!|Ft&aY$WAr=itZ#aHfLcqFB9Si?S;yj_L8B4Gje3jP>QXM*>4q4n z+=Y6C5tnHx3p~G;bk`DU_^GE)pLpr)i^0p!pFHtYaQ?)pQ)d>R2`-%RpH9nzCZ2mX ztc!H& z7Ks-T2sK}a^XFcYgsIr=FUQfyAKd+7TmQ!XTq$n{K=a=&YC>aXibZc{@^S7_*G ze(^m6-@92_SjJ=qYu%tLJF7K2^_p$0N$#3o?C=8RNGu;le)$lQM?luS1<|?M=3?*K z@=>t-!btvY*?lXRs>kQo_b<#pPlAuK%{SxuVYa1?oNsro zF4Xd=4K7xP8w5>A#vqWB_$;K%NEhT3vGRkqrQXZ+)>`FA*3)n3uNWJ~hKU$f)D3+@ zM+j?&`8xRe1$1+2&Z}=pQ3ao^qn+^>A6~$;$=i) z7o@CvdRe#h_3B7_vrIK}@w7aGOtk_{BFUN{NWFmIV68>rVh~^twXzfw+buQYL`u?h zB8JU7PDQAV@&OPU?f5W=s~I^OS?dGCFm4&9@v&hU zA33)1q2U-ek@kV%mfx?;aj6ChYI%c-iEfSf9c$*{aSCU!BMVmcjG z#5jePJTW6?-?02~mSR=x?%io68~g!azgIb=X^Zb^^O~lnAa#_O;`N{A+iOv<0(^H{EBv=f>J`lyj9h4Ro7-k=${UkpF(kO!T zpArIjye8iGS%An;213GVfsvS-+D5U5^&40{ZPn@(r=Y`LGUDkkX|H1~6K9Sj8_7Z~ zaKLeT#Y?t4szO=r1*#+l!!q7ml!VFhUXawK=?p|O zx|v_NjM-&zpX15*^PEQ7RXUb*l6or`u7>K6(O(nWMEiPfp~Kb5$MB(=7a5wVEZ9%=s_1qgKN`MEBO>t}zvuE9~H?XnZ zae5xtSU`;h2tIT1z1w)8m6}+6wVb4JgxY}VPvp-dEbBz5Ju)Oh0=`yMbGb|m(!yku z$SR0$Vr9tF2&MYvRKJw!SNsBZTB$mzqkczVhdRv2^v4LqGz{h3Jm#>fP;urHBi_fx zoIH=ri}K4vPcJH*oRH5VD$fx45)pC;$mc*(dkG4DqL7hRsbSnqNm}$s`;SwUJMlb1ui}kq-2ygUrIILW{d5#QjSN1^44=mPPTu%$ z!!!QNDH#806pf!6rSg9mr!4@k1;DicxKzS&sy8Klg=S&PL#?AHmLMZA3ngDvgb8@H zLMyZZmo|P$#Rs^G9MpS{VH1Lo)q5|@uX=!9qD7`9Ky5DMFvSUs0ib0u1^_KFfR?e2 z#?!vGbt-JY{FZF5s5w!4ejvD58&lBSS-nnS}Z{XBj!i7-Wh`2|1|l zvZ!3A9*up1Q(+()UsM8@);CN4biN~-@Lm9EM)6@(iD2POTOlnEML8K%rx||y3D#1= zYrz##@0GFwn}js_fCY5n|0$b&H)?eQ`qH|!fMCs0^8F4Lf*E$>+W0p7N+;kLXbLE^ zBM2l(xXUX6@g8TZr!#_=e9#=bh~U( zypK!>>juaA0tDrLYK--f$RmsArURUEQQp#vx7YV%QgU?n4WLQxAgH%s4;c&ek`!a% z)dS}A@n?fDM0bHrZH0ZXgkvO`pgGIbkfqO4^}0HCz57 zJ}!TO$U2ct$$9 zr;+PJjJcS>%Gi#uLSv>7g8_u`}h{-qX5uf=OCvb!0trAdMgk%}oEH(Wf{g%fBKpM3%Q8?E~X2 zg)3Iil2bhkcwwzN-AQsJwSh7l4$^ZyXPV5I9EVxzZ^BfGk04%J`aU zR_8R#8A1*=Sh6~m@seL;yM-+nconN1xy)9{HaoLH7Eb@6FpUgJ$hZQMXm9CnX>Z{R zw9otdm38*5vahD|dV5#rF@M)ob_r*E2=xplVCp{6sz(<^{Yji;XfbJHmZrp-^YQwj z(Fq>NtA!_8aM`xvClOF(VKOP-rhJ2He+^`d{X-x2i(MFON#_!Lk4@nh5F;X!YiKH0@@Un{$3{H#4 z1J9SrBKJ{K$<#@UJPE38v8)mQe&Ua_*{Iy5s()!AhAEgRPi~qI+ae1(S^sIO3HNl8&InZP!_|G5dP0?Z`uhGYE!rtJqe+_Y8)h*ZFX`kKZ#8rQoM*tA z<4?tEqT?<8XtHlfyl`i}j{u35Rb9?zs%W9-Y~GV|cK$Ro{z_rhS6lz7ezJY+p<)7r>g zUf?$5<8ujlKwM$IZERw2(S|}V2UyPaa@g3`qO#n+iblU{48ySPXt~Q@A@Z9iu@`R2n&bzO&CQcB?_ptUp%tq*0*HBj8f>&dC2YWYMW#rxEBQKQ`ITQS2YOTgx9z-D7yx z2X1B?x|wF+z!{4^e^7L*?@Ow)fp^P8^GZ*A3SaB%0?Ux}-vi1CbJ-Bq+lFs~V}DQg zErH_@;eum}{Ic#lI1Iue5FG%CvA1nMm#@&DcnHYu0@d^En4Q2cX#kV>CFkreQOPV! z92}El79}1QGwhI^WjY5f08Uxt-5rkOfH)sksa!Zg+_1{JL6HxOZGTei<1+K%F679o za!U>$6I=e2xO+?LG%Qs|92ECJLzxlxhO^=j(&xls(A_9sU2;WD+=tXX;{NciaBs-9 zD|hpfAs!G%kTx$C!hPHiOB@vsg0r8k-UHY~kEDH{{ppT8Szn9mt+VHFYzOZexeHV0 zQueFdhiRUV1vn|!KS46s<-$h^&~z_ie%QyN&KBrcfyH5(YbkfLU)e7B{PFA*X|P3O z2v3rs@iI5_-zV}1MBV{e+-};xPAR0p ze!{#UDM?~K{uhv;0!HRG40}?N*l_L}8E|bu_62t0K%VAg14c6(T0tHv+Mymah*gAE zy+<_zyo7wq@IbGUSJ~}Z@xw`}jXL4wHM7&dF{D)FN#wIv$5*TP9b;cIZgdEn-3XH~ z+MP{9UVv$%ukN}_gKJ?bs}~9yyvwARO(!#~#?6wMp3Etgj!kxVF}3*w3&YE!K-igBLn-+^ zA}q66@VlyNT+QcC1qlyQ2f9^f=tDNNi*UOK-@`Di7jjyfV| zb3Dzh)MJ>+QvMO8Pg8m&YdF@nx&|P36FV2H6MWWr`i300f2dA!PyUcDb+jsd$E)^aA+amoixMXw;9X(yKRF`Q+(`@Z9qp_h2%_hq}!y;U@!r}Qno zk-a)lIpw$Mi!>42;Hg2*xH ze7alo1`Pq>h)0g;+m^9DmtomxM>0fXAWg@4*h%>vxSCB=(0?oh2&cFklL&A(G9&+n zi4#uTu~CR<$nPVa1xa?isi68#!A%`C>Dz!hEz>2BH5`(w#96bW%HLuP)z3TED4Oi=?Ldg*KG`z5>omY_QH!M=jDHQ9J1q zMba1gj{ea$s(h(;v7R(m)cJ`ml55>Cbwqfj*$5j{y#G*muvk8aNP^W&?mtKsXn#v^ zn?OG4*Ug=|!cck}T=Wnre}VGyFNypv2)rc+DIA0NHbhD%9y;&(o&O=_7?#`C>3w`* zB#gSrHK@|VYbyO~IX|Y+ptWI`W!`kwt+GX1&${K_Tz`0n75KkYnC7xeS6i%sBECR7 zjHmSpd{gNo8%9DCjEfzceBch{_nSD?ur{C=_pB>8g}PkCfg?ikT>^GHID0SCL8Y;3 z%0mJcJUE9g7vYPwHg!weaO4Siui*EF?-!>c!U3?r-%ElWJR5#pDBYWeuAvrEwHI)% zlHc$)3Q)7HE4n;)`DD*}Cx=)b3l6ou1K-FEUA_wFGLio3<&!uB1&jeg8pw;b(YJcJ zRS#d`+IxjfSX;W*t_@1}veG@Yv_Roat{Vn{CVv^cevRKitFjumw0@-33_|(EnhnhH zF`%%Rl@cP`hz*%aBaxjgOr@U4u42+!g=cu6t03(fYG zU`d9T*LRID8=dTim)OF_l_qs0Cf3WDX0xEyM_Sr)8;d`_ZZD93xmHXqfjfGBg|0h@ z;q|9%^)p=2g#+a^o<}In$*|kPmA}-5EgjQEKzSeK_!}YzK>CMSe~qv|1?#(Usb6&r zXM49xNB3|h$T}Rz^mQ2C__|->+hQWSJLY?g(|kiCwK+F`BK0`Te7}(OjLW<63T}bG zJF#7S7OCQ_+W4Pn8xmH+Z|oMyLMPv^ZkQEpM#O=MBvf5Ce&~dgxK(Ue4=<$_ftghZ zUJpjnkV|Ed^3^PO zhn3)CYOgw*HuhOf?dpU~VwaS_yF4JRpW;?fcJtA31BvoZ{91AFZl zoyQ!Y&P z{1R?E;Q|%uH_eMcvHi8GBrmEY(2@IzumW?PLMK3cPa!x1f!D;!#v;PKRHc}Gi?Y$# zmhYayt+=F_TvJZ&dnn;gh=i1yAKWNRi!ZDpU!<;J_4S7zQ_dyINk`%2g7WihCX;{5 zFK4X}RO|gQl{7pT*(Uubd4~Jm4>dw@$-r78{g6+|p*xXBsQB@u1uu}{+T<~J4eKWZ zr`;$t?o=O;3)b~1R9L%voBQ0K>Z*N)p~F38Q&NfD5=L6q0v?7q*XXZFX2?*tPKwhMsPrA3O;f#oGx7~7Xx+OXoaBkr^ z0DD^fIXB?}At|usus}Hsa9Hq&b+dn5FzPBO=DZJ#Vs_4S3-aTZk)wQczss?_kNEKV zHl;Hc>3hm^C%I*{r_)>zh)x67&m63J>>727Zc^L1w>)M%w%l*_f{?PJc0v(u*mew! zHNtIGo(I27R{$3VyOS@Dup7lMa7uO~icjZhlKcv@1#W1oZ7P%r#G911tA7t?P+PNq zsW5XQFAOhI@(oA-;2rYvP9j$;pCkJF@068qPV*g0z9z$$WBmK?RD$nLlanB`n#h>) zAGt$`QDMA5w}bj;A2U?)`2V%!C+}e|tR>A>ED1jE*yJI$t9#UPj!GQ3!-xAeAX$XG zRqoh9Wv~)z@+_6@3yWZ;_}84kQ9`Ch@D% RC-tgXy}SCA>bbcK{{x>Cy9fXP literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/granitemoe.cpython-310.pyc b/vllm/model_executor/models/__pycache__/granitemoe.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92d51607e710db100da92da680cd79624a3b2fb1 GIT binary patch literal 11514 zcmbta+jAS&dEfgk78jBrMcpOXmSiKgAj?i-r*-Tow&XaGXvb3AcH_LOq~u zrFb@(3+CAG$NQ4`V4nTucz<#rIFK9+4km|!LmXF$4<|>0BkZrnN0Vd0G4|Ku<4GfE zuzxCkB6%`+GI=U^io?_KiR5H(GC38TN}dj$=D3;onWPys*0FjYJt~mTjJ=nltS#DE8|mfO+nwb|WzYSK{ddoei&mDR zEF28NCD~saVEs?xp;xr5=LE!%M%#o>Bi#v)9+8xcC;U06#aajAbEe)ru=mlygn zX`1=W_rLevJ4Jzu-Cop|`SiQdAntbBStN>>nRol^J2Rn@sf+!kZkE0y`<*C7>xim< zf31x%ZM11zMSsyU^-%T)5r*YnZYP)b4+xgT7-62ABL>*u~5 zzH{!4H{M!!Gras(a6Ye%8^83Os4epO_xhdo)lua6;JNHMScius^a?)On!nl1il4*z5 zfATj|1kwRKkrwDU$A-`Yvt;n!6m?-mmay?Scw9VQWJk`5dr%W4_&mIQye+)TQE74d zBW+O^6;b`L8u+4yoMkaZ?xH28#mt9RQ29u+w4jO|JDXRxw-=VT?PF-?8fr(J9u!O4 zKuV)aQ=eeh@%l6t`*L!3XUpxZkuJBTNE;D0pWnbQx-p1k)@XNRKTR9hRt+47)wIQl z=aH+(@pRf)kGf0C88XR8RTGJa$~D?OL^c*_D^i!Zn$<}5qPWpb8yrpD%9bPQ-a0zd z7_`&$IWD-^#p%M4T#p(u5^J4^^YwWLrCe3f&xt-Rjsr zQIAhYKGU7pAu6guGnb^H$ZigzymE~;cG%(5o4eUSWpV~GTI}BFlqR+V`73x#Jn0oU znYN|f*6(P#HZ+G?Xk;edy3mKV(1za6G-3S05aulhX;x+r^`U`u3#pc{hR)FCJT~$; zw|wMrgoagj7cOkrttD(|Adz*o+fL;(XiGN9c^1x<4X;%Ux#`Oj6n&DMQ{+5N4&@JO zY9d1!rE77P24x^0fT0}&G*o3@p}bUM)6MO44cj62!mtZW3`12L?f=}OzQ`JR-BG)x z+)AT(@sX|XYT&M(m%^|UW4*)hu6FO@nJgKc8FUAyv$h0=oPH@;TTad<>u2Kb)iZ;e z*>b;k=6W0_XB6&5H=@p3)|V=no>@}Jo%EyD;HGS$Yv06^(zyMSqg$q-tG}mvUs&!a zGXH1NDS+CDf-QJOn4%GPABW1#NQkEq?k^E#gNWjI59McXun`fXZ# zZN<1_Y~ps$|mSw2YNjv>geQek$!24C(L$M1av zQj%dDqo&u4itg(h`*u!Bk@K##u&^MXLe^%Dwn(p?L}8fwA<@!WO#Vt3;vmIEiW`Qa z-$AsA<0omV=y1x;a&GgflXDbhrkUi$%N>Q?@)e4%7CWUA#W6PGcJE(6s-z*vC*hDM zrRniBOE)TNZLQDneqb2JeZw@qG%Vw{o^5>LIL1B4H9j}I+God(bLkEW?)dF1Lg`Q7 z89{wX2h5xNw;-U{kqMYL0rMtc-UPr~4DJqtyB!heR|>GNMxLk%b8(IVe@aXP@JnJQ z@&WjDc4yh0iz=W2#4h?`pO^;$RG9#3*c}J*+3ochixQAh4DHitoOHD&yj$`TA>lG8 zG|emIUA;aECG8vGpr3YGBJqi>Lx6DB&MO@fL<;ER6PwSqN3u(p05!Uh79zL!>O=_{yc=;RU_U1UF`!pCx|(uA=PI1 z1Wc|tLx0tlXOyqV-J;yR(hv~2;uLW&Q(Wc>)Jgtn&39}c}9^{gb!S;|!zqC{BDYFrQ1K&cME+8vev=L)J_p=Ebnx=e?;>WD8;kNpma;0enSY-g~%}{g`c?sFTBkeQ_s@*i~Mg1FLn%#^e zRB@52czaY&bH6-Ie!^1)K~?yIsSK6H;dp<^?@?@hJ9fO)Ol&o-Gw^69QuDcUO%bY# zTqLgsZrT9{^_JuUH8Tw-_eP_VbqYFaAd-%x{5Hjm#uFu1rAHAK_GzzKR=CK-N1amR zp#iJP$0L*PP#jU6WYS|K86zSOco~VQfa6yjo&I&b0@v5)%(_K>U?h`@_rTMw2M#dC z&@KKG^nhqSH%k5l?HDelQVO0g91c;~&?{eT9NFVsZ&OfdbX-%${M^Ib@mbBQH104d zl!=aw{*HbmnYehElxQHJ9F1rO1Y~bILwkh|2OSv5Dl6uaN&4t5<9&0}-SjrGQJJ71 zliV>;0&$43Nk$o#NNTxb<0xylaoD)VGHNUXqRa)}njm-+tKD?v5708m48W1e%kax9 zK}4--R=#xGDK6#l1$(x5(iwXfMY7w&F5L zD998h&y$bO3r#!O0W?XN$o-LTl5a2m`VJBn@uYMHJk6MpP^@21vX=q&G79!`JmWK` zWc$OGB@pvP#e-1@Kpyp=qoO8 zokee8xcLi8@q~2?TRtphK0;+Y6+DzurOiIZdJ^4h*;J7>E!slXhqH{_ z-h#p(+EyEWizMZc0js27m9dIpiUF8`a2u-_rU25A)r>N-l@vH8r=-9x8t=) z8BACgDHj2DU5Tu3W5wG_Ae5)3jxbc4e2G9ytj)D{w}$ zy2l`LZ?p;%z&A$d|3TsO2pr4vnH-hW?)VYm8_@+zsKrDrLZqev3fC$m7mS0 zwrBbhSwIVgV&`ee%W+?}M`{A26YB!BJx4#9ctUJO=-cpYBlO3Ft6B(Dm9s+&61BBr z4`zfur}co>n-0>Qp|hXlZ{^oQHzQe{LM2Lt35ConKKy08%UK0JXh|RG9~p~aE@6M@ z4Q*hy1L@nnWxh|jYj~#cOlLDiJ$2zRdMk`BEhxdduol+)a*0LUZQScn(&$Hi8t^Qh zl!Vqy`&j>2`xrx`^QX>{u`P}_bxG@gS7%DQYbx;JiDj@J3!l07O59GaiuS8BXd|UI z49(M>2OD46wK!w&XuZM^o4yKLyDR{aa_MXtZ)kryOR_wKAlRZI=a;|}4ul+vvkzzowV}0TZ93OZ zahqOxJ}a>*YX02Vf{6+}@Q04-0d~2ypak|lI(gUF_CpsV$(vQo#jXTJlciSPK%Be@ zC$|w^isS}G{24hzI6(y(DXfA;Zej+x2?usfUSR@B#Z7-sK2i|mUy#E}1N9--ySdHV zWG}usKu78sWb?o`KGZo{VGo)C`vK_AeXp<;diQ+`+QROka!XP^*7WxT6+SBQf|sZbx3`KwwgiDzH}tF27I1V-*`G)-^hor`9~K9s9dIf`4X zh`gW`!% ziE;Tu2Q`#_QapvyhsBBLNOUx!l2eN~J4iFcNpT8k$C)P^*z?VN!Yq2BFJEu3rS16Q zr34Y%~^UV3n6oLhi=OMi)#g$QYw!8q6rYc0%rM8tVA`1gj^> zS&(}n9&@pcj0QTM8Ho@#Sbb#V%Tn?Y}{MczYAA&YBD7~;E*fjbW< zD(dgS=56WIB(Cz`u$(XHM#THf@oPG%D8N0iop--tPs-X8J9{i@$VVaY4*H_qMh`(< zN@HS0WqjQQ(-G+FGK3*qobzgo82nMSD}3L!oQxHS>ojN&E%?y4tOrmQj&1|?B_#S2#5#_Z`KYuqHf&6MGL~XR+MB+Ri%5ES~f2T(K6R7~-_*x6Hj`g`r3vW<3m*T9JQG&X3_>dopd&ECn_W zVLw%PM5ksn8GlE)x6AEZ!TdPiwMWSyyNPSY?VJ1JBZu^-G&QsUhFRn7Y~8O}tash_ z?rnT+?`irSb;`d%eVyIQP4mRdO``m0x8iPYVySic3DQObqgr^CgK`hTav|sLUTQ}j z<$tDeP!km##7SH4($B(5VBRL%t~s>lH4F)|9*1%7%wa(K5Qe@Ls7-=yo(&dRx4-~Dt0ty+o=Bw5=%ecOw`j@!QGV1hE z*X5$F66)IGenP87pDV)(=l&7rhV5clL2k4w%BzO_1e@7japWh6p)`}yGIzzh4T>5X zxBWX9H@+{T+w{u9#>t1QXsyDi#2>}93dqJRYa6E?&Rf{7S|>$o{cMr>cjbGr$oDDT z8q4<|QFFogxD3I!IlpJkFO2f-F)DQ%tTe$!=ybJ zATgN7hZ^)P6Ovd)Dbj+fc@O%kQDG9Hz2=OAcL(OrZTg zP;fsv|40t+ed1N64AC+?THX0eq<)5;YsR8^BeLUUb%x0gfQi5AjBYQ z#&}3X9@2pm9E0VgmqOl1@C8|m1=zH;H2y$^r#h(7$1JU}#houPSrrV*oKjqb@Wm3X zLom(J0$(4>?o|j6g4JtD#1Nd|G;RrIcJ*aUVgG6I_g8dNz#r^9HY;n*WKUtcd+@|# zYE=-$m$h<+Mqa0WJo~s3EbEXuREWAP5x@AD68s%j@quBBuB%5DM-m7g+g5SeJ>Knn zaU4APn6?Uuo8@obvBBedrtDpWJpAqDb~~ghD9wTFV8O|~T6{?KlyrWAD(6k3Yism9 zs*eU^?Efw-wWPO%L*xxJQk^#99eVBm0HaSR AS^xk5 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/idefics2_vision_model.cpython-310.pyc b/vllm/model_executor/models/__pycache__/idefics2_vision_model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ddba3317665aefd2464e9b79746243b7bb0b32ae GIT binary patch literal 9130 zcmbtaO>EpqcIIER+3X)lE%|SLY42q1PVClcnXk1U?50x3^Mj8-z&1&Em>Y< zkd|4jdMs8IU%mIepK@HUn-YHS{PnMT?Y1QSD>Y_+8X6zs;{T4qlWfV8JtdU;imk|0 zS3}iSao0k%ui2WQmBLbAxAlJ6E{nDv8vTlGih4P$_G@;nU$^U`ZG?^foINM%m2kel zU@!EWcC)`|FAAC&F7;2?CuHd-l2`R=k0h_=Ynvw@s`e>R>RtnsMou{m%A7Y3%6v{a z1ImKe1f`i%UIAs%TLNV%r@RWv3GXB*Cv(bcp8AEx5 zE+oLy9@XqprR3E--PbqEUKyp~8GZ$K^Ni%tz1lTA)2luz*|lFvnq=3(;d*-dFnezF z*L}|mq7CkTrAU%#;kVigxP^Z%F+b=AzGrRujF@S~U2Ba88(}bL3#J7d5f~-fUx-`A z^y#Q#z%?t0EkAPCL*EMe?uO54pe>gLL=s(?&%s&ZOWVvB;pfwL1iP>t=3B1k4ic=V z%i_NEG)46Rcjuva>~G!!S4E-I(=#X5nVJ(T;VE5ByGIeNN+o z;8~v+li$72OOgSJ>{Z*wbQ zK8A`mCf^oBR?_nWWvs#;%;%b?_1sA z0XopLwskYIAgD>u8HO&qXl1hQm^xiDJBB5oXW_s!()vKuEa%zdbS4X{uOWmg#8lD8z&{opYmKz3Mt7KP$$m#Ul zDDp#Y*RWO7R`s6GKo;;|>pWouCgp)El;|maPHQeL@!`N{so^-_P~tdgwHJ7vPr9<} zrxl-+&hdi2Z3@O^WOG2`Eb5HdVe5d1t;?QWi-!GT0qI=UO^iW`n=a8~LG-zUX)%K5 z+tnGWT^Eg*2c%%!t`z#2HU~M?DF%F^q#F0bE}O$5p2%s%aXO*Px#K*M{^w`QNq?|B z2nLrRb%4f8@AZeh{yY8c@R1a>Ob&1!z5-|lP_aw0@s}_fzBE9 zwg)?G36ovM#m}KIWkdXF^5jpEjc*#|T23_&cF(>H$J!I7ucWmB6eo1Bt3#isOBv(b z1gKAZaQZI4msXOEMDU!aI`fkui#lY0C%;pukbGXp#eX47B$d)v+E^Qx#`?HCf~=0@ zeR*t*j6J~gzPtyZ#$6d#Mmi)+gB+q4-4$hQj;l#&Pxl~oo8?y}0C`^9SI4!a1ew-G z=BTU$0Jo!gw>TmaAN zBWUa1+^D`UpOV0@3bfE%VNJ05DL5@{;KJ{}K48>>?bb+rY_l7L`KE3>t)Bh2=Li4! z!78k4382?Yo*v-WYC;8jE|lRk-b~BgQ0%&00cJ%UBJVTW*t7)6?DT9M<~-W)*%_i~ z-C-C)N1-x4Alpw>KJ43-^NuqiV;8Fgr7Bd7VbLsA!;g3l}F5GbBxJ)|C_5bNW6_QNA@)<+-LdpNoDS zbUF0X^A`U-RA&9iiUO}*^)*lPOTKT1HLEP48)x=o&>u~~s9iL$GC70-O6*{1Lx zt5DBcdg?IlUYnrl?vFnmhDmU#=er)9jRIve$K-s`&$`L2Yhh^pJRVvr@O#d~oq?XN z2y2tp!0!mwHggon3Qm4hr*EP={}qZ^M*&7$0WiZIDBJ`Az5p9tI06L0bm@R8fbl(* zT}lj3rutt<0)%zk)xRt~l>dmRWls}MLj|M4iEye3xeMf2h*3rgz?qzqdP0CkbO0)- zWkH=w<{y&Fv5++JgjTZfkQ|Lg4^E2%=maM#E4x1u+D(3d)gN;BRK7*Q2@l9mhJ!^$ zo7l;%3mb7_trgwZu2{QG;g78qeTb&C+Bs(a%ip(Tb`4xweWKXfdf;-=`lNM&ErOa! zFWGl?f{K$U>_r-@*tcwEyHZS$8nhN#ZED^ji+Y2xm;q-yZs^+!(-%D!58yJT<-EeG z7(F%a-`jEqEZ$@_(IlrzG!2@&oD4F@hiq+ns+d&xGs52`9I`6`cRKEnyWtZxLs;<* z^r1}XO#v+83kRU{-$7xPbm2KP02+#H0vPlI?VzR_Xk{J-{EKIbPIZD3GSd3QvrAwrI3s;MfC@tk!}(JpyoQK3;mHiFfuSfYXq3VWXl zS`_<$imAn%nuZVYppUCC4N6ISuD~ehYU8^S7Y$H&77Z z)N*0T0A&g;wP(ApyuhsjP?!+?w=l*b7$L6FbsZP~O%zaCK#5RVc?{!{$06X_9!z0D zm%orQ8C2OS-eO-+@rNi{Ie?1kNV{g_%%Ua<;6LEv1keqDp%C~pFRpx|$wJhtM@9Yr zBjulC=D(pO%%d=*Ss8!h1f!=NT!(%%9sLCCPh{JZeKk*=0a}qd6VA`3?v?z~rVe@5 zy)xvvEacgMJR8$2;*F?-3?#ed?)XgjD2KL%)R(5Dt>LtwSz*r77YPy<6JZwK1Bo|$ zkH_Q@2vHVKs{*Sjr7?Y4BawchP_9$;k-eT>C%9iCY*Imt{v>YJ`s*t zrhO;L{uo1#GIxzd$eDj9^42c{#yD%3_Kz(^|VA;;V}m_l3Q>Xcf*{Yc0=c z(ikm`eMPMOITgR4;x39=U9=6#0(uBLj#J2gMsH35vSyk&|0nm|*N)|WuAQyQv5NVf zjLsd*`;W8$3QP>CsVHV)Xz1C?mGLT`PL=VvTsBmZ^4KDq4TVJ70Sajdg`~7VC7nC+ zb-RBjc64L|gcjSr$%g9$&*|h23eSBF4)ewXGX2^1In4 zQzl$I`*Q8b8g7wdt=-M{_pq2{vtB9U&+8(j5#t;WBNSB%7$Gp8!pPSYNGOTAmxXC? z4j&mw>1%`)lofgi$;b0@Zu>fvD4F~w*hF?v=X@U#>62k(D_wy+91a$5RcXtWGaRehWBNEP|$`pBFHkdxSgUb%c)@Mp3lRd>j;TIOxI~%F^}OJ6Sn75yE)^ zvSU`z6|-$4RL5}wj%6V135!E3#yOGIi?<0SK|?EfVHLA$4k8cPG8{ofcmdxOcgW?U znwG^Q8_mq&q3CnQs6-KE@dDkn%rk@d$}PUbcV zU!VO9gB3$*k{7qUIfxa&9wqL%K<6d2vCaJRw25r>jM}kB^C_s=B0<=tQ7IG5Uj!6zf zgA0ZOI0PRh)bx=K0fFbL9?5}&Ahu-#D0^y&dNyog+rj4rzK8vg^-HWR;<(XJ5{iiN-1`EAX?A#Ow6=S%4l;jhLK;>g3q4 zA5-yv2->^ow?u0@jSBfsMn%C2wP%#cg*HFaO3iN=^=iE?QUs}<>^9Nn>CCR}!T7RZ zeFzL@wC#*yKl?Ytv%}|k?aU6hr!h1W0ruHn(>E0VTOtdzt28=g0`2p^<_(bz&i}uJ z$daUb<|T-nw}^a11S8xRq3VKtGeRL0$HF|RaTQ;Hy#D*xfy|g|3pIrf6z1icGLQR` TTvMBB^Jep-<{QnkC%^x{iu&EzWtL)C~tgGR7`OkmdFTAX2|49#nKL-!jQNq~NG@%K-r3JdL2ZnD1rf&w8 zZv_Rv5ZJyQ6#Zi0_)bvrOM&aV!Hhqn(-@;w_RIJ;TeHEOKgVrWYd%=;7r0(%Ee1>e zlCDWz{Q&*!){$V@Uk;A?M}rlAh5Hm+&jqXgD*7zQwMV-DyqtMY6OJf-riqd)J~GjE zjN4o>gSMHp?FH$IvUKF}HBHRQIn<7=X=~c9CFaEZXBI}C;2y`B3m!NNeGW#JlCkpm zuE8^u@;OkW{%FUTMH+*-NU9}q1QHxcB{<1#%i<{7j;3wim{ccIsaC{uEY+z2sldY< zR{NZ(Qt=#+iuxx~t%7`QL8d@qROkneaZ-x+Q@Ax@(0WZHIiOy)DG z%$_*KGQTt+GkAEz={{$I1SNTzb;+}!Mk7of^&*Y(Uz#w6q&_3gg8T9m?n~lja9?5h zUlA3^UrFV!aN9X?9&P8-wsQj#SNPo*#6@1q`LWf4&poRB9up);yn}mUMKoh#hjkj` zpJxd!iOZ1Saw@?EZhKX{hPKzzwu}7k#3L{8J6{*yVX3NPQi0DszT59HRVwZYsc6PT zsyAqiUlqn}t@7r7)2^?~#ItvPy7g{tybNQJZPDn^CRR+1rzf9>arfx1*x4cK<=ejNMzE58vMCw0E0(@xliO zt*H4yN7P$s%lvIjDh^swB@GyV9WJ z#|KR%g}2*LUcJ@w0uujXK=ON_vK3bE#7DCAGMl;aVqUK`c=2ixlE$`@(Sd4jHuQ8g z^lzZh{|3tcSRg{Xtv%K^A8U_|*a~H9SFNB`J>Mt)duu%ky6fF$_hM96dosHCMsTnn zTn-+rx0*Za-G|YBr@j73s}-!Xm^Jwe**J(gDtQdovo(g*?!(xr)!2l!6C?mhW9hwP zba;j( zjH@_sa!W;AoR-5VSt}*6k z1@Nz#Bx!ZsX?yKU2RF#eHtGNBR`u!N5|#(}gQ% z#0-#5SvWXrXT{uSHn5umpPrA)eV3+K+ zDCA_mFd9KeUDZ%8&=?*);;-a2w~5Jn@93bGvRGG{9o=iRiPU8xWL?lURj(0w zoyd1UHr02j`uEvQy-8z73`W6ihuYa-&yUHHg~^ILYBk!vNrWUbskeyaW||r`ub|&g zQNlS8ikizt83roro}q}BxgqLLiNNE zXh{UCUTt-7u7=mD*$_6vjow`fl6)|Fubd4lUMQo`+a3yk-@l^{?5pN&k5&=%Ek9x=S69W`lW@(R(Bp$CjaqD+*e zS}DYHwILDXLUhn=$+!rNNRq^M0@LH-gSu)%S{RBBa>At{z^(6cz@@HXJe#hl#*fKX z6UKj20)qhrB$}R;3wUBVp&7hr=U&FxPhVwGNi|| zx=8gZ5!ziG1qaN_;ecOH1AmK>9LqW#%Zl0>C__K8{&kd)Hd-Dsr=QKjLij7l(mhw$ zpa_CEQE|o!7bvle{KTwq5FCrRk(KVvgNIO>hDpm`5OaLuy8W|cYpeI}-*Mgo+RIk6 zExm3Xx3!jR;qc8bR^4vw?$o2kel2YFaAOmMV`5v#Pnr$Mr=vQ8)i!8<^?y|Fqq*dg zf$du{dg~iBzwjRH%eJ?*<*9o6zIV;L#Fw6~>g+Zl$DY?cP~A=_!|EnG0&W0&s(yf3 z)h3Z25n=meMv6}c_4~3WgKqQ?H;nvNSBsKs-JdWf={hDF%@}$W!{~7}X zTX+pt`Y*1R{@!jhWH<7zH0X+aW&7rUXz7J>drTiyHyi#T-{GDt$PZ)=c=&y*2CpN11bEYS^m)%>pwyG(6v$8mP+R>&|w}_ZTvcoSnp*lo` zj%w@=*zH5=#CV0SRk+);ebilQ&SlCBt9}T^zoPKw?}Hb^!M1;5na2NFmhq)gaKA80 zHlE7X+?PCJ`riBr=F0cxAe<4VEMQ04_lnq)_B}_~03}X(m&cw&UZsfLXv-OJ%itxu zaYmG}TW7^AgO>RWC~R#dkcCw<;=F_^pT2KKpr!gTjmY)Gfr(W4HymPt_PQe_aFV~hGrrdl|iW<9{Nx!(iu_# zwWy^-BV3A%-y_e^hehFA!T_MM7-pHm0?aZ~m}Nx;z%1k+0JAP1TIv#y1fV*!2xhgN zLo0lX+noMvovVkG`zvks?u`J75uDQP4x*F{Le!H0xIHP3ZX*NcH3{vc;bq$kBoa!Z zqAT)%A#DtzIQ2bJ*cy?mM6MCZZ}q%8-bUMhQTq=-dM7fl((O)yVr0e}y&pfz5(j|T z*4892#W3tABU28+u*u_|3c~tou2yU$kOo%8*eSt(@$wJ@pbr=^Qa4B&?-6;2$h#nw z0)sL?&n)IB(qG7CPiOF*Za0N!H)#frM>$R-9mXi(B@oASbO%Tc2#qlrFq=M%*^FOW z#XjG8X3R$0m_z-5-B}(sz7Osf)Z-tB(9=X;A{ZB*2BGAQKxX%miCY1#Dv$KAOgZle zdrWXrr($h#1iHadvtqLPVuw;?l&4nnn5AMQ+MG>Np{Y}k2F-s$PuyR$knmzzcK~C7 zD5-Vq)S}&g9VH}#<)_xu5Tz*KT5SVY(plvQ3wV+6Bwed5>|-SF->-)^+ij_C1EtAwpLS^^nzyB!J>Pnt->^RO2Sfq%4H}M0-CUy$$Kr5)nH1GE0o1 zr{NRlaSB&>G@CZJFXcLpV!S$sN^fOoNqIhH>}Jkb<)1NR1gm^a0_Eoz;gZEDC7fdr z`-u${Viu+9WaiO zQWsEhwFSel1{6x%%u!H(=2M!qbC%Y@2@iVu*3+HqPd7FbEqnuI%KefS!!9y0R*`J$ zH1#?vV@^a%gU1aeW$!oNY|FQr_oeqi zJ=m!y3ECDiu#{v-X1c(a3ra|cj<;X`L?$r-ml@pMJi07WrUIuOCBP%^fr>DT^x6l= z%POz4>qVXWfc}Ufi_MS@m1Nr5E+phIk$@$ix>TTT%OT!%|ixH|$t+z->g>FsIG&+rimB zvXmqmu%Nt>G4F&#iFXdiS0&Ltur>MMq_y43w03;byE4_OU39J#NA*S?I}in?YHJVu z-Y0EQ#Gn9xuSC@Ljpfe)UmjIR4O=8XZ+m+B*3;eXPd7Fbt#kur%C@J)utk zscG9+)?*u)6{J6uL8DlQ7S#v>B--R~N|-^Fh|u*}%@8RQp$nA1n9PBXu-Hx^ec)8H z)ILKWXsby=PEmTN*Tm?IDCtn- zFI%zQ*||qR^C-0t2uw0-l)sCuold94d9wtyF@7V&$mopWPZAjnp3n(QTMLmY*oCl{k%$a-g|8fSR ANdN!< literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/interfaces_base.cpython-310.pyc b/vllm/model_executor/models/__pycache__/interfaces_base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30730cf696eae68df688bfb1ae87909e3206c3b3 GIT binary patch literal 5359 zcmbtX&2tmU6`$!DjYhI0e*gyaDZF^G0L#E8m1IK*yIvrh1!ER3$xgD-j@2!D@O)@{ zMtrzTRZ^xBwkmtcKafpsu2c@WN&tP~!hEP2(EZeJ#)(Jzx$C495se#|&~#F0dRc z$UFJKc5I#Wm|t)T>c1EaIYX+=@Jm72DJ$OeE5Wcc9E><4s-E*lgE41J@s__Y*zfGu zHK9um8+rdgFz$>ine8784mpPuU+^b_!_HyF7yTo_QRk@Qhx}u~ap$<=Oa6(V>Qoh9 z_D=?r&Lr%PiMKX%=ad-X6;bBHeB>*`nL=$;6z;vv$G&3DY1H9nw~Yy1Eo zf1>emVTvg+-C`5kh5BArNsZt-f==xOSXOvGD`+R~7KAr7J_Vh{FfKQ~K`E-u@I2Sp)qt#~qO_5Nu)AHh{3s)O+ zKe;;h(TDTbQ}c!wC#ij_op@2``l-Djqa zZA+_s#|_&3YQxmc3zz#0k0_nKw$z+!G^0g$o?D7KAx{g<=)NmG*iZFEjFso2<#zl8 zO9RCklSicF^y54DF^dwV_P$>_rz(z3xS1($K$lnj$)6H0Cx3$ zL#;)emfa)?8$lOSas>98Igs8*d~f1%5*7It5t>~sC#Q(75jjKTERl0Wwq`HiMm5d1 zBKg3T`~u0y=e5yoXS=f65KSv|5k8Hy*l1ueIzI8GMgzO#XA*gkOwbO<(;#UM3)Ngf zJc%%lq%=ryl*kN`Z8sE0zQ|)BaVEIjJxlTVuWgR{I{TBMYe`|9ZRk9=N{PyvM__G1Y=;9qkiLVlpAi&) z0~bCkVtDPTwytlmn62w~v6Wvif^N2p73giUl-19tZ1usW?i4cw61K9nQ@dBVv@AmL zuq`iNChK%nV86CBTd%xat7W;aLHMOD1AeJ@{gNLwT|d5D-|GAP!^IU*RVvl#skm14 z;%eZv>AAkQCgi)-saVp|*NQy^ zbdb){q6bTH;)YF;nhB+6w8WjIrtija%EB%NM zS$z;YLtW>Pf8%U1z7K^A&*Ot<{X;0khd?ad)W`LTQPkN>fJq0$bn`_iFRwwkW8ZcF zrv+((ed9=p8^)JfqCW*(;u4)w+`wKk1tlcMSD-YL7yyt7_{=0Za7c}cx4$lMYYR$6 z1*N=#l0B~?T$ooX*hI^qS6`j*9W+)~GTo-P0+;Wkx4Z?c+P#cE>S;{uR^*-WzyY?OgICaf4hgISm279N zwc6fusqgHiG8KADJlM46UcSrQkZ!WIiCdxns7m1>cr^;QynfYZI-LgkY^jwKk4a83 zyLR$5o@To9wY;p8{w$I6L=F?7<&+;2p*vpR;%u#uoF@TFOovtP7{+1zjb#>JFq;l# zsxh<8IWQWynF?hByhWr`$HyK44_&tospAWJ` z`!|R3jP4y-$i)_6j>0Zs$k!u^!-b&RrnpTuPf42cL0`60t2Zd@X%c@xiHjh%Y3fB* zL}{Y75cAKz5^wCO9a5 zlFyFksY`#>iry?!uBVCg~WEbY+wUBJ=`qY;|d`hoL&Kof2Nh zVT_wVAY?2FxXx&17H=WEdSiOD$RX-pBH_?|TsG_3El!rHrKnmuiRZ_1m>Mdo0XK&1 z&8*%v-F|(~Kj)-U%PudyXII^KAHk0Rxv8M5uN1^>&FiqA7clqIx`QU xRB%roXT@)sbxLRZm~Nq7)Jre%_|oxvX(^txN(^`Fm(VOjbEsc)+fL=%{{f;Qrak}w literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/intern_vit.cpython-310.pyc b/vllm/model_executor/models/__pycache__/intern_vit.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6c30ccbcd3c3b42da60b53f95477dfee402c0a4 GIT binary patch literal 11643 zcmcIqTXP)8b)MVK&Rzj5K#(9QYN-q5l}%7$>^LP$ifu}?WmAwW%91^iGFoD0fnDy6 zJ+mMQFRC1qj+6@%JC$-Jkz575Bw?!ZmbX+n|6`s~l~kqp7bFqL_nn!&00d;`A-gp_ zJ>5M$J>BPg=R2pTS1RQ-{GR{U->iS{B~AM;UQGWSTznHp*w!`8*L=OLb#zzPnK#-- z$8^n(o|s{Yn{AXkP@q1?3CP+`KHYiDeN6KC z_L(y82BmI375yr~UFL zhI{Tinm^;u-qrkBls$9Dbiag>IlqFEN?KAw$-KXSl7+P7S(F^}7g4g9mV6l{$Ndv1 zIgyrp1tm-VNtB#SOU|R@lz$o}r_+*U-@K~T9(%xbsOfRGUsqAH-maOk^I8aH}Eg@x{c;qEfd>*bED}8@tL(C@}i&{_LSG@ z`9a%5ul05t8CCCIkDofa{$@|LeJ^Zo2Jxw|-)@fkskeLGHLqO{BhPPk;_Bhu<00N! zJz5VGhM4DL>Z_60=I;bmniMopCe(?Qw4i>%DyScvz z9aVlXsCT30W}S1AzPlbD^Mgix(2l&DL33?A^4h(+FQYJf{qXI}HN$mOC-k})ftzpF zZwE^9tDN`gpPgQW>&OI}tNWU3_&x9D2F6=Yhrp9!+Q9b_A}pY@%) zmg|g4{2c31p7#qV&-q1M=hITlFZpGZ7CzK0%`Ia6XX3@zx>)Y+JIxSp^=ju@;QP() zS{U1_vbw9r)F7G9;xKW9zd{meceHJN2NFKihni;$^_GdVg>wey49+&rc9h+AqTF`g z*L}2WSv%$q+ZUp2tLPgn!GV@|GxHwuUI}^h(6UczzBR9*q>QVwuODZAX3`%@XK;*q zob@%lRc7S@fJA%1an|s<&91k)Uhj5;cIXy+gJ^P9xQ=&+cvA3OJB$>>_yJ$DHWz2Y zK|fHj<9W?4anX-wk{-qtaiKftK%3BQ5V|F~=!H#6Vi;HY zX?D_4&4{fqXg6fc_w_jEdC8-^``V{hE<~OFg?_VtKB_C|%K5K%2J4;YJ2x-1o7XP% zZ%6CB?uCtZyK^B?;^0=WI>5M-%kV-|B*xolM$7%%>Lh0iM_5IY*KOUA{|Uo3ES-Pn zVI@=4En`;C?`=MQxVSBkCOOXGWs>)Z_3y^|yQ>zu9sM%MP*E=72)jriVA>YMjY7B$ zb%7{A09)2ch{-7D8#~r^#y5Qn;+NU6w)DtuWg)P}5X$9?3f)5~dO;5mN0cvUJEq)| z@pa^!p|zt!uyQME5l?-vho7Q|spCveAaS$TnyUjGmjN`E0GY+jwVz!$k|{JzVW= zK7G`3?dvhNVG{7lWpT{l#52qP33SSizovH!V& zRF@R_hPi}$wykj;AaP$Edf748WX`Ab?Jn6~4iX%jV@=+*=^)kfUGENvzU=9t-{%mvkx6IdpuHs_9c?Pe}@*XO$f8xw*@?I<%iCY6KMF>J!30}ehWi;n8iK$VQ!dDS{~O>lM_@G zK^F~B%NUXR10=PgdJ?DDPVAq0o)uqZNk-&4&Nd*su-b7Bw4ob93}gef+m=^~vn4RPTz;UTG0mvKv6N-=yj0NL@Tl>8ZXNKID@ zNO%>JZ#!DWII(b|c%~%ft0>y{6Hbnju#IHGI9xMl_!YwCCxFYq0?)_<_LAmj!8sh? zXgI=0a(@2q40uN&fE{r5N%QgXWF|)(>su5M8naJd zdL3MQhq)DQ65>g!0xuth{CY>hci8704k!vy%ISJ?a7`_Z^a5jIPxugF~k8*c-Pd+Nh8DPkIGMrYH_j{$NJXMyzgZn+Qoovd9 zW}`V=@#)Zdq`X&eRfdM2xkvsrkE;coakLiHPoiTumr%BdGuHJU?(vQvT6!W{lD6#K z@UznLq;M-|<by2&3w?ph^cIY{F-#Ub|i8+{>(2zqHnisvnG-fAB)}(@GUO zU;SX*hGj-=`F}9&`2FX9+y`^H)~tu>EnMQl4Emuvvr$*x6!E&ROLgs(T0*NtnaIx6 zDJG|p#PfW@Sbmc6#JTZfV}~P5nAJ%V!~R|ZTN zOyN+!!T8e5jEEay+Np&EGVZi z4}_1J8-Z%EDSZV+Bvo}JBVy5E0r_d{h%*ZcH-*>FvZCzO`~CK9FBOQ7Q5KR%=7o}x zmT_IUGG(KTRoOgu+387)a4A7Ds!`{x2CZm@4mj$Ut?IUH`q4P+Y6FIFMSq8X=2Fqck z)LZ%kRX4fN;33JVbdvjodkE7W(+(hjkCy@aco`?vKoVzG*9YC}ih{2c3#^T1a8uFn zsXvzMTky%anSYk#7}ojd?(tgL&6mrPKtMk%w6q(>|ltqFJ^;nHcxf_*M=r zosSCJ?3GWlcJO>}AvMzKD&AEQ{?!kekm8E7TGQ2!nEx@7+OZ?;r+SZ#-e*G9Q67_P zOeiYqaU`yt1khB#5>W_IJz6xyD3-ux0+XUjW}-@y7&rNbaAvkFr`7 zyD0Vywrbb09<@@7*5>I$q-JcHM3=`xf=6i_F@@w3j&KoTa~MW$We&whn_iM2A$b_X z3q-<$WkkY*WyC*Qy+5>p$JSPEE5B9P;$qTQv$t~yR~S1mcDN6|*sw4xO5eat)Mt;V zCo&g&C>ne?j$@SP^GA6nDuM-$5V#BhOM)M1eKO8563?W@mniJ4B=L?h3P0FoAY>8K zmbiuya<$nBB&z|dXzHg#Nra@yvObePWYS{tO(wsG#62wx3iPapxP^ing<$TO%T0lE zX#&pb7Vo}|B-Sq-fM>PIPK|GSuCapSyEtB%cA!N1E~5Bi$WSpawUN9E*JfZy5VKO)J3=0P^Tn+J%1iueMZI0z=z}`abzvBow@dDVk{W-zf zu#C@*O!>1j=M>vK0@IXn{=!#~j9@yaBUWq#rf&v_EiY*eO8^=`b+-h77HFpLCjd>@ zo(AYboJ(H&^($|#BtZNY`rb$T_CfF!)EgH3g0ows_2}yH9!uQx3P-WDM!w}Ck8D#4OeiKo) zBy4LoR-aF3H*yzi)6m5iREIzpqTpIc49g>uRzx6_R2fwIp$*#19B6Z8JX>Qpic2HM zCc@Lh#3LoG9Q`Q9*5O$yF2 z|8pk7W+t>?tO_%H8#+K`r~=|##^&5)>?4{a-9IAR6tgK>a)6P1v26bo_wM2d>GC)l zJdRxn!x-h=Gc$^{t2p-k^66FBKxRs+XU62iWE_LC4QXb$S5IRYl;#;1-ZZ3ts>OKs z#7W~C#v}VCv+;>!Hqw2Q?kYmP>I%1ak{E=VFcXoNF+CZp2If~hBF+|!bIDceK!ZECs^nArNtfH$9)6E(+;TRd!b4jtq7yv1_Be2Gkm^`&m|4y>0+k zyB$8yo8qiqI6T%^I1p8gbi4SuI`pCY*pwEV4kT1=NPF0r)+QY>`^N|R^@$b z%XQ7*5Vd^)8LlnenU=nli41jom>HS~*jS77fpJY3lyx=kF)EIhSYxan3@_%I;!-mt z?2)+q6fnfNb2tfEi)Vj$^;Pdjmoa~@zID}m?!Isk;e~suW-Br0PrCy zzQ!q{7TS&tE0;i+y<4<4pFDaZWT#A)l;EZOq81+^Xd#&vQ1A$c z%s4%9(ej@e70amXD&BiEMso3|FtU{R`=4M3U>^orX~fq)fRuTBJ&?nuS>mYW#ZfCj zK+7XXZQq2Y;aDR+DWS4t<5WUjq1QtMg}beZFpEzhg84TXpp=c3X(TvQCIQOtAu|EW zk03(WdBz6PKvT0H0m8O2U0g%Su(K- z1e`ekV!IhetG!PD^r&mE#*V~|;XvE4&_F?<=BQAaiB0QrgM z9+~mPXUo4Xn-W)WXUt9LwIH^W6?qgR#!I1og)t^D`nSySJtWZZ4qt#6kw>(UKne*o zY^(hF$OGFa4&zsdOU-V75T*OP`}e&p^u|>dm|!G#n1=ga(cZTuz=x3DmT?2|q7fA6 z>hI8Q6np!7QjnNsd_4lYJQe$p@lylwc}_xdEI>TsZN`Yj?;rO*x>Nj}RsJkT zLSg+a0hnK)$=e*s8DvJ*tnguC(2z&$(7r{v79D&_%vrDdZH9th)xqDn%>FQasA>RP| zc>>m6oEOwOxQE=;{8;-nlpXhS8wwke_ZSYgu%MZh&F8rFw_e`}s!^{R_Jh@C<92m= zH@(_Z>2)&p2Sk?5rJD-hlkIQFk2PMXVm1jKSZc<)VynxI9ub9?ajia1>SV$>G3wV$ zevZVQ8Q}~*Eh4NPSP;?vUvT~jXR{fBc~DD!e!*zJ0qj9+!36p&Jt`b2_Qcs_e|jP627uhmx7ycvXY!kU2&3Q6W@oaf8ynlvGjjH?M)8*A~IhD<<0^~oo+pJ5Vmnlb4HvA@h9;Re0lw%oC;_^81McHzkks%u`>`R&Jq-C%d5T*)M7;-)bf?8*Z#hY{q@&3+|A;{?RTg_eb3G zhqX@LLIhMTr_w)&nwWV?w(Hy)O%nN_y2<3Pk;H}9I{luCWZT%yT@lZpZ}R7?v>}uD z^dm7RE4+vCIB|1@DvEWH#hy}t2}N{X#Heup%a9~3q_0%87jy@op;Vv=r;Q?<)CIk0 e5<93FJd8c=&f2>byYf=y-O8oPQst$^C;u1AvzyBR literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/internlm2.cpython-310.pyc b/vllm/model_executor/models/__pycache__/internlm2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2ac992910e95bd4f53b8159cff3f6ccad31d58f GIT binary patch literal 10527 zcmbVS-ILr@cGtJm>W}H^`SRGt9)rO&;4wBVW?Aqq7}y2u!K{JJDkLhor5^W;)ZKHX zp0OuA$!5nJoDb(Ascbe`w#rWBA+FT^3&~4fQR`Y;b<(Nc}P6v8hB| zI(_b^uI@eeoZmTDopRZhaGm`1KVAO!S0w2_DKUQONPG`>{Fy9CzU0fH6v>V(6R(6y zq&jM(Ia;JUdSp08WIASKIaX9~3S37GZO2AkEwrPeQ{+57EJbCf%)Aj+q6udrsybCp zo8e?sb85_6;Z!v3Oh+@$Of>7vMsv*&a=^T z&U2ie439^P&LZ=*@I>^y^E~rY;mPQfa|-;lKNG$Xz39Bi{A~EG=q2Z+=(KY>df9nd zmcB3fbN;?-lD{t~u6%o2bzVWvyuTkg`}3UdAm@O85IG0)oL7?avsoC{J((=N(nX0e1ucq3Ytwxes?<9fnE`>p=oomHN zzu!$-owgUI`iH$Pvh|PJNH3uSrQHUtUJyYdyjDB#&vx3))^c4*3$G_h(580jg25ZYA*r(P;!R zUKCli4|*P^Y}2D@%cn+8EqYne=?0ip^P(4F1w>lTnfH51x0j%#GJL@K9|WGC&VJl! zc$Ws5Z$*~^-*2^-@st@qXnMUcaW{h2^5w)0JD$(4Tjf4cX>6IM-f2S^qQGx?N$?@B zNPGwTo219S*z0yX0`G7Eb28V8X#t~F9HR?fuqZ&=^jPGr25t~_ldEoPthaoDW4Gxw zl8(5V7P(^RT@6IsAPF76ig_Y_156+}vM)JGLH3Kj5~#k4TMIN_5A?v8mwdyQoA#$t zQ}#{Yx>j~fzu?=~B**$x(j=#VJy1$Z{A%rU=TDtKcVQ_lUT*n*(9T3>a>qbfn(VwB zgH20ov`}t?g*`PO{XNlckOCZB#2dI(-0^RLB+|BYQ@$n1Qlj+bzOpR)@-!4S2MC+quEJyF-rB+q(RATtxe~H)nmAWneR8Guo2`wyN!i$(ocNFyh_HD({F15T^ zJcpE{#Sq=ieIK>sY>fhc0RwNXvuM4tuz6&0;r(`ab%ClawBm(PwPr^wv^yJ%^+Kv6 z7Yk^)ST}4WSInYoY*mPY+v~br(OD7uC`;KmmD(SBVJ~<~h>j4`lxg6BZnIt2Q!Ng{ zrl=!RJVWGJA~ej~a$0cR1~z5vy0@h--#d{+-4oqb_juxAgC)mLN4?9@$!Oz5*t&G0 zdo{V-X`fgR!{|h&6@n{4qX*f^lJN;%wRRYtTI^mGC-HzMamNQiY*UeKS;5cxO4I0{ zI7_R_C1rDFToM)sZ`+`09b6h1TFbMzV|t0cbW`ffx8zThEoDnZ3Tt{x-jcy#6**r< zzI+icxl~uhDYOyaBJu)AT@x?iFO{!2WdHY$P^dZM?0eN(5tGal3z~^mFQ=7I;rBssVzCpQ(4b7Fe z+%Ij@6N@yO6_fiefwORzrs+v#aK&M&l1${K#RJkdy+M`Ae07TIPWDMV*7^qMR5Hzd zQg8Gw1z+o1zJ7zsW|G-Fm!2W#CDe8<*+>2HZJ;;ZH*V<2pHKE9k4J_7N!-Q0iT+IP z?*MC8Ye|r`6C~=(xN-f*Kff(b;J>apSYRlss{|;V()g05+9fXz#Cudt74P6roF_s` zuYOQGPdslo5+E@NlA1}EZI7r9QUrCJi3E=b-JytyA5d{^lsl9*6$9W$r4|Vzqa0%` zvu7r1krfkUJ=5xHaMfKWb)Dg@*fDd#6z`&3oFg(vHMNbk)%Agn?}!7GH+X4CttfFN zFGISNieyq&DRIE(FL$Tk3I?D5rx@&MMt5>?adEmi4JyiOsiWlCY`Dv$- zCFjw-0%7ROE7(Ev(iX`(?DeW9UIRxd?4K2NnJ8@g7Fl$88#W=)R}9E4V2ZEZP+=ng zUhva@rp}Y3Dg%CwQpPC7EluQ6JB8Zav}iB${Cw-!B zX=D9W`3JH9dyc5!palHyKa@hkK zd7o;2NaQ0TWJgjR5TJ1xH3;vgYQsxXZM_w2h!3dB$3!lIIHm})PRO#08j%Yibt@Cy z-{v}Tu;zwA+c6;sZsc`Sc`cP!Q+YkLxPL&}-FCW=IT~ni4PC^v$!%EbS%`2|eg-0I zDY!GtMky5wJEo%#!pYRLm!CNY7AP+ezrTYcg2kn-= z1^s_h>)YHz5j_+yN*VOf#gEXxXc74_5r+t&-+C#V;uXqRB@z;eh|si)*N6l}nnXy| zIz@h|4tqGMk^S$QdWI{Xp>j;GSuv~74mC_FAkzLlCN&32bzQuT>PzDLM3?F{@m(_1t71#??qYBMbcHh_~x zJX-2OkvO^upfyWAK`_lJF_#R!mj17hx)34sAB zFW|P3Z?bDvOiD;oe%XieL(R$qITH*Qpy1fzk$AE7&&(dl8MF)=`){_5a1OZUGt|kr zucQWDXu@6SAvNJBVW$XQd1s0gCz4t0B?F>a=E&%0ShX;$#Sneunx&;?5cuxEtC+zs z0AHinZ6o^OVhm|z*p%~9tJ!QbS2weLE@q1JsU z4hBt%Y}~5?Hd$jfXvYNRGH_dgmaOd7l0|wX?M$(UJ``OlU_D3(>Kl~d8mjyw?wH(U zC_^2Ke~;9l0rh6EdaK-37IrH<3yQH~*1izb{sE;&u$r#;??A?t9{kJcLCA>Jo}Q~c z_?JLVgV2%X7ohqqc$o#@DwAkaL%^v7wN@<12tLi^Wo52W2}<-2WjGO({VMvbfS<&F zmTLwR#03>rknlzQscZU}w>g_u22yFMg zff;GY>*P`?*JXmIu7-bde)Pm34!6n9nw1p5BWCug{HfAZNt0f)`Wk{x>aUaxOrjN> zHW!YR3n)k}aZkYgB8^y0CdpPT({r*OYq?vrRz}J0~8_i zYTlN`AE3@{HB%!z4Fs$*87J{htye=Yy5xIj?onsi#wDe(S>HL0Lp8AYYKWj{7@q-? zwcIopvv{NU3H2QlVKqV9sXjHPYH0vYsSCd<>8v75*z%nsQWR4KeZ`|=w5Zlb$FU%!=#@C9b0a95??Osh+6gXM;t^1pE z51He%&}t(hjX8;TxKRYTNj6!p+F)3JB2YG3_G`$kE7_R;1pF>_mQGV3JnMZFG0`AV zF?9|c{lu;@?Ny((ZCV&0=a?~QI^EhMFrAk9VR5#Aaf4Ld;M#$z{{gwQ-Bx5V)rbGq zlSQ#_Y?*B`7=R5h7^W|6S?vlN6FHve*b6`wd?$ELJ^<5}1)!3}RrGD;P3d=L35x@y z09*la+b|g>ObbV5_96a-IEJ;YX9#`mX@0sHzY73E5e)S+Wg8QVrxzH>lZi675Z!>u z(U!&}Tp8nSF@g7R%827PTLQ-<6n_x3D5(y1IS@YsFG#nDPpM`JHTgV+D?U#7*FgqY zpO=tr+n-TUMC1mMKPNKWWvSdsbr$o?3E0DinY6~wqm@;lpn$C)B%n}$pq9sU6<9X1 zX!czzhtby8RY>jT>@JBIH~{L>lK6cZMjzxs{9mM^U6zdmH;dVL#Ibq%Q{qVRWQLBX zpg_q-L;o#Qxr3X}GBRt2y~gp=k-f{}r?c~jhRWI(1IuUSP~U*%!@GV_vTC0T>aT%& z$3+rKSNty^&0Gp z?)=hj1$P`T+|cOV^NaYQZKQ=8*+vQ{x`f%X0~p3{+mwQR1l~vR1pz`>!yA-Gn@?G> z#WDQrD@6_|7Ti)&rkWhML@1L(t`xx3DWvF5vK0i*le6GoZI?~qTwcLc=R*xaU`=cj zVK1N!&Y4PDfs3O}f83bx)-O~TFg}i_mfWY43e!M##mHz zF}MZ-#^a~MFCKsH5ZqF64m3nJiufy(3tH8|O0X*YYh})0Iq@}TY)HHG)brsBD))5OF8?h{YjK_ActFw z{Sv%03ryNr9dr{@EVN5!SAf03Lo|S%1UQN(Ic5Haz|X%ZNNujchmcM&A0gMX5PpRY zYx$|pRkFwL+@xuVI?KC?U6PxuqW#zAks%Bim%Q=kWI$l<|KZ~9GXHG(k6^&!Z-_ib+FvLoDxtGjOSA7Og+aZm2C_xg9!LbKP# zmlvHd76hPEBe_ZooP{`P*Jfkp7i2KRqi1~MQK=2$l+}A@MjNv9>ynO&a;ndvD*O8(=(1eq6L3Sl7I8VG37`0 zKhg!+_dI+g=RE$1{s*?7kN}IL^TH!)XI?CAPX5|z=x1f9gdAJv_+HJkWnncS7V|CN zOIi@E1!}YB+C5-X1??NB%>CmSrWdpufgn)r*zBL-L!lrRN0E{0S?8?7o~R>E;kO{*QMt<=u!JNv&m4dSx^ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/internvl.cpython-310.pyc b/vllm/model_executor/models/__pycache__/internvl.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72b19060a55b48e391ea369b0537bcde41a9e897 GIT binary patch literal 16336 zcma)j36LDud0t=B)6+XMdt!mbK`udn;P6=R5J`yy0fHb1fdZBk0Z>wl(r9+OcXtMJ zEMNEHSkGFJ0Ed(vSXN@YoU&A2#4aaHoI7$ZM>*^`av~)u$CcP!aimHrQJA9?ht6Rh zK)&yPbM0a&dAItX*YAG+`|tOG}psF4ryjY3e6dS<;C6s4cx z#z-*I$s28q1>=%->JyD^!8S=}>XVJ_!FEZz^&O3=U`o>2`p(9#U{_;zuv^}}`kuxe z!5xh|gF743!L;P%>UTBv274v#*Y9rZ3-&ek2m9rHsD4l5KyX0P`TD($Qc#j~p?+WE z{@{K|7wZo+9t<9o^l*KqaWFV2>5=+FjfaDWB|Tbyr15C*DAHr$c>P0-$AZTij|Y!8 zo(P_hyovgU8y^Wi(s(j>Qr@@KpK2Tm4mF+*o^Bit4mXYjN02`mJQHpYj)pseXTz!B zShzEI?t~HU3U^;K!rf8+svR81`=0O)yx$@3&qsymc=Y^&HDv@R!aKw1Yew+V@UCz# zeouyXM=yl?@a)HP51s>f?u|YcmZG8P1^zGCQ^rCuyf3`}niHJLnc;Kc1JU;I!N`wZ zK#3F1ga@x>g45wc(YD&QE&9naA2svg!{H<7W42TC(ScF*h{f8AsP&=nG1Pi#i;*4= zpTI~T=lBa&_~Gy)sP}SchEIOg44;bb9GKIg@ab!AaAwQ0!{HH>oduSj36BCxN3UAJ zx#(Q(JQr2-Y}8+RCETvcdQ4<`Lv{7Md~v*OtgPCJcjniqLJEX zqmM*SUd>!J^?y(f%`-;n_*>+S*^-qG&sXM`qOiQITFa42E~nY$ib|@LddW+@lS!m1 zbM+{7PF3S1&7P_6@SC*Hn&BfGvaXG2Bn#jtWU0FuO+}X>^QTSYSK1nm@nkXun zY35|3vKS#T@{)=YRjD?k@K~$4P+dgc=+PvJn(X9slvKh>Qo*E#j&V}DPC9aWrJhtz zx57%j)5=bZCud(eceecU@e?P{oPGIIY5rt$c_l${5?xBt+^JKiB_qZFV;^6sp#Rkh z$LMrmrP(u;2Ii~MLMQR!N`f9Zp_AxDHD9U5oKq`|>YTaM!isy7G}H0Z=T4nHdHTiY zj-KjGgjM^~?ag7LgXG01DJQM-QM0-lDbyX8u^LfWtt8Res2R5ur|ehbavcaSw-(+q zu}>vCod{PNjmzaKFg8ObjCpJ*qN2( z<(5j~{v=bsoA=U7B?tSllC(Y%RTr0%_*APB0!98py~0f@&o$GX)n+v*U#Qm`Wfd)A zj*$ZKG>E2jS0hrGU4z?&oJzeMHRhr)1lr=&%%N&y@$jm9=%D--4xK!GqI~Sd*|Wz# zdG>J0UiAO%FD^grf9J^JZ(aDtFYG+{`$rak_SjF4>nFI6u^b-i_Iewi=j3zhmx6sJxC;!Yi+diKq1>XpmQN+T+lQ@>0Utkjv# zm&>oMRO-4!xtK*dTrO9d%~m2L7^6-mX{q_8)YV)5rlCHBY69MQ;b78OKDb<6elV#3 zo5_PuG**@x4>v9ztXJm_E?-WTTFryR|3N`|Il2_hW5<+!jSp6ZdN0&xmM^OkdVLVj z+a7>#H&aR#nJkU(W%Fn_XNP> zObFoHJSn^T=OB^8a|}=X0zhIUre&>}YgS^d+jtyEOzWC)!%VDNrfsy11(ej>S~j$= zvyOMeztdBxg9(4uG~%gE?^CR({)9H$(FV1Y{e< zC2KadUq10H*v28Q)9ie!c>yvK9BPAz2iD0Yrrx1?r7;&)4xeDp7QhZ;^{%;NR7qZDZz8y)|E{$A^*7wb5HaHf^~DHlxJkcu}Pp*l`k;{IqcL^wAT?XOEvPpFVzc z7NV~EdgknNsd+ZY9&0t0TXB@Sr=-WgeL0FDU;-`4y4{|}5`yfrCy$-O6L@D^n)rg; zET&bdqXD?zuTqYdrMbi50-K}cdh_4B=;2N4dyZ?L03fN6;9S$WpV=TR`9 z0qFh>A;EXq_cjsT8Oc>`KlEZeu8s~Hf7gH6p&86cRPq(Lf3 z`#Kj;zYn!D>R5Xyw6XQ_`-HJJ)E;W*+WrECg||^yP%d%8k8;AF1o#4`{Fwu5Cg>DH zhlZZwXZAo&ZOj>m^bz`f!#HoMiB8%)pV8?B>lMQ^UYmGr=rt^w!bXT}7SaHF-h{d2 zq}I$sso6};YHHTrwA3pYUY#QlZ6PuzcW5zcMwgb=;U&&+62Q9RLee|8ig+w@hw0w* zE$8Oy0Z5(s6-by!OoBOxq4V*8H0t&v)A(S4If4<|;8E})d?%t9bGK(zk4P1B zi*%_zLGUqxQv~FIG~Zz;P2F9V?a;TYk)!ATS+)^e!-%^bgL5d3cLI3!UVFrLYzI8# z+{|b4!jn~sug!ppD_s`J`5nZneF3Zoz^{ z;4gyTAv>~HsZ?gx^6fmz?48D>L1oi@+qvOwQe@G_a!M!WEI4b0w!iMQ3+TPr9$L?| zi#JTL_MO<0p{r&)d)|s)dBON19yLoNs);sR?fw9lEU-#Q5CH^3y3Y_U#$qr6 zZ*V?}W0>}`39)(=by9n-bt%Zj%XPQ_tXAkrd%kYs_jIgBT`KDI_Y>$Peh9#~ExWtl z{LQ~p^k^>>@jr%t=Y3x|Xaw(Kw636$qH(c-9>^wqM4uc&A|bPw*Uc}%e1vWQEl@vz z8Zv8{*Np`;v_t2b*UnI@fM#G?wq5L-+ot-@K>ErUs>Xo^?Njn6b-_5EwZO0o@an(8>yeb(>~(Yc$4)pq!Aus?P_VXT(4X^^zagHFeQ zi0-v2IbumELCPa&oo3>sGJhVtEgMff4eUMzVBmH(@;;WyTrM&tl5N|TBmYH|$6PD) z#{hGVWn@Es1O-1tY=Jksq6&ga3$Y?N))pbw7h8Idx?&59+hYD$TAlM1c|$FtWst~r z5l@`Y2Js{{&#|Udk|p=|V9%CI?#idXjb=UhFG$=y4Z{*Y@v-tjf^!^ zq1l?KyI8p+p#@o=M@GMr-t|5Tb}&ZJbr%QY1iy)9@fZNt&3{Ymc(@ea;B0zKS26D;kP>o(>HJKQ^y^5BsAc?qQ%y5eU!3{|e1x z(KfbLQ8bc0sB`r?N;WK-@~5|GS{S`WW3hVUMSTxr*`$1n3LCm$cW3eP%f(Yx!D0n@ zKpsJtEl{LfLx))1kKt#40f>GfYp}(ph4&TmX;zvg*hKrgK49t8Dc?bTVS3Nzg1P&$ zbm(U+{tcAQj(pV&N26Rg7Wv^gqQI=@ME-&oPK4VKIn58$ znT!hEIv8PlxC7?p(dbFZ!^5o?nqyBp_E-KaQq~`c?f&05FUbNA-nG*>9R@ zt_;7h5+e$w$$|<%eHq{n@WeE(eAmUk`{wE$TP}O1J7k()h$?XLA`w;UPoU{MSJC}r zn$IyjB1|b0@t7oY7*DWZw_`o(I?7A7o+;-pt3bw2G5x0r*!5;2o+!`5L54S@*%k&y zV>x*r6=N3w&JWz5)%!kR1zR*tUEM`m7+h7nVXqjWBD>3q=8#~NDzOrq(W$n1!@8=a z7vdL?@{sO~O~K6E1ryap9K%?r%WB`S=&r^GLZ|Kiyn*P1_w&Zr9OPxzv!M%D)w*G? zWnLR^(~i&1u1=n0%;3R$Uk&dsJvogyD&7vi+SHN}{-TiJs;g_QT2D@|W;>Nio|?xr zQ@c@FR#%v^6yjKn4pi#(G#5cYBA|~T?oYEY3q!45nU6|2jr^ZMruta|?m&>!sIG>W zQok8pB zMMvt3oaR-4)Ms#7;#jdLgdR~+MeV@TQ`8?tPTN|CI|~~SGMREZE`(OwxnL^DYs3Lf zD6CA|T(_XcTw0P-2uLyZ05#<$*ub`Z)j}vL)6OPt&An==L+xxWiw*Y?S3pz&K7a*J zB)?XGp8#KD!dQf#(0nTKdM!Q+pQPB%f>0*chTFp**t)o$(>;iP@qBYiM)hkLK{YXI z9^;IxjjoNgM`es$QV8t?Jzq6JU?9yen#piGQyalE${^|5xU4>d)gwR=x>(z`_JnAw zue8V4$J*PZ#8@A1Pqd3|4`exs{E7A?^0z0b+ujbGWUuSUNjf5-e}=*l`)8C~%+b|} zlT8HBtKqcl{4|5;PfkM=jV5Y^0kEji0w2#KKB-7?DZLL6mKK@iY8XQp2r2#&O4SX3 z(qw8wiKTfR1J{1FR+3*}9q}J=ic)QYH=6U2$}_h`u!leqJVNj&061u=g229W*X9QFQd7i1qYihY$V!AmI&ay~@HX&kA%AuM*5Z594CLrx1CvvWk;!d5F%=^in0BZG zr_dq8!aamuP3#)dY6?Can{d$#66#%IAuklMAJ zFcX}}8jcWZL$ESn=Uhh^LWD&g8ax|%U&R=Z=Q$lK>zR<4o#=ns0VE@JfuLuxbv;Qs zw{M!kl-}1t2N6arLn^w-V{Cj0;FH)Ebs34($A1tKb=$3cO!a5c`1#aYPIzos3Bxjj z+&CB3q!9;29p5eEU{H=NEp;DSq(wbL;TrI6si4`C9+UcW1Z2e^x6+JXTZtk>G=w3A zLC9}P2$P(qeuY2`0>;;%%ETZL_IiLt!d_%5&0f;*>wFdD{dJ~@jx?|7MKARP$@L** zds!jBTa3}=aXv8yn@5Nigy%_Y!nuhWH$jca_inE4{o!~_vpk7K1SZWj8W3nJ&7^NY zU|^%jf$bg=|ed-*{ps-T0ay=+8k#hH1XV{EnU(=4bH^aTwCd`y zpr#^UAGK3(FOY@~qi4^9$bt}aal+N59m~h-QGAS?a~tB4r+5>nsC6MqiCGq6(2x{0ve`(TOptW5FU$vj21 zb<^$T`R`gg+;_Y6+>G_E<@)bfA0Es@o&7*ar!OCUx$f9Iilfcr+3+{od5f2T@~&%{ zGjRdup^kIuzyZueuw23nesTT-(Q7XFBC;WLuN49h4r4C#uOZA5_|j@98p7qEd{}^6 znU@?lD%6U|)2`+)t^sPm{g&y!_+@J23x8!aJppX@Pe|Q~bs3_eMK$@hKDSLR$B?JK7WNqKk^F zPW!_BxMp+*u0wHF2XL{7<4H?6x%Y-8^gbQlx1n?oTDsx=;R7hU3m1|0Mt4X1Sc83{ z%#v$TPOx7b-g_Vu9_`(MIdyEQlJMZTKAXC|(~JK7j!ss7p4W9c=LTsW*B_d=eIkb) z;_+^X+2*jPRCYF=_*(!U;P9F?XL%Gd5SKM5AMu-IoEB~{B_kwX*-LValk9~3K2A&B z=P$!)C>!ECejy2KdHmw3d622u(84kCG2~`yg?4tkME2Lbrh^hh^x=dV>qSUU?*`rO z;ba6gM;WlYN(X!lrQ>k)I0Ecsz-{PvtTu_%B%O1_?bVNBP5tw*(suQe%=j|FHwgX# zKrn*K0P|hvRV*?Qc`vfU8Y|@CzVlGIUbzf8dxiO*XZ~={%F~KC^_Sx^L@{qGrK2qD zn0e9=mY@z0bdGk!UQY9aZ|WX&n+{2kqO+gHI!+*0c+*JN_jMnQ`trlLnm`DNzbUi< zU5{IfmGcoU{VXTAsl<>;=VnvSM(FsJJv69)$aVcAg8s@UIRaBe)ty~2+#~j0 zh+~75hJpI=44Nc3Y(O*v_Z2K&9bvHVjFH$^Nd~w&lGUMFg-}d(sS@J`l~Vr{t<*m! z*aiSiD<;iGO(W&M##OnfL-WM7R;O;CW2bvX#PqCRq-Bm|uw!`QA^;CX5OuL|FhZ4Y z;g}=Xc^jqlHMFI+qm}63ZWu;(OtxUtWPbxSf4w_dYC-qb>b~12dpi+sI6L5UDHmkQ zpG5)_)@KJmDm-wTAr?F|$|2u@?CdV^xelM>N|3z{)BN@lhFVaT!_^?xKoISQb%R&F zaA180MiEvm`P$m}tIYUKfYfR>aVai>eTsX7m3ojnMIPienE^g@Paykdp)=Aq4o7S3Q>rhDxTn5&udmU%bK07~7VrjzPs7fhxaF9uciv9c8`_Ki zvw^H>nURTwtbZH{5H4`R14??EA#RNP7^8%QlID$O+$50`5{8;WdmKCB7~D5_r#Ky{ z;dUGDUhtehgPr0%0F(g2=Y)GeAGTKukww-=lpKK@)Is!-N8qRf%x8Jnc^~GaIe|1J zO6#p#_(RceBLjH-C(OS>@aF*nL<6jeO zlpEA+l*#Q=Cgkz+AoScJy!RLk9vZbkhAJ@_V+% zvv}bKP*4MXoB1hZ9Y2*4dPV{k11;k}$)xShf zsTpq24y0QSNAo*=5b;dT@McZeXb#oT180X3Cr^# zldaWz|F|l2|G44a5#YQ64DR}NaIWpXkOyLQp_7nUwfYjj) z47oq8zQgz8QCyM1#}cikW}S`{{4N`aXzk%(v-rCc_r4UaOY%IV$1pmIBGD=vQY?NP z{V+6zt7Hf=ct}AcnjXRkFtZpp&3Kl?T$zV3l>?0hli{Z4u0Fg4asJ>D)1iUkH)t_R zHVRSEVie=lV9-q#^gG-uUaSN`L9mjL;!(NkgcD3%($hC^j0B~m2-#X}vnZF}MMl@m z{Ld_u8k^L`1P_}SaPFKQ36kz1l6*ADBY<4Apsuaed$wBm?M>614hrIMAS#ET`NrA(&g+H_5ZtD;x(&Ok$cxh*ZvvMJfCp_8aenFB|HQMek znLmm1BOLXS_a2}Us8Mk<_{?KM#QTuqsS@sw;S|Zh$&#~{Wlv4;E9%;J*`mm*9U8a07$Ujsw^4FfDY=Uwi`*b@YX7vB=#loMDK_@DXk(BVHcD zWiF_m#xT_dtuPt{4q{D?#dCu0KdzO#TX{#F9@Y2JTRJMf5VI)4IXLgk%hm2+cRqi}@~!6czftgM&IgZC%&hi4ed*6d_mpNY>W_eK zlLbVV$O78Rv|u`CCqNhvCaILVQ`?6cX&!gw!*Zv$?gal2OR!9SV6+)${}*L*oFk>>wbWC~l78 zq7}V2G0hZ_p?-~^PgxQMA?K*S5AdgGo?Jl!G1?pm>^<-Xg3dZF>oQjB6yLM<;ZqD3 z7rAatWc>VOak2mxhuVT+>AP9mvHkb0G3OnGbdk2+_aw?|y^B`wSXt|je9!-X)?F-R z4A^u0_bgP=Z!X^UWo0&c13rga?9pO$N&OkjE6pveG~tc4>aj*ODJXSu4h)vm>lF-< zMRbT-Hc4fJ&kp6Vm$`QXq#50eQbb!}_b^4h6y*5%;F+^WU&i-OOdg+oE*Kj4q)sz(V* z1Y*hk5mV0-yi9NfAa&+ImvXIIZVl)=wU4s+M+n*kUm$p$;0=N=0R%;@H1VOHTm}nv zbw7t3v{1Tdv%ks~Q@tAfLnk>Vjf3%CQUCBq&S`=jy^_v&-E;TO>CS2S>`-%FUqDET z0sDja^n-u_irM&b<-Nvzr_*b P^ZC1cTsrjezt{Z#55ED^ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/jais.cpython-310.pyc b/vllm/model_executor/models/__pycache__/jais.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b727dd6d2c328df7690f033c33e2d7f2eab8a79f GIT binary patch literal 10102 zcmb7K*>f95dY|bzFc=&J!BaP-m2E+`sAD&Fyk*Ps%4=J;Ods-kQxiKA2t6P{&H!M~ zfRq?8hbc#Lvf5IZC>=2RH{<7mR8K~>mGn0B{`d6 z^~dh7kNKwiyBnQS$<^?j|HVHp{l_bs_DgDvJ{B66P~tCjP2-xUahLKYdG z5t*JDS)LWyo*m`9oUX<=o`W%F=tOxhFL74rMg^}B6}_Tt+hHjxdu3VAg=100t3>18 zcvSVO66b^y(WEySO?gw%v^Onr`EVwh^=4(=4G%;Iy@Rq|2oFVvy~EKF?}%&{!=uqL z?^rbF&B=BtJRY6!PRM#W{7!VzJ1Og9;d9aR-t)3v315g#d8bex=hd(lo%T-4`b79* z^pf{dbjCXq&3p5@_KwCU`P40qPX+nqvs;FDj!*NMTbg$sJ+u4(dJbeg7l1j)4*_#1 z!+aN*!~6&^M>5R!fH}&K0dp+F{1z~C{5UYj2@{;&GQF2MyQbAnY+St3ZU!P~H-a;r zc6f8{y?QI2i#j|A=Ng@;TTfco!(eW`l`PG@+gn_0wHM#6H-HFQi%Us7|A0<+p=PDd zl_U^#j7`m}tvE@Y54%aL)2@f9^=YpQwDqGl+BH38?RJf&`8Sg!Xw#5%d|3S;Nb0_i;F?xcf0m3ntKW=RVSatGpC+tP-kFv>ETkW7O((#Yq|7<_%lg|2H6i6Jq+F5KR@kgT52x5=| zRQ2Ot9UrxUZR@l%;#r!zR*$d+A}wW=4|_?smtYsl!$m)MH>mS;`m;`>etm%aVRSv< z99xc+r+Cn;_rkE`jo)@tH;6Yz9~+HJDDf$PK!en49^-moESsE3Xb^1L+~n3R)3dk@9?jts*IVyg z{n49^q_rk@Hf1-?qg+UB5hOj)ZWvf_@KD`Exr`E50Q!2OE$jDKpKa@t+BNOIzECqn z4n3aR?6t8f@u_FVt+0npFykO>isR@OCjjni;(7f0=Iygd)IHm6bS5Mn;YE$5)K+ILXn!TuBj5$X%>i6M ziD`ozZH`GygP39P(Du*x?Ii(O3jUUpSjs?hY7b zLYvfn##W&kcXtkA%I43ddh0$*EkXR3S}}MaQA227Q}?wHN{Pa&z8dg_|rCxYli59+2qt8bk9{}LtU=?zk0Sb`2kz_jC=p)5!jEJ8n6&9Kvf z=6qsl2->SOgNX2|jj$63F%vtkoc48$9G65v|0T z@&5oM+7^UoThqaQ%iv)j;k0B0$zq;b{PU2s`>GMTw3ZDObBdh%T8=u~1o_W1hkHrtrAr1l5Ob8*( zOpBx9CEla~G^#d<0lhAG6Z*5&ZY9u}g{2nfL0ejXZ=&6c{QAHN@t0tJV^P9PFHb$P z?K$+%=dCEsE!AU;7J~F5JjhKmZ)*#(NYvbuKWzCw6aeM+FZPi z7u?s=yze){dK~-yu0j*KAD}L-0t^oBBSL&k;1dAPSY265bwAZt zQ+*}X*V2Ogq>l6f_v^n?n6ZWulb$HslX?~VXzCkNd-pAK5AL&th5IE@!&q^ez>5S9 z5+I!;Xc18*K<6o5B5;O4o&dcoEoG$R58ub=6rDVqlI*Qk8?_1z7iwS2z%Iud;lB8K#7+CM$|A{2J7_MvO#JW3d>AL z4TFiG!i>A55J{<-;>2D&R4u92Tml6QrMf#hpSW96=}N_C!e+$lt;MkSsdx)7t`!As zt~wzSId1~Kbmr*#1OHUiX-pX)H5-1j0p_OXkRLpHuvW*|gv5>qkg_L2* z!9&bQMG7em!Ty0eA$EEus%+DcVyOgIrxr#Wl!6Iwa+|fW?+{Y-Z^F~Ohx3r)N+aQg zij!CrA1-%^_)c6WzzImvG}WHayS?%rU2{SFm`2G>+1;5_!2K;sOwwaR1|fy4EPaqN zwBE+p;L9^fR0I_FiDGJiA4lZzX`)DP62C`amxNu}^U<+|5>Em^_G~42ER(#WM+J?b zFW&PwSxa0*1zt;@$o&Ca&y@wm5<|PPM=ETLrgJh-;t*g&D&U+zD&U%Mt*46$)!LAd zz6poGN=z7g>a{t$OEROR05Y(Jll1dX^t(x+Z$VB}e^E+?vG9Oi4wGaA-MCiTFBiEs z40$^cGIUGxS1HI`=!ht-%6QXnqwZs7D@0Pk4BP&BsfMJ2mhB62vr;1pyEXf1Mpj7q z%geWr)qk42{5|ar9VE$16>?(kRE;ufRehs6+~$1}F|-Cx8n7IeB5ib;)+KA(*Rju; z5hGq-hKxw}Nxy5rIANb>5RGM{;4Eemp=1N@T;EpxP>qZNT`7`70${x8V3pO1sMZFk z<#y#`#B@0#h+7l$eZ>X}d!o!2Y#koA4sw zLy#Pe@9x+@k|^dBUl4ejKs3?u3zTOYqskM;h~_*YBIK0O6ZZjjjS)fyhJ+mw5i(Su zkik+R1Gj@*(~)6Ajynh&90V6to<~ZJ0tSMGpg{jIrx+A@3G+&*mxFRpB776BakI$B zc;%MmjY$Jm!6{8B@%(_2ZVTyt$1kAzC57ntiLH$=$b_MkgSjJ~}AfBQ` z5d&(bsAe@w$S49h^28yesKNVL&TWyDc&ogMbSVG%2$~-!~u6yobfvP z#xY~OZK4KW-ycsVP@ANP{5EUCP*P0ZH-C=p`x}Pn!uXsN!4%3g%1lyRo~5-h=K#-1 zm2ogR#GOT=_pqeqNERn}mh|w`N-#)=#mdFr6WHt7j>yab*~vtskp2$i5SxK0TmpXA zrLO6MIOD!9R_ULidgK^sF(vyEH>b{PVLiIe>lgnEt(cUnsgWVHA8(wLhJcQ4DDtC0 zyqdPb=9ff_X<~^0No4KlhGRey`i8BfO^3nJSZeCRG^))Tz5h`9&E3rN(*^cr)Ej+@b^+l6zUQ z(NAt7Y$CPf<%bzaZzihWK-|86Fg)UQj3&J$1WlQw$Y{7$c2eRPb&bo|MTKB0Lzot1 zOrky~%_Atus``>xrX^PZr0Q;pqqKxA>E!9L|05qc@PRJTB12mz&spj8ggT_AdKa~S zLy0LN=jLo^CaIke-?)bJ7}3iP;+)5>K^lu99>g_9_2HyrqmHb^j?(&4dWS5K!iFHj z>$iBE7Kgh=t^6ReiDh(ui9?#1pgT*O`cM?(! zlsYJEvA47@suUTWkOD9^oz+(*U8VT5qzn((Y3YLv)goIMFCBVRx#24$pi5mncZt-ljH!{^!M;uNaUlO2n zk+?}2wQJ&@aKF@N*sdQ-&IV7nOV_%4$MWHP=pD7V{J4 zGN)CW-es!|qbu8zamJonoA*sARK{XH(a6f?R_%7puECv56L(;pgyg z-&sA)>&LaPy0#*(pW;EvZ8;3}9hU?eLTvAlMR0b2Wv~jf74Nbf#Pd3~S0(?O@} zkmWPk#?1cj82IV51k$&(opf0^NYU-Niz6;4KUI=WA*>`l3u(3u1n=<8l!jyC&(JTv z1{fTRB=Rm0sjUukAH`{zZd`f^{2|F9Zx4y56crJFbU*QXCy1Y7@h1o-q9wrVNhTz0|U z7)_(lxeRyn@6bp4ACm z$s6uG`LFZWk?-FaSM{hKuORXylvfZzh)Xyu(G50=BdTT|vF`iW6qOF6{Le-%TQe=~ zy)}1IleZd)SE1`uzI)Fdr>=BJk)KPJ#1Sl!I&ey5Gu4wchYL0kC&Uc8YK~GKKciuV zE`CE0#qhKnNovxvUP)fj^X$Stwd4rgfpywz=LcAMvn&1zE9~c%BF)a`#y=*WIfqdd z(mFabkb8$K!QiPna3GMYV~(lXgu`ta*uTikUv3=UF94&<3K?pjq;Hx14g8DW*tM9Y z@A(S?B=X`f2@oF(VqWnTfu9q2rVH|?=sil`ynxD6GtP6*az5N&u_C!8lp%9W_Yunv zS~glI9YJf?UP*I4E}R<3c)yHxYQq_Y5>r$fz^MGWq*uC3TlNUGA z!SV{SN8JF*)Vo)a7>=u9(Yg*j#igUGZhcBv(K{{&L=-;ygC7jctJr8BS{1)WkvU&^ zgCujk-izNhL)pn)i${jgd*NGV$ZN{%=PJ^N^iF)s=z-}ZLn@Ef`|h`lR-qzD+MFk>kRTrPN^)!*toG;!PJ%pO@0i4m z=#5ydKCccgmY%ie%CHQ-Y6)U-7}HWqkw7{@hN3d0O>H=P_&tW0BlPP8-XP!;Xc8cY zNV43Bd_PY(3Kx)9z{g_#Mew=`K+Oq<9fmj;^$N-)F29Qg>i8?7E&muUg-pk;j#Xc) Lp01vredm7xA@F>( literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/jamba.cpython-310.pyc b/vllm/model_executor/models/__pycache__/jamba.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..753a1d65b9a99f257bca00a105775d2c0c400212 GIT binary patch literal 17458 zcmbV!X>c4#c3xL?b#?WDMxy}&1g~jw4h}dZIG1KG?(FcKi{#7>xf9ybtZSQH)nEgC zF{{7>ZfI8mSG$tUm1Ui4%Qi5QkFi5OB5YZ{B-`ObUR!pAe>g&ANBH_RzwNL!Gn}En z?`8D?@K}W#QCXQUkIKs9eeb<|h3>>eK84?NzxBIUf325F{ZA&we>M`Y;0m8NQz<{? zrhLOUTSnV-O@r}tE8VtStDSK(ZQHfmSvMIqmrI(9cwSTi|ym?@%FRsvqGC`J=cETeZKvI`$GFg z_eG)YYW-OICHEzX&$d3-{=EBniO;pZ(Eg(PMTzfjeW`uIJt6V=)|cBayDzuD;(i6` zJ^tR-EA5l+$@VGtl#vRufwO#?SHql7xn~0N)s(-_-+w#h@82*t4EHS32mHs7ek{l> zpW8^guYywb4}x+qp}YplA^&ku9uHnaiPw>S!haI!CzBFyfbx`o7?i^a<;~#D;0^!j zUpL+J$XoD_An!<$cL9`V{G*^8B_%i+oZ85^7yVos};1o8(r1%y|B3!-~rCL z)iCfcbk8E5edofZi(S=T$iy?JyRFrB=k27<`DQ1msd&$YKrIFS1M{Z7`sTYkikIIB zdaY)?76pFNz|=e48`}%vv6JV!OU)>JTXpL}i2g2^ak*Yw4Qnm0-tAm}&i5LfcxF3g zwTA%;P;x2^T0uQ(UJtymUhA;v>=-FRDK9B(#FekE)-b|rHI8yJRA!vLRBK~mR6LPH z-dc@%s}Tl#&zZC5PJZqDW$)d$PMRo!>JsX#X?{VR^pytQ3?{@38 zD+A)$_Lac*o1G=JGF$JqdtHpOcOz&nU5&g}x8~zG(S@^_ZTI89yP z+LJYKq1LG_1q#Hn^cfEtTQubmbDun`7DXyVx#`zCSeNa9)%en|Ld~@EZlb27w zCa-kKyZn}S`sCY}zxK}Ac$dcow4`ditKOAbRKF?<=0k%k-i#++tA($3I)S>x0gj88 zR(rj!ibAf*I(NnRFO9@2xWbna1S!`5BR0PV_UontE6DifiBw>3X17wV6WGgHP|`z6 zE}>W_Qho*-(vHicQQ%;=!*g-28G1d1L5|`)SBw`*uX177*3pX#`PlKiPOTkyUYz&1 zBd}W#FM8g!)mlsEs6})zwyt!$t$5P&YMo9O4PmK-D4B`6s(v-L_1LN7D5{=C@Igv_ zo^b{rzj{1s_m20Py<<@gJ0&{yx%TSS_H*qU$6L)S$9uP;tKH7=>#bJ%xZab&&7i)D zhIKMLzRVe3?A=nw(ArU4p@SfwDw-8z))4A_!^;xS*=!(Q2*ZSI<Jc;ubesht{o~hPUP;GWl0ihrC0tB5%>OLI?K~?&OC-eeU{fXzkQN2Vq z(Ct)TSp2cY=c|Wgz4sJYsZhI(>d8Y=xzSbC8&_{tPga{@=WtXVa*GB1N_?Nuvybg~ za&)yBqGGjHT?uYg{bnOUhw7Cex)B7Oq+1*gzF|ukXEkcJ^;D~ z#op+*L84A3?MqLO92xfQ$dU1J4HqOowlRJHk2-hQ;Fm!pi;J{y6O$IDHd33$R?0~A zjeg2A`zFG4-&nSAw{XwkZsVTC-N8M_yqOeIdBh617jd7!y@dND?&WBzpIUZs<#CmK z<4zjX3gVUJ>E)TtUB0=6XEEi?M6-zDSFg#LK+ z1SpUDx#&qhzhN|u4Tyr#Q#$1r{NnA@hPiG@se^v5ZzUxLJ>{E}(VGeB&0&nl(|FoK zbOdyc(=&eQ4oCQ?P&lpw%CY+>i+&25ZSvwm+Re*K7bfc#x&lCw06S{dLUjbQ`5|Oj zh#TqzZs#s9m=bX(!j)#vVO&qi~`6X80eRJ2{~~(xY>&{4d7OU2N*X;98Z0L`Ob+J7?)pr`5GHzH_NHDBRe?A zqUSX`&B*f>PqYA)t?*?;bhW{|OzA-&J5L0U4-7oK7cLx+GvO+BI@!2xMSY1+&b4Z{ z0_8RRn{nZ4(+7vpqc+hdC-oAR5|@ThptFmEY{G?G7SLD;R0q-#<)S#r$7^FF$LQ7w zJEed-Xg~5)t+NuBxWU1`f}W(^N!`A*%c*fEAZ3X(BQrI~2WTy8g2Q~7=_$SzH#P*H z*w&L6TNDS~+-fJhwi*O$fvT`nw%H->EvsEp=cXj$tRHD=2~jGwLbUdYr*QR&}rzr8zgBjBl+Dm_^XME}T_y?ZidZ4!sU! zM-Q_V7tsZo7*gWGmGMMGj8@ zr(Q$w61bM^8pCf;z;MxbvB5Sw_h)4d+wav2d4IhUFYx`cwPf zZhOWwjiO=QE1RaN+d#Ryj%_35+#7|(duzKNw$|f`!KxH!izgtDbv!_#2>2r@qE5Cw z_;Exzd8cuOe}s*SZ3*VRW%$NCHmPs==`C|T)6XoMzJ*lgj`^OsZmip2LFr{{6AWGM zwl6F%iCbF+YGwN-B!JYap-v;_=vV}n-v|3gNiyR?KP`F2iY2A(JK1%kvnv97!_y5v zyJe!pSow|2x-rT_JheF?Y3Zr%_qr3603HO5GA;th)KVIfN#;)az4Ur+J--g9*{t+) zo6}@L>p7%m`uQYB@sSq$ z=;!j@zCEO(6~AyNzh3H>1~kM|oBJg_K3bFgNkBvKqA>0cKf8DD-iNsEsV`$qkSY4j z_JXCpfs|ViHlt;ycJ&-y48G3bCm8Hu@E(GNvbw@}gTWGmp#<-i$wVPN4kb|6T7n?z7G+>H3l3eu zkdZ)(ff(@o*fs?nuWxA11jML zdL8}}g4s;Pm`Rg0<1Ygu4rWO&6-OPLa4vtVhq zQND*#x(z8$R+-12lcc1z|815`p=w6S64LhnDHii-)Ib~7Jm7;9oiEpc0$SG z?#!neMbWXKMctlop-G`s-Y+n0Sbou;fM!(?&8kRYFLv}mby_;U)^H zW%j(4xG3u^^hA_Xe1W(KJ+%{2hM>exSRectFXBq$3V#y8n8a`PD9vk$Ki!)|oHBF< zcYwuE>K9f{)~1d`WY7X60G3qW^3!*yALJqe1EhRv3zDDhXOK(yG5a={IP$%M$g}oE z;V|bQJ<2X$N0(F|!Q};8-bE5_Nx@`Q4?GowFoJ|`0aiAM3!)0TWy(WKstcLV_b?_a zimfaZ)vIibps&^t#8Vu+2x@w3Fof>(*m!yvRb;QIKgk+zFnEgr*Gp`pd5T1nOk|T8 zSjcKVLFrN+p{ug{y6^HfvIJ!@4uz2ajuPP_f_%0Jv4KKJ9(zph8VU}!Z6P&Ca}QxZ z{@=B!!EzYN$HJq~yC7OsojRrI!H+@$uhY0fvNOzMYTeWTgn2Om@YK7Y`eQBuhMNGV zEd#%&{9S_kyNZnJ^_jQ{7}B{g7b9g>E6^zv2p?M&P7OOV2I%;>YAgwrPH5r_E0t z%e-r6%uj5)^znhm6l>kbndCHsO`Soooiq8)h$}7I311>NB3~jGa)b*(O(qu_T5@s& zeslh9&2MD$jh6B@!IMKlrYxLfA0p%=%UBH%`V80&(%A9l9UEqT>aA(L&oPzUiVxZ+ z(Cdc#bui!3+XZV2P8t%LIZbzR}3b z{AGOy^9L-!$wVGlQUH=<7)R-%&?Xo*@<6FcAPcYuHoFN-$5cE!g5Wbj9Y+-EJhkIw z-bKGE)_H`Gpos8P^;78PwpBnpv2e*MP>77<3AjA?2Eud%-qJTWh%FjVaKJ;QL~vwZ zlH#Q$BR##U@)@eCMd4m0k>0dQ31#xJ2T$X0!H;Cv*a3JpeTa`E zX|`^oFM}ValIb^bg)V}A8X8|pL6Z}PFIpvZJAghlOfrAbBnbxC_So`PKeJ_tM#pI% z&xZ*SS{h5G8|I~ydIilxCHX#X>Q6EFc?Ne#O-8Ok!V-gDU~rtlvj}2)1@i4uIL2KC zR>^js8@FUDE^t5z<~uguJ1GbQDT1lk2@{#{OJpKLcC)?IMALBR1X|&X8Mrl}k$zQbGO8GN0i?gE@Gn{`T_0yDgK%xD2D9uuW zb^4i=jCx+jlJd{7d?yFF75NG6OQc0P$gomp4{X0^iCl^l-7b}QO2h@RiHkjYn1WS& zx!|X#`TSBu`I_}uL!RPW;C-1pl(n_ZoK&`ha!DTFs}fCvF3-Rn1_791(!MQ{_YBJJ z>XWyEmx^5X%h4=&>l{ouklxUKRygTy8Z+>8V|fp3;Czd{_fhrm2l$yW1Lt7J_5BCyrGHfLu_5soz%iPTa%{T z+$buNI*w7i>$TQukmOfJ6$X!w)9u<#^~-#Q!$v$9KVtiuHq~ZPiUtQa3)LFOElX+| z1!LzL=wrswu1-+EiY*%9)SpIlgbu=}G$;@cG#euaTX4n{f*ow1TUHA{+@cTS_vgtJ zejJ(ktRtq$is%M;=LkJ zaEKUj4)z+Dgm99nl?w-hBa(oy<8A0afzA>(1TtvV5omxe0AR4O=4{A=&@^yS8%!^y zAsd=7K&ICn8u3utf{`!R#|rLSTNyFxW%>oFUqpQ>!H^uAXjxH0f=>7Y#;-B>RR+Jt z;LkJoa|j@{i{a+iNe~423ycYj{8_}@G-g`eMl3e2#l}i(Tz3ncU{70FVq4#LfD4WU zV0re2%k?{)D6Wo@Q#Cj-ma(?*h@stHTLA6h$on4Jg!Mq6MeZU9}@c4an;C?)QKn0D=9M>J=B{rE~ zBY03SMi_7w6kJisJYNo>VoO!hAT*fp8G(Y zoRPn&jvJ=Lq>gjUN>?puRbk+) z6T)QmI^J$fW)Zxb8^b&5uP*F!G)t}ExUuQG1*GWfBgc@tWFTSl#&BMzOOQsV93Tuv z4OI){{1~~!K8(_;%BK@j<6^*G>4(5B2&4N_;EK5`IZ@KOPCR$={CV%p+0$>GIs1-x z{^ZvYUjF*qXD`K5i86ErCjS*r&V=n+k8TulDCp*~5a0wOPHMI;JHkIGGfnucjcGr* zF&G>!8}*wUjl8_FyakUFMDx@&VZdp+1!?@@I??ZNZ=OIy`796bV41!L%d^;~ixBPy z*ywy}*+k2c4b8$aT@iW|6oY$f)1$>bhI`3@GDH|MyEr$%y>Q1Fz&3+wnSH@^8dvzc z2!vl#+rh~h&asS0;BUi_6?%>*RapHwtj;|0VFX<-^27-Waz4u7Eb1s$D$Brnyc=!d z?(E@OE=Y+SO1#?7!N77~nYq;N6wZh2ehH7|S=qW(Oxo@Pic2 zC%U&gB{-|6x7%pI?Q9zfHo<1s7v)C%O?DtTgZbNhfOO^WFeXqw?1092Q4s$YpPZ5A z6+z#fs;??oiE$`3@a%Fc1CpH8$blD5od(5t=&DzuNC*$Yv_BlKJ}8(KIUwc=%R?%I zhJG&(g_Fn1u;blx(p-%LwA1(EM`hGotBkF_0lIw`U496++F^jV76iUd;NXDzB#`X?8e_LY4fR3j471b?;k*^d3YZ4Db-*P$~%lB*Y*$k zhmgA$KF0}dSYn^#(*xOZ_xn!;2hg&#!FHwP!~WCAtBSIG5IgXhSR19!b=7I{Y(0NL zq+WQ3LXV-E0vkwj*J=wSHxY{|0?Uq!vn{Pcxn<4N1Rn@Hp*y8TTyI2zHYeNoAFw)u z1n_^BX<^^w+;KS^I1X=v{&5o$3a0LGuy)${^f& zRs_ILL|9Y&q(=T_mRjh;ya)~5TF!kgWy0r7&X(wTM$2HfZ-p;L#STnQ%M%a^s2{^7 zh;TA0gL$%CRSa5EHqHurG0)?EUbNb>!;!#BNt(qmJtdB^zYP8?9 z+DCFW)ZWa{t3+>-u5Eu!-BTZ>HADOYV-$|(Ow2J2UyefqTRu%2<>181!3lBRfx57$C8^HZKz}w#bJ_tRBGg#xU&39EsF%?CH*DHW{nU#23+6TIZicp&ejbHzv}P_o3wcVM+9fnq~~P9)M$Cmu?!lO`Oi8V&=E?ZIi*Lz3!{6!`&FyLxPPDI`O=z|=AgTx&49N%IA5qLU$v5 zNVa_m%f|&t4fStO`o12s*V5xMB3i41C5>}B?5&}w@eMl4R-~hDp+i45(fM~CRK>Q& z-0SQmA5nzSHgSj=@1ZUgXA_UIZ99o_Hjz$x*MO`Ix!QOiHogdBN({qs0nDy&?a06; z$ivsvW@ZbRhIL0f$q%3bX4c^*-_Kl6sp_>kANCf+j`TB|@Copt5w2(8!${hTLc?*8 z(4bd>b}B;o4I?}yd9WxkuWzE|V=(PPw}h69LzU}>x_Ir3Uy?ePuAO0N(6drHv+4Mg zTZXFBH3%i^$jA91_2PbhIY*>~?rQgQ8)*YB9G#c@X|#XnJxkgz4xjL?YtgoLAJj$$ z=w0;7cQVq_Jj&~5Zza#(x;8g>HpI)kKNS@=(t{q*%=bN4__M?JCacB8aN5pa}4JlCQd_U0Vqf_bPGBhpTrRmL!V6 zA{VTo6kODs^$38B?|pn-O1}GvvkD&p;#(m2^TDsb-A(lI*nuG`ywQxV%Cq7u_&2Nx zN*&^p6Py3P zgcqeyKrO<|Kvzv3|3i$<+Mf(e3UDqlX!05f-4HqMTA2z=><*mCL?}1SZ(BcH+kYQ3 zAMe+KMTaL1z=w9&*WFosn#J*ik^=kRXh3HZEeTnoiX{2&)KIM)RB`7=q=BlElpgcD zDRA`mG7$R)IoyMc9C7x|cTbC2L;j}@GoWxL(u?{IV?=*NB~%gP)RPQ;$Y4K%f6IW_ z1lbe|=nt8^$zUWy_)LnL^Y;)lIBNW{wISs3+@61M+=3W5Za#t2|Ho#w z^pSbQ`UG|rO^p;OZ{QzqE<^58~n zYHJM*CS2{TzlSvsKcc2MJ|`b-3qNrWJ)(4C{t#n`@CWyWN0in!8M1u&G=9j|#3Z9F zG*$NR*n`^8Q?0*GT1-}fOxZy_}X~cE#4I%mhJbIbDFuaU_ z{$9HUKjBaf6rDkufBTGLhAzKFIR4HUGq`IyXlZ<_l`b2VZ0cK;qvhk3$;vTviM+jH K&%O!TYyS_Kxe&AOM0*5)B?p63Ycaf&jr|fgzT_a-}lY<;G;qOb@!c zd$MW(qTORjJMs+e{3qM%^%BeF z_2$To@}0sYe6kO6`YEn3Z7!#RP*Whnc$iD+2GmuE5TQoH{EJJ^fAb$% z;>cYcWgZbnQRb*Bv#AGffOkwB2k$uXqBGG;n|AP~Ft2LWN4qb-*;B{^Y>&wfH*7C*rA{bF)c_mG1pO6Suz05q< zx}8}U8jES>y_rU`j#`;@sgdCM?M~Wgx9ZKzzP8>0+x}(?&zW`Yb|+e>>Y34Mffg?{ z>v8>jyS3Cxx7s?D=kN9*y)7Jvc2BP9;PSsk+fyVFC2F2vfgY)%}~}`H?m`2o`0(? zn<7ja-3TM`uQr~HE_3M4+Yw`6+2nY#UnGw-aeZ856N>;*VyDre% z?2VhFgG&1RxwwA&dKAhiu6II|;UR~udQ9UU=OLoq5eb-jvjstmBhjd*(KTMj1Vx=j z60S5lpFoRgc+1jNB_|0<0@f;jx^kszL$KD<_P3NIUs9v1T)A?&ktFEpQvEhYyAQ2F zVq(Q(35g?go1KX3H=;0#JL&DPF*1m@FU>X@L&K zYZUZ?Q82}rup(2~_&Sj#+{licIZb$xD++jqx?QS#s4eNDC`uoc15cDiw2S}WDZ1Y zsqud1$S7Txtwnd_RVYng6W?S7B-J*xd-|59YpK!Gd&V-<)U?*Kdsff+HBA`bH-vfD z?rEW!T0JV!&#{nWbB^7^o8K^HEp>$3Gf~U?zPYJ?M^D{N4K+QX;VlabpBU)xez~>` zF$mW?VMn%CvABMt1!?BD4N7emZGK4P^C&N$Bk}@B*RP58trl1N*p^=dD=3bX-&H)Y z5^VSG{%^K&1r&*S%vj?lZ^swsyGLsCZ<8jaV)Kn;epqa&E$3V9TeWH-v%!TBc_Cz{ zk%pn1MYqsTBd;L$6RR*7Tjsx8Z>~qLN!gZin(`d<8!ok~Ze}G>b4fk}rhJyjSBUT% z^1IS&eynGOFkEcblOzm3*6zQ4Dvdj*I*rcBv<@wmo_rx*Ux}ZNZ=Gs3uAl1MPFLEk zQ#YH<_>|I((fiRNc8gSx$*Cr3>ssfwJd4I(!Z$ev;uj3v&^>&KdOtB-O7)*ScU&(S z-HEERxBIPrAopN0)&aJu>0Ui<$q@w$JMYr_s_+Yeo zk*c|HQ9g|#)gmn*X{a#F{E&>udXv(nFvR|8<~b&|T4pAxl;mQ`10dK@?RJx=PJW%9 z%lXnQM$IOqVK}%8RE0(;p8_G_OdbRAG&QrqG_{`-`$*&@@Q|mzUSoqp-M;vG#7n;Vymo-jgi(=$NZsbLUC1Xy5HYy>n z44#j4B`S(Bq)SMTN1m7v#?pS)P?KT`8mcU&qsr=-m|;50bYC6LES@{~&D zrE65WRp^z7oAXW+FLed_p*jb&P3qB03q82y;1S;fm-_aRhR$8}p^3Gyh?F;YqBebD zO;Ju6+E@!KJr{H=9jA9#rMfhBr&s8Cy&|-+4NdLbr7{!gWS&bc8-o%qN_ecxr$k|+ z+=Iqm*X8$m-o5E9P53tq`8`qGD5o=d`#yTKdrq$`N_Q~_;cU7OIXq(gmhts^K3XYr z?ta$)l?6$bUed0}JX-KjG~#7E;q%{r?|z}S20>t(;IfmFIi&Z;-ltGi;rrv|ns9a&Q*ON$+X35HMCt6Ya{V+*y z!yz-}&wwl6BSPxBIwwh$D#gI+e2N}JBHtqNr$90jt#if9h2pNK&@A63zO$xmyGz~^ zEPCMPaxQi!8L`>3GTpAuzb9Q8uVIV7beAEHwI7s9rXB<-1nu3FU!t5f+dwZ=o_ zf$@8AH}syWub~0r7o@$DxoL;TCa+U5)@}nO%&$$#DrtiF6-TGPNxcN>>(l0>HK{v> zuRGfnqrzNAhx&t(roY_Cd}{dqh{>_MY05MDCAidm!&7xst{Z(Rw7+Pt zn#(4_IpePJp1I*}cpC+jw%{ANJ#WiI4e-FR=^1;>>{(m(2KMDWr{{BvMYLFi<}?>R z*~SOXunB3bx{|g7e2z}Dk;({ZxkO}@$PFUou~kcIVPj&nh_s1xKmrp2Bi}+Q)7LWn zMyB5kiu}^bPs$wiJ7Tw9r@Ca?{sF!TO+&?<$68J*qdL25zWb)tmDf;oL4K3y#RX+% zr{pC(%F9H^PnJa@Z-Zp^5~7M!8CKH1@(L01CfLr(Z%~SyY?iIe=3p^%RJ0(=ltF`( z?-1F`>Rv@g9p8im99Gv;-i)Hw4@a%9kz3yiTVLbHPQm!UhHw1HDH{LBC>cLA%9a0Z zELgDD7VNbJdrdVgXY#(JUP$nFe8{TMC+UHVSZPEDu&pMnvlUs94Qp%Tmtq6hX5k_( z@Y(8mqQG&19~Ge4i%d&Q%dpfH)Bzv0+=#jKL{@#NgvetYmblE8xB_{a%#_P^AzB15 zAUQHpYopY_K!34At?c9IE+E@Tb4@Fh{4bCZ`>2!jlZAx6g6waa8#bp+0h&D*#_!UbRKCC&V>>mm zptLZ_x3UHEZP@^x0{b&+Ek?Z%*4VTavI=&aLPJ7|#V)c@%BI-OiEf8CDKb#r4#F7< z213@=#s5z=4QQ{~;hwry4autWM+O&W)Jdx2yUiD=JW2A)BCZBJd==%M6$ayFZ*>^K zz|b{2a)#c&Bx4v(z$ziS$;MP@$1ycAv;EN0DZO$&M}wSdVZ>VP64PDQYJ*0(he7&% z@Ok(9t_P!2f+=!zYuhrNZHJ7{cEzISz9B+dH~6WqDG&$!(5Uo{luvQdeFp}}Lw!px zZFdi_G92Ap1N$TQ(8ieuh^HYCU@STqTmM68`m(8^SMAx@I+`r?U~2kiXTxZb2_YxV z-ZD6#BKJzC5SgnPmUIhqMKh*m)%mIA(6euNdUx)N^Vhd0=0a{m#`d%! zG+)X*P*kRj2gS4UHjyq7wh=C+9&9N7G>ZHH--MzA7z&#PYW;)^gadW&AtHM6 zLz6``x8pOBKP7*FMBk?55V8@OOE&C^1-qiKCL6yl+Z8+V*it~JMt)=>T^Qg`ur07N zK%t@%v|xeg5~4z&QC3hX=zy_HXa`)>8sjqK5&z{qr{SehdK!FLYxJ;n-o@Og6 zHrukEqg`ynGM`mQm;$0!_KZZJ{~>!uqGRH_%bwkVtqF}gI{9;m^(;Btvk<{pt4?Q{ z5U>_(+=h#Mx99GoSWBfjV%tQNm7GLgQNEZK@KnMtMLkreLUrz#K+PyZ9BWzWIHO== z=>&2nKh!?dKQulxG4qzNKk#}sESdv5<=!>lqp?ikJB{y*KuktQEE^uMIn~Q*x zP88UR9T>EKAUz1IvA9g{RN>h2A+~3w1rE75_yWv2*@noGIocIOFH&&U|4^Fdje?qF z9OMpKy`$a1_(<{e)e46eyOi}sl>S)fwfWdm0TUOy3%-ihsRSb>nRf=ueO=Vg;&4Vw z=+w*7`f8(ld?*V8En7R&1c=>C&LW{|BFbX{NaQc0Ivi;}Iza5gM+frfs4w=BNqbcf zjhHcC9QMd^W5=tsJ_@-|3%7Yvq7qRme!p3e!(Fi(r~stGS4u|B^Zkej5nF* zfjR!n14L=*oEGEN^ULflOZ2 zQe6#7AF!hxIo^VHR! z1KEMrQ#eNYjK}+%NRU6is>`X|+b;H;4Yx&313X(Gt*+2EyjF!B3O$))%*qz}oVEeH zMnDt8%cl61@F9ECxS)M|mSR`Jije06voYAyCZ4q)xFJw3Z_`#0?b>79yPLcPuN#rA zFh4RjF>~m>K+o)^xEsZu3;kd%jBGn&B<=Iokbe~~BkzG^HulDHB>y@wevQaSM7~Gl zQ6hf>q#yF~g0M5QMOhT`4rgxINv1b4n}wfa;ay9}{=v6s#U4j}uiz1CVK@N1;qd8` zgjJHn})8f!IMPWHGmU^4Ic~Qqb*%tSbMQ&0hI82g)Q`oG>&Vwba@SnY$E^K z+KZ`6u;!*A1C+%`sIQ*8>Y*2Iy;s~cbsQD7s)O1|UOT}^3zXjEw%McI0I%QTCPtOF z2oW5Peq_{auDpe+4Y=)3V)~%l*A#-6;%Ugdp_uDCmqt%s;# z(xhj~>vO3N(m!wg*Btg^$FMfqYqpQiKl3woHM^kc%8va=Ds-?PDOeyyhTW)xntbeu zs0k|1WW{K%+JGHw?4T2BcS42<(8g{^0ljP*$)o+}RRDQuX;YIYu?zLDVi&>KZuFvSLz8)ALqaR?))NM37F5Z z;dymVO$C)1KT@5Lj5<%EJ$aM}ds=^uQm;`TzB&;k-x#N?kXHx|E|4eFhZyVJFCBH+W^)RvtWj637f1&i#Neg(h;n_X{eb2D| zaEImHs~u(H=XM8}6v`>2l4N8g$|T2Eg_;ycD6D@NZ{8!GO{7fZQ6e4@4nHkQk=)6@ zPvjpGd56gGg~|6(?$9qHh5J`pas>&5q$Oo`Apd-7*bhtlr-n{`YB+n=@U2hb1Tjtt zW5qb7gAf(2(x)ZQ|M5dd51f_M`XatNKJ;$%NqXqFA73gng}B39HiY$&5t!iEzpe+C zfT<83IJPJ(>wyEl557xx8kP-pr-9H&;GL5n*+D@}Q7kBCDe4n*i1Z9`P?&U43b7w9 zM}0IF_=S1;6R(IPKsg@)wn=g4QT$f$dknv0_&qM>bCmQ^aRSlnIBJ{|k3|!!lcLJ> z@o0*EQTB;wT0Dt*GgL2{C2lm6Bc&%r4Uy?Saf<6a741iflB&1lh^NIfh)w6jvpaGR z0AqE;SHyG3J1Cxy9*Pb{4^s(sriz&M1#udAhs9T;hk2xycu{-}oOy8uoR_$@j9um^`<4~>WSq0y{uK&? zj}7=V9r(>MR+Y~Q)baW&^hi!^<~G%-YcRgseGMk^YZOC|7A`PVlK-qvs&2P9w5PTE z#o=X)TDQ=*sI{(_+iXgyu~SNozINZs&*Ws*XlrU8PHF z6e(55Sc3+IfzHm1TWk2<2#i}d@ofhqFfi2!N=p%Jb?!8x#MAgbgYUC}x3tt^z%-~J z4jqXM5f*7L(l{o*$-f5~1@8eUz>fr_YdixC5l2`E9zr{H z5LUSIX914`bqwLh$rC*UtjyEg%9uTE|e%rcfv;m`nZ2mDn#1-QZOz;orp?wNF!0g`=yGWIK|gzL_s<{aBh zXCbqBNgxOO?l0i&;1B*1emI2Zpq)-q8DSul2zT+6^&r{8=MP`59@rK9tENah${!G+ zkX^+ce+8*vYQ05p0G4eetpCeYW}XNI>GCj0ux}Xb!u3!90*{6F&aY6BB1it}{+ilX z?Dx$fFB|8<;__Pm=Ab-8Y&TJt{#5AyH^J&-vuxyts0&ZXzeN6Cn0Ns}X+kE)cOCdy zfHPr0>CgMjvk`9V<4ETrwhgQn0gv}1r^E-k+a;4Ra^3mdH{qoj2(uM}JEE!v&1P#uLX;bw-WZTK`^Gi%4!7q`6^IIlv zIN{2!ayAV4`^aZ4nqO~HQR=@2H*?g=kGKS{6T3o}LjltOWuhCU7Y3sTQq7}R2*c>; zch->q2F)z)9x;uYM@*)_KkWzvM@Fm5e}%mMK&cf;0}YBTz+inSYUsf5XdNY|muZyL zMG1&a?yirHFqh`vK#3i$IgOI$l75rtK~fDkBj$3|kU*}Nb>OhLg?JY?%6i5x?N~5biLfZ)c~-UlN$}XU`Y2MqhtrR+3)dL>Tw?A7cd*&F2dDE zfPP2Akqg~gGu}m*yJk?^IWm+@joy33JyQUKO;~E9_LTfPXh!~B5FnrDDb16_^HVsp z9AWx8^6byh6p`1Bh(^Qph;4tn`_vZ%aWuLafjj$+U~!(H4p<5N3Z=-ugf8T6X?jVl z=>D!6!+)S6G!QQL4h@0A1yTXDp+SFIu??g2dH2u*hr)jTj+H}GLo7|n)GFQh8>~rx z2mF9~CkGhXn@9Ou#MmiDoc|t;j+!0UbM$vqg7)<22NSY3Gf+^l%5ULWgbWcM1kW#2wtm*Ypuk8V!(HSa^TzzG2%37 zEXHci!M*~C>FK~y&Y?|@_{6~`)7MUtSn-LRw~*<#GPk9Ufp^R)Z6F#rg>}gp5DbF# z$~3}NPOE#BID?1E)DPgr=-V^5EWCmt^eqDb4-N>o=;+T>p(djoFdi&2bpJ>J%+}3D zbSo%8idzXzWpN1!r{4d~UZXYjkVLWgbG|8++8pXT!40%y?darpT_B)P?NYk_^0tJ{s zzW_DNX)>Ze?-|`=f1EePs^F;e4YXH1x+@Y5W;+Vg$;Z@xIeQ#>rQ$*T!;Ab+5R9boe^})B=AgjQ9kusH zq2zzV>wS4heU@^E3rgyL&lvhRW&B$r4-@%!L}-VoxcUmEhQT#0)t5)u{|LGN8PP-X zw~+9leX)d}<1iN{)%o19Jk#@WHwjvLD2HwVdzSy1M|ZHd0X}a(WKa4SOCHIWtjW(Ljkx+&=7ynYFXF1Q0~7}y ziMldHicL)gPadfZH7OpY7_H#>C2JFuc#Ffq!F|DmS0$+MiO?W?;ah{e1i?YhQU9dF z7v*aLcZp;j_puU5yE`c6CFmax6j(yaUw)qmyU=7`7?=uQ zKW6FL7ys|4ww^Yc3JiYWFo$@w`oA5%`WZa(fR6hC1cej4gMw!tP+A2!gaPxmAE*(I zaa722geK?_M9rt1%oeYi3 TbOnhO$Y<8}w5$ExfiwRPlTv~D literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/llama_embedding.cpython-310.pyc b/vllm/model_executor/models/__pycache__/llama_embedding.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ddb6544da9489e3f89f5f58ecab1d7ac4678ebe GIT binary patch literal 2667 zcmZ`5%Wf1$aHi+6v+G%V{lG6s7!s1ONmvKvK!hR`D~h5>V2E%oa~TcOz3iBI(cOb% zYj0TMl$>+KQZ5|w3w%glbK;y!AP`d3vkS&CEp<(GS9QI*s(Q+9w`0J#^v9FY1K_=? zXRV!Ow)nsfS17q<^t(^NI-~C>?+3|vd^=;xzLpB9yc?BF%OcK{vt3uP?fg{0daG|L zQWOB^H>G3+@?2*!rm>9sM7cN1GGkoL;5RMqZ0}VJc;_a|Bbs+OW`!uZ zP=0g#MlGvaJ~h!YQ+00DRaJ5+Zr;SzwJgr#>m_Abf~FrG^OJ>m0thoglNuqRW@xoZ z+oG!vjiE_x>Kyo?-L_z7jk?TX?oMmXpe_7Tk9m`Io3&{Ns2!lqyyhZo2I;IbBx}ZS zmin}N;DkP%)u-oJmlEKe1)58HbpF5&=ja0Kv3a(D9C!+7gf7x0pe@o`G(K?t4iBKQo{}a=s7sjj+F*WO$2<37nYhG+6n1a(m0I^}t zz!YBrkj4Y!k$GsChJ-szjj{Q=`KviLB1_uR*>R~!$u9)ft+BCV?RbEB)PTFSx3>Ev z^e5UQ2}PpXMalCxgJ?ZA`fcTi8iJ|3C`yY|MiK9ULgnb_@HXHayH99e+q|`{S`&Z6 zvw3`%MJ%u69-MnqUlCL)78q2hEm$^G?p+Ao9r4&y?I_YY04kP1;`1LjWL|AlX?0No z2a^{s=k+MRl<#h2>FtebPmaoB<9?Rq8`_qL{mhbDmb_VtjSR~ZOqgWtpxWchAa@<6 zK$AOm&+M5l@m}qpIU&IZQ-@VMEuaZRDHW<+l|m;qpm*;_Nt}$BP~BL{BFZPtM=hP? zuc?_=W;+~Jsvt+73J#_pke%Xn-;$KxwN?YH zsn#w_?~J5Kkas!|M~l#!zc6JX^Pxt@=Ans?)zl)QBdYwYjA=BH(W@H2xXv zT}F{8zh~|@{ow7^I5;Xyt=y|)c%1fiJBq59$2mYbdZ)GgQ`D>DHL-3<3!LU#B$7B| zP(h8A7m@qz0P+FsUBUg&0qkEmDR%D=xc~>Y`ebXXzl9gydVzw z$92hbuSK5uUgvM(+dw5ib?wfJGp%(?%j&Nx7kVMQb&Xlt2Xsyw*!b`zF0pdMS!_ZB z=mE`7h;}wSr#Ij3OoJ@mV_XcHygkx9&!5D6yx4TW!mef&>|PB|C2L}eK@2;upF_bh x7Rt%60XFU0y{@lMJ9i{mblT|l;CvJ2)amOweiw8mQAyN`4owRxj5=Du6OV;Tm-$^ph6i?RUE?%DlEipq9)Gh$e zF7%+m_#vfS<xCc00@(%`Ox6J7w|4?w)J&g3Q81biq zBkqx4+MN!Lx<`X!?y+FTo#C=k|9Ei1JrSIAPjY^fe=0cbo(|5qXM(ft+2EXeE;#R= z=dv;XE5QZ#Lh!cxc5u{NuEH3HdEzEAm@8e_0mfC3$(# zoJhG}McIUyMA@VqTzMC@-xJ&9wnbBH7jHeW+$)q5JLFZdQ`+*Hyh^PWZQ+PrDE%6j z?v~pZEwM-J1!h+4lcQ+8UmQU0eR<&N*Po>T=j4@VhMatQU4D)K&&%0o2F5-p${4#$ zW6Klrno-U?_vayg1*orbwCBTsmdS2p8;cWJhr zcEi_OepI{O5T36w7^*Zof8);W%FQd+W^diT`OUaETW_`^6i4!56zAvWt}`OU0EHzP zRlI6cS*#;B`i+){(N;YgvD2LyXK#4{)~DjZPU3@Bg#J)}m=PdGGCL#Qbp6iU?b+)e zymxsnnE|S9n#JfUkfP>A^0usp4aMWv!io?2RvL?+8MGOhcxataSeEqTG0_TwN0k~8 zva;Avl`04oZ_(ILey2ioSudENB?aC?uGVQ4HSS@^cx#OaLEBuFA!d*?0#!$EwVKU_ zio$h*#)U5K#tlqtq<4$-)HZLNTFAQ?PrPDsouX~s>FIp`1SelLL0z$Ifg4&;<0DyH zT8_fGh9|Hkg+<>Z7O5=MV>GTsmHWOQRFqr-QKbS;2Ta3lfmEPgK+FP+=T~H~Acer* zg;kP*-d`G-OZa}f7eJ<5L!?~uPD-YQF_Ds%FfXKJX2nJ-eIX?*NKz&qXJ%$*zfpTA zea&HUu}PWAea~;nFt#E{NNmx5&3~GWbCpWn3uL7d7b-N*mQU$orShGY=j#%c!4%@5 zO2wU`BGOT&OhrDFRghKbY}v_8rco8d zm{=#ql`fgP1kr5F)DP7R>Z{`b^Xdd8lV#}CR|xkuz*eELb>Kd9_lprqm-3$^VrNnyp`v_E(kYNV?|a$ zqGus;8<%&2jU8mBR_1a}-VW`)vURF_buE))1Fz^6u>uRxu6A$YH6XWm2E zDSY7=K*6+4(=d(P^FrFjf72*FU)}XOLQW@>D|Pgm^rDu6jMz#dQLn;uNq$5n`rgGC z?gnV5o~GKyGh@v}3bg8^Kp9S%w=n1Va#~$QU3HDX*8qT+rSu#?JlNerNRQ zwkmzUOJ0_P_o)htz&l8he1v4#c!pCCijiilc9 z+gLHz%qZP9BkQrLM!{L;V`D8{A7`8m-1aEj!3_u_%2E0ksb89FR+MjBkBuKvABD9{ zJ0r|{raIn+d~F2}B5xsowhhA~8W1R_R#d-i&wsw{P9021$BPt>79pp~R={~D&I-Bc zK{2bhFxS{xZ2A7@d+*fc!zN@7^}&%yIE$J?9MBOvw)E({Q!d7Xbeb$LR&-}-ka`&G z$f(A}1xRc?jJ$ePsv^qOE&?S22MLf4iZ{(SqFFlS1DG39Tu}h`gb?rk=?eSouH9cvTYk0Dzlaamso9-uI1X=w zqW-|f`XU__iMyE7^3h;B&zxh;#~D@+pPFt*Pp~CZsR?&mpWa|iTxr~wO4VQ&+`*d4 zCG}1GW1dj78B81XElM#TI7r1Kx|>e7bs-mUafLdph=BV%0G#d>a)*(ksxjwA&^r0sUv; zzx}&HE(d;i8AlwVR?sjmkY&UMR@kRq2T+)BLlK&!Z9Ik3s5PlbwNl}rfM&gKsG+t2 zNl&dnV}N6zM^Fcxv0_0Vm;%lVI3?4zBYTB3Ng0}ixF=gb2@dIE&uQ$E0exaWHpWsY z-^;BED+6eCwQa8yKTfaZ*0O8)`W9}fOKyv_u%4va_ItWl@QKJoCF-TCYtT<78bU9A zG~7;uOAX9!q@5M^CmC=hm5!qHyV0hXOGOra!9`*oEJS1N0`p*QJ|1SXkt{UWjkGOF zf@FODyV>xNy5CQa>@j1Gy+0tmNPiWWr^yDPeb;)5l^9zE!Z1 z4GI^?F{n0z=Ea{F7+T%sk2bWogzdg(dI0>ZHa$m&oVb}VIdT&pDlEK+4k zu^pMWBeNRwO862$bbQPDC$JkUM1VR^k$dL~{=pYNl6NLsXD3`1GkP@xbd zlMLG;M9Mbo7e%v#^sq7dvUp|tMXLmHQ}@vs&$y2l$gU$^VC~vO0uqw~Cy?B*?RGDv zHW6+Gxh)9EY)Rn+CY{SE=v;$O=iT5}gU;7aNTU*#r0NLdC&3B}@&Tuwc{Vwr%b9sa z20@3{+)2_)9lH76BdNj-nzEqv*Vu;hj_bxFFSYK`p*v_8-q?MVT33^q^_^4?sOTYq zy973ZFwW zej}Y+7=a#L#~^Ry4RNgN8VE~fq&<-m8T`-Uzb%Z#0=o-Yk$W=e=44*vMd1mQj$04| za$uzh2LceTLs1OEbr=*SSpqI2hY-K9#E=*U7I7%9J1R#y^=+|9jG<&yjLS`OOpa5$ z2XUnJ9&L^#S?*>rA-5!>Op0x2wN-4FLkaFJu>-gXtYik!BkI9nxJypbfAqT>v86qj z({`lw{5DYu;z<-`Dv7;f-;<2{me|kj4#*w60&a^K6vd7#cc(biSL$$l>adtX*)DNJ z?v{Hbuaf#inJK2lQIzdvXKEi5-tqV?HkhwA)VqzksOgZ=HCab6iPH12-Lca~dYtnP zs#v8+Psg~6i*SqUQP~Iah7y%APg!Pylut5mx;7SRp61!U_xfYi&!n%u89#SOk+Q9!VrP>6D zDdS=WaRC=SH>+T9s=_as6Y%?Rx?-NL@4xagc0;NsNj14oRkcQ0R&oXkqHz8y83){-Q zSl$1|4NW||KAlMFh&G0Z6ixy}Y)1$K8m&N@afNlSNFYnzMY{s4AEy7kxfM1@SMriW1WqeTB$X2 z4Z$EVRC+ND&j`AC1z{nP`csGjEFRbyZR=iI9TRETImBr2wti*YB>N~UthW6Ns2kmi zbhog73A=+lE37^{{}~aAb+=ns@S^H68@H?bZ!b${-62gjn2e|0Cd#-ZmA`?;MR`Kbb4!owiNkjiJKsl@FOTn4gSdNVc z?qGNDil_ylJAlvu;#4}bUWQ6)ql({50k9fQa#`t(hkb^=*;p#!2p+{zJPJimo{as% zdSO|sC*QnrukJLq@4<0LRF7D$`=jGHhwtn7!hZwMCay?5gFDP=Y#!2ZjLG0tn{n*& zf0B2`8R~3=4W|H1WG$MH&G2xPf!ktt9c0=-Y%Vm*(;Q4&lY&j3ARGbH6Rges8R{eS zk?!f^I{L_>4}^CT6Sg2sw1=rn?R9H7)Hyar^YiK$X0I*+tR9-JR~t%^Z%k(0MrN3H zt~GSYc?aPk*0WVFDx2$3x7F#FyU=#xdUUk6n^oubGFXg4NKUxiX!*ifkWQPd6J)Z6PFqj@TlDl=(^zPO(MxV*-Z>P>3%sT@FKBKHyBy9<`+z@Xx41vgnPR zwX@RnR45T173=Jv4M(;E{~N7C;^89fykQfX%w*fo$h|P3-7IpMUf9obb}tro;qzQ( z_3bZVzc-J}Q6}YK=tQY>m0)5Vo<=D+#X#Vs)JBw<6o^(Mbe=NjrBOCUiXlRzxovj2 zgiO2-(VEqS`iN3=vUoZLa4>WHi8L#YTRsPH2$QFTL)PHA*c-NS8Tt+p(g#EEORd{6U&1> zF~aQoDN^$sT+yLRTDbX4FPM)Y$R{uCr?k?}KK>=rS_U>Oa=d$&)3?}ZS|FWV@cm6} z;fXIN_KXfX zYhLRD(q($(X}$8lBE7$N%jdl~-QC_IT!#|J$ySg5K@{L_;ynLD zs40x|%|^(UE0Xu_SJ)~I<3X4u^-9pWn2g6>Wq?&_8@j&%B$89;2{H(p8ZATR1wym< zvh~BvDd`%59KSaH+(1Ar$2EWl1#<|u&{J5k$Yt3@%(e-O%Z$es`asa7w+8jC zF(hnc)518(B5-G~71{;b2k$p9n{T%2PA7ovG!f{hj59aqeR%oE%=D@0Lc!_$K7j7O zCm%V{GVactr~&nlfE&eE5n~8}md@RTa(5li$KfCzIbK!AA-P{vO8QGkOXnB0jv7Z6 zB%&O27{GjCHgtNClSD4HBnhx(YJ~s31{y96iTaoHM&h4gG zl}@eheEma*8F|WiAf06oMc7PDcBkp1(sIKmg6g~DWLgUnoPoN_-~j}}&;nP=v{7hH z+sBT{loRS4m(phF3umNcQ`SY8bm!p?R{MnA2o+ydI|p&S)}U^^(7DTmyIY>3MbHf{ z7_V0UL&-ipJ^k7(NT$=b6`JJ#pDpO^Z^Kr6L?k2Lc21o+)gi8?mKUL<>bK}pnn~GE zwWS&!SoBf`Rj4c(fC)TBDC`C82lyWs#Xd3fsWYdYgU3%ghn(Xl%e|e1*z@IDbvfyU z?Z~hR2OScjcMegX1>*6=zJ?gSf4jR5Y0_0^|7+87niuh!ezfe+1}~5M6QC%+ra)I15q3D!ucH*bWaO^iWKV9o58dkv!A8gG zw1Gi;Ocu2sK-0g6Ah#rT*$*526F$%Pv*+*fNjFPxMP_Mw;oW3NdX6*5a}M?E2Yu@a z=kPCZ7O9^R_yIti(ZPi=x{K0lTvxy92vuWSjDou50d1xK4PpP5z|RT%9f4mEVDtU& zDMhEaJJN}T^mOA%^7XBsd=DavoYaH6s>Q; z%k>*ewJ7Y`FTBEkK)sF2d;Y)}TXuL>wKIN}OHxYh$~yKrcYz;f?aBh3zJ_W4ES2sT~CN5g@alAK552 zgIOqPMQVt^HUc*YOb~cVU@HOIEk&%V$iP%Y3`KWQJyha&-&Y8?3h*EJLh3!2GV@@( zFF3?*n$PXD`FqDhMWl{xL^>drEZV^uj z>LIiNUWbMHQkb`!#M~{>BdKYAT}B29Z-6_%NE`!rz^HkFSfhj1x>bz=qc#y3qq2S6 zgjH(t*lJo|!%QQDR=dw}aDI;C9(Y4tj(&IECNc-QyWdc|8`Y#(DYM9{=kb#6W5#la zbkE6aIW8L(TnJvkYm*`+js$`Lcmc`fQsR;ng%t=2pq>%YXj+f4bh=Yb*FS{zw0K@ek(|<$usa_rC%j-oP*X zNK+I~@zkmksIIC~S*vP+?&^Wz8bQX*1g2{SSvMP4t`+3mTwuF4_tUFAZV&qF)qK$F z_Hvz3Ed+gTAD1)L{$Riz2nOB3V4J&*>&)s~MGRbGEuO*yZlxvQ-@pcDuW| zoU85$_PTq6eeOPfwyXPt1MY#qah>3xdywmTs)vG?+?TkVuO1GLxJQDQ-Iw{fxB5zO z)IG}OLbVvY>b}b5zUphih&#gN{_3${)Ez~6z#FU{560ZFVB8%KCfteOgnJ@5>7EQu zxu>{&TlI7>=}rb`+%x<b`YV@pgH`A1mIlpIg=4DLn7?_TYIBKY!b| z{VD(3^V+cDzU}Sxr@ejn?Z@u`ehz*IeaoNrr>Vz$-aF*I^s(Vy^A7ui-VwCE%sr;( zE$Pz+@|~(58`G6AinG%ds;I`+;8DO@Z!{J5uC zYm_}%kJ)-=>d_7V!)8VJ9v%lSMv-5mRbKO>vR97E7_a9NZ8n**|5~#eRj$>&a#hw~ zrG=@P>$hf0H?CZrx;cB}yK#Q1)@afiBP=z}O;2CriVzbFzS}Hg{N*yukqn^4*_-76 z2oZ5_Qh28sVFdK==L%piU5^@XyLM}OcIw(YmoHAIbE50EDaZ}(f7Y*sbwM-Q zm9SLBft2d=pR06wX581oEH>g-;~}pZ1P>Bq&DTX~4p@zM%G%KXutbe7D0 zFGo$`m%_@jAMYmYb7P|%%`HmYSN+;Tv>5NI5MqdG=KK)rONRuO2X8hTjk<`!4uWI5 zU3dLDcCw3)ztenO)R+9Zs4lRjfo@8Wo7%B;Qy~*eJp7z(B~ANi?Xvv5t?lh?9;E^IOP?^9N)5oKralBD!9E-}pT6AnOXf6ih!Tsaa z%AMnl2hn1^cAW5hoRM4dKl0~rGD1Fv$Eze_N;NRc(DLER4nya?}0ro^;o0JE8R-)78SX&-GHyAp?d;-NT&am8tXnPOI)p@*{s z&JiHYiEj`%PvDybQoP90>s0p!0KFAj0JJAt#eS$Z?yHN+LIo1Kw8)ei7t)IR6%Pz- zL3o0xanAjAt$ko4{A)X2`J@F5mzbF{GbzdjTQOPBBqe;c5swe%rnC4;{P{+cbd zrR-5wvMsZf!C$MDZCQ5>@fx00Jm(^%rL=NG%Bs3zw=ykz7i^*3%1Tcw|Cl zs_j4vjY7TEq<0=8&>0EJ+}`#xBf;ICVz3-GAgxM*h}s=$_u5pO59i9@iTN@~HP^2B z_uB`Nh0-F1cXL!JhaO;6=0Q%(O933Z{b;RLY8WD^>7oES{P;+TWLSsDOLS<@jB^!T zLkGahTB@bVUtZ0t1w7H;JpLN0rHZ#{(A?sDX-=x>ycB{7N%Y7kUhW(zANoglC3Z6L zV>0+N6k5uXvZ6<7OOLcwO>B?!mcC@*hhzOrTQMTDrQ^smwSC+ti$1SJ)|NiGq)(5h zMK+axto%e za;NCU+ws0@7r#@Qx_0sEmC~D6F3#S%aiw%~%Dqy0>&n#Cw`SwLFKB*$>hkPcMOBnA z##TffplO2gN8N%&A}gxjg(do1cqb$st}zlc)qEm6T1%FUOBjc7E(k!HO%Uvyj)wL> zdMAq?@BODJVCPDtuB$5~A0QW68ksaJ8ORG#31(!pj3@)?@ERnB0WQs;AC!rweU@3t zLSh)LOq7dkPyfu2%7w%|ztw}c-i-3kd8_yNZ+QkJgpo*@T+|0y0I5;T%)}Yq{3FdR zbYY=X@!UPnl2ECB&lm7d{G#S2Y{Bv}t#`M}m!wQhtHrxON3ma&@rMtV8HLd*-lIBZ z>q98T1DjS+^Zbv*LFzF_;0^)i^w+5LDgib#-bblu%kw5W7xM&|IhiGGVo4r3j0d0H zmus~h2ZSO5_8`OuG(yKFVU3iS-=@9-@E8!SUQh*(ViC&n>`daEu z(pW@=%9*}Yp*Er>|JGlTj}1 zb@3BrUiAzw^Rd;+Tqat!YKM6gdx2BYLZ&{uNMKXz>=zbRWj@Jze~&6B*9gBHtz zGU2?pHJMx+Ml+&5jCCLSOD8%<{Xom|tj}QNbL${le=XYfY^#^+qV=9=sMW)?ZqLO1 z?3a)^53`X@Q7iy~;x@o0F-E8+^lrm1+z+q-6n?K~MU8;Emb#?k2Q+`C|493gEHIUa zfL3FvXeJg#hIs?ZRT~|1^-8etNR=eXRK)~kpC@;Bt{yZl{5P75XxmgY!+1g>*?4Mh zEKf+gWV0XIA-vJ^)jABZ@B#|$p`O``fnjgr_p==UyqD|>?Ew+FVI|$e0W>~BP=kXH zMNNE&=h#4GB-p|SB1Fso#P4T>s2$4k&{PdBRmF3dwH$J*&N*Tcjgs@X$2z2LoR@Bl zv>&z#1C|3lcqKN-@?ca7!hM&EaF3cf5+n69^FGUtBh;}%fW^Zol~^d`rBDc{qDEku z00{**v3Zyd8dO^+aF@UmK-{xf@jSS^yqu0W5Y&NBZ~vDc&A&v)a2CK;`*E74W~-15 zut~xFiFB}@N+o^>%v04bX zvCV3LzoA)Jn+P%h*c>dX?rsvg2@b2}2fhfmCJo9#!s&Hr;KItNr zIyxZJ5&GgUQPW>0&^f#g>kUG#QW%46+EP@%?+YEP4@s$*s;R>gA`!g-;k{w$?IR(J zNgjszw2;9u{3^PWZrb#3AbAS^J4S5#);Tq&dZ#@Nkqw>V6%^GNM$7_#5!HZTROf9L ztXa?cxYxD(oR{8j?M9DX zh$Yj?_HG%$^7eWA(LRh=^6nR`$?*=N#~$wx;>pin^-JDi^w(%!Lm z)Zd3la>n0}^~t@--S6j^b%A5i8TSC<(6q{9h)C1!CAQkJ=!4$arq)BijhRHHU-C}) zhy5e|5cQxj3CrF|?-bVc@+aUo_Z7(hGx5PRGV*3!AXe&CWcc{1UqfJ-$}_Q<_?-h? z(Xpf|KSyEPw!@Q0j0sM9A}6ncm^Q^Fh|%fC%AaIfq^!coUWfG!7F9}MZ$Oya#G)uc zxvxQ;ujL@|p@!FCCZIMS_2Rj()FW#F`^Y>;OMf&#b);BV>3I-BV{i$gqqNNh55|I9)Q*?WY)Wo8w7p#zRnBGVa-~M!29j1Rc{vES`dF%!DPeifxFG2@pfY`EIt1 zwHl1>>t%s>9Ab^E58{j;G@=J?4^Nq3YK30ymF8XIGl`=#CW(9T3AJwTwgA$_iE(Nx zRLix6CdJ@cPaz{zzUw1?iFvThPQ1E=I4B3KGf-82b-ruyKcIhffa|i`HW5Qarbpw*$+#qL*+%4;g2Ek%(n1;2n{OVWI)o2@so0I{j=)|se zE-QX&MlD-5BB%3WRTiL{4g%fd!Rz)kxx zPfMsGJ?H5^fsOio1g9~HF|&MjcF}h_feuR7IQ99&{vpy29T0f(Or<*%i*zcX^F`mN z^WA#Wp|DD|f+$I>jsoqOh|)}gEgT)O0r6d|@sTe6I{vsl;2MOTOI{@i-QM=JhzB>s zBeko7+B?LC&@@|R2)f9G(p5UB9T*V)J&r1IKsJcjj1A-I^7XGixXy@@&3LOF8?@8h zHX_sbZqbkgbQ<%EzfFL)CRa<|lQgNecSh)%;4&?o{A)uU(ySbUz}oWUFSyuE9QVCHX9S<+-sZipY#BN3+fTLk#nI zove8=jb)_#Wf@OV#g4yFG&>|0tBbLE|HXv2qeFPz)83-}kcUw~pe&`mGr#CAV)O2O zIGy1px*%Pi!;eCWA-Pr3nfH`+jpQ60d-$3t!xTqZ=Q5e?aP!69mLdO=VB$Je+(Ot{ zi?k(J0P_e?hvN~##|SYYP~3L1W$-%f@h$X#mCz1Dlc0;T5Q55@1>+v?k;SlPqpZVD z&3=Y(9vpFm@_2StG%%h)Gh((yjFqE+R@ZDjo-UmP&t|70?@5Updjb*YWKqC4Qej%M2s9Z$~1p3}X{Ne|2UkQ?J&Ky7qlOYns z5o9GvOqn2oWgu?gZ7C!v7$Rl?U@ee51643?|1Q|g-vDjYjuM;9kFnqAUv1txFjngB{*;e&VP7SG9%FrpTJiWcfc{ZBWGuIK(_p&Yho+AFBm0inw7RWY-dF=Jf zTJO5Tf<3oR`-0OCVNX#}q?2jBReI`M>c=x;pb61~^yl?qIA#dvWwCk$5NOVys-FHM z7z{sD*0!O3XdPxVM0OAIlE@~evuwxgz}`x>*2k+!*5X;|YP4h(G%xau7)9ezdRx6W zm1HL|uga9G!hW{_w{pg=zAY2!YPhgKAUJ(Ad z1G||NfK20)@r-*Glzt59(n&3H?pLBkCqieYGM4+JpKsuwo8eU2=fpSc4v00BC_@et zdHG;hj*VrzHd2qc>O*sFwKT5N@MIpb5Q)vG&a$1-hQbAije8aU zzMF*u5Huq{wz2l8ve2wIL(xy2NGl+V1~mMTN+v9N$n02)>aOYe_bPL~n?u&L7B=dk zUo1!=#YWEGqt{td)v43~NOFx`EET$gb)ZKimmj1;b9*Sw-@XE1^8-h!JDEr}BHf*g z6qPat&oosE1uD5q(UwG1*VCNvkc6)aBbZLm9&xBfTfU1&BW zEp_v;VHr;i4dVIruOYO)Y(KI2f_x155@h&?{|Ert3UFvs4YS`?kj;nV4QZma4Kq05 z7a((t4>Bt{Nhr#ag%%@2pws!rw-PTslSfdDSLX&7lnwKC`)j6eH`;_ozL! z=%fVr`&9SG1ZZIK4*(G6f{P#FngAOYL>H{yWAzSmSgg*8pHk)D0dR*PXXftGMb)l5 zYVMA9?PgoS-PNw&%AM|Z3Y>E-?vz6ElsNPtH^y6}-NoD6HybzM_(PRYoiTaVX^iU_;09j9CDPt&~DxV^+v6lTr z`GhniG&J}CcXU?a)04aNsrHHXiH?>mx3CjLE#MAoMmhl(u_VbS2tV4pD@-pmcRWM<}JqiSF#qJ z0a;?ap~LMB6=3-wUiJ`)L`sVevja*&m)Ew$AK^`K-an*2y38-C6?aII zaNDMq^a^RhUSTKbm@?&F+BDgQreYd4Ur3=$DVX!0qEb8nh;>}l*s86GVclFAW-Z5W z;|vt*0&aHuU>FM0`)v>FpV2U+U|pMeM`i$TQ7xe#ru3gs`JWT$ERxx3XbW30dV6TW zd(k_OAcBqbo<*0^;8^vuLzUM}!+gAa`WHL9ILC9~#zq+DU?yV~vs|KuLj8%gx2hGQpGy z%TcVg>|rT4nl%Skx)D3^U=D^MJvhMEdl!$*_}`fwwQVQ)_YU0gyZ!@Mg}8y}M0I%N zxW5cxSwnU@WMlDmim=;Gxe9;Xd*GDkc{T*$8aGHt~sK& z((>#WM_o&~gn$FCmHO2OBhqNaLY%Tg84CxZ5P5}j=`^3fCs;A@8j(GvEgwBV!GfmNma5Qbcb@R%NbIy52 z!i5iN@6!%|WyC}7*RVi3g;1jCb?4&EoA~75;>_&1L$|xIG)uRxDBnVz}|Mr%}WRv`C4ZN#GzI?CjV{v=hPz4>ZyDHly3YwCA17$|g`p4E12f;+2bQEvDV6=3&n?AHyHLPz&$vMu$pSztJSX1<+-4uP_72ZzY zr4Ragd6A*~Xi270iwma%yOi;H~0(2X#*dtxPe?b*= zA58o!0)Ix}UlaI!0{?~p8J+Gxa_^xtVBFijVSz7<{3N@ZIsu4MNvr53GBHOngmsGC zvVL4Qm6y0FGQ#%?o|=Y%+V)K{x|3>TzfP_`nL+$pj2;hg<{q??+LLL-7R{T2^*R{KcBw+0f41-kTzm{_S=U#LW3I{;6Uh<{Ij_n2-}{3p~!v`I2ltqkt0 zZ!C|c2m0mD!i%o&3vzu0`Hg}cN3n|l z-GC8j6!>?l@tCg;YT8p%*PfULB9%tru{Lb0J4|f=amwFr(U=Lv|$~g5^`-D znvJW5h-S8*&70b|~-qGkK7Ja3{JOM_IIn zqt#T$${G?lojU$5hXd&DfLti9!jMx>&Ix~J=@#g-$x;4fm265u2k-W9B~&3k(3TWQ z1e#P!$uG8}M(iN4liFV4S9n$F=a-}MGV3Vc(n~&%X4-ZSeM#^3m8W4Eo5;;Q^1R;9 z$X6y(w@>y>KdTiKPH=m<7T+#3$q8h_c5SX}Mirdpe(Ge<2hPGlQGQ8rh)ORJI85LO z0Iot)OaR~VLBqUEwXaZZhEdK}=v-5N9W43lRqFB@fe`}72#f;6896VBBbm(}r_uz0 zI|SwkoFMQ9fj=iOMc{1$zf8a*z)W4G(ytKsRRDKS`(xw8?2@V%*HRGUbcfSM`m{${ z6w(HDx1~+;V`{qZ?e0n|paK-p95b2%PS$RFRvU` zDZ7zJxU^qIPMrg%bemg#f0$Pttsm`}CrfIS-%~ zhMhQWIUXmG6DNl(&LpkGIn2S?S63>hR4%@xazy2pT;`O6D^U{tIxHMbxlqv~X z`0?=Y@RtAm`~RBxyrbcF^oRej@T1e3_HXpC_D64D#}%D3G)-ti4>XtVMxcj=Yv@!? z1*y<<%`okzL(8?ojGGB<*ABC8HgsGk9COFGogU=eJoiy>3w$3Bi*AwMnZZOj=}v}I z?o>GKPIFy4mnnV&PkmT;@PHQeTI3%9%5xy}xDggf1xT+Rli zaF@G_%TBO6e9C<)+~e-y=doaKxX<0k(%TKJ5h6bO-)JcLd_Yj+3lYcPx%@D4Sa2E)@qj6BlhC$oBVd4 zza4$<7Y9&tmPV-=qAU*L=`GCRX>ka1IOG=>zl>R&L(O3^i<;S9&3XUe%A7dzK*#-p zIO3mqVE9{CF8XhxX6wpV{Id^qeDjPrif@k6H~x$MOUvnHUH#olLcgq)pZ$pFY_6;) zmWO zX}8cYd*yD+7pLmgI7we^;8i)56i=Qzd+}O5!qdrSqgJ0!rq2bHTa`KgPJFE{{ARCq z@^XDXsJGU(#Jna>#IfI?QO^5uMO5NS*+|At(qwzz708L&PRdzwY8i zpm$>JR`I%%4eLjVcsG;Hj~Pe3qJB=a#Ju)Po-4R0R_>Nk$G?#2tv#t}m1 z5yqhB-|?$Jxm1tQk$@P{Yk(=#{AXKt< zNRuz(G1*T%w|@CvP10u7z!JXMYFjI7;4EiHi8ehh`R zYsbosMPn%im|8Nsrkv^Oi%f${>4qVUrF1vdO~0(6MGCk`b)-|m?_?#Ywgd3DDAi;$ zEX50cDXiS7hwZS$$yq-60WI3jD@=yuiJn3fQRtGf}Bo<5oeXG2ezb zC^1#ca^ybc+Z8vDc_^;*FjSHwt>H!0N&sFgTk>`E zAqf@oO8_uG(3%BYBu`Pr)Y|r5Z{T%1H09A2(U&9))Yr6=(IkMKwy+zPZW=jUwqY6) zmXkKJB}9sJ@s?~sp{uQEvHk!A85>(8Sngw zx4ko$PMkk|*}Hi9l6URwsnZwADYZ_bwxp;g=s|fZDq2r)JBONP|1nGl=}L!iEmJ*?7_`oD1$GPrn&1@PiviJ&Ah$ctu5JcrfbIbvLP?T*)H)asB_Wj+QPh_UdqN} zT~nlmb>CQW;@nEU>j=nAAayLxEf%_C^Lp13w#eR3E#-vX&DHg#d^g+8*Nn>=_dJfC zyK(U`Jx~1adlpVS*)^B-?`t0zO9e=n0cs^6HQacVvm`jYca;SEQ`mO(43-zL0tPh$Ow-OW-g-d0dh7 zm#N|$0T!h^KVGfUa&A!x9;SA#l4sF4nN)+Xm7lKNH!<97s~#xcaP3B`SDFA}vjIYk z!=J-(QB_GQjOHIo4roV~H`Xpx; z^hq!q-t_7peN!;Nj4rLtAvshhLo%w8sPF#@C=8hJpYYJtSD@Iy5)k`BYim(npma-@ z<6Wpa)WOJrs3+BEf?>gsXbToyG{KBuwiu=em}CxYXNwftX26Ir-IwfcrfY*mv$4}P z7spTo7VJ7ap7}uU>LR^aLqA|RFlhQ6V`*$D7ZR-6>oy-#u?0yj-RVzUxY<$8uK z3o=O0pP|0qCGZYF=jn^GS@k1O^?ZrP1?l(MwH}pV91S^EUceXS$s`4-$~tBpH_1!X z?G*qw(-4a8)6qf&R7T7rw*n+t=615VQ3R30WHB75$(#XpFpO0jcl@ZHS-|ITFVj~E=LE48)P_ue$f~0qYy!CEMr0$b8pjr#HhoKh+ zwF0__lru|mlQa+$V|tFMt}lAGkmy~g3*k4spmNuj(NH=wX*iOF)^yL9?29CBn@l|N zwW1dmqnofiB^&opiNqq4LODI$WhtoEC-AtfnuM4jFGd-%ar3q>{|ar{CWh6*mL$-< z3dFD4slHUQ9y+`Uv>k;swxNb^Xe&m3soqmWv+KJw-_~ln(1mf&c7a@B-qT$3+t@KT z%}&fZhx;NdW?zoTFfu%7gQ@qZ3lM9|+KT=_(;>}ZNWfkq(;d)UOhanX-GaH4A4b$oRF-EuBTLpj6agp0=SIRQGz(wO_E$Bs&E(S2Rx=-~ZI1O7-dwU1!NKMw&pq2?OM(3-f^qds17;bTd$GfGe;bALuJt zerK#W4?q%S@H9Ca%-A0h+|hqJ1X0dTsE6)`5o<8=dW5oGL}wt9Z&#%FcMQk7X285K$^2NIPURqXbNCw%ozw=S zv+c3F*6RQ+)DiR{pCNFR01<;+0!Z>bTB-U$(5DwB8c9uVtkm_@z}6PW>O1Nh7haB4sxeRxvzm ziY|~ayblx0Gs&P@Mg+pM_#K&K{dXA>Oy$(vl`<^$8}$lN$|AZ`$PQ3)t#kE;UxAQ- zWWn7#DyLDMq%fsH-Lr@|5P`{W5cmZG-z32MG0M*WiTa3oV08?|Yl9dpwmowXJ9`Ht z@fhq47+Q9y^Dh95ur?usu9LW;-2mhYc9j(hN!~Ts9)-2pD?tIvGiU}oY#BFskHEK7 z1+RvNN%H-jL14;{*3moDqYjrxg^FNALmug!J)gUbK7~8$DV&);1<9_^Pk|TtJOQG5 z`Evvqnd@l%=kekvxFQOe*+8tJ7gl%VB-OLET18>xH_>~FZei+uqR;AC9U=A}*$b1x zf}1sw#vSghu!LT7m=!X@zL$4xKP$4rxrgAD>xeOb%+D?6{Q^N1L!OwjjY5(x*3N=O(?iD|r<7BlE=#@|5UNwI-j^;=M~!gMzx zfiQst!WOX=Pn)ooS+R}VZD+WnKR2<{-z-WPc?)_{vrzNg!YPV^*d=z~OS@ZF(>N%-J&hUxBRQSlU*?ntt4 zKqc2Ga(bpIPc|E(uJQ?Q_zft2D$hwWHD2oB?0F$=3jUmRnIum+2kJUlDZLG_u- zBx=T?D90CzixVhMuxK&nz+K3OB@&Jug=tlrm3@=HOSrWKA!^%w`LG@OQe8HOh z0z!W(!H`yhZaD8>qRIan!zH74E!7!;it5Bdq(&&#PEBYD7-*cRg9hzU5wy~hM?idzvTc)A5TL2 z7Pu6K9pVS?YX}nV5ym_sq;cwfZQ1w({Su~vupuI)u+UBQ6}e|3{Q+VUh#nATEoHIG z8|b}r>;vRP1dBPnZcB>)v zJWS>+;%?<{43r=YLRQK|L5aSQ=(pu>Q7c-zTF!4$i6}MMHthB>gVO>jvRyD8WHan6 zRJPGM`Twx2#JUOFcRqTA>WIL#{{Lj?qg-r@|g1%lD|s+{ipn6nqimXuB)>oH#mdL+fsl2WwLYzLyW0QZkGAGKx> zG;foa2tt5RgNBKY4SM+5L5cd?y>`=j`MuqxW2KQs)R3~{U%zpvq_T7e8aGsIGdfg4 zvATe~o*&>r`4@*u>~fUf9dxraD9J;>&|K%>*+vziUKq_K$|kPoo!QbG2!+ubgpm7I+rNoTIjz{AzdMkpmCE0=NKzYEZJoCXtAzRjrSO*Sju^O#N} zL`nMz+NegjDxxFX=72r|XAZ;~e2~|l6k}^7`|BE~NS}{Ep4wmlm8@IWC*Ea| zIY8tMnF7iSAdQFA-WoYuw&z%^FawMTbptCW%DIe-mE}AT@ot6;2tvnZ?R&<1B%JkL zI*j!;4we|R&m{V-^?cwQrvochgu!2ZpZ1e_-=K9aKIsnj0^%b?GE}j{&ij+0uve$F zHZ|85vP=uGOpxIzm2--1E}^v?3e)TKc$>fv2#|4$vj7zwcZym;J!Z~VIRm7`sl^`w zxG?PD;`9hO(QnAF(yJTvv0jzGO%K0A?FvZeRd3QMEIL^dxs(0p;W-y~Vtwjuy;q*h z^m7CwD7j6zeFmdY{4i%sB5_U`0$4RL&COK@&-Q9ARX}{vBolqLXc#FZ*H=5o|1Ud{ zWEr1G|3pa^kvVqoP`r7|WB)Bm^02`hUf4U`l}ta*@u(8l^?ze1P#}|SiVg}CgKp~* z!NnzT;8LcskNGlH2S@!t|1RatY;JKFe5HfKbmFi@SQ`*#6d}oQS|WoqwYEsPHv})K zFC5LF;3c;t;Ygf{WOxRlC5yNcByM*8<5Ig(>IF4QEyQr?p#+p&J8^ij0$AJ?mYl67j)$}aEbD;4HC$(3=rdluM8_DZk8$o^MN{} z9|tgpBU05`9p5u$j+HJll?_aBMOZw2Hq8|yFzH6a-srGko?k`k_?4nRgwB2+TOof) zfE}t|L&=?{Qx9r^LmHN4YDB7nQ*H?OGOPSCRs0EoKPB*I1pb@=nJMxw0NgRfNZg?d z-Hw>~j5Pwe0I zwqMft$pm|Q*qi}PkEmVp!}@CXCj~qZt%52+Uk@NORZPxc6eb0diU@OiL36$yM}4!6 zm-(0Ybp2ZY8g+k9YbAEE)8x5yUU)+89UUsM)Q~@Q>YUITnXIL}-2Np0hUWLUN&YQf z{9|9x7t)=>PdGu`Y`Gs@Cb$>jS4`mB=E241sY1GX!oDAa$yO8#IeGs*H$G zejgyQnl~um&S(4-7qffK1|4TZ_fS(-c+B2JwQ@TF4$W?+5|N(VNPzZN5|v95(vpzy z)1$ZV37A`{Toj4>eTQ=n=US?Y&Y%J|GOjZY1;@Tsv8!6OSE`Rb(MJTzvg4MIw* z_7*CcC>dGAejl3n#x$Kz3lqjJ^JBADV^c3_;}gp=J~lGOf9GuHzl?+Q7T(EcHrg9# zqMj@~?a4@*S^2VT|Aa-g+$LYnE!-TgVO z`=PH5G!LQ=cjx0;^R%K_I+`qrW_J(Jv+)aZ@rLB8bHH-~ngf)_l)hoHyiG3Wq zh6By7f`7G$=7?0Iet=J1<&d7vrHTacFEZpinVmSAGL6`iX_|U5W#pON!N8Nw?nz2 z9@u=}|9Xxc02F8isQLHxd#~U5-|v6j=tp@Zy3fO@S*!(9uJ4`3m;7x zhHv<0)$sV&s+u*+vrMK_)l|*)>{{AO*BsBOWxPzy_1s$4%hqyUuGZ)ENj)5g1GPbKP|ED;P;J;7mUOy0QXBO~YwNsqwe{Y5$#be3Y8$}-r6p2m!$L6`)a$r z-IDIFj@9;fdn8?`-e246?XB(e_Q~@=b-cFU+b`+C>I1b0y$2;dRDG!Su=jB75$_Rs z9SAC*(z&lWT(tEOY&^uUr%6qEzwD)xF8Sfb>Utc{` zoA4%ThrPq{yrFuecGNpsJLVm$9rupcp7oxsJ?A}FJK>#>@{QFG*FNHXq;}FfSv%#O zs!e*6rV;E4p1)#xr~OT~;l1FU@i%*C{Za4SvxdLL-+I;Xw+6W@DepX+$#dJp;!A?K5l z^Dyc?5@dohTEJMQ9sfiAqbR)~rH=(W(eC5^6UdqB$~oXai5$Nx=b---asrM9V|&_v z22azN^C5o%bDjtW=Vq`fv%!PQl@$YU&Of|j;_s{ek>HgTOHzk}Pa?Hrxf)zR*^cE} z5UiMJut-Iens5o(aZM@!*k@Cr-au36XuIQJ=2N#G@yx<%{L#>%qnD+~Kp8nQCReyXKe` zZm|58h*K|b1>Y(iOJLF&y~&`d-lZHb7wvl=TFq<7a|l# z!Rt|+J$dq!WQ6RmQm;g%YGY<5PS04MJ68!yRcv;tG5wYa)EIN(fzGyJ zfr4th!C$D=E|>OHLn^_gfk-dN#+6CW09hLkJT1tYY)~g?XOlY?4Q3J%{J=$xvcwTEla`c!4x(^>Bn$?l`N^$Xz`@+ z`OE4tG|nXpU4WdCH#eHLDY>^_;=`!=!In5{1X;c8SaKUnp2m_py-P04w0p_(-Ak^X zK)31u0CKh6vhgs#BbiIs(ADA+;~P z6qZ?Vu^cS9X*p8JJcM7k6X15sG3Ttou^nr1OeTD+h29CC<@MwQ>%-2ZC&x0kpJ!)^ zx0={-4Dbv`+S=nB_n5g;T&mW;0VEv!INR`B08Oiz8a0;E_-o^@(=?mL7Go*Xbed`W zb(@){`>L(RBBN;_&)i~+8duDvY%|@=ZiLdh*;vXUo%796<8ic8A4N?QIcP;{_TjC* zc1@>U6YV!0^q6m^m--`f+0xIx)wFzb1n+J!R?MYBv*4%D|1?@nHw&B5H&TvH@$F9j zQhzhw?4P!Pb@t?27UuR=7T`NS_C@~t<7XfeQxf!{6(=naZyyaSo2p4B(r%O{> zAIfWikt`{+3d%p3NbINpQnNk<<}UxZKcvAnGX54Gn#P>5l#0w|DzdIvYF(6Srsiz? z(!P1kTCyXjnF8{q>)WJG26gU>+@3nwq)wl2MLDLwWPHV1av~t&74t`c-g9}NcLr!> zsi&K1-xjpZBPWfVBlS$w?>o&5-(vl!0CcvdExb3tp8U+U)XUbAyOfOvS<Nl5x!hmXZMEn~^k-rjP)9%>yR@QYf1Eb3)EP>eHvFkOap zRQF(2Vtaa_T77%x`Fij=5eoIgpo&B>!HHrR0Rw`W%Lj^MDfJRw@d~0`Ldl(P0Ov#X zQDlw{#r?XY)_y@-s?@`%T%QUQH$!pvr2%HMltUBE_oIRpdkQs9u_$p%SUa^6D%*J4bMy;1z(luf9-&N(e;@ z6gqUF9JYx-Yh$Te2&KN=;b6OZi^K*xISH}$(w8*kjrP!(G-jUbLY<-E1f;z9Ve2kW63>gUGS#rf)5! zNUEC++gP%f(ouTZ0okS~VI-Z_=}{7GYBC-Pfnyag@cBlgO8kYIsGdVd5YNQJ4u!;- z*>VV#SE+gCb1<AuoFl4F^WAWY%(nH9t4Fm)P-PbqKolb~A!{w)1oMUb1&g(@ zUeE`csSVT(sL5HA!i>ORUd2m-CejgRD%%|~mqhDrV9&CjiL^ybBYOI<|X8k`}4v}a8OBY{M~k+a5i6Kq=Eh}V^f z*vHRIoH};4bo$tt(u*gK9(!KZQNMRsw?$_?BR#Zl;>D1QpEK2E{E0I(I$<8h*;)-u zcmb?=ioq8=gpKQ)AA<^C(N}Ed>&U@x3kj zxf|h2jf;UI6^@xo@a$t)nz6wyOce1jAlo;2{*>Ogv#Nw@>U9Dlg!%-5$hmuwiif(k zs#K}_!Rw0jpatGOrp5^<@W%Qyk}opj2M9zwNv}d^dTx4f3n=&8mU{sn+nV?pl+X8i z=zhDl+uMGK9(COi)DFFV)rRz-EYbgM{B_>RyObX{=TWw(A4U^#j<&ey zWNSGBSA8oxteU!F^ zTi+?IUj#TL$VH9EC0AO^*$&+0g*n zx&)OiMYKl#P}B9@Yv62BI*iiKL?bts`dPHtLhrt4wAm-1r9N-HM)Sp#CctSa>{VN4K$*10 zAMjF@+6-i@AfRBA2$0-Cy1k|vwfU#v>NmmyUUdx1w%?<|Vc&387oXAkc*h@gX%?=U zE+>pGmr0BttTtc(g-;{VYWJo&+317A_%3CU%qasQY|i22Vw2IH%RMN(4mYZJI>Bi_ zjpx{gE4J1{4C)P(;M~FZE*EUQu{e674o^(QFUpXLbi*7dswEUEVF2sm6of;Z*Nzaa zJooSfP4i@{u=t>i=_TF4COY=?pJTDOezkH+ir0K%j3q;wAtek-_ykv1JxD+n18r4A z;Eyol3c=?H$Q`^y{U?5C!O!jVhINzOJkAq7BKcw5H(T+2Y>o`M!#6&|W@WqY{{VY= z18;@r0CLtKaKW*1P!n9(bYK?=6LH_mr}1RF@8+EXMChP7Y@uEci^*lyvX=r_j`~qF zyhg#G0@;>fU&q7uU?-MO#sP)`wqm!~3Y8A96$iwb0z09l_Yuf0*H58t2223#1QO1L zg=d3Jz#$Rkn_20}UI7mJ=}lk^bNwK4=UQT$fskpRS+w9l_ORxXmd{6nzDu$Ou@BKo z6pJwrW|2>bUGHiWlTpA=BX&SUwfQI>)PGyrjvtO$-snyK>Qcq`@`+Phgg!|7uduDZ zOhANLy#KWJ62k*p6pDmx)e@MMum~?=M`mi|A6D1+wqROEBO}fU&c{}L-pkbevQp*C ziWshC;9W??S;1wJgDSG*Qv@3bgffWJTI}`rAhws&GE>l5x)wGhy^3I_>{EXY0O$-S zhF%`8pXru>4{s?5?+~G-dz=9p;p6B;BQ$L%qI4bz3%MvrjXa>bW#-BX9Gs zy;^{dsJ?)g*B~uf8Rg#*{K6LjZbI6jLsb#2pp+~oJ zY-rU`-vpJ?lU=BE75yx(@dL=#7rDB^?Z#}q$)qtEvt~DT&#v5ZLGS715IpASQ7=Bu zg&alWTu2uSrcF?&7*ra*h=MgdpGFrKN*0Os3=J+ZyR@oczG|ou=FLG}*np*g@sO%h zXO$Ed!HjHB@(xfnd^}JFsBtov>cT$gp6y&8dMg%GQ=(1nwY7YtKW&Ma-ey`F39GD@ zF9#}YYl@mka@ub&+T)p!y7ACWZ+B^kJK0TvU@w7)1mX4Rj_Rj=j747}@Chc`KzQ^p zlr0{C$k`y!|5BjlxZrzG4K6g`p+kKQwwtr+rdrIFz5&fWrUc`Z_?|-%>AdHWh@uc$)#IxKvjD z=Q&Lm^~qFuoUl0dy{DGEJC^M|$e>yvtJkG?{j>}wS>~yrT5T=0toKjx4O#TBAa#|a zagcD0VQYJzVxAI-d=;rF{MP*A2@wBs^?kUU$7SyCmwR;6LFXxSICv^PiOW4hFkrXY@`w z{!{+b=y%M2W=-iHthXZ(F1B~Se>m71>|iYKXrN7D?xLY|$A`j+!i|1!=%=CX4GD_}gxrCT*OH#9el z^f39gW;rA3Z=f$t<7A=gZxc{_#rbX_D5M*&lSwA#bDKM>NBGWH`A(r)uFotmP)6)K zwZhEo=`vlK2qq9V5~l;G;g^;0|9)*$CEmt3Ew3S9t^8_$_ySae_~^upln|#NK_h7; zjIlFfI-%9i5eNqoUL~OoZjX z#EFof;v^3W^y2s(4m#;i4(Zp3Dyk&>3D%7x;lnuu?<9ttC?zT1zM6-)GLIUsII#F| z^nArMjn{x(m`r*SqotQUHvKg#!Uj3e5&Xgf0D47W^uf@3(|9$dwnWyO^c=w>SC#{k#U>Q|T+A6KeYo>yPvBT-a+oj~?(f+^u~+}Yp3c8O1K z7|1q!GqSz>XWY8(NT1u{90n|`SoHdrIIcDdUUxbu7@M=PdFl3uxV0A%yF33TCrGz< z0gm)GMjrY9Fs;~m6{_=0xCUn^i?jHB1i$bP0km_$H&$?1DQS3rk%rv_$9qcBw3^|1 zS6iF5{!8{Hd8RrKcY{R>DgrWJvBC!AhPy_;$t?b+-Rz0IPnEZRz(;$*E+pP$k;x}Ou9KJRNc zd45tI#eCHxhzk>8h*)JPe&Tn=fxK|oZ?eML zW$QK3?m`rVLi7VeWP2E{xP8721P@8ic?sPaMCgvIoxi5LI+%hJ7=Nu4&yHcWLU(}{ zv+BI-x;Q;;m%&GI!%Z(9zf-b!{~_I$Wvg#soRI$NpK#t3Dr<;`Y?vIL(qH{kfH5#h zstIz~c1KLIQZI7Pv_G{dXW|s+9VV&%8Qc3h!LJfb5VYChu!X08j;zHG&iJ&uw43+b zmhq{7!Crr(-M@^4gNVC$+0E_sadiKRqx<0e>*m&OT6!BT3*VxbC*fO`@GW@tz@PFy z)zO3>fjis{t3r5_q|@LFYxxqnj)n7Qyfv0y{Eb{O`8QFl1~cVqvPXi%EvlO>Ukr*7 zl(y+cwc5Bu7df;gXzw7L@IbqTA84o8SQuIhqs^11w{~BP-Ro`NI2SJ-VXl7An zo-l{USTog2_=13KER1hS?t1g|N&yGNJCx5cO3&sGL=6Y{s&&J*Tf|N``1eK5U0vQD zQ4V276(~E{X!D{WF*tCsRDe7`VSbLAZy`1qq7G%-#j@!xL)>tar~w)$2aFxz)S;|x z>`4DwO~ygu@1TPEcL3e%7w63ba>YGalf(U}zFg3K5t@)WNAa;zEHgh-@=u5}4KR zvNcg5KZ#V!Y-#tQ*Sv?oHYJl3rV+wd--@>iUuaJkLE<1FlF*?$ovt>4Zf?}eCl`Sj z5BCI6`R`+-@krt%=?)NH!X2-h|(Z!;&|0Xe065jXu3E#z%%S?9L@M*LpLp* zCo$Qps1Ie8W)A|0%w>1Q&_S?J1rY`-A+r|3jgsE3q)?jIQQC;yE;O^g(Jb)oL{X*e2M9&QdbhcUAOM51KdG3F!95kHS$s=vn(aW2uw-00l8 zCfgcnj!4Y;Kyz59W-PzoFI-E>N>N=l7EBdK>*qFL6nGNy1f0e}_1s1TMPLNTbvVzj znttl@Fa$qmEN?>o<`pA1jc|-*?+9U?nqWuM)V0I&5XM zIf@fVSv~CYdKs0A_20^Qp-(sIv#WN~)o)>k2b<|O>lm8!Y+mM(<8EOfUM7i(Ts%m- z6lZ4_;2WX(uH$j3QSq1>PRS4=6>-6kmC>vS?vBqv19G0wPa#qsnS zmmlR2XdDYujGd?<+WP==E^r9;#Y%7qvH7@cwh#qz4)cpDGYgG{Pz|yYm7^G^`!&R< z&8uL#^16h9s#7dnVPS51p+2Quy}W4$7vy{cOB-kpW(qb-?7$xcT|Ty=M(p^wPB;~K zS=bQuaJ~T(f$U*yomLXWB>sIi&Rq4M2`E82aY9LCbQjMEz2OEFD5Ybk^<}Hj>*Ga* z)+L6VJcxFqrrF#K!4!Zu!2RG^X|mIv+fs|XK3-~QHz5zYi5K|d2H6u{;^Wo04)^Mn zG}}u0K3Kvoduo4@9QbF@o}0ln1W083heXzX37qHK$W2)(aAuag=caS10u&z?*3B>i zS)c%+%wDzdrj2V0Jj#Nf7Ee5{;}fE*ZTlT7ZNK9>;PJ4Ncq*Hx@+38_cP+>HQ_Ds0 z3~VO&MA)V+uD5*dPbd{QCbW`r-nCu(hHXJ49=%f;@qSZ>`ok*}H;3!dAR_Az>E}T( z8yL=l8D>6g;C_QC5u#Q{C&w)ZYtDW-y_90`GsE#97!X$@0lv6|)6A^cO9;?gcGyzW zB~!&yrkTazVy>CP87=D0SV$q-R5N_~vpxsy1B}^52%aS&czuzz?8-r8Piu0f1|P*$ z7d5hAlzl7mhF0{%QojciS))^8*A4QAv-qhu0id^-On;W?B#g|w*n>+FL-w0^SN(ZZ zTReTIyMO}-Uea8n26Qa7_lbc^qZ^v0;+$>8z+v^Et*n?j22O^E(f@pB;93d!Wn}2_ zA7<(efZryvb9WthdYx6=dlSXw6D;r85W=ti8}ojTfDNnv34oLG`D!IXJlan(>rV)d zGT*!)q#T=5>NlDBF9Ez!aK))td1t!o(xbP&m8+?-bu{g5Y~}Y-v$w8`H%M%^IL!6L zWQv>}Jj1tI<6q`{A3%8laXNXB-l#YrXuU94$A+tdr061ShbPb^WE{CzP{Qume|)QM zTek~^yxO`qh9T!>+*6^h&mf`c$+w_T^&t(hJ_Akr&6M6@@!yGJ^;zpp?Xi0dp+8cO zy6T%LDq4tZLp+3SeAf6ZVqKOIfbj-Jf7*9I5xBo}#U%Y8nAUp3dIKRGJ`^F8h-Z!^ zpc8je+l$X)NOJZfKDAp?jG!Y87(qv831=j4nln}xGHQ%$(NGD24~68yQR|{if9qMZ z^9lCaM$-S%K~h9KtS&zPXQMJ%{O?I*+z1yxnbq#rtOlV=;=Jcg1e-F17Quc@(>yY- z*-7ZB<@L8Na)`Q-=-Jv%u+;zIoJ6vwbWEyQL70dLpBB=W1e5XJG1e8DTDub8;Ytjn z(828<`yEOY$^-jdA>DWR7Nhjy6k`5+H4l=rmo3BEkb_9x;;YZ#69fUsiRZyV%OCh2 zoB4g#&dGRmRH*pJIT~0vtE|CGKwSgnZpEWLL?=uBZ#d|!D9%e5c@Clw6b3=a>mx3% zkvNX?_TrP@$ELQ;6Lezq;pgCY1i$bb0FXw;%Q*jQNh25>b_9bZ91!_sA($nRO;FV# zogk5H(wSI)i2Ou+5g{l_ZUrKZ>pYZIOlheF86&OWk}6~)RPRwpsucv0<2{#3I{(9h z1~o$-L7DfhmDF+$wP83c<)Sh^2R4>alBY3s66^M(H{Q80VUu^Dw6A0fYdslA( z;3|S3V#lC;n?*ka5Ig$5k9vnW9Y$Z!ew!9nYsFCzS%6^5Cha5BvS5t`{6|EWU1)I_ z-rkmfH*c^1c0^seBj$ifp`A}X8c#C^Svy%WBiFS_1{_ZB!MDWA^0PLKL%*g9sOmVlUxcECQQ$` z#P)l6&7Tu0yutiGWOaR+Jo(s*m&BFp=J6Vw1AkOM+4w2JgTz>`y5hAE{B2Fr@5DO?hWdPt}*;AzB|zdLPtN1$33K@LrAy-v>oIal6$gqhrKcgB)e~tp>Rc1s6C-Xc$Hzy}0iV2dJBmGne$oht}u| zTvYy!;3qaYBW+-bA?A@wyJF%SZ}2I!#`_Se#rY~eUsOt3YmFKo$HPomf>^@|RMS9k zYXizE95^)-7hlL&eE1HBzNRN@a#H;Rwk(VHz%3)DX~(Gy0SzFvnP8cKYo-JJL_a*l z$6qJN5pZA9sH?1Av34UGYgm{r)2#LqJ1;y&Elwv&6?`WX8gPT|`O z+uEtk-R~8)=;e2E~GJ%@|>GCy9Y*G=}B{t87x*N7r(ndKu>Go-1FV z8n4$y!|_t5Pn=XV(!9KQoWa6ie}?)3WIx{`Z}JXy7miD8Z8%=-`Ph_Xli~Fg$oD?P zU+xX*;UfZ)b}x_c1q z{9c`=<$Lk0b`ji1u$y2E0M}Nj^-bf$d!gFH-20j9=#Rf?7O;=m;{^K&=){vNv2j}W zMb@e#o*rRJRD>sy5`lsb4XP@$YXq+o{2+laAYm!T`M4SM?)7l4Q(TmUbDd$Oj}Xif zGzdb1iv*VmgnRukQ$I@Z1%fXUd>J5iPRR${BqZJ2()v(c;szE3!#7LjVZM9oI)(CC zqa=HKJDaB@T_^l<-7)k=+U1>_{@%uRrtTtfQ9>mo;dgPr2|kkh#qL2MFg`W+G*-k2 ze-6OKhun6{M*;YCrcr*SDe<84nRj>-2!pX`4W^3b#uPN2lnqmIFg4geIAUkP;NW8= z1P9tC%oI$>5{8v~r|i82L LwD3S>boKuMJZ!W_ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/mamba.cpython-310.pyc b/vllm/model_executor/models/__pycache__/mamba.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cba92ac9fe6e4ec0254a95a9e2054ec0503bfe1 GIT binary patch literal 12729 zcma)CS&$pYd7c>z27|$|SnN%TBzO}PwG>6k@l?!Gb0Wud-7|%W?^)0E^CHhQ!?XN?C~!RI75$PZaoqIE{)CwDE2830 zib0L;hiLn9~*Sh<{WZ<@ls`%s(!U`zOQ+PFKCUe^Q*}_>}jwe@dM4 zpApaai(-*!)81+SS@EoYMx5dFjQ5;>R-EPdtoOYCf_Q=BbKa-?7sZPlpZ7lPe@1+U z;|tzr{m+Td`RBwrRdI8!x%zo}TH1mlUUIcBD9#>d?|sGDyP<8U;x~|f%GrnXJ~zMm z@!gPHu+}v&i3r6apFpM zxg7>?$ZpdO(3!dxRho@{(D3YLw{!c9W4Aid^maTmWNn6b46%~~=KT9muq@G;RcDkYKE`(dviqGz*jcbko+w8NMDCD(D< zon;Jtw%PT2-N1G1J8pYKTvSZPRSXKr-A^lBq`t#vDR*XSB$7K8X--! zA6$;|?ZECyOmP@lv zNMGu9y=cO=8=X!U6=CrMxE7Jc-~vMLtJy3O@M4>Co?;?zMZ*gbi%# z@R=9={)&IbzjNAaFP-k)3s<_G)3-g(KOM7>d)ICDQE{9MPW!~Oi@kgDIBKKX0~3Lz z6t${4t1|Vm>7^X>3oau_rqs4Ktp90Q69Dt%+4Q2fX7Akb_hM0Wd2GE0+EcnJEReQ14kZ!Hr zK>>u0+j9|gLN0qEaNQdB4K4IsS^LVFuhp&-Pj@@Dvx_e*K36-$+qfrzb4=~FYUdAe z;Z|4H?yTIaov*cn&hfA|Wc_+PBpT1C?Pqqh9Imtj6s$FBH{E+Rr`>9~((PcVOKy0_ zbvsG7XgX+w9s^u0Z2R%3+nw7D*#@H2u0J178tS6n#G^r?NpR|LPfwm4_U+`!@p%pE z{5>)-e?ayoF<1J_B9g;HwBXBFv{2blHq|XfRR-!nv9*DQFf&kBb-Z=Fvv?bL=kPZ1 z&f}dAEyM~NRH6_T2g<66Cl6Z5Q6FSLEhAlComj1GPCD8a@=?mJgjK{)^J;z>J8l|n zu1+~RmzWM`h>oX#$DprsVCdxQEVn{m@{WOj?g900E}Y*`)-&P4=ALlx>Qh`kvqz!w zNbhsZ2N`PH-XG`KJdoztQkh;04}wy2^5G%J+ECl-1{A08aGY`qPVv67p{?tjYrm5p z=t+*Wr!+rP=uL@xa|EfQXziGzfIrID>KS424rR;}g)U10P5?pLjo^LEzw*9@psvbK z;dS{&UE_!-1vlG0n=;uA#{uwWo$y}IjY`|3gQNv*o?2`UIetnNVB)V7o@ zwE*&aD*7d##g{1K-t9%%76c86CiU(Jq~uxpZl3diHBRHD4pm0otSDkg#yo5+xA|1wLJbnV-M5fn|N7Ht=Z3o;O z&y>Y%AIHl>xQ^2WhVq-P?6{t&j3wgFXniECVp0aAs7wq8IB|QnY;- ztWE8*p`uc{0~qq764Z~54N<{b=}53P3C_|VtzsF^1w6q&ff2Cn0We#tqb`669L>pW zY3tcRc2#q9q_PjRuWIY+x&a8ttm>NpI({1tgF_P6w^WqM4KxTvrLW2hh?#LL1mFz- zi^ypqEDSRIO}(jeZsS32UG2;OxKy;RI=L+kImYsDWY^WvH^i0A5~sPRalhBi5WqPg zc&*@>q-Qw;VV!-D`D$i8zizDq6gR7b{El%J!ztvgI_8EZe?YNSYjb)_=|HnuFAYcY zi{WT~0r*u1`hdWEc3`FRH7E>Y%IX}|FfaxsKq1XSI#TOFalPC*6wagkXspHIsJX@g zf%n|N7*bJ-Q+Qylmj~sPhPbl1!0GWhm>5g|P>a_X?%sd+=+UG1@jQ~B#!3UZGEUpC z>+)@+M2rZrrVvf806qz0+R*%&7EOSA9@ycF=8wJ+B`~ zn?8O6L0uIEuyXpLzDHgKRpdk9M?h+Olx?o`0VQ9g5??|P>9^bN9k3%*7p_7~CF+dr zL2XXVBsop_3HrJFa)x5pC}4!n20gDG%GZc+iGtS=L^%wL^l?dSDrYHplLEqgIY+?_ z1oeviEs6^YzCyuJ9~BisDkwxlrAp|_(5*y~Av8w3Krq^lbTo`Mh0REPB~mYls?!dd z(4ky=_(jCnO7$}?iXt0PvFHoLOfm=CRl1#mCkG|$4$PZ-=ymW8g4t|Uoz4(^SQ$&z zas)C4zy%OO5cDf84{AcwKP={Ebsca+pj7;L*3>jZtvtf_xHRX~P|m{NOj2CWe`#fn zsWFUE4zBfoibX4f5~xFeq-$%{F~~}~S}(|_X^u}(@C3Uv@!)z>knX)h{yxUA3S93DuUt20+#^?w15qE62PKxxtFQ3BH3}6p#1H8`ok{U4%W< z2}xK&y^IX(fWGW(Q!~jkMGvc)tB8!NnXB45DVUjcQlIqoY?uWu>QD&PX~ls8>SS1* zGQu421SZOmVh*4V-pt6;Vcr3b|5$O1pJq1H@2H`*0kpRZta#zB3 z`fKxVb)b9nx)M9THWHhkt*sqgti9gx?$sz8De$9gWDs<^cNXhK`7&yXSCV9m`~n4U zQ9!ytR3sLHYakZ*tOn$r?wwdIh^)68Uf;bWWmig)Qjvl7;asR2F`JSO#9fs;>gIjG zJ7OrF{{?x2LkO%aiDv`5G~TJ^$JqG1XU&gIy`mPiwby~F8z_NNQ*>` z24qd9LKC8qmWi}S=Ku75O=2hc4N{OWvZOO9VY04;k_zmrO~dCK|?Q z9JKlW0}orTW%R!cj$Df^cj|4zS8*fx9n2N;6bXMhPXXyH%ubT_+8CUSnCK=x{4<_} ziL|W#F*K2fMn?Ns)5{-KW+mll;@S2qB&mQZ9x)NvsFJdg14eQUuu>N99NthMS{Abu zu~G@H);t_~mSedEr+_?&BUh1WB??=XQ+6g`T^E_@N?^Ll#7%S2ZNk$huaIsWb04d9 zQL#z>n}h}9edD8t+Y}<2pLBf#nu#$cBr6Z=i7*p_4F}o=QL-T{y#XNM0i|+bZ11V) zyT+8lW0WGZTn0Wx?gW&|jN1$9r7>P1H8fU838tc~PYwf~?WovtA%fDSo}i4h5u<8s z3v}QWv{7B#>vkfYQV}GVYqwf9XCO*FJH|iUR7lkj)5~30=f6P@h&N-_DXL|-O-Pg! zAFb_Sm>Zw*AjdMBp9R7=OaS5>*^Sb_Pc%3-D;1-75UUZn(oig*i!F&&{cPilf z67zi-d|yqyYUJNf7@qes?dquDb!8)E`p5X)L>5vkGI)Z&K`_Se#OdJoRr1Z~16|Gz zba;03RiigeVi#rsnMUO0$29jx7_yt88{L-JD;dkx19yEoxGXgi{S6nWR#r!d1)2X7)K@HRQ&W2-pt zH}1j?kg(vbF>;yuFiOk4s6B=zYt*YK3+;c|jV7tHiNVUa6hOH5$Ro+Sjm>^xU1fhV zpU4<-F3EsLN^WE@)V1p77?=&0U4E0sX7K>HPh_XCScqW|u7dV%Sht_}n3O*&M>agv z;73q%SrIORi&HxsN`pbdH&%MVPO#UDP%Ib|6YHE$n^J!boay9u#k2-d9>x#sEBL>o z{LR}oOQm3kxzS8DQFMhZblRkZ^8ukvs&K5szJ|4{hS(UZ(CT<&?Omtsml@<2lywW1 zC%p@UEWe8b3t&k)q6w?D0HI|7NOSAvEwUnYd@&hXbFg=FU@h$$bgkb*iz6_X+d&bs z zS3n-=PYlSU>g)t$QXfXf$|s}p?;zh()X#H>#Hfso^x-%r8$e*qD1V^U2$%H)mx15F zW$gn!r~OJdtN$yhrpNHP1S0){=g~%r&yKnbbKzZhZAAvL%)6@4Va~ve`H6;ejNGy+ zjJz5DojP=|95gV~#TEiCn{KRwnQj5;64FJy%Z}NC4u;bZ!etzR=S}CNGwH$w<5W4E zax3m6a!$KdX9lIF78Gc2v(DW8teAG@oduXqGtM4&wxv0Hou`29b3pug{AY=_pp0Ua zLau$ze&kxfQ8BerbN3)WVlBft=p4eAy*MB?iGo(+ucrGHN*a7vtc!ilF?T;|V~ z*LvJJfwDE0D+jUppH2?NE_dYxJ_)^YH8O93W64s-@uP8zV^9l{ck)G-Kg7HXqswGb zWb}(67Cui8xaT$;QHcRz1dU=cadwYLFvpm={t)#^3KHoyQr8IPaOjvGD{qr?;~Eyl z>0voX`vuM@I9I=j<3ctD)eZJ{Cnp*izy)L;j7-w4U}!>@rL%+4nT9Uw11QQk!_Zgr zFDe?HbKnGloO)zZA}od_P)Hljv7QV`#|CF5(S8%Jc+Kg$0gOeQ8pT#-4ccm}eHW)T z%@sc8mD@*KpA<80htBFTy_Ayy6ghVYHu*;wC9_6kE;X7rm*60l-v&)g^*bcxphb^K zt8Y>6;bcbzpNOQ@t>e; z@jM^1=Gp%G$UQ&~xEsOoWZ> zcM>OKApaO8>eeSqus^4|*Aa*^b!K!>FQ&#@NG3fNbGt~YAJHCIL^W1U*YrnNf0zrT zgOeu?wzg(HG_Vgy1u?KgOxht2b$xA~&57}ykVvm>t_oszl1IF}(}>XWuEJz$saOz= z79=%^G@j#Vf3w@W2L~{emB3zdQ7J6uzSD^BBal`}GF_Bn^jhlEX>Gvsuu~5I1o_^h z(a|0we@i@uwbPIF{K-v5)@W|G4NgP7G{6gZf+7MM9NB|gI|p@$E>vKvKs~V=%bjiz zwwvJX9(-_aGFCt>D=R-mJv&G87oh!&MnWwX)wQ#ajo{bUzwN{^^)PVHN!ui=gRIw8 zbrlu|pT>gc9CaSjZk?37aXPpORxxmy>{+O2F^^;=N&I-XvzP5MB`+dK+)Kh5T_{Qp z{#c1hm|!}@p&FF>#OohZch1C*QQgmJCMt+vZ_;trLvjn~4^4gT_+#_3JHs(E;L;Ew zC)pb5A=Nf_D9sURTRcbc*4?BPW_y1tgBi$-w(g%IBlCUgN+TzN{7VFp5##obkwoR> z<^&8Gy7%;VG$=9#6T4koJMh?GwwI8b$o*K4sn^_NnxI%shcA%UYuH;={u#cd9ok@%3B9v;;JiJZ>Tw8=15m-9|u<={A9Nt)2y5kuoNKA3>zvN|Qh0yfKk?DhT49ln zm2EnPU0kz9iAY(?E-vBxTN3Bjb2uh%H$zdO%Tf-X`N#KrqMUSl9`5`^Cd}gCPCH!T z5>XCl!)IChsC*&Dt-qp<&w`g}bnGdlsp(Oh*@02+OK)EwuXAh=9-(Yxz)KR2 z@>L2*xJY75N#H6EQLsh99tz&2AcG*v1US9?IVD#q7^xDhSe_-?I|#l?vroI&R7~g- zW}Z&|4CoN#^`pQ3!A=X)fGw%z%D>W%=pV+%W2ui{`^d;zADZ)JUCZoItw&}FHQ)%% zdZZaRB4kgYp<5pvF}dyfW!Xl{QDJzs)(s@d%*en+98%F~B$5Cuc53plpQR+ZJ4BI> zhM{u!q*D+YiyGbPTf}aUFWD}VCQsb3*jU1R;5djURuni}e7GigZHCJtD*O${uAyu% z;jWy+f>Io1^7i|Ji?NewadgbZ=T73t6Y_Eqaz(LgN*sJrR=&E%S7-S6i|@UNr=L_} zr_Dj8h+(YaHL;Sxm64AGW=KZwN_3 zQNA#=`MHSfXtaI0DJcn8#5~iWg7zUSrt1H=BOOaK4? literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/mamba_cache.cpython-310.pyc b/vllm/model_executor/models/__pycache__/mamba_cache.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b677ee78a04487ef883fe39df188b49d81ace660 GIT binary patch literal 7755 zcmbVRJ#gH{6$TbR2cGy(%91FXf+gG1*%E2_C(e%|OG8<99E)*cImL}NFnEh2Pr5(u z;wbR~&M1~LapNL0sS~M4p(3d=$xI5D=~AUbo65DDOeQAtyZq|*4SvDK2%oHS6&hdhkK3U$d}E?%zSme)WtyjWXjVXAh0C#jVJ`?QlDCtC*a-4{c0c z;2yqtd@LHPGG63IP+R2V(HNgVjm;8zK9lc{1o@Gj(J)Kf3`pBX?_M@ zkLG57jz5o@r*btf@Uy5nma93(OQ<=Xt9g;Xgqjold^RuR7x)rt7RHwOYy9Qtq@EEg zyQp^k3V-#XQ+}E+Lou(XGdIG^YRxfRGbjH5tqgjm9Sj44|ptmkXf;(ohya0p=oRW9dtwbb5Dy#RZw#X&$8DxXGCw3x~E7-OdURng8pJ4L&@aAR1FOT7Wov0Ag; ziGv!K=|mXE&7jdUF|zO8RltLd`^UKDWfa^@tSwWVNGuLm@7VZn;lGm@kBr25Y?{#L zBQvq_%y!(w{lGvA6D`zp$?mQe{p|W#6n8|^k2fM8RxVpY`qh?D@0E@aFnnKhnrEd? zV)nx(_oZz3N-azMLCT9V+N?*-AY5;@WL&EN4{eMaK~UJAOrH|bW+#$rr$O8b&|Gc$ zXcdx_8xaq(29l~vEL}iFJql$M9B7gHL0d%a5Tj=+46wosObUmkf|!Jq#5@&8QKasj zyU@H8Pf^V=6y>>`QiE*g3N5;rrCxLS@v}ChZb<`sj>kN+$ZQjTdv?LwwORN1gSs9u zMunk&?;;Z>@k_w^R$<3XAc-yG=Z0x)GICl^%vGf|;D#-|oLYCH?I)&K zK#!a<$1bf$&FEfRT>cJ?N4CrE+XiFqUU9FxGD2|>TZZ3YErL-H-gp=&5y_=X^;RXU z%gamsaZ79;UkYUEbrqh}3hQ-2I}j&OlpF=m@`N^iEd;68OW4$-kmnMos8Cp3tbFZq zUPePn4aZ%^#&9lJI^cgKYAP>^NlE({p zAiZv3aYSeyBjSXEq2*jv$9WqK<0F4Mlji;`rPU#E>ovJlM%F8lqA_?{`4-b8b{s}LcByC6$vQ*^J zAS@-emMJ9;HT5kqFM(%(G9on~sh_|=I}UnW#e1$tY$eVk`!V>5BB3t6I!Gx%MM*~( zIqRLk7S(wDcW{?NQb?>jmiQ2@kcRYnQ9HUzPWQKcH+@yg+IrJJ2Z{)GcwP&jc-|+8tB|3pwu#HAZhBP0kbd6}4)CLWWGJ^H zE<(1er9yflAU8S+>0$y+#A_&0yH44)yA{?!hQ$S{I;A}YZKh`$!2lpV?Weeg&(hRI5C$0Uh1LG_t%f0P+l%`r9yumx53UIV0vo(Q__8 zK}ADo$mtg^5iTpOM*H%oG}$By*C5^5Uop$w1&6#oLPy__h2=;|e|TjxU#NIoKzKm9 zE>&Dg$08gEL;}ga_#tLbUHIdu$zfI%q=C(#)6P8mn$$5(YF=|hTgQH0H-!wO2}|x7 zQ+aW8>KRn)NVq`=&R{wmJ*niBd`7vr-$6@j3N6Gm6@zk9Wd0%Q<#Q-(%Y}c3upF0- z1MZ&5EZcMe_Hj1O7R&|HW8GK2y%+~DL6Wm@t3XLvXa;BFHc~+iXd>T(K%x!=CIzHL zJgg%c+&V%eR$IX>n*cQcqLL}-U=g%|U?eV~D34|Sgx?z$#GCm1d)ygvkW(q-Fhb6o zBer~)s~iOxxLoBx)pAU&?jb#V7gLjcOR{zsa&~vSFMW$?(_@(f4UpyK4;2R735C=? zY5N#2AZ?(hnw-$-%5By=je%{R!f6bh$5@Y%s)`GEW(c7dVw-xw(J=Hx$b5v5`Pf7_ zq%i0tg^bKiD(ae%IdKJ(sX&7`Z8hDz4#-s2`)sGny8KTVbuP`NuFQu3@r2 z;FdlL&z^=wjFVM>2jRSgoXcJo9`4)!9;vUxOn*QvJ#>>vd`VM5l!`xQyhgE(1gFtG z<|rsv2C7u}Jqpi)Bkib10QFX$SVIZI@ZKmN{0r_c$*v~NS#wm9{q;=q&`pRRVa(nl z&*M3r>tz#)qh`{k!@%KX{Qz};qx$zybT5uD0hP`iN;U@8q>KsUkYFXzSQ4!EGPh1s zTJngI$GLS95y=MH&BVFG#7WeHR3@xus_=LOG?MCH$m-2r{iZ`0SGh9svA&6mtC%Tu zwN{?ELwYDr4+0ZVdyz}e`1&@Mr zda{3rO5$4=txj9gi7Oqp4jl00jY&ndgb2|Dg$LR#JlT}qb;6W()$yCsVdkb&tKDiN zY7m7GKc(U_6+_#iqp*MAR*AI7if{}#i`(sM9DOth7j&y9{L3+hAx$2c9F%Bfm%;X0RLF++9(7Y!$koQSdtGACuS+{u)A-qv^sU(JRAFvKKDakB^0vVis^id4%{1*W$Rt$RFLl|GqM#5j-l0Mz2xe_q z@)RNdiCZ0EPr?;XE2j+BPOt(mA4yS$Q((BDe14z-@C^Mh(hM6>5TwNb&X0ExN`iDe z2sS%mU4IdCv^Y9^ExR|>?+t=fBu-L6BsaBilqb&8EBQX&eKP@RrN-6!)T3ERQ0xXxpHEcPo#9B_Y%2CTty-Ql)mx&@u+Q>ZiFwSPJ F{{gN~r$7Jz literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/medusa.cpython-310.pyc b/vllm/model_executor/models/__pycache__/medusa.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab11d57f9adfee01df7f7bc66cb1c3a8e5084e08 GIT binary patch literal 6280 zcmaJ_&2t>bb)V_^*xA_+fFMYbjAKnH*2h}nLX>4Ir7*H7()vg!N)~CmG8NT$u-%It zV0LENJxh_OMa2o^k}f#-l0!~F4yn>1|APOJx#pys&$dP6_qunn07+}Mrnl#H_v`ol zUU%8)gchD_|NDp0?Z36G|D%u7kBg6=;Z=udxW(<<8r!iwW-%K(u`_mK*Csg2z1Y+L z{@KacA6(yZYPDSH`RHs_r-PbK|vmP4|QR{CGWH zx2^Xr-sIsI77s;Z`@+5xUj(GZ+kmv^kmoqNYxO$cfJ&?9*1?CBkm+VFYWH@gsyg_1 zQe|b4=CyZkI?2VIo?WlpF1NBueIm<&P^y#wb?&BPjLMIv)nr=rSbgQz&)>W8%iH&o zUwwS@#=FTUH*Vef@Xq_m-4ElR*WFK2ndZ63Z-4ZGNI8~Y;9{6g^D6mVWLu*u$;*@r z!n>;RX2sS=qDpyMrGT|R5`3!Cn`JS~wgyD^>BpacsGp%Dtk~vO%(yKa&hR?i9r|Bd zLz{ct|Dqkcydk`8pF7kJT^{h}7jE46(sHdhfLy}5^Qll7pQib{c{$jrTceC~Q6wtc z6?Irl$4Q<(5>nN^6xCD~cLvTWYN+qwrGNDkG?le)9ok2hZB^{h(S+EXeGyddjP2XE ztbO~)o>_^9QUAz7ufZ*l;oKS6SiN`{It{$%@v85k*}~M%!acjPw(YO1ue7}#IKOB2 zNw@61o*lcJS*oBHRur*+J1aygzs6Vp_1aFpwq+MczPA4K!A3QnY)rDr)hd-+qPqIl zcsd$i8-Ko$XPX<7N7bk-HXh{pcw=00ktgDz7)+~DnoqSc)|%;09@XLX9Og49$CG!~ ziOK+tZ!y+A_K$aect)oEB*}`bN|OF{h&avFJLoJH_k!9}(}|FEu$U�aqPd@{opM{UR?(W5BlUmyk2c(6?LkcF#yNOr5fM#ZGTeqywT6=5;g&x{7{F8E@v-a%! z&di2h>^nUs5jrgSeKb8sK2LpGTzi@md659RAsXw(u#}&tk`t|pb_?xw+qHKuoTYI- ztJfrv73o+cNgXB$GzNL0-%gVIu;=-RlNFWbzV@KV!Kn7{i2_b7e?VJy=CT@yJcqoN z1VtD%lIx&HlGPLi))n^srX<(^@0o`zmLBy7G)vEn$unK;neHji^x&EPb_1Rn(EnxY z#5UXaj@KyL<)y-QlYm_hPjfJIB{B*5&8LidYrrXvUCT zwy=)zb$$V3YdS8Rhwd)czKKHVBdGq~C{vNPsVJi`I2J{vqG}`}Gb!RS9afR{nP@1> zu^yYGFzdIYQB_UU+Z!9He3(7xmvU<(-BcTIy!raI{^5y`|!Ap_fmx*eoG8Wj>x%TXwYbMb1NIpktsur{AuTCe?<|Bii=w0Aal)MN~J`i$*l+=&3O2I476lFsmO^Y1KPAhIO5`&#d z3CD?sSx)?@6I9|MFb%Tm6i=1*Cos&TfzDZa!D;@}s*>5}v=S$1Dmg6%X(f1~x!36? zx*jb*Kw%kyaift+3`Ru;dAtVY7=j>(VOH=3i)4%y^s!^4xRBmLY*`VRh#ygsRie`5 zLVm!X;zLDgWcXMhq!qwvEc=v1UYRPAP3dM6IMBg?w2+y0h55e zuk3BM?JtrrW;DLCwgdb(@gL$Jul{f8l@``AL@i3*Pp@rP9b{pbd;89wQ+2VjMXP7d zz5`4voG4h;soj4%wNq%RWOa%GPJpELCLJ40pK?zMW z(a}uURH1+kd)uj8u;DJ*h}RJP5~5d(HwApR*ivzOLS9^KX&2~uvI%tc`DsDmGwz|; zt=FGrJd%jfY=6y+XmqMQ%lkiIjv~>3cc1;)MP_wvX1jm!kLy-W6>p+Yoro<=!SlzT z_Rh)6=;-8Wuu=gb|ArbW%HpOLJ;Dj9!%eClRU9roxLzq`wsp$C*mtQT%%E~daAkPHEJSiUZ>`y&Y^vDkrLXz-L*p}{5J3c zJFr(-=!bu5H=%cM?mrI0-IXVFUw0M-Fvt?>jLT6$gluT4U7}!F-7N%x&nz)* zNp=s++A5X^-zL;rYE%pT0Ud1FVDZVC>bI zbLh?dBiptBMZZGT(z4J7kfXodq?6_EP{sT=?G^ss{*<6Cylw6rIO`UzocS}vvCD{8 zgm=iUYg|y3cSq8(X2& zpH3#C;IgN|Suu=Vq$h=nenhp;k7&s|)ch1puW?HL`&5K70>I6xG(Qv(CD9z*Ze*hpdh2{*w0Y@Y{9IWxE&V+B~}&#|gd3 z8BJ1bq!WVHxOv@7OHdZc-hpHPrx|ZVR4-2Y@3fJPE)5GVr&muxgnd z^d8^aYtCASLDlBgtTAg|whlYo#j!Iu)%TZu?j2wh_Vq8b7Hq;dHi2`v!?Urx4jFty zMuNM8MyXH#Dn75>0nyn6Cq%1Fg_u@u>0WHSL#ij zR^#+x?BhU6|BX4IZVH4~sysy;DseyAlI3)wF9TRJfH{X6Ry!Gg7&lKSlmATk zb;>!(JGF1AL$l@~93v#VjFQh9%=D6d1z&4AATztIf0UBkV7D_jTZ5y%IFWG7 za;g^D98O$jDT_bfT4x)>R64ar~@KN2mKRC;kZfx zQNB+Ng~!GDm4dI4P$ISKrynA`h+0!fPhPfd+ zn|LU_pRnb_8WCIdHOs!e$1=p)8Tr-NNsXb6wnO-~yW1?KmAbYJ@9cKQ`bx%B6aCuKftfCZI&^-@q)>32l*{)>ZRE#w=ZQNw9ie-x zwF_6O;#Imf;K>9xct~}0@zs!GlBWY<$`>8esSc`}bjzpbm}KB(l^82|i#GZ(H55l{ zhD!%Z9kp+^HM<4IFDZc6ExngY^!BZ}`sq|ryF^Re(U*mMesx%Tdb<1-?eK&JwrS>k zp879LP!zq|@UJ$KLV(wY4F<>Tw*iAS{&p39cM%CAU<7xM+bpESLB4qmA3es^%lWyb zp2pl;moL)pFH!R{n!1rByd2>Egwltp7bzvmA5ud;sXa*2QN=hEMJeN7cd19Uj3fiE z9feZxH}v@`HA@B~`qZ@w#S!{aB-X%k?Tg(jFQA&Y*)e16ae#X*b|pZMIqX~C4gbQP zcTau&pd$%G?^5j}zoYOryK5?`PSbSgXdMUo!l+*q`bI8(l`aPRXDT))I;X78Cs3uLo{?T_#G!->zovxIZM+Z~%Q`r??eb#bAs5TzipWFPj%5tR*Gk)<| zYcx@G7+I8gT4(V~dTxbVNQF8RNl7Id6K-SZj#~AN6KeJ2;z(b_nR6P6`ji&6K1qvp ea2@pFzV03JNk#BW7Fj4+>Z)$r_L_6?-2Vf2Mn~cR literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/minicpm.cpython-310.pyc b/vllm/model_executor/models/__pycache__/minicpm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66950512f804088cafa5ce815c13d8fe3d8497b1 GIT binary patch literal 14884 zcma)jTa+BvbzRq^tE;OY)6+8;JOvIwkkF9AKpgAxuz=U8d4%bkz)c z(AC{@s|Jr*J+`8OLOC(~l$?j8WS}H=xDq=rC$XKx{>{oq@{^CO`pR1Q;%9y=5d?+o zeQxz*21C$2t4`fIb?erxd(J&)pL6f@Ditq>-3mde(n7`kencxt{KviWdFF=(K-2`n>;n;nh25qUZeQq&(AkKKg?H z1u4&VelGf=|3xY9>wGDC!G8hexnO_i=c5<>7o#uxUzY0wotL7o_+N=$_Fs;^>VMV9 zeJvLp3=ZAN1&6}Y>ML88e-@m>!4YtdWSm#Q`Al#WoTC}%95_z|^We;9ob%uu3yy0m3M%>;A)A_wyKsyZ;2J5X5pW$6;ch=8#gG=ij zTosqs<1oN0d8_c&r7KH46)o85p-Z7!;fHqCqweLbD;M$YO_fgn;@@~@7pwMG*zdGk z%_IzlmEP*z+*t`Pn7Y_oX(#bz)oX<@J{7F`FRnK+fE$otubT~yk-Gd`)$4~C!@`wj z1aYXek`>-uPx|Z0f|VXRf8n*WU%&Wv9v|yhs+Dn)i4O4CFsGvAY5*)canxymPV)73_^CTChtR~kwC$V(Xp&zGf7m8 zdNXfyp-E8~wDJ9KiwxrrV2yI4`3>tQ2|`pNA^d!p@3LE^@8vz7GJ?X)BnJI&jnid$6i$zL_Qh<_D@Fb9j6^Uc8U zt)f{pgMEP&+JTKDkHf)H2=k${S~!>sT$JpCx#jYExn(0L2Hu^D?*^r?xatLE)F}m( zupCrFd)W=9g4!M1uLRS=uahp@!Avj%{s}hFQujKdeqP#6Q}v4*Kb@;OV`>#5O(!6p57sJnb~EI zR1`OYUiRA7w_dZ^MgFNUQ$902ZtTtX=p(%Fz@nCIjL+{6L)n= z?xD5S!YtZZbpp2O1^K)AdwgGQvL5fV=B_%^Ngdo;z6G=VEn&ZN2~kA7#g3XIY&hXXpELtL)qRu-uJCs{>7JG~UnMf8%r# z^-uTP{ZmO3b~`!srD*+HbSAoay3@XTx_>*l*6W_W(dk5|wQ~t?g{^gTUSGziBYb?T zA1(H8s~6Gyc^vUCfOy5S;mBXva7~vb(>6?lhx@>GT*H;JZP*W-hrC8f?(^Cyp#DS0 zat!a`#?06NEDi@d&5K&66|uQNW?0iwCZsL~D=q4o(r5>*JbEC%jQAMk%1jpgTi;r zjAy-+le#wQzWQk0K_QUVAF1!4zCXxX8MvdiR; zA71~QzJ4ydUdXE&IH^sPk!fNK_Zy~e*bQQ$poOA3iz@0hlCP1xPVxqbPwPR0DmUYr*Q9?n>cTSbI-s%do!Qd zG;FIdN-!=oK83)fi?Ubz41?6bH^06~x&#AX4id0cL*bgNOLN4Qo*X$^-yv zc7Ue*Lc81V07Z&;7Ce}WYfTtm`$jvwsg1x(=$@J-`6|gPBr;_=M^znUp`hh(31#2Y z2IV>+f;{h6fc;<`067SYQlsI|h)Zeg7^YoL?E;kymsGaOiq0tC>EwC;4CB;dnzD)8Z z5}FIKZt5#2rH;lwsTC(mYs?k0E19usg@B@si4S^}kBfSJ4kb1la{%2OtVk8T|Ijqe zpP82Vv1ywR%)I%L>zMbQLiNMyBEIgS3CnV{s`zXR$BthqyNdaaA^s^EK`Agp3+Ttn z&<{`#Fpm=$%bwt!Lf``L6a@F=d{;0`5o%H%W0;GVmnOV`mq`ntOnm|GQcEB^j1>(R z$4y-GFQG7Y&4~%UHCLe5vv8_3L3g3O4Ga2=I};o@Yv{10o=zMX9_t6_@{g=7~Go{Q*hHWyWZ3Jae4I!@{xlD|pv%Ot--B6IFA zOCkU^tllLVzi~W~kR{1|4~Gn#n==dcN0XDt!6Y)B>iyb2#hU!89Y2bTzxcZ#<7tDS zVHh~rm^yhJ4vqo>2ZjqySO~?{VJa2D_rNQnT*BGIxg40wb(vn3po-~L3Z}yHY9*)% zofcXTs|YW^UG}m7VoD}o4SGG7)-oG53bQ|`okUFkL2e%7Kz&M!G_ThX?8RxB#p^e+ zE02+60}wuGre!I|EtvT3O1eL4-fHxFahuU6BCaedL<21b?ir$-nALv;pxPt=PC`juE#Co|)0)52cuIOR@KY#g}YM{7a`w%1G&h19fj{NeZ;1CKl<4|>2 z%~Gp4sV)i4%EG}3%Oz%q0NVtD2k zGKEst`QC_RKS=7Xp~l!kIz4?s1eFdy2U_RPLDM26HK=hhn(Q`2|NP31OGTHK+Q>U} zVJ*gnlzrEqEwAS|=st`_K>`}JhlTwU)QxFMypqQGb%TiCGiI&2%`%XMa2x|hL=Xn? zJ*@A2)72K4kpPU0;C{LDQ58;lh(=7K{L#k2-E((jr_di-e1Q1 z%dlA1l6n(8Lw?AD%0ON*u7DwtCri{C%1V(CM5Xy<1VD*eXW2wpc7t~kkcEmC^Gz~1 zKI#_9Z4lprZi(MXjkVOco*FlNucz88I!EA_WcYM0B6ak6!e5Awr6c((9C00_T9}9A z>-d|)A--w|q5&kIOoK#7%ZVPw_PnoiY-eqZhM zTG!Jl%~@XVHiVHOG_|YMJ?x(cw;dnA%RNWiKUXXst(WuAHOt1vk<4~Y4)@4>wea}p z$1*X&IUJ&$2|b^&N*`^k!Uo+HB#_VtY9136#%cjnd^67U)AY{3OLxI*RAdybB_w>T zupckDI_~&%i@#7+`_KkBcgbWdy0pmb72D{(Z)oE+2B~6nzJqRIEi4xwLV+nI_n`@Q!Qt0VgS7n9 zjT3tgykzOdqJKmHgvufS8p_8A04g0=qPP@-6IjoM6MN~8C z?T2D2!{h7u;uz%z8&7_+s77;A?cu{xSA}sqK&qidxeb3TkK{-vGJsnjy7)~QZx}43 zZxAjUcWrR;G+nFaD#GS%nO>{N!X3`d=Jw}sS3g~l7!UQf-K_K}PX)PUriaWWh-r_` zBPRa4=$ra`B(yYsi6}5cCb*wmOYNvh8n3A&l}vJ$CD5xi>Mo!9O^^lW6J5OeEmqnk z8+yl=D4P=~*hLRA8G1km8QEFbE!*~h;B13s*V;I|mvn}q=iBT;ZMXZT8PlG7m#gOF zRa6h10vbDQ!m&^B+}}nOmp%U?3psdb!+t=Y?Z6v5;*F~xxplW{)_418W__xK`iF{b zv~cYB5z+G({|}H!KV3Ai(zSn>>k(g#l`Wm`!I^7a2wMXeetUAQt0>$oLp1pm*1JkU z8UEPA3Rnq7+9kz$S2dX8Dj42p1P8KvHK{QjR(Zyo>fux{gSXaDp2b;n-Ef+vuqOV# zigmfUJ9!_WGFHxJpnV6^awfh@Th=#YMb(Af1i+w?Kf_4kywXxMMjvs6{p?=yUK2i zuhlWWUHez*n0^To2pW=Pvt3-VSaDmN?K5~Ped1G=d55`U+Lp0{Rp003?}PZB^jtc# zS8Vq#=NcW3=MJ%rX~H5PCk%`6u#EMbVMOPCT+7!H;g!L6A8zcMRJk$O$)l~|k7XXB zPtT!{qjPJZM{o>{Q{4xsq*^R^OH|!JBdOHU4-E61 z?B;G1D)#)J@CJLTegJZdoutiW99_jq+uVau%q-J>aZ;PoqQ-9RAl{MYq$qft!x7Jc zh|5GelP*w0m03U-J(8CCN2sCwR#E#c=r7VIXUYC&AbxQ)({}lwbe0RAx)Y;1@pt$U zl%I!$!N##21*%VItE;~mT1%!$yr$yal{Y1mc` zH8XIqlx?Wez}YNx2_Gc2yIBZwn{LYCKtRBXl=S!EYvx#{FZq>_l-GZ2_zeb zd)~mq_!R&*BrkR#hs8oKJcCrU9d07!+X0?Ij(+ z)2A<(6J<|Lf#p}Q8)Uhy0P(msr~Wy3X&%LuP~AgW{g~vRvSt}IvGSW$e2V-Zf-IEA zveS6w4|t2Y0`-R^|AJ&>EmNbN=0#g~Nmso1uTz+WrEZb<&%<&b*MIAR0*NAr|%)#MNc@jw($RN19 ze0=iY_fW;>rxCbl2*S^>?@z)IogruHp!%VCWSCSSKymWkM{^a^!xJA4A&Q%&5gb4i z4916LxBgSb_q1^A_)*OK#bxwh2vb&&5&p3RBP%eIKQw&{ocxasADeOlEJcBn4~i>> z?-V`$1y3QTUO*zjmHc`IXXe+dI5UAT6?oY)RxOxD3Zfj;gBheADtPi~z>ezbRIpFz zTv+27&+HGUg9CWJ&ga7!a>II-e&BY$ygwTpmS>KH`%uHx&n-K$(a-khf+u#|-H*hD z6U+z40ErG@i{HWUP$G%2>v6Ml@e;OpAkWI= zY3f|h&N`Wz7G>E*ZWXd6C`_psV=GHFfYee>h74J@5rdZ&I(m)3ukE&9{lD61LorMJeUt@J35H?< zT&LO(@5Wy;IwPd2?ywCul$NwVlg%jVACmnKNajfhrqmN4{{At(!R|PX*yL>#MLLtT z!v;1#1``7FOTyHD`2y$JH(X@6co!7J!p6pm4otu;1+ zHnBzw>1^j?Bqk!{cs2~^CML@mq@nfEk7erOIUMm(5S(eH}*;*{CnK^fBaSgn_+IJ{PD)o%th?zz2Gn{E@F?p z{MU{Y0cY39>yNPxW)zYFgFJHb*yAz8I)-8=hj99n+Sjg8WHlDp6+HM8egvJ1=OQ)r z&^9*??ePhjuaj2NX;B(R7^C$I+JfQFMuE<2Bp2piS96DRvIMQyF1RFZVG(+SeJ%AL z@re3QB%A=lB&T@vKLZ2cQ}&VU^5Mb323TGK=d){b5ShK2S7!o%NN~=8L+H3TT4n;sykV%XtUW)-V_^y4 z5F>+Dw&B*XN9+;}I__Uudmg|M0Cme$S5Oyk)SwfFzceTTd}2{}%QBEs@4h~;@P?Z3FzhT5|{Gpm-P2I9hyS)SvHX{rUdd+^B^g;G41*62Ra9J3+Us zVXwGD6XU{dC#9u@JYt6!!>#^^YQgPYjdj8U0om=hR~L^Uvxr@(d@i9eC$lg%=4nakGiSZmH2 zvw8)<3-jg~-@N%8&d5`ID|dD&2>x7Lw4 zw)Cm@QV0K3z&Z5tQL1qrw-JKcfQ}!T`Rd1rwja7b+jxEtOTx{N*;dGSGAJFkw=t6~ zgd_G^Y?2|U~i&wFmcEQjeMDK*|27x~Zv30u)V|2Rs+rOf||vOkCLxpjTm zR@{i1{XVRvzc1VDneBGY*eUWgdiOK`Z^LFUWKUz7ZD16}>ulH6yBOPn-uNgzc!S;8 zC7tr*81JKe0@OmJ%dLI&FqB7icRdOh0aopq$ zbFpyVb{+1U29|uL9z4_hmi<3Bp7<1rW8oOV61%f-bXT0@&-L}F!BJwP-gwyNMmx47 zW0h^>i~Z?QbUtDo9J7XyDr-&cmGqS;j8jd~olgW{pBQ^R$$I~pgn&c+7ZRI~e_Sw}pCJ-M>R2YyMDlpJugN`W zmhJm{MQ7!L_aW|O8wy?9dtiFQD+gC6T*Y@Ud`Wn27Y3!Z=CF?C1(1thy!A6r#Y z29m_w#~vJeOsm*ph5r{)glDoHMmEIDUP1ra$2}$}W&(#F;)_l`<_X#UpZyP!>??R| z`(s0?{|87NVf)7))BZ37U_c;h?LYsR+8X}TWXb<;jV7D~HEN!19OD~L?bU{srR;}S znQ?BcC+$wGXhioG_#h>>M8O(Nx$L$i^5o)1Veom zB+cum#HurOEs4<7xg=&t))r+ip{yZGhE|6QoJ|u`TqZlGII^?a=DbXwAHM`*&$-44 qq?pPut2MJ`95Bij%Q)M@!(vuh?(+Vufz3(3Tt8ZWr~cNVxBoxLSs>8> literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/minicpm3.cpython-310.pyc b/vllm/model_executor/models/__pycache__/minicpm3.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9afdf527cf85ea66cb6d900a701ca95818f5052d GIT binary patch literal 8052 zcmbtZTaz2db)Mkf)F3F`tNro=VNK9N*E;*xF%wQMT z835ZeAh}z!^s#+lun z1~(#;n^A!mRI1lPi(B$-a~p3xw4)*~$~q$~MGkjl-VDo8g;%0EJ}1kCuo}(td6~Du zT2$wCnYY76w7?f+z8Ee>OMEF>=F8CvUy;00cq%&0Ps_X$u10708JRDKXQOlcT(rj5 zqVxQ`N8N%lsNc)pi-=u^7twYGy%b2YoL6(lY?#C_q)CA$MpcwrcX?ybHEeK+px z2k2J#@Ww|QeGxUatZ}s;4x`=&qjv9OdQW844}(G2X?aQDk2v!m_HP|;gmG$rIrOl) zeQZju|#*SOG|Fh4TQ6c^{GJ(QX+z4i? zHnTM(=CAsCp!r6i`zBHW$wIP`iby3SCoueSU(W=~6J~FU#DDr1M1E75|if`mWBa_Y_^>^N__e+5Bij zE=|t;Vf7V7*?6d|DN1Ixq=;mt?T+sUJ$bO1-5W;kcHsGOR^8okdpH7Fbp1{wIdmvf zPVwe6U)puUzVKY(^>#96=jKeQD0?N_kh;u~`MBkUko#;Y@@~6>e%ztscY|m%@O|*( z%xRHWx}!BZyCfy!k_jmvie9TSb4YW@TSyvG+{URU${poUJyKL9WvSw_R0q{k15`)e zSW=KPQWMln3!sJ60<}^N)J}?bl!HpDbnUdV2z1+F@oPU({!T@0ai7*xPZ~5U#>Y4fWv1=IWCX0It1r#g zEy{7MnQ@jdj?$%;G!oX{HJ6lQ<1HsEqgu0p)pps;T&I%L;9!MFC45EEH}BI9ttMx{ zli!H9Bl8Zvrf{G3>nui4+;hGHDYG^NU7;-`JpcCF>rKY>&5jp~S5bbTW?C?an>vRG zB%atKIpWUjp=bK07Y71zsWdh2ko-Y*qzH;8m~x!vT4uz<0WQAfx2VOY#aT4S_hOB5lMOk!2lD%9m-CODY%MP& z19|sxR<`T5LNAV8_kr@&PcA0W;NqY&cmoFq%zWcJ(QrF@GrDy#>}*~f>?Pa%-o=|? z7+uWuL~uK34U@jei}A$>=hPZRm(~Y+;*T)zA0x#nNU>_F^eaj#^Q?xF!}7eYGW@Ks zYq}0@o;M9$1^+2ysCi5!3wd3w=Jk&qbKHyNEpz@Ovq1eub&rbX{;6j$aeXXNnLP#~ zobVVF6X=dOv#{k7kg_rzg&U3ZXI2aQy)8PrXpp%5b~_G|Y+3T+?Yj z*RxyPVi0~=opPgA_F=;(AXz|KJkn5a%XyY)j+q%s2Tod) zHFEZunleT&9xnR^M#ncDuB2t;%E+Cf9OkK@MP+6l8vFE;g4rr*Wis38nK9=Ms%fFS znyLVPGsVn4%-LVEoT7J?N^NaLNo^m{)?JciC}@H@%;+4oY5*9U;!SwBB9wRm?|k18;o6mWJB-MSUH26FbK$SG-B9|$qafEyo)~}0#^lKc(W7S5^quY zO(KMN+#dE~d_}Myi0=~fPl=Eg;Po*Mx$R*b_|mH6izk?^{$LNVDlW^hdciH2B7$~nC z;5tEh#&&LUGdH#}^(L68%G4cRl30TS8VnOYN2{7)KxN1Z@eWNc$-Bs91)Sh5Px!w; zA^sbXn!OC5U6n9ssuk5DPz888PbzvvEwYMQ1DC1Ar@H2V7FiMHB7XYQQ%nco(r*<| zTYS{8Y79U1tooMPP%8l7{gr0{b+m$}AyzT}h9LWSW20Fk<=yk5z;&~tOWwsWq`c!o z0KG7;5jGlUTBir^MoN$%UsPz48z({lDHY)msSzQ~BK{mCGjkJPl!)_J#3_%|St|%b z7!1>m`T;ee-4drkNR(n9#8Q}(e=Sq>-$B(pVH*1yTkx0}>=85BSFFIkWET4ZR{tMZ zk^Oh2#QqC?{?jb8-iMPPDk z+KnH|lMP7FAO$HQ@WzKps`;u^%6bCJx~n4Pd{Q?md;+4R$W>{N7VWK z{!7n4KIuD5$9x`|XO;-!jt@kd;(Ci*%ke{bO7gh3kx30-83L42c7P*3gxv;x&U$3S zpiWv|sWKb%lFo)8Z600MDHuBjA^X-nJ+ITH9ro8cO=ji&j$B~KR$%xz;u zA|SE_!k1yxCNl>%@j(#A3kOZ>f7fbV8e^i`X5dpICn4;=BE@vEtP(a|ZA`R9j-uvQ z4Z8@q++Tg}VT{zo0|wnNCqALs{+b9ULTcb7%y)=KST9JgsORghgRAz!$Zn@Z9Y z=s!cnZ;)a#4*>M!M?5j12u*@{R(WEXtbP)>!TZVra9e=^@u`G#)-TrIm`7XY*f2z84H77;B1lJQ3CQs7X~C@3Re z#yjU$f(qq=vV`f1Kj&BP8hp+#KsIVwdA!Rq0G62p5wttEPe63bz#_*FRwFl1~zgN1Y^Uo>z$;&2ETOG>Z?Cm14NIBZSNBpy%gcmzjA7w7o)YX`Lsw zF<)h$DkNGtvuk(+c1*qB zp-Cij6rF{SDtpCw)GW?*FWrRkLxGElhxr% z*a&Y2{v3j3c$X6L`bY`R-(lC3pRK?fq!}F4l~eLki}0U_r!FH%3XCivENA=L{Q~?f z_@BG)$$4vWFR7z#$=Cmp-BCZncO5p+mVDvyH!esdC7-p3`(bS4Ry(0ulTC3A`DT)rqvc9nK%g?C z5E3d*bpe4xt`KNf1c6$v5T?37=HwTJsQnTtCVx`uf|03j!eBRz;v-fE%zQc42G+=o zH%HpQu_|8(8siyFKsfp~lC*TEn!rcMb&GXWu5CdN{7U7z)By@NBywAmzc?;kpZz;y z3qGVf9LT^Rw`GX}lDshDHErocOSf9O#<|x^&Z_uZ5c~_*ztzjTJ5r+Dq`!mzL3km| zgZ$4RH*MopB&pPMD;5`u%o2($!v~SkY?wQ_F`}P9#jGU%0my^)FjZ)iPBam}053CS z$WV~+bVA&+QjVmu!3QLE1Yu+2&iQ)(+*o{O_G$#DvJaU^pU}7yTY@hRyw(m}f7HX{ zjB$$x1N_Or7ZFk(AWDZSx$aTDxO042Q^EN;D!xl=q7ZRzL_oBk=ua)nnG;mod_60; zuHSFLuQd@E!8EzYN}`dwvQjpsyh^iWlPaVjlTFDT{D0SraVZdmiKXQ83I$o$CnmQH z2}@b$C0Ubix4av?Dmgy>92A|+A-+t8z4-;nmiyz5In4XsE25lqlekQKpScwP_rE%ADk2bo2Kc5#8oB6TUt zE^&a65dMjkNGTyVGloegjHL;XnKz_7NRM2aB)PoOG0q6(LBuczt=OE!J(DyRQ|}I=FRJOpTGD0j@M}Ba+w%DkNn+VnmzW#SnT)t(EUl{ z;aPnB?}^7^Zp?|fhHKUg$BRX+y;!oCw>)V`d;!o8^ z>Z8u6_|vuR^&QR*@n>o~>${v?_1(_y`aRA)64qBM*6(%ht?zO6)bDfdldx=UZ~cDf ze(~pO57ZxY9u$ARwy!?sjMX1<9;!d=JS<`Twf*%2&Vl;4GcM1C+QIrE=aBdZYLC

{__x-ctUu*ECH~>s$LhzNW8&Xdd%FIN z^NjdMY9FsZ>pUy|(c1C)3Fn0Px7VJlpL9-&e@E?9{dwnk@$amiuD{^CApTvoGxZmp z7wa!MFX4H&dr$3b{hV`7{KeWt{k(I&e!;m=f7y9i!tSkItY302!N147uVOfty@EGX z9r6aMuS}bxG3UzZn7h}#|1&Z7ey{&l+<6tx54aEF`9XR91fKV~V|X5u=hu+`6W(i* z*YOhG*6Me-54jKDHl5dza=&{3DGzvKDCrG%+}q7av=(-dsn@hS9f3jQru~H zCzhJt?6QI1Ij@Z0dzY?xp8P6rdf7nki|!@lzNB;CGW4J0yXK|X*yV-i&NQYy&<@ zpmMJ1mTOUg@i2d8;=(JJOBYX_K6B~v#n-~@na13Ffat(`D+tqP&z_S2AC>i=ZO&8z z|AJ~xdA{FN2pW8OzKrS@Fow-Wr_9`mvhQ8={Bj*VRbi&%JsfVE@q*G}x8zqEQ%%1# zhp$(kL|@D>c_~Vti#+G&12hG-18qZrqlE~7)GpsA*2OKyMx zMby@Iu%5qCtyy+Gwl#CMR<4)Pd#{G4Y_t^G&-wP)c(s?jo9u4pM zHl?hBG!YZq8FnMwa_*J0m(QF#fAaX*&Tyibu0ijNWU4T zSG`7M0hOBJmNS?XRrlOVIq)uf4NQR_W-nniaDku1K0G>bV2laXhNyQTYs<4_-rFT&%+a3IDU^A#L{PTaG+glis)d3H4kU%cb9 z;7c)dG=}g*&y#&D<|eVZQ{mQV3a?x&-;kXneq4_?EiH}BUfQ{HM+RzK_5 ziLrE;DwP@-rBW%(luGrcJ71$eTPj_fFV~`&WU1sfrx@IBKS@H^Un-RwjbbDYizI-TLv zzLN|)1?M8ZK8HLL%VHQzBa_6}whW2gd^ipv%Vxt-wJ|VyC=%kwP`C+n-wt>7bpI47 zql?wq9s=D_A(+16JgFDxoPjejmfXCvAKuTGko_vYeijb9~tc?ppXJ@U?+;tWj)V4At4ba(At$Y&)U#ebyOO|42<0PL~jVZ3b_J@9c< zM7o3B9F$7x6%-f_lz2|!xaB!3r%IRrS*+c3ayngvaj@|$->Xfl3&^Ejro)A=F44J6 zhsE77!gQ%LRV(`-OI`cqHN<@uUq1zBC~ccrqhKy(o&yE2TU}uuokB=HT&%@s$l{$4zWE z`|ePD$XM9jJ7{ZXPw@=heAK>#Sd15z#b-@y4s5NZ7{`6=1Oi@mr?a z1D{2ozxT!1rwuH@Sk)pGziRo1u^jn?60ch7EM}$?d&|UAC-xRf$AZN(W(YYjOQxO` zF4eL4gd##w<8ZKOk&?HTg^K1b7Rfe zV#$}?>bxJUF8QB8Mt=~_h&f`67(30pna6k!TGo4O$9rwo(0Hq7w0RJ4k;@wW0=m*N zsz5TMv6iWSubO^VMlTBIxL3JOjjFkbHQPNmohO-*q1Ym}dNq#CkHg_MMy+M*m5$j< z26ihMBr!K1W9dPvn#K$yr%m`WT>aP|shQYfI_PV~Zy6_JZya3Apri~|Kz2Fyy1Cf5 zm~Ew(a;PEi##((?AF)=JF4xL*9O?R7eXZ;e4x(x7tfd0Tk0B(+ZlziokRg;MX@S?S zokhI^)xqkP>QF1~+HTTK-HtD9b+PK)^m2SLjrkvLAwOz17E$ZcHa>NwFc3C^){J5o z=kdE8&pTRq>~9kUFue^U+e*2a+p+=|`&<2M^6q4LizJq@>aIeJKCG|4uJv`tkkxX> z3=_f{$kW>^_+0)YOrd=4m`>khIRN)CT~U63y<3*^R-+6xiA10ig{$68z+N3wFZ)-= zc7xu|mO(z=L<)6{PMuDLPL&SNQ&powY7!=Z;F~}#lg%bLK)~@CFSLVZjm)A=pxSVS zQiMt6>5^xemnbYu0HD_VFs>R6fS@vXrwH+bAq6`yA9!IBhYvW2s*|WSN>{Ht7JjaU z37%Lt9jhD!jWAhK9(d@$Ng@s;DsU|37%OP{)nIkpVxc+duXf8xgFVLZ)Vu~{QrLE# zM>WqJq;8>6aS{Yw*pmFY5gMh?xE98n^TFEPnFQ#*QC99xBcIRRJz{1I+q8`$c4Zd( zGGkgo4J`|Yh*2aIS+NakWf+GF>7;enloaM&^KL#dWMoZ?CFr>K&4l${KAAy?Wh@N! z(9)>lF+-J5?bwzzU@6R;JoWr>FmR=_6MEh#&{Gy2nT7OO9_bS}+*O8wJoKWuB^31? z`=p#{ioMd9QkKN`;xWuc$L5q*tF@0#ITPMsVgZ{I@ZG}K=Om+ z)D<{WJR94eI6NJa##Nw?-73g<)u;l--!>P`0EFB%aY*b75-lUJmy#fOspYmW-SK5Q zzP@G1J8uKkZkB^Y3tKiZ0a!5*D3WaD)tiA=ngW}mo4fwf8_{wjSkY2t9w z!a<;bN)0Tb@bJQF{tm+ZJ#b8{YW&^J$oZbZO6H&a9%oj3A1Gpg(AG^IpW?~b{%}?Z zURae+P{vsSagE)IlhnDf`IsX41T?y8s$uv{H%^~xT@qd*z`=9N;LMRce+CECQo_ZI zos3aJhH+0%C>Q0v4JIu1_OtNaF@@gg1G1HibNu~R+_wt zwLrnw__11Zs$BD*8Si8Zjd37|B$9JJ!@k*y>v|21w$noR2;RcDkH|zIBDaJ;SE~eA zIX{BhoLsvW4et8dkI1mLv0FE6)Ss~7c{o`kWetJifLjK_P(OvBbz{d4%480~gR>5} z3}Y9e+gscSz(&i6Q(uGwVNZ^A&Q3=Ipng(96_;a}UJYiEy0x+%cQF&mbepG-R;KI% z{{tkJkxm&|L)~FfNt`7l?qP#)r!G+fT5akRcg$1=5a|pE zYL@EI^7%&KBj)cQMv>yL9oHT1e7$~?=UIu2d^{Mm1*p4dVPIo{N$N#T3i5F5j@xli9Dp#FFwDkY+1X0W7!`Dazb$hpM*moH zB8*==eL}VY&i=$yv({8t&q4#l8AnxP77(j!dP*8(BdZ+3!+68HF=op=NsFc5Vd6%` z4Q4}oRz%9WtTj_NqQ^oP+C3092ZAHdqmhT8wPLUAj;o(zb?JIN8t0y&;jnaOjqQzA z9MC1vstl0PvTk8GfclB%Ef5yFm1x=16r5vL*;Q{ zPO3ePP-@j>KG#n7Hr>T_Q{B%V@=#5g5I5nVPnqf$@EB$wv;v3^t@jnPaSj+OcP#ZqIB|c9u zVfM6D)7D=`GC+4BK0-K8B0zl=4m7wipTxk90bQ+tWc?FH{bM@62nT|_IL2T7AOn7i zj*u*3MJ14qxKKSrha^aF3*}P;p`oe3TTNFQSkSdFj&oA@I*6U2wi9}3k=g z(5f~ojnIZZh8T7mg2o1PtN%XJutB;dQuROLBU|cdoXRCYEhBV#s^Qo&|GpC)G(maB zcXBvhR+o#D;iFze>X%qfAGLO+soD9)Ro@v{jg5lTXIUc$5aT3+`41W3(&31PJ3nlK zw{xgpN3sl9GYk8Ga$>?Bg508+NoJ@nFf8niEdC?~Sg|~qG#f6@FTH<2;qT@x0mLTR z6)-m_2|McpE1`a!jd=l14~^i0vG8Tv{r>_7kT(W+i-Q}iiX8PI<&X~wG2Q~$Yu+*- zDG_!Ba+6&_5^_Q;E=o^`SX~Q}tfP!~DUwc%#)5P;0}^}G1#xeORUo@*If=`4(&ptk4+5LR ziA*@sKcPrJ7miOfdz90dbghIfR{Uc5dbuKs=wi?;Mlr=;){B%{&B>}a6)Ze)hJYnf zh}FSw6(_wSw|TKS-ByfWku+y`cX5a}`VM9iqOW*unw57PTI46Uw?uX+m=js$wUv1??rx)cG57 zb$Pu7x#z=#u#1{G%w7awt<=3!N`W7|%4ka@G?^<@I_cnFWmk^VIl%;(IfzyCTxea) z7(&6f%I4xK)^%~4IT|kTrD*}?hQ)1(-6yw-h*gn{f zE0h#u;VWS4(FKV5Eo7U*x98IzJ^2}o^QyYWiKor9=^l#|Hh{@c(m?+TrH$>SU=ETJ zD8OBibJA|w%V1Ar5Qp{c_IWuz^GoOi^-9*wLA{cX%G9azlBZwX0&*fX@(y%-I)Bz3 zbhkjI(;w9_E7=?fVf>K zYd4-Abob%up01HxGm1!8tlm4Fa>v|<(3(Bbv*|wU?nlo1VD_^Y#x;ARaN9lT9zyv2 zQHhUqdU?3rc9;USM^g7R(uN4+e{VehTTAm~9D1norFK7>5wehej#i302) zP?~4LfwuCI<{RgzD_$sNiuhaxDJf3^bCjooqUe9RNk(z1+$f554d_XwF$*nO;JN#Y zlhC^(3aV@n8YxFh_+dhW_;CrD5MDeS5ag(VVOd^yUp5Y$6Sq;$hJK z@aX4r9AE!8;b<+|G6WJ74nucFg+v^fA2O0#41r=|2_&9O&oa~)r6itH%Q#R<=^(Sz z2gaPLi*z;HN5cY?EMg9$4Z0n z4OX)_coO72jaaY+SZoO20*(kMtdM3aur$J?W!r+0TWrTD@+Bz}FKrjK2^3m}mNFCX zhSt=)TLz+i6Auy$Wb_~4AKRfm!JyaZ{0N=jrX!$-M|_wP-5{7MU>P`r@_1cLL?R=F zFn)cmF67M_AZl&f(r5|#EQ@=YC1pehO4YO5f<3gLo%% z9m1SR5!I;25THhwt56f`qE#@hb}={^mEzn}FJJXwEHoF~1W&kX2-7C@;JS%Tx;p;v z`N)BB`ow5On+Pe9xvm~N9IZtX+5PZjl6JxdUMT>_q{QqY1=5-^2SHG6kX#eUB^w7B zj;B@@wy)bSt7^`)Cg=ook$Hjy_*wwV#oGRmSBDHvBb#r)gUJ^)FxW3Ltd`dODUijFsxuw<5jY$ zR)ZU%4;nh^r@-QZT}=d7^uR_*V554FSjxF6Hx1O2SjsQQ$ZU4yw_EY0ei!WTQh}=a zWpio3h4!gB=t5J3>(WwoD|q-vtm+M0pzV%4NU7@r;% zhvUZa611@g9zh#9(K0~~No{e9zZjj{am_|xZ~R9j1m5ML?bxthqVmP6H5)Ss{1O78 z%%P1BtQ(+(k%h6kI=+hdpXcgG!DGj4vjFMK!lUbFp1Y;9DOOp(Re&O_kk))|QNP1# zv)YhA3+L1iAxJOYKcbJVRez1nn{-aoxkiVZFdSiZ(Y|X-J|PJ8eD$l0W{0M(ss;EV zmzffVq7+)!E8Y#7B2W*ct|Joi9N9B}2)`n#hExY%$Rs5~ri1$pd3D1e09 zw%$*r7Vh~rBfk;go8lSK{`A4aXSUb>IXGN0qUJU!)d2%RmA47Gjs;f2P(`RdAu7|M z=c%Mk;u$9cQ{g#DfhgoV#_I;8J}H-ym6U{GlOZe}n2^P%ZW(aXkjg*~)XGr2p-y3; zLe-1SE2jfAB|Mi>ZmgC0Ea1gb(uE2Z>eZg-87+3TyA+)`2{61dExnh$r0@vaCoUm~ z9&sS!NM;JI4RwRgn{XU^R+dSao^SY&n0pJ}SYOwAc$^8Jpz|c1r!fJz zCcZ(v6m*VV%Ce?t7wi6I=)K!ij72Y*uhY0e>5(lJVU|j>N zng}3(1-p6<>}nd!CFGSq9{U@bl?CzaO7Tgg$RGvS#7+tr1z??}8Fn?U0;K4}la%3R zbvV)@_S5L$Vg}#70LKB+n~)Xm2N}&S=8!Hcr9g&=QhpSrL}5Q3bMuHrc)Jwj&f=N5 zv&fx?qCV5gV+Uj)AMD3(-?cAYTSOknJ>m9AiNA~z`*j#fWV;aFDN)k81?-PsBlf{g zqps^;(GTN>wd0owEXY@>Yc$+}+t7WD4Nj=%F`1&v-iR28+zf=DG)x~gr{P<8=nO2~7*#~gu z1%kAgY>-L8u{Uj0U}N+Z;|n~v1aX2Snb^ij+V2oUwG9zlNiae-4=!vl6Xu3WX8oI- z^>5L6m(F`|oScTa8bbKkXMf75|3QbCr{{p#Aiv^Da`HVO#1~|uz6UX(4Vx0krq-+i z^}UGO1967G9UhuN44MT96}40`k}bY#r4~llPex=H66ShfR5&1P2kqV4!``D{qsWU~ zr=W--crbtj?gBtz00n9uslstO|32b{(HXP!@k&Lkx(g??HPf@EKVhDdcNq5I@&a#M z9AnowpP2+onI;e}K4lOIeDv)aiq^ zo>wiPsaSV#+nPr^#b&aKYy##t9ZT-jChBBiGO{}D5%A`bFNv>=D42IZd)N3fEg0BP zm@q=q1+0XV<{Fl$MaAV(oJ`3Zc|6Bq0kw0Xcx7KIYnssm8-Co@Ecb4imP;3^a6LV4y`jeeG!&;IZ6HD^1QG?AXiK zp61Ef4g>R)ZDp%TXybA?6B9Vo!0;Hc1{f>iLbb>eBxmDVmN}UN=Xcgk25Bf?qkOj@ zB|yt?+Kr_q#uB)$k-8QccZK#fG4fKM!!TiIp?oZ$s}dF(S34ov6f88Z^RCAX>EZmG zPL!4>h%2l5Y>2Bg%ScA?!kjE@QMh}cZgK~YR~pw#)5^QHu(d0Y_jZIZxjn(K-;9Ev zzZ~XzFW+R;uc6IOfj1vujNHCK;)mNxFP(brQt9&f67J2MdH%K1x#JfuoS8U{3mC8p zg)#)p3bl9;kop~AL|56^pz2W+t+MEd7)hvmIXZd0(P%{A3=7I_*XzSRLSMa<8|8Z5OB^a0BTr$ODV3vORt2&hKQABSC!wC9B_sGZuI9HE`$iyby(A zyqc2+cxnKYH5F7_%^NVgmT(Z&y~q%mn?Nv0LOosaYAyt2!qKi7*F5=Um?cDWztASA zwrHNfL2N)!d$UZDu(dz*b97kCJA4RYpEe@Pr*I3d=$+xsw>k{VJy+BQcOuF!Qn`*? zY7N;}{|P018ch=oh-GgiU}{V7)=gBqIcfwxX^O(d-ogKKck-S>(pZ`>|MVo z){6$UZT#;cYtOL~N5Cd^gCfaXyo@A@8P~Ss>j=AzcBow_VWTPN6(kD64EwbSs)j@` z#??0Qo6XLe;b@<0f>n2OXap1Yj!!t$C0*&rFn)jy-N}YNuzt8U%CKIEi4ry)D^riv z3mXj6e@ECCkyp&2aQHv;q4oFR7&V9n_nfmI(uO-x8<=P9R22~RC60hh)x+ysuql1M zV@^!SdK`Orjm@okhMlE8N)bY-G@U^@qjX-QbDYizIJjPk)quOZT`EdntLz0`l8XU6 z6ji4Noi?)$(4k~kaiI!OR1ea}iBJ?RC`kBYieOHCnGUYe$Ar2lT%wPRf9-QZ)MSIo z?$IW~UuSw;pN}a{k$Q^`2TI+hZ*@_L#Q7w{{vDhfNa6nmJgFFXg*1rsdu=OUkRw?6 z=HD~-0KDGC&5#eECV$@?Lz;IP0x=oIV(+J537mo@uw`f7HFq+GHo`X3o>@(qLP`@U zZKQ9?#@=9}jBcIp&Ur`u1=yEBzPVD_P|_t_rS&h}V|3^=5bxM?#cg%}FwSTNX8Q$AXg6T-GTDAx;s5ySlfmgce%R} zG9o&*Q9!z4*mo{k#fM9WwDL@k8+EvlFOKD_np0PdasBrTLzg9Q>>*5ycr_E z5!QP@hO~F$UFsm00nuHhRn<_Z){T@RU&eNq!)-7*2y6PThIiB)TAQ(!_OiuX{O=6I z31YNzB_|0U5?2Cga)(Q67B1MS)$0E>QbJNs%7gF+@TBgegZEW*ZzS16JSBgIZOg)A zYdd+YfBS>pHp!XGhzBlSqkKX3z7g@3QHBuELVqN%39 zH=KmAik1hSM46&*Y?&o8+_=-IE+7^cblfB*zzEN`Xy}&G(%yv~7iu0)CC9)04m@f+S`t@8ZkG0UOJ zLiAbzaCI>I6^$-#FcjRIK&q7~*a6Rob`E=`D+E_hM29ifkK4DXIqU<JqgqBtCTbd);Pcg!`0M9er44V^ZX)KjlT~b6Go447z2X^$n;ma|3Z-Ms*}5ec z@#Lgu6qEpBHGPTCDaf?&YK-fiGw3$uVv_Kx!2FPcjipr)i8Km0$f@1I#~Pf_s+VC> zZTZ(!pw(u}2yNr+Rmk}bW?)yfBWvO7c>K2UGl=UHS2LH2M5R55i#8UGXy5VX@|0NfKy1>!U#hLGVRlm{R_qjIm3H|s`YB6~~8 zu)`<;=`r+7u>-cR5(+t8Z|Jlsvbxw(8hTi$m7t&P(>M2EZk%Ommv=iaNxmG;2@C!V z2n_0r!g&7=bwj#mE#W2}?g5IBg9gFZGC=_*4G@oDIxS6v-oA?)bm0K$hGy|1nb6pv z@NoLRtDx9x9HHxUj$r0swcu51)Wq{XnXG6~B6;9qa0-zit~3prF^_Rp|C3oy)A=Df zPjdRdlRlZ!Z#XQpMgm!HqeT!d9Tx)prMk#tWhGs*`HVox?)3r8~Q9?Q$W) z`#mm~Jt)hvsfijkhRtDIMzgIvcTMlVZ7bF>h2ncY+33l~8^AiKT8m6cre{po`Vw9`hLFUU4acg)oWu0buZv@tNqbS+wgQ;Gye&i#9b@dQSewfuq z0DKQ?-VA_`u|6(BFqf1st^?plG+bH>z`6I?>*w%&XFr_wbyJAi#mI#6pe7G zBiGX}G=r=bH|P6;16QamK)?HBJ|d9$+n>xl zO1LtlFQgfrlNtOWPH0YQ1njC-H_WRhaMuugBFyJZxu*q#ISst{iUn|Y?I&1#jKbc1 z$;PWXlJc^L6khg_h84&Y^0EgQ@RB#I<^dN$wB0mc_RvaI`#V~y6in?BFv7E!YtiFoGeSO=& zEm9cQEJv4pK%U2u3K+?sU9o}LbsW!5mXqmYyqUZ*8tLRcn%hqP%`TCVO7gGj2AOeN`fds15tc9 z&|x>)S;EoPpw8PyM9koCE+pT=Zi=LQBg|W4?APgVQ|R;I1bu%_xchNz3xktgMIk9z z!xHd9RQzr#Yw?v68VkJJE6%0>I{z--@3E0!xo^Za1s<;xEJF|-36^&$);onNdp@M? z{P?8GZNc(KgTU^+n*rcR^h^PA&dCZ))oIrPUlA<>oa*9_fsCpKovkG?AaXy}avms>NRQlXTsX$Vb81uJfJrS&` zeRM>_!F{M-X2J#e1*{Xib`v7jF46JPKQ|h^pM`IRM!(3q_OGjJLlH1{Jdz95P$${= zQ*`i_VAuoFk#2}{l66vW_!VT(_z;9_Ej~QDkzh~&+V3Ijrua~uVP!AUd5O+u811K7 z_b^HV@dPpLV6?kds_;$NG<9wv94fKgozx5-yYZ8Hh4EMDyvg|g+h|E-iQhoUWBr;P+zc(LSJ{S7 z(0PrHL+3l_d=J|tQeVMB8}S|lM8Cp~a15S58z9;T&Qsg+*dJu=ZdkPW6nr==Qm?bs zZ_tt1-V6)<0q6abNY*Wt13B3Y326|tYYhleZ?d8?ok=?62{wl=|Ibj1i)PGmMqJKF zFKWqa)bi1Vb|XoRWY*yopqL1++^bk41hhZK)l#P8Az%ugo)0PIv-tXC6S5a~ehNw)UO=Z2*wj31=^Ftwm4P>-}su<^uQ`nC+mAAiGK`E%k<8^nwyBRh39xdns)HdtYyDyS%BrWSPj3$@RIrYfX z@%2@_UgvLx&n!ib>Bwdmr$gFvC`A2BM6S{mfH9ghmNw#m7mhur=dgBjf5$jZ zU-OMP8h>>W9Ueclzj%23$bMNQhsTeOWwg>}iH%vN^JO~Rfcljge-pm2P`dQWg$w5| zUM^iZb@9@f^AqYcG%4Ana9HQmNL4|V@y=(@iX z88SA+0(jqG0T((mk=Z1!oxxBXTxg-aS=z&*O5Dc%Yl+-5Y|WpE(g~Ax6@_&*aCLHK z5%DK+v~V8^hD}hyN`foKA8dC3_duFJSh_88Xd$GSYb?+vR7c>)TP3|2?=AJuP)L}b zo^QZtvWWpC+{LRVZ}PtoXd5UDMr7|KAfB2^#eJdeugFy?YA_Yaldz05y~G>#$Mt&- z;}_1H)$cwb$LVBYO3;3}yyjD#0bLtjG|sm$@G^tEw47%BcwxLqbEw=E+bpL~0wdw= zAgh$oA|j3AqKmUz8Alu}uT^m#mVDr*yWVI%uBE->>uwT{u(St1B!^rtkN%B;Sf4q2 zKP2@kz@`FO>!i87oIM|sDZ0ljx0=Q9%z5-fQtK;j#BotAI1Lc_{6#a)Seuw1&UmdxXm zu)|SfjM}<3N-jRi8->=#=|(-we7I?gBHQnB)i#O;q`*q_&l|dIO}v_DJn$|c-h7{i zMZB*LA+dUd&S5$tIDC}89LRyzbbg$VN9dfQ^Aa6Va{gudzK_lXogb!Cqw`aA?$G%e zIzLB8NcS($_v>`-WAVR5-(LD?(kMDgkEfl@^(1PdTHVNQxkJ zg6N*;mQK{_`{JCDPN24OiQc(_D|;k_8`0G;bxf};g7lcrv(Xx+-F((KWTbFCt9>CX zYYu^@=hG10Tiyv>jgNeIu?kYcwmxpH6s$tL&{r6>1}W&w#|v@$H6hb9gxr$P4<<6O rEYGZDU`&)*8SEQ;dbk+-uGn`K9v&G(ww_@A3b;>J5j_=@YMo#SdF4L^27<<0QaXTu-WxpI({0gU?s2WfB6LHP2aoUY0 zNwk@&QKI{u>nMW#(gjrdXj(fBd{F;3T` z$Kx~p8IDgyi}6|iEXSv#FU3#zPsC69Pa-`dW}~O#r~RknXZ&Y4Jr{jBe%60Be$Ia` z{)+zSR4W6NKSbkl%wJpD93Wj*Fkwm90%oiPI&>8 z1#tqD6GRE04xibz{BMZDRjqMy``E?KN+`olD?HQfM7I_$w{NDhi6!WYFj{<{Hn(AA z?!`2e&E+V}%uDSg&D>XeX}jBLMwxYOs|T|6-44=A4LvhD9fXytvm%6W?8&)l^hH2321*@{V)yqy%3QZ+l8So&3 zUK0b3!YH`Wm5~UN_I8L7xEHpPP+ac5jJWgq<*Q3w88-^qxNumEh_{|K+SY?ELb# z!luY(zt?Rwmj}d`E!Qf!^lh=B?#0kYU{H1sm6c zq}7bVpwoNC&x^e|`o@*zxk#Rdo&^>Ql9HQUsblltsgD~!;w}STA zNb-$Nf|X_~?aEtOi3>)}TcJ!^bU4RMh{cY{_ntEa0Rs$Yy6FS@f9v@~GJ9WI>$*l#-!pClsHlxX|o?xUfR(K?CAu zHbU76qlI>|z+@U!x)##lZeTPEy=Ia;&J|bM;Cjf38{vWs#a1iia$P>v$@oC?LDJBd zGA%20w&FldC&{d|i({FU*4jdZ9p>0ww>SwzdaH-jCNW#kVu_MDX;0O& zaXwetH(SNAlR)t@o&uia8iG{Y)$ZtbHC^i$`dVP51-x~k_bs9I-JfZ~__-kpw{7H^ zsnyr}2J%hhn!@bceTU0fC}ZFDP{tM-Hr!ddux%}_f?*nFW-hmziCjcq@+<{UAh@>e zF6L8ic=AyqKSsgh6r7=e%KKHdkl@w?oSgV2h+|@@2y37k`x2F<78_1xC0jTXnHzvn z+G!A|*6PU2Od5-vpr|wGx0snp7_EG6@B2Eqr)R|=XhqoXAh@S}bn$E&_s;g(y)$W3 zLav>8Hr`r`pNMaqjoQm+d$-cHZs+XvD2mT2SsdOBTU+2%l}ygY3QiV#x8&0p*Nb?P zFCy@Ywr-k+uKup>er!5})Zp#vtidD7rSRJEiK6g$~CHVGDy$|LFNU7 zkXw-aK~@d|@K2QII6)w~EszV~I!Wt9JSkgizQxQYXNXitQpwwv*-HA$ZxXqZACOiU zML2(>q0=%;8o#71ru9hZ;?QyF$=L?fLJd;tQR2G3lQig3v-)=xfPO-FNdzE zh{8&p1$$LYK(H4@E%YGRCpoNhI2A(ZQ*6Z((_#jKzQTgOiUTp1)$%PEDl*WM4DJ2e zX-!kxz^5ZClFVBJfG1g*q5$GN6*q4Ny>8NGeaItSA%M6|n_0O5+VB5 z?S65?l4n&cFTF^mJH>vHs5$L9qNR>N+f?VUwoARZ;2pHeZBl6qPjQ-hfXG~Thp7_9 zL}7|@%6*gyDruGLQY{**4WT3;3-VBoTRurO%*Qg0XC2prNC!eA? ziAN<=s)+d$DH@A{;R%vgh*}>}hliJ0Y-d>G(hNF@+x&h zic;ndW+jQWGFweV&QgJI5oIu+FkY5))H1V~rX3&|OF9NI8kCw3O;|NPoEamQWb9(e zFcfb**9aAoHxPIwTc>|rFC+By=|bJ4I0PP3AS`r~|0F^H6(1T!ZwxpLhjJ+g&&M{W zC~fHFkGJRdbLt30iUS5rSZZf3){40=tI#|HXr^qOv@8sA@FT6l!Nag35CLdt1QdV} zYsc%;`a7|=7;3#$b(fNvXb6n2~)7wo7p#Ncq>EuaQypjdR2!C=rh?pixwmOFOe z;}%P3u>^@zSduTGFX-+C>~6!6KR^m_1hXZTzk#^?J_Wx?0WGQY5j4tbqkc#P!X^1z z6a)zT0`^{>LoCxbGkqh|ulpsQv{K}77+9ejlkF_lN&NpKo`ja6>MUT->ewYy-#+x2 zbFmE$Ej>1Y^JiXcCj603d?R5n2Er*cBmxksUe$UXK-zK?o8GAmjq7it8R( z9SF-DR>Bf&4wK@j<heh3D5hjM89&#g zF43|woctFe<8VUfnRRBUPQutN>{uy@X(&?=s5&4(Uw0tLS!aeWU4Te7Z{uVKZdyd1 zho^*x%9rVsqm0s}N#ADS&v_HVxlKLQ2oCm8NVFHysa-=x{lYp#`TFeo+=faW5@fU$ z=e6~D;q9T7Ml=C|9~>qjFDiprqq?L779GuPZ;{mGfF+!(|k=0Fk0Kqa3?e~K^yN#$Kp8iqeNu3^Bi&OHW+D!mcj3a&S!tuWU|io?0H z?v+halb9F)D7Pb2y*v@X0149W2HP-MZT}L;8cm=qVY!hV6JWa8gqkKB?nKNELHc6a z!(oqdyf&NzN!o-x#_r7gV=s|;@#^2LDWBX_>F2^}`h)nVTiJiHy4%-46e2Z9h@FOT1!qJ;Z5Q9cq ztna|TN-Qet&~l(J4Gh{{`>IAp@0KPX>XV&A77*ZT$LJ6gA>Oy{8h|FwgEp4egmG-b zdLppeQvy)MwTZf70IE!-LjkIe#me>zgc3^uZ*efd1@oY3HXQ$G`zGylC>J5RQrTW! zB=d5SD<|?M<|waIK+;xrC`c&iApkJ4YI4B($s1INYmeY!h-tq-W@k?IQUuYI|aDjIg zkNso9IvXfv=O|{_)sKwR`?1Xh0-8?!LrJZC2H!{>rC;)61bgs}5U&0?2xqdn<-pu> zfpqYa!H-rH9-CVwHnGYaRziM5gjd4~d~0S{McfT1C^xLG81ou1Px;#5jf1ak5~pk` zE9Xmonf%nb(Y2SGvr^QR&4DRJ-Vu(z?&<2F{mfN=gf;NEC zxm|dJN)aqo@xac>>rV=o;jo@g(aNnFL}yEDdSI=Pp~5B$*sb1`t86Zegnf&wnZb|Z zH1#Pw2}$Twdq;mqdk2#g`fU@7YAiL3k!jP=`8eFuS!1|YP-2pon8A$Mh&U*7pO2dH zvS_}5I}$CSJ`By(?fcuO$MwuN6piu~Y+-^^xsw?n$U2(FX6Tp2z^0z%44GH;ArrPUgx+ZOq6SqShj9Hyte3^ z+*RFoiu@BNVUoFIkcDJ^!6XY|QNSx$49f&pqL>?SWRT&4x>yts!-rU1pAdBpr@|V& zQFc0<6f>w-r+VQOQNwx;ZL?zTEz6%4hq%ssID-;JRd2=S>t55J6-Ng&Tn4KWa{xwL z91{=07(9f_-uduwc!cUvyT~*6(%1BlGH4#d={}VWT*nu>@_chEX-1bWUtnJX?&jb! zA{SR?Z{+XX<>hh#_9&BrAuU+&s1)qZxpM z{vPb&6l*GQIgA?zk^=={!4f7p16%m6wyWdbP=`u#59$Wai-5{Q+t`xb0wmlO}aWrQd2uAV#(sw3RHx;zI*nu(IlM_(Wi*r> zukWoFcS`+I9|}vqxLeS1rQKQY7tq&NcFO(o8zwuND^QD6kKyL`$9o*Mxyf|Z`|k{U ze>3m>P2?WbqXlKFLN_Yh?;_Wu@xME4;cni--OagSUw@CdNAz8uGkYKY87q|ckYYB% zOUrO9<}=$hbZv7kw{jWfYvdnNu#8|p_#+~;#t45*1VO={Qt%TB{tN+bXQo;k zy)>X3a>h5lBh87Td!sW(oaB;Qy%C9BdfQX^{SNSLvE8{ItjKT^9{?=l;@`L6WNOkE zGcJF=0gD7y0e`$f(aN5$a~c)CqsOOc{Lzalm4E60pH+{pX;_YSG4jk5?N;j7=;Bt4 zzL-feN8+ocW-H8GSRcuacDg3NK-0ZL!7&PA3XURZR2A3Lq4#HDxwJM0Z6)L-qaOX4 ze7JiDlEKDO-(X-Auw^g`zrr6GiTsnJ%;OSjJxlpSshs?4beCD2%cIJi!R-EqMz)WG z%hYP#X7abh$0LbqF}UrjPGof+(yQCXzS?s(zU-^ zFz>rX@LDTh87a82(Mqv+--8>v&Sq+S~@%2)8p zY`w$2*{Syyp z6&Jkto{?{NWgUh2V%LA{K{fcRlrhbYGztIo18U?>cfP9N8@dO!Ka!a0i$%U&_D?>b z{eeEX?F$F1WVRc9gKtWEtvz z;h>72(aa=qj$h%jNw~R1pVLaRh5gG1)mJ{E#qp(tvwQHN<=~sg14wcKZK@j(j~y@g zEHX^w!$duWAhXo?83-OB>S?0dm-#CozT4qDBlhgE;mpi4#>>pr#LMvNDY^0j_*pr} s&Dobbp{&O!C;2u@UqLja&=pc<>h)MS(0s8mc5c3IlI(aE30CZ9e6mvkV679 z!v6t3s(;~~W`C9Tye0AcoP*f#$y$ZL`qf%V<%3L<0YCPplaoLQf;~8%z zu6Z@CThVMh=go21j^^V9Zy`SB9gB~9$KyqBk?E!AM10aa$z>-x6`%G_$7j4V@mcRI z)5_6Oe9k+^WjA^#e%O1M%a!Pn_)+gsE?1++;`845_;K%Xu1`hv_zT__;wQW(xIP_y zF}~nk;POng9AESgGEpl3(8kPIT4%$EmHI>5r(f-~LK$|N;e~D|y1jI@eJhm>EJrs8qviK#i|a;a zznX@!u^NS$ex;qHnf+QXZFf73C^O#Z_dqtj*Fk-yu4basL0&1wx!mov+G}-@mA;v# zVTU?pGo#|wFl_{lw1E!g%Z=uGSTxr&cP&i)Ue8~X-F`1SLp5|yy3*%~`Mm}v8HJHA z8=cMUkx##Wvn!*(PukmIwsiCe7{`}k&~JvAhV}Z@>nmLu*R^csyTASBwc;^X+MTc= zv+38vUesIfGRDZoh%I-)_(f6mO|!HqBdh zdm$Fcy55Mfk}|6lh1dFNub<*hb@YI%zZEuuY~jsrv#~lPz8tTHLD253;VJV$*lP5n z)V~?F*Va=%>NbLq9zV@vqS3^P&%WBhrp95=#{0a%i<`U$;Z@Myx!&*fx)Sej4J&c1 zozPyy?Ighvy!LU3zS;S>u^IYd+)Ho!?TOKfy-NI6qnUQ)?X1iVqsHw}CQah0$zQps zB-c;~6;B0xh>{w(fe5ugL(-9qP!EkoB`^cE<-D!5)W8btI~C6iN};u32PWk$J#d2Z z9o;Lvt>}v9fW<4BQ}E)|E7w-Ca;G2r;jLbXm&?lQ?H~v{IeW|;cmw7bcZ+t-Z!!~S zrVP`*>@*4LhCeEsNE%WSAxo8AWly~iARDLy#TNq&xi%2U_0$+B8#)AerW;*BVZ zFXkc~-U^$2a7SKEF2*_X_`K%Jz1#8$JmjlL$>Ycz%T)OJD~rPZV?=bTQE9 zlpXBoo;EPJwE?Pe9qY1E*X0F_B)>%2Maq_u)lGSc%FiJ4%$t||Za>XbAH}D*IN$6` zAaUv+n5=x7`f&DTlrpQ;l{XtQcoUVxMOIZEwW``ky1HGPc>AKEd=`!BWnzg=BMyB( zb9_S4ene&0rv;CSI~rs_>S!o$QC?DZ!M5l-o6cb_^E7r~ z2TD-7hcWyGh74zAdp_UErCy=jmI&*Xw$CrMlchBoLPF$Hy58s@3zq5lFJ3)*X)Fjz{vOYL;IN$ObSl?3LlA?bIa2=3_}HBkRTYze@fcvt1^S^+Fg<L~$#*aY@np7-*Dbw1)t#>tf857{aa+w^iehG^ zJyt{>zXXpndWmmS+t~@#tNU-1ZZgD`{rn zA!}h~rd^tpe1)dKSnphA7+{mmD)yJv0t2z4DFj58)l&0({`YNH>`mP4NMM+eow;) z<_=NJ4RTG4z`Q3;D?4VO?AU1Kyrlf_{7wm_@}RV+>^K8!P}&qbW%MY|0R)D9cHDt8 zP;^WX>|Y?AWcIv2CG{u@9pC!&)}uN zxEl1eX9Vf(UI-cZ9*A$$1z$aDGYosUU7v#PUvKnaqTgtTH|2FS%G6gpE9thy#0o9a5OQ(DFsPLGzH`q-?Cj#~PC4#Yak}kwHTm`bVT8 zO|AOTnTI9aZ6Y1`%@nod1`1=XSP~_zqy=+@p$xqMgV{o|LmlR^5t=ZZ9b7G3%YoLK zX1(bK73fVnsD`BtCz#@Vn)8{k9F(cla)Meg3q9(xL0tiB&S%pF@{RmBxQRk}ZyHwN zN{$>QVw80_D@o>3ar4Ho22S*Q-K5Q)phLco4>bdekxa_WZITyQ7~HwM(+UrNB^0Qx zAtg^jxr|kFZf%olhK#bcO&0aQUO=#6QzXZ}qr3$iAj%1{pAELlN#@Ogy=lmcd8uf< zOszYm0ZF@A(4He&Y6TdZ`kYX9=_xincSGkf2@%M`FU`>t9F(YkUMo{6&}Qk5J3uQR z%AfmEFY*b@Kp&WaaSwCyr_$+yO1*(ee22V+nY2bDanjRSTr0aOnF04`%(-+P6rKaF zE|LvwG6S1O%?*IkiX^Sf$g+k4e!u$HTt~f8*F9_rHsdxq3zASh(^nfw=vCkXcEav0 zKS^&#q!WIGdy@E*H|~ITV>f}I*_+B$@rW5DvAb6rQ%8KE5kmcl?%KpQ?5mhtxnpLb z6`3f2GU!$yi(ExRNwSdHTe#Vi-3>`*MP_#M$^tcbfhfcG3FB2s?33w@H0{(&IjZqg zX;S(7&j{5CVjW3gJZ3MeV;TW zXrLHWACw?X`h5eMP}wsF4v$#Ih-LOsgkDQhFE zyZP=!M0kd>n6eHsPs4_@p2^g$Ox?`X8?bfgm7#PHE&&Jdbpmil8sS&ypL9+P-rTH<}v7QYdt>i>x6b!ZG7K9vriidyJq?SZ86KA}5k`{~DF ziV5AJ22+zh6?BNs`och<#b!N0UPi@3xFxg!VkOQU&dZ?_xM&YLG>bvk7PLxGg~lke z#&E#`)7j7|e<^Gtt|+fiAZR5wgGU&W!#%C#@JFT{$*hKyq{t@Wo*0#UF~B3(bXVKa zQ-V>r=m0LFV$y~`Hf%pIn zJcbNLiywiS-ygjQEVIi*B@983L<@UKeQLt`rx7v>5fbpN^8Dt z4Csgsa)%m?4aE^|IndY1L%3;|bZ8Wv*9>xBXc4%jaGvaaN7(B2$)nb=0JL`nV61W& z#>N&bgBQct{~y6fI{yoaEpD=9vEv`+)~rCJjMYc9#GM(P3Q4wnIRPN=PQ#mR4$`j7WA7>;cT(u?G|i(%up5r&4SgI@gDF>APu# zdLmT|K!X)Ymf@NhjX1-AX0IHG%YeIhf*hBjB~O*RDn+^=0yrZA*T|Pat*g9Aa6kVL zHAGPO-7lk@hf$buiDxtW`KS@E28|cKN{maAC={4T`x6lpI$IhBXD4GOdB1IInQa`LRIM8_`}-`>*cAw!EN~D7E)?#SU^)tfO*D43^yA8+0-bH`8USR3V7AWj^Q`1@YNM4E zXg?Rb>UA<)OH^mzHeq&}kWqbQf=R?g*^y-WunW^_OQ<)FxxEdVyhhnw%Kn(LKSAcX zaCLB~Q8YVE^q(NBmvaSpj|dFke?}!TO68wZHsa7sZD$5^3LC0NLHf67b00%58_oa+ z9m}2h?+D--H>OLwz3FAH%&}7^f!S-X*xFQ3yW{{3zT+zvMrV z4dFgeJq_Uvfn#qC!3@BCZk!gu3^Ndgk(!?d+Gu$aCMpFqhOj>bd>4RUK|ca~^AO-y zgQ+`)2iPydA3#pD#3)b)W)T9J3Fd~hU?$94)cF2C#0K?d_KjZ{sib}3FlF$iO01|`-T@+~}c5`D*3 zF7Ib%@VTB`L^JtElriF3Q635MX47`)<2W8+z>z5{NeVMFkLAfZs!ZpxMDpYF({p)@ z$X1dckIO$r4Yt)KTn;-BN7$jO#Hy#z)-lOTuOg8RYD2)tJGWI+06<)5MO9P*i@J zz#jlg0?83jE}+7;WnoLg!X*D1AeEoRsq(MT0=@*l#HWp)0}89vk#1$-yg~0oW}5@M zPv6(Y_K5->Cg)#(1do!<@amCj97P65^oPG8Z(Rl{u6~YL?VBIbL5m}iZ29qRm@X`LR^Ki(|4dO2RaVH=C|OdfVzyh z3sfMKCEVB>syw%KY@kEwBA$mg#0kb89NYVhvbr(!%HN+~c>3l4WN>aM^E zhFiGGbq%pWit%`e-f>)xplU?h%xPPbv?6{dLED}!qK`m`<#y+W-;&|hwz0g54;;1^ z4?DEW`AI;soK^A-7=G_!XF3iAQ*WA1wS)2ZDoJKZe8AIahM5hcF1guG*X0tPEB}Eq zI_{GvkquS-J1Bb#umzi&!(kFXZuEK(qzuNUY`~YMC&3KFle`LlZxa*;ADP1~_|V-e z7c=B`HqL1PgY6(QxJmx_Lqg-yV8b(%pAsulzzNQ@MX)8v2t@9xH8_G4Pz02*?NEc| zV8az0Q%T4iA%3j?di(4Vf;Raa`TMj6&(az!9=TyY3HLHIs3b@2gSCjoxP z_*#lr)~nKYGs~l}3`eqgExj_|DZXW(wfCl(63|I+dlje~u=y)1G!=61_*l<-gj=!3 z#<5)fB~IaW(lcbHkhdyKcp=CjK2D6~|)MvEL-Aa;M{{C=e%j9;?33}Ltqq^ry%?Kfv z3pBtvX_PSJc#j^{=M!36lB{?YZkvQ#efn-wlIiX_?91VMJA4Xp7KF^m2VX*&{Zn>! zGZXO(eE7p(OK_Z!hT8X`T_(kxBm5oQ;alp*x{QDhjBIm4&*|H@?yGLy`tNuD*S*szm7Ekl_k8u|&))rWsnj3vV(`P<$MFrm zmQJO-l$-Jl&ukj5f!}n~Y?-cUFwJVFTb66JGH#}ob+fIUn`_yw-O9W9mg73Df?JSs zX0zxPrHxXnY^IDn0ow$^rc zduxZgL$2-S&ekq>m&DEH4Xqocm8yG_q~x2sTQ|EmxAwSuTDQ2jNSf2UwRM|&TkCfB zcDXJzr(1WpcSyY0yt8$edsl0(ySFvt&PZCRxvzD%d$+{P&3jt+y7x+aqtpW6BtF*sc5)_kUQ%snRY z?akw@S$9_AJDSh7PPivp&$-VTDgSQ&-Gkh3zE1AF-hIfedH4Ip#c%N*KVIpiHi%HcSrfs`ZOlSp|oPFY0CQ{GXe9F0@XA?0cB8KgWD zr!YemXq24Lh^JgzvmyG1kUG~gVsp-=zTTZ-kx})aKR*yHE^;W&w>Ue&0 z=50>;*=Zxn)Rn5gGHplp(a=}*Gfh9Tjx~ZX${lMkp&8|l*O!(W?S;sGb}4Lh+DOWr zUS2}M{OMPgeD6?WE{rnIw~@v6j<)IxJ|bqjJ#9uKPt*hdIB)sva=5e{Mw18Y%R#+) z?6|)EWFtI#q8@~(Fty-^wGM94syAz4qv?CKxqACTJ(x~M`2%6-w|VzytQS4*hjp(W z)}vA{i5)Y}y{S`huRXjMS`0d>CIeDis^eLkezW#+ zM>V}#&{*+t-}2Gf6VIQnojm;H(Nm{Sz8Dpcw($^XCiE|cQU2Jm)NDt!(8$EWPII}{J`vyHSflONRW$b8GcT-9I@x)7I0+4n9P2DJ!r+AJ z%=rQOi=@hP%XJLKN}U4|Ka&~dPSsm@D=I3*5k11_!1hCjj~saZ*y-8}&mKJRMD4_Z zLx+yeK3O|;)IA)PdpA7(lwbFv$rn0v^)pH0;no@7^YHR<*DanuU&m0^UiKRcXZfgg zPabtt?uSCj3qj%W1A{o;wk zQSp?2e%Wu&`G@dE-!?#>*wGWmkDcH%pITm8>Zma2zfH6^c8oNA$e#m+sblq5 z@btDMp;9CDCq`-|zrPEDTJXD%&5`5>KUm@PTK5Kv89BSa&$ zTD{%wghCeq%49;${3v@`DE+OJdIZHJc=xG&VQXpMQe$avSXY=0dmm~opKaaKdU;>7 zac1ArE8*Erd*6j-v$app5C5V+w~XfWWw5U$Xfm_(irR-ZAH_GQAaGJe!!bI+mPlN`x&Ms zj+fwNy2b?~*zq)8!6oB0Lu%zN8>?p5WX)j4v%4lr%;4JI26hIm?f3F*Z+~h*+S;FL z@HfYnPo+XbmkiBC^OC8a(N_zSiV~^C^rE$z_MBBq=Nw9X`j*v9H?x>|$z08@=DO*{ z>|$=!4()EPo0Y44*Y*l;TFB3@I$d0CPpuZZj{Fw8R=3bCPNW`6ty<@w*R{RkWXf~S znZX-)u97^LwCWYNr@Zob${X35S}mbwX(F{+4xMg!f=_@a3Jd6|z5>DWd8S8aqwJFE zw3b4*jLF4WQj4#rhOcX3=bYar6}qFk#6ZfFr)OBLMJOc=0-{O7E00LZUP-Akr5c>= zEH}OC8NZ76Ty3#qsL6ql@0QlQw5Y)=eb7z;^=c+{XfzHV-0hjE~J@>y&a#VL9m@Yfb^)S=fRv_RPtx%YjM#QKgGsq8S5tPlM0jx-yhIP#`M{#AFwqXM?M$={aC6d^JC8m)jnq0B7wox`JQfC6C zizsax_BAW5PV>H5<%i3vJx8iZK16xGT>+Qd5y3Q0EgGwUi$!x0SdVFCVp^p!txQS7 zsKqmDC{yLK_+NB;y=&^)v%MjXH*4I<4Q5WSwMc4{G(0>>$TM8qmX#P^=7J z7v+O20^69tGpQF4jdFp1zSi{HdYpxvhEbCbX5lLj_BbM4%=vSfzy@xB4SdZAfoGw) zU|hmu0V{xtO!0II%1)&OdCV}~H7^k@(z7cM#CIK-DGSQ4gA=TVXY1{1M^&*-*PGR_ zQ|0^D{feill3BztR(!*18UY68lZ<}~0j6dGGV6zD(pzq|Ua2KG!(I?gUPG;50)bBsl#?V5-;oz{yknBMu#GqWlo95$lFv7G92j1^$I<&fJKK5J2>~u!P zeAE70`o*&U>J)<)8LWG|arYBQO}hVi6b%SMB-Etqwy9o1(h!ld<6Od};~zuBGpy7q zRf!x$&3V&Hd)Aw2tUsA9R=jhjn(U?bFCh60y^u+yV%-FDg~12?Vfe95&qGXk%bbFrFsZyC5TFz6XT>`2+u|tkG$d>pCS3!h!7`)Oui2hVi{Nq z3rt9A%>Yx7fv&~4Y^|ByG#H_5H@gOe^T@y;71to8snlu?B$f5R5PcRefMGXRkl%SQ z4k(>P8P>JfBGC}F(@a5HhGclz98HDUH84DvjYS)?i8*<)Y>a^nGqXZzLV<5z!{_a* z2&SDV6P&G+1HRDkUyjVML(E#TgJ>g`~u6Zq=mctkfd-)PTwqI9d#cJs^a z0Ahj<;mnp1RP!jJ78smG5IJW!n6ml{E=T2hA~^zmgL7_SsqQ5szWyEV!HVTC)sDA5!R9ME_YkIy1UJx9iE_gsKEVm%g2Mg}i9om%*z2I%gX7pkL!iy&3qZ^?cx0B?p|Z9#CCFp6KD~?Pr3Oy9S^2 zE(|Ex*t@ElSDTjc&<~cB?(Qpx zK$51IhF=qighdPad5=6*W)?_#5pAni7?4pDpm@trD~Jm~EQ80S&?^d+nmU5?DBWm> z0tc#yOE>$x5LJ|`Y3fotSSVL(if&~(8(D$hoL3i+s$OQ$za?+;mJ#J^wYg?JfRwCJ zfPAZ3MeytRYEjlE`&j{l$**Y?-(T5%-3c?3bT=|C4zVO0%-+gAg9`vP);<6w(`*EG$mX4-teVwqPe8S_0eYrbpd%J1Azk@6cbHX-idDKwH`Y~Jt+p6Nr9 z2g+vVEkR5mEf6ycY|Sm&9?%q_%@7LfyypNn^MacW-bgXJ<+zA4ClsXO7BJ2C4u~e_ zBo!|`mD%Yii#J0Jq}>Q=3*e!>Ar>T|4E_e>G5XVuVhE-TC}5FO2Q;y0;TM7l!i+4u zDfHjiN(KOFi|L8fW#&QOq4ps^A3AFVF2|QaQ?(+eo=iXRSt7`k6lD(mawxt;g|iLM zgZwYD>^S9=p(U{4slUn!oP1Hf_lQy6PZWevo;t&p4@p5@ca(wT+kwK15B^q6l3!pO z(vZCA9#Kk1{wFd6@`z5Fa-a>FuV^|(7F_=%meQ4(k2LE2iC9b?3HmT!-N#!K^M>95 znJZGsL3{&B*fKj!%uc`@XD7!O%o0Wz>=WL_qNTP0^iqqNwd`d89MS-K5FIg_K^y5= z)j%tHkaLQxvg$D4Agk|WAdmmOh(Q-vZ=VJKEwh!}oOIH4dSkj?u9n&J4hjaN2pova zpRx=Q@`YY@xfpXofR-b2@ zywY`Z`AuZd-VjV7uv4aGT@_i#EMK+Dj9EtInqsNz&)~!4z=l6bLy0UJ{tM5X(C@>;NBoNIs&oh;T z3Qm2l4s!yhy!z`5$c?M-VIbt20h{i%HI08aQlqjc-g?xVtV0}6w*r1%#<|wZSDRpf zjkPYIIL4*dHYV{c8WnRiT`?@6!l>a~UAgna-(e2~RBu{socq)`^DA?I2@#p?!thLS zP(?|=*&T~enEOc0T()Gkr`8}$0VgbZsbG2_f>~=e#HG|?9^beu@*$#;iRxM+CPBCw zL<^#^7SkOWRrUQ$5Zq`o)_Wc5GB3|D=mE-fZasEP$ud*hIN0u1c{v+1ltPp4E?w_> zW%{#P*d#9EFutdPorp}Nfzv=~$D+0$jKnotbTxn^bLEbY@U9X(a@K*7`T;)d4>I_3 z!Oa&@Bg@{E5wTOy6OQ*lhIt!6hD5;65Sy?W$mA`ERtdz|1R029xfpC9e!ZR281Cu0q8fCJYl{lil6}>$T zuAYo_G^RDbV0aSYa0qy|pp2l4t2u|7*Jwo}eF6I{OpwsvQ{vH4n?Je~6PqwjHsh@9 zdCjfl&Rx(Et5t{E#0t)9{JEjFRER<%y#5r5S*fMt=dm|tbuS7bh?LB z#Tg5(M0C|t85VIjHyB?I=dy{ma(WFEc9N|;)N{cEiR zr4rh@oE}ZVN(QCWvZ1~eRx*=ifR|2~Uo3Y)zI-;6aA03U4oVkbL9@I0{i*Y{uAMy3 z+6bg68`q;(=|lL4j7DVf8l)`=13cJ%HchlX zO3EdFTh3ICDN^S(SXQ7#-xzbOicyty7t{^@do)KojCHL{7W1a-LfT9Wo*~bT%K!kv9?y%msAPd@%1ud&`MX|&NSSOtF6FiL|;T=Cn`paj% zVo&zVQ~HC$k|pwA#&v{?B&C$1slib=tj((8d~ig7U0D1fF@vby)2xts`sfY}r{I=| zda;OyRA6r{8-izq{X+P~I0mk9kqFKjrC9e;-Xv`VqTorO{b)G0Rw0)ejxSC?lbhb5 zxydabkT=;A7_C{(2!AOMk>+j6OVe8iI7xDf>PHYnD6H30%x%l8@(#5B0_ zg++PlXjrAK?&Nj4)m(}tOc_nO&$xCQVGSLKM)cAmT1X~FQ{ZO8fm`-+`s`|Cb=rQQ zew@vlON}T`K1XgD*%Fzt7+|8KLCDPn(WI`YX+-ET<983txS~i#*hL*Jxlf}9^ zJTV4IVl$KcTNX6ij4N;~47gi|YZxQ7?mfD3{a|;T2jqS%tX9LBakt{HR}GAVTY}Fed!z zFERFu4E`B{>Fhelifrie8y?JrO6suCW~lTA2bkQxg%y()-t4sr3bknmh(k*#ILO#- zO|PFr1DgQOUuNrqI>qaAkef-Z9|3xP7HzkaUJi?{=Jo`HD(@?n+WXYsoKxDQ1e5HU&1>p!_#U&Mvn*R z?JDkgWLC481fJx>cn^rU9`u3eb7Ti$AYFq->3dUdX20-@!DEOdO@OSsXh4e; zqK_hn6*K`y6D`*l#6M9jt4JwlY*o$9fBO=<0AA~Ng=gDs{>oo@dnt?3B zlwj7_d*02-WFg9CP3LOaTG{zQPY{)c?yG0#1|D$;LSDiRJU~cDJ`alrc&_o-L`cvu zFQn8KOnqY>+CzW^AOLR|&~iQpKLDxWnOgou$vXy0LK3DqlsYcHzgQU!6(E2gj-r70 zUuQC(K1#!enZUxoWvXy8Jr^YS)r5fMe~28ntsv;2ZvoJy{Qs|&oTHs0~r7bL1Eg9T@B@?o6Akj6KF5oiD`)^RO;1+|t=K~%!O zP~!z=8)|9tD_F6H^kQ(tBXchLFmN@x4y9Wy%80y4S+^LLL^I}uBclI;SP^T)#==U> z6^(}zaDu}~h%IW%(A6aBZ$+!9ht{TsuC|5SF#(Ll9UdiXjG;)lJLA+{(jw*V8wBq1 z1DZAULl~ALK!OYS-OR{z8VJXA6QZ#!pPU%eHx8iBGptB`VqZHP70E>u87n?5_F)})X&eDw=`k*V9Y3MkPKLCo$Kpl&wW z1>?AQul3YRgJq%{gc}>uAPwfLf66vW(tTm+&{|X;xCAdJ(;^z<5X7*jpEfFpO8}-c zfX0>lg=Xgrnve4<@#R1O>-$MX`JjeKdKoDj9i%U?_ zYJ*IAt{x_sENW53a&IsYE}k_8fQ=&+ZGr25^Oe|A+M|a*=Gc+4C?8@3u2m=--nQ}C zH?6Ji=8f)Sa6f`MGQY`Z3nGLZ8I8|loMbezba0_IW7UTC09I4Fr9dr3<#&$KKWsBt z#gcT$_)wpNg@%X(dn(jhG*w?FBMi;KY9Y+6*_Tsm`ECIz&>zTn7gtNGV4_Ww?dRj(C6q`@3FJsyCC_|_-nj$iq?B%_*lQ;REsE6e(!9TDCeo9s zDTrs2snrqaLh#oenYUI)!xA(fh3+VzCqkaXw8%s(jPq2v8ZpCvTWQ=#my$tF=%-9AtYqhXCLD8{00h%iy ze*z=2B^kTqzR>2By6{ZyjoD=HS%=30WiWE_xGio??#eq3kMB@>g+XT}FV!9UtT_({ zgMJOUfBA9-DHH4agfX$u>xt{MfgD|fZJ=yMo_7j8n~-_NXU6*l=TjW8?!7H+!6Il? zbVaTW%NgB;Lol=!^@&kNT>iEG$lA4ksj6)S6S72g+H=cF!G)(Lg1vf{Rezhoa|})(aEk-0QK0cp z;AWmBeu5=LX@EX*i%H^@1CvEPAxgDzdfImenz+ra+{bYvFt zELj(@0^l!urI=$GD5ijk*{c;xOSH6$;F>rIWtFsT(qaN5Xqs2jE>{;RvE70bnY6+- zt*fJ0WuRsN&eEBTYY=4OA;W?&Z;AHp&!Kn|I^p>um>|p?(wrQ$HGO8X4||APeL#Addk&2EL{CWuBHEAKH*SL zW205PmFnwn{^3ur{n@Yl#A8GHlQbIdiL|%kqA${h<{JEz*05j_{K3W1|B<3AP4(u= z<2~Oqy>boQs@e$X=iI;)28q+z`Sb(F=>A5XRIl3XkTAK9)%s<0MSUGXG$T~*CS`2iu$65?ot2ydDhT>^la0#U|(Hi`p z;l`jHkKjB4&&>?q1N^hVY;=(eZ4sBt+*W7^VXUOK2|ir_M`#blBOYFzhuB?e%E8{c zxpTh11aQ)z|gkGSv0{}f*CT}n-zlt6bpevIttKp-$#537oiBs*zjZgAOvST zbNUBUKa?hg%&uhbs^XnJI&EPq6}LP}zcp6#F~H%!il248j2zIcs@LF4s?M{s_@<_e zMA9GZV80-m`XjcOTQ8xCTq^VQ|MFI!AHLsfycDSWL&R#ln~jK#?G?5?YAdF0n3$wg zenj2ao0#Hz3gwjPjne;=8{A+of#<1DGbjB}sSnS}^rgsIa`O;5oy7t~PK&=N<@90& z$tn6puf*7V(JOl+5JF2b#mgApu|$u~0XYa?q2NH`s=M;&ac(r)OXD9LM|)c3>OA%` zRd4H+s~!!i2b#_5i=E}_tnYib!Sxhwaa!>eX}~6o{&pHSEp0Bh5JU$!7q5o8pT`uE@q`x zQGlq#YkaXM3?*oYr4bhPFkFowM&qr%f)XEb)~D)U>VQ^^dFF9+X zqG|=Tp7jCNW-&Ts&&e;$cd$f*Yf+=D@Q)B+*Mfk7JSlslheKxQufY#nt3fMub!6AilxxBS@?kP&aZjWcIqI zX&a&(_72E~%qj#J_y(^!?HQ?$6$K3IxCG>~-cKB9t@zY$p z;pSzCxKCrctcmFFGxr?^?=ldM`x}f2k-pCuDPCP+K(bL41n$V;*@Mp>I()Kr=&<|> z9imO4sNPFc*I0q7IAtJkbM$S%o+&NG|0FYiios8_Ny&tARgr0q(wr@uaL0d}wD=%O zl)=$~=Agv@hf5lSOs}si@V=sxFufi$^?_$KvhS6%kosf$%qBF~)4{WFRXbxeE0#gE0oOAYOMZ{1X&e=7ZgfhE2UCqL3oVChV%=;1>vJHP?EA}V0a+P-!>u*>!N$&gvZ3Kg>CcKHEVZfT1#cG-3 zDhWwFJFgjC`XCmuQo@J>Ty`W()<&k3{36p(r|g%o%5ts5O6iSatt`u0IU;LiJkG=G z8#m#8lSs%%s$8*na zSUiacvUu{93@n~pXt=cy+#_bXA$tHUn3Jd(rXM66IZy(Akc)6thPw#n+-1$B(IHGAQnyZ>xj*na=Nc(Sctb)8BXM1WbBs^MB@X+ zVG7r!uQK~J2E%K}Z<4(EFqBV|S!utY#hkQZMNQTP`XlMIDRX67PcosC9uGB0L+jQF zqRqfV=mR)v+(s4cnqP+AAq3}EQ=OM34d-EbYhA{QekX{c^ z!F|iir2C-CG5CJ;Qr?b!PxMcP(Dy1+r4a$-JWDzZdeE%?1&eHgxBn}7Z&n_wc@Iw^ z*}rGYhB8R_&_qxxEqVjW`?r7SdrPo(bW@lm!m2d;C!i{*yBV^Q7yc;lYN4rL<8$1F z3WV1pX%M=Q@>O#iLDdq+91W_rY|=2_vFyq>6l-n()(TqTC-^=D39!;S4!3Rtn=+Ek z8X8&&v2gv01rb`#i>?E!Iah6L+2Ak06z)55%<07nh?S69=9l(#lq>opyh5zxj{<_n zJh`5)KnNNG1Xl!t$3ZxiXr#YJiP9y?P|v3gXA$<(gJ+5*3;~iMq$Em``~@l35E(#9 z?E%!Yp$W?A%^g5NY~I$M8b`*9&GVgkXl1Lz7g zHB#qyOKK@b;dGK3me)pLQlNlC(L$qbg_;1QBnYbh8~O>klw3V*^Cc2ckHa5?Nt|&4 zC(UGUw`{Q#g&rj@=K{}@(Oy4$(2(F}R1{l+t`fK#SW#`(5ktKCFQ^n{VM*{|$RPcw zy=+FH_SYEubp`^-MB~UJo1WM?zQDYG$EB!X`fJP>M$}Qf8&Os1{U_wLvvl);Zcy{x zq6fUDZ9U+Cu4b#tjbQsS@KT86Az`PBSc;3Gg)Mx0x^{4L zw+LpJjzd7&F1U{_lwx3co%6a}2IU?>855yX1t3Jgi7G?7QR*~Gz_3CJDzW!Yj<95SL1^L;ytKjxT{b) zjj`Tu;^8$B9AT_SB9Zd~&)hkz6!yN~!>n->Pk8DHR+am%gYZdakh@k}5x6CJhk-VR z2GU7vM)WSarMOGl265>mGb8s7d7WS&OsSo2qNcLr@f%h z3(}qs56hr35XM^^sXmzdE@0e>d~8;=p{4^%!bgP?9Sg+vL9W{d*KLDk$zC(xwQcho z&_4Z{>6m|N6)JzC*v=f982a=Ui!~%~wpe)S1rz!KajybDjor@x_bh&$-lCy-a_X&$ zaG`XNhhLN`^kT(ctb|w@`6K+si$>WWz8e;o9wUB< z&J3CjrvoWjaTD{G3_5j1wlL2om}j@rQ44ZlP0#EL(^F!q5zPtL0!+d#ezc5+gB?@y z)>M(Po0%bu_HM+Y91ee?LN8JKuEXX{5cwq%(FE!ektvJzwIRj#5{3A{$oARSk2t$G z!m7i&;d>BYapVDvg5{(P6N4oT&KN8jqPeBaoPEnyAMtKjN;xjmj^-afVl9n}n{bkg z9Gyw@(<&zXfAX9ChUFwyqQUV=*u*W@MGm!{Z;qRL+%YJ+YU1!7A37FX7d6yQ)*^qS zMf5%+`4Qj(M|Y;4=d(P9`Wh}Q>cVJ0(~wdA$dJLd=FrU0hf5_t!T&&zKt|qxEBK8) z-V&PwaLj^MbOS5^6ujnoUF0xu+#|A^~S5?v~Zo-M=f}2F1R(C^?_ebYVZ_KLz z7{#2ACIds!|e&PS2Q z7)3-g-QpZH@BoIob)eg9MX}p%p-p{>$VuzBcH?jkdBs1$KJyku^m!#dXyCg2)perm zIUMM)5Db|)$tQ6kBZUUPjl08;3-hLja{|N|3~pZH5Id0$RXoz6OF`U{IQ+gvJ--{g z0nbg`&3P2>%$kujL(SkcG0au){48sz|B9K4-{50Ur;wJB_+yNRCLJkXM}2ydUy?(D zJR2uiJ&P9da#oosr`GgKP~QU;z+whF$Y8cdojinJC&aECDC)5f&7&oZ7P36jpuYD? zSd`)01uGfy@mrKMYqi*~(@V{)mco*5S7fB8SIcO9M5lGB4!{z?p2B}^w{rIr9h}!A zXOLF)8KhO}dF3RpsxDo{iAPnS(FGiVhmrVxljzfFgJxZ6U3a>>vx z{orjtW7s~Y{vCfz^{?(>U0`!CLSL z;)$;MB+<=${G}@^n9g!i*7GmG z@e8;wV*9;F8=&aw>9g?8;YmLOhZHWiaY`Zlrg47t%=9?DPL>qTj!!9W98`N4d|jGh zj3lZR_EgjBMf{H$`(*@f2JU4K-mjHU=a^cNb`Xm|QsGe^vdpuqd{0yhQTafCqsrtA z391^fSM*iZx|6l;X6!D;?qlqJ1a6t9tI)I^pPhJ&IrpJVl*ZvuZiz~CX492Aa2;Oi zk~&^ja!6}|>4=Lq-ZiSpQCM{E*4?GknwEjs92#x_3d53)y_L-%+ei zqoTY9qn_qLo)svp`A2s(y3J8kOT>{xY|DNl>uAO!3kw``StfghtxaJ^%#{3TvvyK9 zd&SOCf3Wh{{~yh1wHx(1b0oFo!zGfs*@O57zlwkdF5}V_-S)g(e#A zvpCrbI@labVDAAIv`*x>a`K2 zR~kz)RiX;%6psA!4H|>=QLvQw+>xYie8@V^e_mP^_pn6z6|I{Xv3IP~b=I^aLT$c@ zQvvh1P$735$T6bsmYB_smCfy%oc9;ypd6M$w8FR}>z-i!G#F>owWJXCD#F>Q1Pl+T zK%`%&Cf_WL!r+aI&;rBvQV#mdLL)~HQu<+n)Iq?smqU{WKTe#1!CbJKc6#OB4@i2RLq^Vq#phC|Y5Ty_Ku8cvYJtjXi9_NpWOqPsfv zEO~M1r?$m`yShzg3u|scAA`2C++= z#A{+N3H>ChIdZ<;s1PwXZ~*6Fdc?I%0IiCqlQK~H+Jv}-&;go+&%XqplV7Nu3M;pM z7}4va=c1hUX&*k%xQIUWp!nx;u>b=9lAM7Hax*T2*Tk+y9)ksG!!Aze%E0_} zYd5n7vlpBQ22Zo#i4Q#A0ulsj+>CuSdJhTC>A7I2d(Yp_S^(9Y)WU8^5I3BFFew0~ zMyl)JG63I{E>ENGm5N{BP=XByv4JF2Bmi)9NzFV51>p*RJNs6vG`ch6cK^tuA z5Zkh6;u5ksb7gYC|DFc989nSPH>^*O&rOW;g+Y|b6qSbm*FypSxd8v|!yV;+9pEvl z@riFiln!uaI*xE%YSwXJx{y96L6n9ozrw$<0q=t#%3<#U98{FLg}siCx_AOP?xgHR zNE!{m*XVybg7aE&lK@A+;J)SMCG!9AopHDv`{9=4KT&w`|E1t*R;VHWGQlmx{g-k! z&X-jlxTh#1MRdQ0HLFpPjm1p`XE;}gB{+u4rn@mu0pNV7z$i#whbi1P0Amdt_;enY ziqBYolpIraUD?R!lV%WbjWy5ZuJr9US&- z=L}U%is+`zID$)fx`6{5RhgHcV?YiGuN3d;S=RVG)B58in)>}ry9v3M$TNNlSJ?7O z?(T{p$)KOrE~;;NZli5ESIm5JmBm%rwx7qr?$^vJQ|UH9Uo12B#li`@@9cR%?lOHr zdyM@!Z%fHB%EAap|4a5&$&+xsR74OxLX=n(O{DrvN^&{wxcozmnRa_-ez`rT^`Tlt z=L{sg&7zaKD9(#*1~V}2&z)Q9Kr(Rm4Y*YWGotRpk9e=mXbs}B)~+Up=w)>o{F5vx zO1$9Q^ogU#761X?= zpL@)B-0&4X;)aW5|UkU3zDs$R*kdA zBwu%Q{f)&fI4&J8e>7{TtqiDii1VP;!wEWS8&md3BQmnJv{aodc@l5?*4$%v8Gp5(X7m1_r4D( zt(_}+@1|#}Q@dIHW(Io@xO+EiLyw+-r;4G!Cgd*Y?B23josGI);*C3^%hG8dt>WB| z+n8FCR4`HUkA2kb?9MbZthpvch&vf|WHZ-1R*gX2g@h=hX;hp%W?16^rc&d}mp{vt z;|%sOIL+YO7`(v1W$+S%Z()FceU_4=-}Q;>0*_~z5h94P$K}WeIXYHOpHyFD-j^8s zECY&zva?D2!Nn<5dl%3!t*GL6cPM`WGYJ0^L(ets@WHhU8O9~g0y*9I4kt>7Yh=Hhwu%ujkQ#pYe3weT5EY;3ix zvi_0cBHBrLX>TS{mCp8PnDUlcF5Qi?VVdN<+gwF)q$2b!ZWg^n$vdLxa48GXVr5OE zaZ%1sDt|mu!;!+s!ktJ&Y0A^h+aK~MF4sRUI?>Lw^H#RQ;~2|h+2>;3jYg>o@AIVB zS7BO2F&7wEy}fzo?vQszX`~AAZjpDBUhiLi@)zB|J^A0aI&_7nA7TeRvJJqEz>JN6 z#b#hNSi_3zpBP;;wqxf(Bd}vPu01dU=M%#=0v7~nl%7^U71ehaVY!Iko#}=Nv`w|E&pDE zg(OzR^6J1Zx)=o^eImX8icAhU^>q`B`wEza@T2`iw)6sgD6w1@gnP2!-Kfj45ZIgf zs_=(`@8P6=oa`mEjlXl?A97K&{oBzl_vJ`%U-h})=@*jc{$7-h`0Q}lS4kX5V~8m4 z>BAfpBjNMBQxG40+3AB8Po}(*w0Ya_4zIuF3x02u2rhj(=gpt`ukov|w<@+-RpL(%Fh1<(+gCb3gJ^9*LaC*b>CT)|lc~Q}k`4EkZ&WnhVylzrkT_oOy9 zPw}F^{*|7_n4#Z`4TzPw)oPZG91Xb;^H{j7hhdT@Dh$iZ-*EGi<$2j4RmnK6QaX_S zPQP?`HdF_tTd7HD?%0AurrjhQk|%rQ6; zgYE+>X0a994?*dHeZe?2*9^irafmNR;LE@j-v$JqFk+6Wr+z`#z!hifGurU8Gd$ZZ zGpOxyK8%yB)evjw4;&%y2~}Fd;(lpoJjyi{gPJ-RLOi7=r&tH3xB$?y#3g)6b3b7F zUGW0-YXFyFl(lXl?nfftKtnD7*ya_G#6|H;+dO>nYe+Mh)?$JtZMSM9IyvM4qT_`j zcrt>ngymcq-Wx^fWTX~`anT7w%?t7Sbb$E@(>pv(G4SMg&tp{38MS1tprRR9#|haA zGZ#Kzn%5&8jGTL#la@?TdD_m#jN8PPuB~mY6T1)Q;Ph(T9n|8wh7DeiJ>G~d?(ya& zqYHNx&pogMI4M5Q7x*G!b=f5YV=P{X7csV^t@JV&w^YubBUf~5W}LrljLBG%Ih%@T zvd4Wb$YjQp0QovFAW5#R%ggU{Kpyx(Mb-rF4Wrt?Z*Ja}{{2Mt{dbFOCn1}&UF`CF zW9Gi}{3kw2zN|a?N)6>38yk_>Pxjh{=xq$+?#7K*uf5v7{^RR6UiCEAwutnS(=!%A zR>w}HPqOAmu*uP0l%(XKec0>|-}n2Y9Y0cUoZF^9>h*B)Zq(uJPO*{fWLLYH)Mjyd zl4qdwV8|0yS?tp7XzyCOf~%3px6Y|#9gKESi1YFybVnJ-P)9P3pBe-9sTneSEqpC} zZG3Hfo!H!kZcr_!aq4&Qb@8p?TgSJKZv$VHL0$YBki~4*mOuQN5nIRRUzy4S238)> zOj3F4$WqOdxv|wV(OX-C@?-wIY7Q3A#va%gk_S9{(&s$Y=jge1&m23|{Fq%d)(kQs zSY=Ty0ax!fS;N)%?zz2|)rwvbGX0Pp)ig%!9L8$?4)uOwd}2b18?pDGuGTQ8c5J?9 z9K%FaCkBRzn56m4@kPA?jn6%Ee0erL|IG1K?AH+gs?Un3ht7&!pMy>si|7OW7B7P)`jnTj%k!s%dHGXR0eZ&R_~qP@sYpagR7eC75yO;n~!eEo$eqSY5#r3($<&|+hk ziFx9r)-BRID>h>`yPl;P@xGv`EnAR%FWpVpn@E&35S6YcuG1hBKcGP~0bR+HJ-MHF z4b6Kfvx}|cVoYD`>ks=PCg{U9XVk|rDfgKjj!n*Gi3V+WjR2iR+#v7f_rZm+jy6SlK$LB1Hbrhx35cO3&mJ4Nu+av$U3*tv&J zJpcBVs4l8o|MOPs_rxNq+VcjDJVd-5CmmJY?R6TV3kGW>(`ixcMtvkjp(1Z0%k?bU zpG8w}Rg?ap3ALGIMY3#G@v%+|w9sm%2HtHQ3+U|40%OIGuuo|ZMKRFSEp6JV3O`wz zmyV9#ODoN?T60prPxyaV*{fx_Fy_j#7R| zH65x@jQJjl+yn5;x_*~P*Usuzoz+d3*^HU)8S#oSt%auh#hQ)RoCCIZwqmUTi*VN2 zl2y08&lhSF$h^l3cHQ#6a2<@B^|K|*HrIjaF^XpKp3Skt^O@pIqseR_VGENO{G@Cv zDMm8vCvnX4P}5nKE5!YYWK4O#tVsp;9U|D*nRbVgk?BLhG*McI!~cm2r6U7l*A~ml zj47}+{u;j=#ulLUHxBts9VXidgeg3>gg-V;>@hnvm2;WGWD`L&jXFK!$W`tbe)c`% zm$!~;z^;vJmy9u1IjW;o*DZ!tW84@t#?H(!%VQ69cGMg<2P;R;*gaVtH&0E|xc9v=W#qN3!z;I= z{c{0ar7b>U<0_W(5hxwH-}iTrCm$}tzaUP-V&k^L1UCl%;3m{-^7^sLqc%w{`^n2Z-{rO$E~?aG5&}qY!bLlV2frok))}a3CQ{W ztl?>+k>oMoFU`Zit;8yDD(ImAI&40Pq}wG@pXwOX|4D9XlSM|7j5I0^htu>LIW+R$vL^U2MFd^e&;mDwpm4WU ztN8S1sFzJWN5XY>RB6*P!My%i6;DiM>1Yh`3t)el3v+}yHRitof}~_~4m4eXU2@5- z!p^Y2w10Q_{CNsLJ+ZYSuF&FtK;YX1zC(blM3waE4A$M&e~3zpavicJ!XiKpRD4W; z!X-ifJxWXBrV%w?A@HpHu!Vsiqeyailpn5p>}N2uk8O*60eky7bJ*`(*Zb|2Yl3iF z&kM5ArHdp6cCFe>AAokOa_qYPrP0pw%89MgpN97NKMnfe!S|jq(oX63oNRx^_mH6s zUjBv|6St(DlqN^;gKxmDVnSjm{l|O~idInz#f`z$XHBo}p(R7KCqg(#|2VN@or2qf` literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/module_mapping.cpython-310.pyc b/vllm/model_executor/models/__pycache__/module_mapping.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb01564ad0f845bcd738ca664b54f44316a75c51 GIT binary patch literal 1909 zcmai#%Wfn!6ozfrwX3hWOqg&RE+bG0B~2EHO*EPv5<^-@quKPPRB5|AP3qPzcS1(Z zD%rvluw=u6S75;d@BrSj;uX*gaL(yYXC^ZOm;Be~W4ml0AG^wSJ7SOye);v}cb~C8 zu<&B?pzsiK{UwNGl4mUEL!OJF;8+%!le4U{r z{-p!9Kn9Ol9R3O4W3jLenVNKzrFCt+@l<6JZ0=Xn+F0*NF)fP`wtF!-PW*=A~dudUJA z3X#6HHrG~&dDUw8%ST31sJ050wPUdh=k`|Fy;b&Zl>^Dfkql+y%7atgkv_z1({_&L znVBA42X4%5SE_Le#$=q1z^50slcmLEo=(7_*vvLYWl<=svrSW;D}8OBs6y!!>;vm- zW#+mVx!|YkgnNJi^$2qPDTw4!T(R#w1J}>OTd+$G%1Ple0*32HIzPG#_b2`X9x-n) z;?VCpA>-}x7&2|_0&ciD)BXfZhAo)kmqK?IYyzE)zyaVRd=NV}n%2{zHfb?Z`Yq^g zT|AtydhaaF=ISOE^}ARbn37KKA&x|`#9e-uKRtNad;4Qu<_R1Zh`NL>$Nj^s9Hm+P zV1M<6*wydDNAw3MAEIDH(s(h~_kd&nbuFxadJboy*70@oNQ?~Ws26}@NFXApQjr)hn-cVQ1SO_D~(O`~3~n}Q7wmsiFE45=Egv7Xc80`vVH{x2sMwVeO} literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/molmo.cpython-310.pyc b/vllm/model_executor/models/__pycache__/molmo.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5dbd8beb2abe07de5ac9910c592d8ceeb7a8810 GIT binary patch literal 31073 zcmbV#3z%HjUEkb0ckbMI?CkFB>S;+H%d)MtrL`?5apEY7^{{N&(#nx!#lEpK9__um zn$4cUs8eO=X3;G+ zN8FKS$t^WU-O=WlJJuX`$D3uhEcJ}Wggb%0?8ansox4u*vW@l44ekbsJB^9vMt7sc zbB#^S&F)cg{=ELs8Qa0Lnr1_}(XmgLdr}>!s znBoGww5z-q<+OeAa!o`JDTl zq&GE=HjlZ-B)+*Z(>(4Tm-tHy^{oVt}dEkn9#c(~OcY0GuPx%`ceAJmk&V$}V$ayHvnMckpZ#Qyw z$2n(_GwnT$oQLC_y7!1bv9RDh>YwxWFg9mSi zgKkh9NU#5m6*{Za2pq@RpyDCF$-4j|`1 zoD=wt|1#b(^bTTNhwym{pTqb(jn5wc++5Ln#yf&hE_u%)=Q(`Vs#WriddE=ff;V%U z8b$B8_dIG`^gibAk)AE@gm)4-mn7$ue|gUGUhqyMrz1HZM^7*MCI3?xg}Et(`FhE7 zQQFNJ-kkRdl)fx8`ig{~^hf!PdX881UcH)em!+4Q{}}o`lo{n3YN$S#|6Q%dJE=SIh&{wrH4L`Du)`Kw09;+=b z)?4$Db9^zZw_CMFlsUDui1OU2%Zt8us6HD;nHO3p;!~8*2e#&YKKn?sCK0p6vi#gq zYgTuG)Y$XN4^^$+^1XxY)?9s_!`dH)ev4J2@m}s#kADm9?AWA=UX>c-B{V@6wT(=U+HgJ@M4jM^2tP@lsSg(pp>! zG3L;}6h^tDM~_KHz-c{MYvO6G`QEf)T0gc_pFQ{VQ%7Hj@+a$!rTr}rZ;r~@6UR=@ zv{iG;iY5=X8%xdB^YQB)#k$l~wE38?=D9#?l#PGvxznqvkDq8?9L~Z+Mvu1V>tXP` zYR~!sCK6fYk1f@(%$*vSI-YtXayXf)z35||vho_fibmqd@ud*2jFM9C0mq*4YhJYe zbbGdTCdquNdB*oVygr_?&hzJLSmEkLzdnC9tTx&;57UXqpIlm8Y^yNnuRn4N&DuF1 z>vGvw+#(alUO0N{$g$&x_8;v}`~bxCXMA=^DL`#?u~7>FbxkbT#?x4n>Z#+;JvCE3 za;Uoh#EJbcRZqS2{8Q1y5iFW&`d+;j`lkf4f~a`XKfmO+X8l8ib0ZoNAWSCgts6IQ z-mL#Qg0~Fp>bJfX!7M&Qe`%zi#3x|jr^q|F<_jr5y2ad!yZGwq*nBH;wE%cXJSp%6GL+5Ukd{OHnrP&sQ6MD=O%cW-T}u%uHpm zUG?^?M)@kY&r-v$R-wN|Sg3S=(SC*`z_&e2 zvX^$V$gyQo)2<`&4DxL^C-E%ej+>WwPT~cL=MgWsMTr*?2OB5x65^w72?WIU#t)s#EoPkI}?jaO43o9nzySk%qYDk|V*isVa8a$<%dY2KXWqk|?TUsBqngtiW%~^`TZhi|lBm%BNz}v_K82 zpJabF0@JXJ{LRkxwH9Z3sL9Z{xa0Yeacx5cqe{JOE*r~c7h_v6HmAC#{+=^Wrqs)* zGG)pD)p5qts*113I2#!kR}Xbyon%?hpk{zYP36r=qqAepq4rl&n#@eCR~(x<#Na6g zoE%j^5T)y_P(8yWhoYWm@G%A_8JuEpn!(2zyo4Z12cc3flb>KRGY51Ys#h6z7D+W! zcv;P|yoW1vj8m_kL%@dwoTz+i(kvRc{yU~=l?e$ieJ9i_f zECs$-skbV6dMl^EV+5oqzM7ti>{(%xq9U*-sEtm|re#z*V4UORcfhv|Q&&f;b7rvN+0<)Bn7slRsybng^;mzPZk~`nWzC$rZbsIb zdM!}r@BsB9f~i8336`(`MEDxE)%`4#1?37x=EX;471g~=rLiy7l+NFCU4x-oU<>w% z-IJ?Ug;NFom&KGNY@>|jdI>Y!H5PETX*lnevJSqJbn)HJsGe2q9zCAOxD?soHd1l`n1s%u;`;(dhKV`>@uX$3Rb%Lk9G_fuLeqLn(&Kp}`XmAZg< zWW>Yoy+f46YF?}LJEB zH|j8hrx{#EKx`wEIO~Iz92`l=Dl(;Y#LnU)B+yA=Iy2ULrfJ@=)1~*etye6c#b@X* zAEyH1gyEf#OXnWCSo*jX}o5m-z9{L>Y zuFc$COm~x~tOV6%SaHnx|!Gwz|11 zDfKTr8}D=%F;Tc>WviFHWJ5s?r#e4#PzwMRVVNs+ z$X|XFU?_}7&PiY*gut^d8u1U!0!Wu3DTK!8AndQQ>SIN{E)QfkL*U(GZM#quOKc z)E3su_0o~UO-hDMD121z*FSe5<_FWw+NG$-CP|ji5>Sxy_?*M%LP9#)N;pKzZ(Ru7 z^&)ZYKXvMMk(;UsWFd!w1vf-jkpF0M2(Vq>WuZS1MEyt1Snaj{^TmWON z+_8A{J=l|hD#7PH8Y3(89q2u{gdm@FOvkW|NwaJe&644md1KNjn`K~m2ee1iPt(3# zJl~_Sw!?lL`5!YauunGHTPEo4&3q^@y~M8XKF=c8O4J9j}{ z#K4Cb8_FJ8e1uHjKm;rd(MYaqE(7H(TP?6Fh#AXZYkCAc1N1@yo))s5l@!pEq#UF& zD`}t{Af3-bW&;{S+XcvP8&b=;Zgy1}dWy?{ToA` zO8Q(gzgz(7DgaTVhG&r}d^59LY>lEk+kldOc?9Jn3nOt2({tXmmWyGjTl8{Q%$3n= zspTTxVXRw}Q5P}lB1mF><}GLgZVIjfavJReX_Y#*dCZXf9iB4XoCoH(4N-m3wP$Nf z0W^I^lx{ClS!67BCeKuY%Grun*}bpA=uX6^rpDFh(1H4P281QZGmrt*Ri;17;0?*V zQ1>sY&m-#Q`$JHFiy2OWw$$Hda(mNnwiQ5mZmvFCr?#X=s;Q|XPozY`%!98D>+?(P zr9k~Pw45sG?ejMfSASD_P5{MCfagMFUOKM`?yh<196lG^?D=YNwzlZI4$z3iWLS}n znSkEHwJu=>1|$VxhsZYlSA2vMbf5#U0pFHU!oDdPTkx+T+=lQliQQC|63guDxFf8` zuQ!!W_zn@oN~>5q^(j19*gCM+?RGII zJLK)mwE3QCneW<}(mUo0)(vodHzE1lu*~ue#g?~leB^!jr^qWpW?6s22g!mCr^kdH zXtZa~b>5fc5Ab1USaK(BO!Hxy=}OCEbig6z%2tDvD806WA#G}et^M0 zL;zg^^#}T?eJJehq)EOKA3TZO0tsi^ZHAvrC%1*_Fz9#I2kF0uf}8lPEvl3i%V+T! z`bz*epl)keRQXE@?Rkpr6zOBZy&ysfF>o;&6_|%+oP--nwa&qQywaYlsCbX8-V4EB zbu&;Rkm>~*s*uIts}sz>>~4Oon3 z+s(y&_j2*cJKHHR8aM5mon5Qus|U8zdx0WqjbI-lN%N`fz*giwbMwbUe=H`cj2M_i zK}3U7AUGx%*jqo07Fxn3y9#5q9!owm1_r2fqqFgLW4X;1A|B#Lw$S~E%%`AagSzcd z>Xicc4s03-Gr&SxU0|fnXSxF_w;x4|$c7mWma8eNziXnAgoTEz0L{9g20*{Jk9KnS zd5!?|#{u7ky|iKV3k+_esgc{#mfI5)u9fHwZckPI5vpwFV5l%(tu5GIP%NiD9|pQ8 zf2OAlUK=A3_+&aaB0_uw=mq>%cIG!92n-6W})*>Pz^#1x@hX`4>>aS%oG_e+k`Q-ph%8&-U_O0s6fH zbb_Ux=8vVt7?Ids(HntwV?+>132!(WjcaY+fU>Xi(5(u<)mY;|tizwa9sT446aNXy z_&!AXD1%B1&zMM&W2T~_YK55C2v?=Z$jO-~d@sX;MrhVAF?rO;lkn(S+BEMztytW(aK=jK*hPI97fB z#8Zclyr@ldC!=u+oQukzt6$PE6NsugnuNZ-u1(g}UI(JGcJ0_iI;uy}Fa1FdRoDO< zJyshny)g;{a^_TKKqeGPFcSa;`dS$-L{A`EZ~#%*;{@TIh(OW;4ggPT*f(vk2<7-2 zG8Z+W?>QN;3^&Kj&gNU7s)ukjlKUwR^A<^qQWm#^Y)yCyQ7+=vG$|tiUYLUHB`K`; ziV>P&dIhls7#TMKYObu2Bc*hWfFuS9&4hB5vRT{)3a)hU;iOL2K9^Rnpl90y`C7^1 z*=an{x@G~&K#c)qnXV0D2U*I?LSD+CM@#BKsRguQt>oEKat)|=_!Q9aYmgGVMp#%V zcGHmg@c&u72gA8Eq(AG{ym=rYFaM^wA~4=(iH4O#KZ(hDi^#>zo~h3+;dA&DaQxpz zPGr#n>5kMI_4yX`^{NHSvYVsj8(Gkx6RTZ^Fb0Ll)TsLBY?1{zque6wKRSR10A#Q8 z3N99sALW)>0c^#7$9J=+)?5nx$T;j4U{CSYVjCuC6vQ=T8${{ZS{RvOJIZ(=95z5& z`)nJ5z|GD!0zx-BgcXmGLT_hP9~n$Mx*E|IYMw`%7F*2_M)#mYN=pzXjLi@%tecLt z#as_aH6cw=2*LL|>$7F^5yVi64wFl`hXU*}Sh@H(~) zh9YlT=`CEC8^tsgIbbk0-MSz{EXL?xfyMAR8W}q=%r{{+ejHWa$6;>gFjtu-Hg7Pu zYQdEPtwX_;^=I)J`XkrGzkr&AVZsg)JuyhyVYJBjS*V|3yN`_(z%DRWI3VA-g}j%S z(Dn=b<|lUcf(Zj8A7ow1OA?MsIOdPQ48odum^;d-RT8p43K*JeyJ3nCS`1t*CO#VXup6q>YLSn% zVw*&WNcZ zFQy7<85=7Ks3W71r$FxoSw6K;r0JqMf*8wns)X^6B6k$uWBAr)2)1P_j-DpWGBADm zO{A)DbVWERA@~i%*Q0!c`r&Y+%*SN7N!sZCjfKs`2B@_T`I>V{a@L1;#ZTdrFop$~ zyTp{Th54R6!QQuq+mMpc<9jPU1sF!sURFkPcUVCVM=5UuUdfS>Vvef!2se})#1IWE zr)GoFZeBsd4mBDm2K;+#mxz0XKY`My7WnEXP!}f1H`wIY8IZX+JfypSjoDim{B;EG z1Tj@#T_H{x>I#dm_KH_IW~gst@#_raL7zn|vY~LJuUnS-HbmSFgTw1HPXXUy5iWaY z;nt^i@)^!~v^hf)oWx+vUl7eW*tc_$eNL-T&_7iv?qq+gVy@F_mAg2Y`#6qLKbM{y zY8Nw;X%iwxPKyeBioTJB?poTQV7I{mwim9)0@?z*?9Jg?I-9FRPa&%1p+4vTBzz-E zCX|LCYDIHB)1o?r+8zI98u{@*Shct9oDGsll{!T_$jHFE&P`LaJ*?-vTLgC2EksTK zj_GJN&op4-)^?j7{^)Uck#RmU&JB@FA_jgA;uBmz0MG+dcg|AdA;{l~c@4S(5J?KK zI?k;`wvLPx2;o8oaR>%l=#`d_hY%Q~AE?<95g2LMdeH((dHTml3M9E}(gOuL2$ANd z47a4|BOFi3TnAG*0YX_M*4cX}Q!7}R5eqscBi zuuA}wisfSpu}l(Mv1wnpM!p)g&?7h-$w~Ke9HzTDER1z6vkZ(3J|{0vEr1iJD+(5e z{HXde0@$p617F}d>A0%?BlG_g11^hRhTmuGzc8T2T>Ulzurb0}{sA+D_4x0MeGP$| z#vT{dk?OCMaltLfOlh6G`T}bYFPipb;7U=#D`hKCC5}Ppf$AMA71PZA9*E)Px;Zm5 zHMS;)Le+&xqA*NOZZNMep|7hS$u~6a&am+jmRzROY#4UebJ1Fm1rn>ip-J7 z5iq9@lJ@zGSrGWoCCrC5&57NX^ki+$qo-?l6;*EKJU~YQ%R<@H(Ab-clL?&98=gVE zL5~KC69s5tHl#I;GAAUlUTID!q&1^{5zjd+MtXp2Uq3+l=8lUv6noif3-trYhj%yR zqw3P4V0yKKSvH)Js&g?7$a(FH;MYM062u~8;D4Z^X1a*(HH<#XSR26~bNMTXkPH-o z=KHo($I(ci=o=_|JelbpvKCQ%$Ro)@M`?TnK|F|vAPC@LY-QqEYVC=MDV_kcO1_vr z6J32;{ZF(~f6U;2AxK_?c;C&&@2|Jw8hfo^;$<9EEm^T+k_HNZ$8;h9{2_Z?TgTYn zULGo~xxMbZpZ*BV{)iLGwYmL%0{bt!sQjK{i(7WiI5PPcTt$%VoScCZ88E&2U<>K2 zOJ5Uw!)Fi51S4kpp;QpRJ@MGndOfiI^x=cQ6176^S~UaBaSmFyBh51v zZ+14z#u}X3{tjD%8bwZGoS7xoO8ywDGtZ4=cxnzbwqp&mA7#wo{FP^RbH-kT8@O1>&?c35;_2;a# z#$JAibG99YPLA#xmhhK!*wFv>yH=KSn1>@sXVa}SIJDhwF<19@`fUj*mrJ~TzX=Wf z6VzMBvF_nmQzk8-w@^@Xay8v_Z1e3c+446O+pmcMoNUv8n9@)?SUA}J-VpG;HLfp9Rsukssz3WRT{p&!Ppw` zbA)R`mfQmT^q~eWMc8|E_u)5SXZ!7^A)y&F;PAPv27JK}Gx;jT=nqF? zLTukgm8Ur-O0R$ub8UzyeM7P4EfA4ICjWx(N02}SC_*`*2#E*xt<=DG03)LE@v!z+ zaoP^YHgsMAs%WwLAdlXxIVb^Gw+N%LrU4^yE|o$oql$=!I}pay0>?SLalqB6P>V6Z z)&!0p5w^4zF0u8*E~_IK#8vRl%AgEhl2DfB8~-$dA<99IRg+4YDit$8T7Z2xK6z;} zXxt(F^c)o-V95xJv08+3UlvXl@D)c4^Kc;*yzg1;9{Nzid)%biXaOFTXpt2oP$7ij z2v3AJDpdBQSS6;CdXp(A_eWQt##t$QfcQDUu^6DBxWQ};sY8Ad?;>PsA3!11ch~GZ zaJ<#HTzRC@R+UF9+LiJYL;|T#3r;+4T4Eatt6SS4sv(!Ih2izuZ2EJGAfCev5Yf^>`Nbj;9-^PufP6x}7 znf<*(Ms=Qhhd4G5y@F$mnPbrSLJAQY0LIm{=o^F*8j`TNu)7%>vo{-^Eo*>dkH!vp z7Jzxfc>_&FzfN%%pn$_D3+4h0y4q>pQmi4G?#R}M^R#07lc>HL6H(*+I!69=E_cz4 zDb?`Cm=?>%ZHJ*Ud>kn{FktV|3J2)H{%fB~wbr8@RQL`&R%H>1nT*vB7JpLpQS?2; z#uJ@!`w{5{{~7^Uu++=o476n;6s;l=HZ7^NX=erUC8Mvj;Jsi02Z7&SV#k$zz-}LK_yn_gSAaepo!^o4iXU-86bGhTy{bmY9tt5 z&!rH9KRz(BxnRJVE(e9TK~Nh=Idow4U`R#XJOIwV1~&lkAb2+N!C6UuZZIEmtPMF9 zHC77ba!6+uil9N+4FFzpHlUX1(KBE+u;91~x%w;+*`hF3#>SymCl1_DlPG|Jv&v|d9mVMoT|*y{dOnw&sLD! z>A?$^+3?>ry|egSR3ApuQ}BwwVTjb!I=unJB-kJ67>X3G2#I$_0siWBD7$f@)^mFa z`1s=(5)aQkwnzONzRxc)AiJc#f*?wR%4lC)tLfKTimr+3CX2s^L7LAW(b92P*$@hT%~0&O=c|I5Wl(OGXWG#@Ey{A`025IL)@*fI)!54!212y@l#A@(kC@nB zClD!T%GgtMz}+I;C+QeYE1C9ZJv{2_w3xY z7G+ChpO^7WI71VQ53sIfCZA&DjrD1B3zdrez@PtyVq31_(BEjBx_PachJAmbyhJI~}vPxZ@|K{uFJ2VOsdoV9lpTB^b}N1C=)P z6jKxc@-)>(5(M}KLrut$Iztt@Hh4N} zrbH3EjH7kSPKzXxgnp9{dhyBvQO{7M15pPE>vJwwOmK|r#iC?)!7qYn=E03-K_JD- zv=ORIkV@xD2E-FqrSm)G>Dk~}XywUMroRs+zLP1$coOGDi~^@-dRC|cc}1NE;;K1E zs26_>Zy{QUVC-uw-^th%gWgjkqpn7obBPuD$XI}aSel}9@Bw@~Ii7sV!?0U~ zstiB?=1vn62u=9UzvEmzN>>rlimwf+jdGM$%*`JFBMAjX*8+oRY{WMd74S9%TbsEh576FCuTj_O zf(UpfT4%|ecAX7C(sR>|gO-Sco6%35Wc^sKE_v^jw0VC`nB05}@uVaJ{mnjh>YFw=}k5Ee%~ z;53{Cl8=S+-h$A*GF)DaJrQ!k)vfRlor0L+Ng)%7h??cppq014_g&h}y#hZO8Y zXm+yD9wH%71Vvy9z*pcBigc{=lr#lVoMiTR8U+#vmJcbK3Xz7Eshf#`0l7q9tk=!Q zZeLdbZug^(JSOjY-VxgfvfPq9gddhIUgQ0D~P0rck)(936 z%%?tQg_sQhG2T=V_$Ej~!#Sg}B_O_cQ|AnI&l_;j#dd(htSAJ^=W_{%^SU($qCg%e z#O)Ao$PHpVGqRiok-*t(@Gn5vd-2>M{)LQ14jk1S~gPRJcQ?QgaFYa*1L$yTminEw7 zgHwM&Q;l#x_4BD#>Is{wtczVb_6 zSSB}GM?W>KHHoi~k}Y(cvpCv@lTV|z7!!~H+Pb}*sa)^r>LAmTTwUs52lnLZ!<^CP zWJ#K2frqNT!sXe9CsNf37c4=4G-Xhh;(2Kx!M80(J;R4ta&I2NKKWyupM&^_t(<6! z8%C=b4fTE$0IE55bwmPWXpj+T*`mj-t)fky3sO8(h%2%%(l!Uj?g1yyvhjYzfg_rj zavUoUhBQM&Ej^IIWs_@z3_t)NBTo9zhgb_XSURwdh=H-AlLdA?-JWwBVFADgIv=$= zgr2ACK>D%u-3Vm)=BQiYVP<;+@bGd0J>&^!#LLjH6m<#6_DB~;no*haX_L^#%+z&lN%C2TARDVmnP&s<*Tp|=arWFT)8B}ng0h2wA) zH4sjCW3Z2}^Tzk4x|3eHyRME?b=W);GpE!?xD>>OvQIkegt_DG3%oNShR;gpxr*R% z-r-YYZgVl*Rl#)-fO+K2;&Mbp2)%j}fsW8zp?utE;?rmRjIVfhP&$dyC!Rh4TRbcl zxJq%h-Ds;$(OYUZFULJA;dDg@^CSj#-4r#nVqA2HYP<(N8G$%hIB*eaN1Gphn_(I zqa3G9?|++=s!3g$Mg8r1f!p;I9)u&*F)p!T z8iisiJx198?39zCYAR-*;25e+FSvpVHkPumTf{qP1Ka~u7Vwc{$q92qoyOZmqneoU zY7GI5Rm(g}x{>^muO{OlxT%2%7%VD_L5i_0Ep2xYwJBAn;C*r74wwMa6)6L)iStDP zf_$3&DB;nT!1(HNoFO9%H%BT1h5-XkG^9~RZgYb_@-H* z!t=Cu3FHPp#Azm}FWFlRXxQ<|8w%Iw!ns!{fk@FqCYW`o-4ma`gqk>4NaF{v1x_2% zQVzU`m>f!i?&Mx562wV?A>@@W^r7v+#TJ!hPNm?3?z+k-{+rj58PKPXH$~w=B7Nw4G32Oxna{xbmlFbW8%S+HetRnwhZ}Vb zy-EPI4Q~>pHsazljYy-hCa-HH>cmcv4f;G*Rioz4mQTI_|Mw?8@pAC;$yXly=+6B? z0B#Rwt)7^Aa0it_E!;?pyRdNuDGjtBw>VoBjp?p6t4P%ic+w{bn}X?ks8yp=b4ugk{m4UW4vCk5G5=)-uP|d%lmf1&iOA#I@JjT|q}z(d)Fo(bU?KoM#%am~ zd&BLKGVZWN*0A%50wFKBDn?^ziN(~Q|Td6~quv4MCTeVe@9F`qVx z+1c-V%E!dbVmL#8L~i^Gs9)&2r-dEc^0}TSZZ#9aZ91*yi3O+*KyLbtuF3 zm2g8+)7$8;i=Vt6j*y$Y&8WMOoBh56RMT1MUC26D*NyMdj)=lxQ=8sm%ztsN~f=H{f#^t=N8| zQ7y>H6;iz3RdjV~KPtJ!_zL9kGWNat1vGKf7jbel3n7X}e$>P4WgGj$b^cmuuO38R zG)5yL^wYg7+B`&bNjHj<{f0ub715uDr^btj74h&18ZRe)I2ipRt zE^s%4zCH?R9A&n77Nh6!F|fLQox6PwHwVoJvlw&lkITppCnk6efzW&0)C#&B(|(D= z`2ut*rOLF`NF6Xbr~`@(>I=$@I2Ksy+#ljDI(j#WavfR;q_wvbv`{=;dc_PrgoDtq zL(`Tl-cmfMO#defAnMF$Zs!1oqZ#!5h;?>pY^D(xu2Qbl0|*1S%N9(31?VfjQBH(~ z>CTS$p5;5%g3R0}TYQMnE?mZ(UTiPIt%>*R%DrS!9$sX)Nv?)#rw3HaLJ0PuU9z$b z!b9KC`y{)Req~X6D;wPntggV+!`RGQeCt5bLvsEDT=gAjT?BJW6k3E8m+Ex>!yb#3 zC)yR%`z|hZq%1tN=6CKsxqMpYa)=M)cE+#!X7GxpH>GA^d#?%! zz8^Q#Sc-aHqoH{}< z_lVF_g#GX5U`K#whJOfeXf$d1Nqk6nn4Smec{*VMSFC}%#Ej~ekoBQ( zM{njz1{a!9-|L_rc8%17Z5B>S)(S32gU_Eu6&5{A(6?MVAcwPYZA9`caZ%zyrd-ko zIB9b7dAZC@j`d3&NdT9`-~r^Ic5;Qj2L)62RWTw%iGmz?%vn*aF$;Lp1ya6I8OWun|&{X!wkNP z0KAZYu`14exSVaN8Q?bz;G2O%QGUy{aj+b}qkxN#k!#gMzv=}H6qlDm781Q74 z7oXwgWY_~}C$K5rrZnk%|A152v+oV<3GOt?eS-hbBEtRdS#|7zA22?j+&OBf`;m{ck8;Kl z4sn20`XhmOMjktWI)u}VUowW*V$v$}*J@mE``y z3`-njfvvUZKOJLS(BF3~r?c^vl>iZvyZQKxXk3WCNDj%_Mk3eb50C5L%rJ=rrjnB{ zlLb_Q4Vmajaeoxww$8VW&)dpdTt)^A`@(XpdZnnui=}Yg1%AhY9Le9|+fmh_O)K!M zyZe^W54TVq^gg6M<%sza^f{J{7${65x@5GtC2nA#>_u^)ZlQ&r--yi&t4C{V2I>1b zNmqFO(TJl$~48%UQdHw7MTEUFO_>eiG^o#vTT!LzS^Ng&cr9ay}?I_&oyT z((H3o$$ z`#&o=6UfPiPzmt*-7mnC;5Fm?Wb3rIPU?ILPt5Xn46qKiw?Xp0U-Iya29mdldBJxG zDT6Dzx7oYP+w!J~_r@vN5S~X+M>rrIk_#`1DQ~Ov@+A+72#n%Kr8h_*?5(v~Qg?e5 zNlkQ3%;6aNg5m{U5v>19tbureHH=itIR6u!apV=edoVA*2I88&fbWlE)i%Xa3 z-t+)|<6{bUw`lfs;CiF!MyonEtwFjou8ZlF6KBDx(Mitzc6jrxK2SET&)>(TatO0i z?w9hC00P1 zT|80g4Bl%z9p8{A?lXLI*6)n*i0{yWSU|RR!PaWrFXLIRjq1>vcScydcRaSU@zzz> z6$PnxGSg>pp+je4V2sJgq7fa7e~+j0*ebDgS_?(JR~i~(GVPCivqrsa4C&VjxYui}o#FCKYGwH#nbzp-18W!H%aYIgLFOBT?i z5U&jrQAP^Xr#YmFK|o2m8SBk*oU@t=7CC~rEchYP!WFzEJmKM1$D4Q-4r|vP}#vH%zMx-ja3D_f7h?@IDTU z+678qIG{~#i2SFH4VRRXn;!=0^x%8Yi)&`;?lrxF$$Y9!48((r6o} zc(;t2mYsjw+?nJieq+#rzHMgn-*9%Ie+UM+`~ZGK`T}Cx%D=nAQ5=c3nqP%3syz?4 zv3d2cFsdk{{CWS9s<8rCA6z$2ov&Qw=Ld>xXOxrQlY>`TpvYm#%^)yAUgodW>G_rk zk(rc*kj0_Mtf*{NGS8fQk*jzgbcu=d^GA+~!Bb9QxsLom)pV<+#@XmAtWq2xc7dYT zo}1&n{?ojQUcJe5Nzx+8@ro03F zCfOd08>)T=?64=!lj<%u-oilcMxl2@+=J*K0(CdbDh%#n*@Sc>I1!g+>YvC`geUjD zHA-Ut;OYJkD#T|h{0lTQ|aMPgRsRO zL{Fezlmo>)3M>J?af)o}I~lyk;Aa?Ah%5xHQudGRV{&D%T-_}PG36R#xs6jFJoeb= zOALOT!LKp+BL;uM;B5x)Frd$?w%zSvj81FXqe;9ZzMV<&)Dc8aEu0vp#mwX01w4km z3J;$_U9WcfaX0k}Y2w!vw8_hz=oR(74&05sOsz8RPmdtdKCYJ-aXpR&V14kzd;2h~ zR6re+12Y)u9;xF_xG69e|HCm6_v@Hjj7jKd$B?_xpte>!EQ;Poa;XEttpTFt8B3e% zaU;)$a;1E+eAnjnWvhIwyik5{Y)^Sxx$7J$f%?)*l%M>~(z<>s&Kc?FC1YruC9SP+*)H`$;hm7v09JDy6aK{c5UrjuGw zW1JJuB(uRRm)-bCG8fDx^TB+w5G*jxix-ol!BH+3;$zA2;5e84_(XCtIGLOZPI0{$ zpH9vMXSiI7pGnRJXSrOCpG}?%p5t;Q{aE{AU@vkTKppJ4?OvleBF9a`exfZ{e zoDa??OTkidA-DkCO!ShN#d{=j*DkgkF(>A4TfwE@(!_#Tyse4FsId0(jv2fH%u#U+ zm}5iCSAaP#P5^Uai1{ioC&ejXP7N`y0&`lN0p<*0q8E0o;A=wvK&wCV8J*8^-B#Ya zStJ|R<49Ro+i9k}_j*~o+iAqg{;=Nz+Wu|_^}3-7-^jA4Lyc;BT)Y-#4bjLNbzK#& zG@7f?m2RiiUa6a^v=U`uuNSV!Zoj8aQVreHt_+i|h~lue(-E-|Wg* zglT&#!X!S?HJVx2>R^u9E8V!Cblx8hb+z4z8d6Pv=eNIi81*~d&4Vb6HFdSS($3QN zWw#lnSQt>X@AMn^#H|K>=I~Pt<$lmeuoY63hlTh0S+Adge(rnSX5;#33FwlzTibyL0$tT#Y!fssEo7d4bwKTvta*wckJ&^|R5sfPv5wwFJn<;kM}bv>{} zcB>ct7b8~ailg$g**9*N{*=o0Q;YJ)M7jlt^Zbr>M7L!-g3pl@}-Yqp9GSl-} zW(h9(=HK$kfs_yB}>y|E;fNJ7B@UU(M=6`Yxwb)KHmo(ReaofA7Vw&Q)Bgn5XW9{VG1g+N-=UH4hCXAl74r3OYd_Yx zh7C#IqFPoQ&NrTu`m9l(Qb`+_!oBBUj52VZY-g3-DWc!gfvX;vH5e_*ZK@~`sJknS&uu2UHBnufStUiX|RU)1%;lBTJ4)*oCG#2ru&AnNH@us!7P$ABuo7Y3V8nQAM2JXg=g&={y|Uq_M;#%*hR%VTHnF#zmHRMLVc3itX&Q#C_T+gQ{5JvM3uA}{K{W@rAe5R#dH*J$)58E(q<<+(jQ76RLWl^d;>>ukWRpyh0A!Wx#p27N? zWaUN=oyFMV{REXCLdbWu0fhY{W7~$%Xlu~1eO=BEY*tZvkwi!r`aJ_`%qI24VnkwQ zub8lH<~`#hbKBkawnf)((cH&u(Zz};ep>^f@SMP`36wTtbv0QOL`#~zZbVN`8GAYLx8#X8&vux zfo~BY+g>lpcd2}pz%>HP1l|J(Ok!p|RQiU}*Oh)_C>-EkcuC61|8w4Wkvb8BD7V#> zn++*8QAy_j47UPinAK0?Kz;n^xGjBa_P}8dJKZ;ym+zNJUOSB>g1q`6$!kBRaw!a9 zU*fz)F4Bx-VkFrWStCHqDbE2YC-=6bN0{FuOnJ!7%_xpX9L=l#HjyxIzl0JM(&qs@ z&9K3H4-Lckg<~3DI9BDM@huBA3pL9!9-{qo%c(t()bBC#Ou-R7DV^8EJPUunac)^` zlr3SyC_BPIxZuGw1GX&Ti2{spfsL>a7APtQdv@*W`^&kUlX`5DK^;9bCB1B9KSJT4 zk%eal6HEToz=RR7GyKCO=xa6%odGT;W94Knd2|DNhm5h86+l7#_S5*R+|6bARm>4s zD^c7B6aEG&`3YP`NlisJdyP&?Vkyb(mgLvS*Qv?v_j2`DrkV+#RkB_zi0)>GTk1~E zm8{R`J5_Nc)}#xs0~1Ui@PJuI@==aikr#6hr0FvNzU_bkJ=Q1&O3+IWD-P+GyjFU+ zHI;AgM6>8Ob2UMp=jfr$rS}S6W)ui3d>9s1eH)f|7aj)0nv@58nlT>X*?oPvZptP) zN6U17WvJ%+gu+q z?AS0Ja1Ct`lN{5grlhM5aV^CU;8}LL?2IFefyvzI?rR}VMU26&5cB6~OAfN7{5mF( zZvrT%1LGWCWa(vnIK*7yk5gs~p{Tu&4acjOEgItqy!KPz(-MI1IIwPH7pwZ#^k^5x zeDx${`?x(CNi#DVDG_EgSR*)cusZl!l1z)_78pqEJR&k?x*^Rjq?@TnZ3dKxv0S&F zG(WIw^jT~e)~OT&jO6etA+8Y^TL;M;JK>!3)JiP%4|vlB0N>3mH%r(<&mraSk?OCG zcVxuA0d@X;0+RtJtk_1w-{bvqy^Bnh>xK11CEV zoJcFTJl4~$@Svxmmyveyh0m}U73m*=Rw+W*gg#}ID^Z28QMm;_x>XWWq6%F;#k#zT zW2mukT%(N1veFLuZOX}A;sfppu>*+;i^%8 zrLu=k@>K#P6ZOT3?F(wrC|MM?BrF^39~^wrG?cKAM?kOq8*Hi)i`LO5=*Kk;owT8Aecb8J-t~OG5yi)nt*bWBkG$UTKDo|dF6G}l% zMk@#sI)!GMGJUOD*^at*nG(I%0@zoM4D1mMFdM4Vu7jPMJ? zvQYE%+7}gzG-rwQ#G{gFeByw@c;r1|Eqd^PRH5C0A4*O?D>PLYx%w%)uWI&ygODdF z(_GJzFpZu93S+pO4^FjHXwA%m~)u#V38%9mtO zgEG>At+q@v4&G!2oU%EmCHpAL8wAMR4oaXz#5N=(9V|)9)eE^9`vXF-D*uE^Txa8wA;a_6Z;4iwZo!nu)h&kt&--0SA@vh1L1 zefrD&6FLO;;gyNVcH+qAo(k!_+?D5VqSN~&-b(_Kf5Dql2%o$1aOOXsIP>JbS3WmR zjtqx4bm$$&cu;a{KbJJZW5#0&NP5zL2H0;r#NvLp{>eEvWaf?@%DefLZ-cq;qat!| zR#aj$QYLIv8k&x>sNA-LikRXoT{W75p|SFQxLKGMH5isEZX76|H)@@xe4m&VN6=Oi zbJ0vR8yz7E8WU}Xm=_CZn`47C51u=w$fbOzE3Y*AX(PUR?WsN+>Ef}^M#g($jNq3H zlX+>GrG*TNl_Acc^bq@P6iGX4lqO=YZOz}IKEd3bB-t0` zrunsi+(r$4(Yw^)kfM8n_g99IN@qb!-XCQzD2G9s%rT(9<`3@f*vv>0N>v!8FjDym z%{CFuGpIKAt1D7fV|7LGHIxV9587M=b~e%ntVBje@eU~J?mqb^pZ@S9*9bT9yv|dV zTsd5m<#C7aQCh|_w&o6f#>j!A5!5tq99>zAvMdL}Q(_qMV6NchjpCK(uB@@g7YC!` zHsl|HNZth)9RRQKCqO7CKej^)R+VNqfjx+DB}1o*tGVIYDU7GJpE$JkgY@zq#(r$Q zv}Rr}MM0yT-ZSWuW3*OX{xRA{%cM~xEi)+cMMBnH$Gy+twK_SAy+UiGDM~inVoGoA z6TqdMM!ES-Q+nWfhB#onmmi~&VRe*>62?846a)e~Oww5-t|;I)cae(P)8ztq#JGX~ z4Ijn}f(c*kpqN5MEe2-fO1d|qP~Z-+jjNMr z%U-&UT&*e}ybp7fCECp_s8Y5@OuP#u9>=nki#rfs0a6^G##9kxN4v z?Pu*cWr2J0&OWBbCGl!d=JC^LqaRT$K>`(=W!yevn}$64QhvVa67D)V4Hfw8jpKqk zO64(hRd&7%(gGZFZ~&xCkmXL_5)nCYC6WEjlQaz?Z4*cc5RdK?3>J~|z#+oJSEJV( z5DqP+*MVnc;N2c zadtt9@F8&w#!@Aroy7TZDt3x=ew9k{kgB{Um8U%AB@eD-A5y6}RVlsY$8jWz67zM> z?%e@|l**yDG21gcyED_>Uw6;+t5sLQ@93ZX)%lQK^j{mAn%E8&N5CJ(qcARE{fNC9Zl^zPF-U zJmpQrb+69%b~GI~yhc3Z&G5Yw&BnXDU0g0jbMd@4AMf^d#|z#9^W11pyw}^y<#Kda zywBUm;>mj;5-G+UBNzZ_T`*U zgL8MVADsO;=V@>jg9G3k$T^<@=bqqRaPB2ecr<)?)AWu7`dOuUaQ(B-_d1~nd+qRW zzZYFuTn^)YD*C;}xF3YkV!Iy?T50!O6fR!wrso%*TU%M__Et`|+F*p;mGfz`^ft*t z)6Vqcy(^jVOt+n8&hu#~T6iimPIVL9zc@&{{a!1|%$L^&V4Gj;;eNTPW?HY;)Uwj? zG!1*yCYv4=PlstMXr(RGmru0X=fe~IUZ=Z~xu^Os9hYTPRaU~(9}N5z(O(;62k8!7 zC4J%Z5d1+)w4x}C{L6h21%A?95Aj-;+O^uL-|02=Z1zMyT8n#U@}{S{y|5*+=`Vc# zmF=vT`j@w|(AU(d{z^AZ&WL_HOz>i`8edpzVL0oUyM8bKR5f$xrJ_Fw@!9s-R*abt zSv4=bxRwsqQhdywXP!NI{EMev_FsAN#PQGiXO2Jf%=62i^Upr-J)6}=FF*ZU*b1__ zSNiSNxq|s@d@c+EtPlF%6@;DET9o>7nf$2V3h=gtxOL@R=!-CJ4g3q2{G{EALciCF zX;SL^DR_P~^J+Hpd=G0Ghe5ZMhA;EPCTLnK1Vs++ZD_tii*a^sFzAakp(V=P;_l3u zrh#uibGn-(=>Ame3O;4No6x$&lE>>5hj>?ZC~jQ{{V*P+SN!h8+w*lz{0jw_|eyxxITv~xd$XwuB$hc#2PARQC`(n^;P4#IV8SyL%E@yRo+sUA;Tv6 zZJ+mM@M*2J6my%Te%K>P4l<*?7PK-Q%jQ)_l_X=H=~F{*j%BKNY-OasJ0|{^dLW`t-}o zS-l5VD}vHObU5y6nQ^Y$O4>FLMiMGJxO7~}9*|Vo zgdhynp*GZq#?TyEH!PT72!(h6)evAaSsOqE9Uqd{O?@9SVI*PH5f7tYe1ga$L>>k4 ztRTG9ZDaj;o!?TklJB>pR+9KWFSS^rrjLU>iz^|Vtl5U@++M$HhhZ-j@66QKGe?9; zH&}zkv_zP$iC&uqQTz-PXkaIBCEo%WYJsw*3YUg9uWVq58(OO1PFbZ(3)Fc9|8-o( zrn#Y~<|5Sj)Sn`jhH6izK0Df%6b%z);BQk`F=0oQ z^kw{x{=R#@zK{xVA%Bc44unN2hoA0zT0i02fFB}gni zx3ku3OXToe)^nfap=pRG=n;}}@f4Ad6Cv#y&wFNf`r>j+1TTV}%z`*3>4Kxyp$iSI zrZ&`rAFS`)X(jR}o2EF5=U#q!d0EQG6bS|yOW)62p8#eJ*3-`_zJC#GC2It&Os2zf z39>imO1$)R z=O~(?sq)Xb*-*9Hnx=iA>Dv36QM;v8jrR@QJGeKD#w|h5wsCFwP33pVG74L4K(7DL zcOCj}K;O;Kf<`-`9h7kI25P6q`dPo|TI!K7H!U^2toe%ICX5 z5cU|0v*|4+p5Qri^Lm*1tb&Q{rGA^Cp9Lqg1|sZqUl*Uo$KIjur1PXI$te)2?>8Wx zsm5AwLhG&Hf%EwTEzqt30DL_)U=38*0$HPvYRuFc>YGp>-_C`5-C0^2_yf^jearOR zZV!0PWAMs-25-KR!c9Ua&Va1DOF{o~kDqEf;xXcrMG(^*IBXdJ@dIDEM$~si8!?@f~U(+_U4Hy;JRPBZev!PtaTryt;UoGsaE?&Y@;$%z!gJ~*tTn;x3Z<-ag z0Y%-LO&5|da>3uOA5xU%x0OX0b)MtQ$~BRMF)Q`fV*flG(Il%-5e}uibIHr~)AR7) zvkI5V(S+-l&Bv|R{XswJl39n7mwT+qSe&y;o1FbZv( zt2)$K@wHUnR5mJrdBryf^@f!>sQm>knSVj~CSf8tdmzX@DZb*vGhDTXl?$dgB1?Je zC2HL(4J*XWc~1~8wFA6|p4p>p(i@$$G(=1b9`Oxu>D?~MFf*&}EWN9Yk~6xaKHb2W zp_(cTkmA>d>gp8!PYrF*dOA&GS*1Q`4A!tTbcSV^FB3+}x<+jp=}cZrZ_q|9Y_xFL zz|E$+@;+Sj3YQ0WQ$gt({`+(3Jns1W@V|=78M;F!aJhCjo4xX~!1O8%?3qK$>j&L< z1uXpj`LA!5mM&u1$tX5$K~mwBxDi|OskEQAB7ZC!O4#n*t{Qm1pNtEyz#^niKIgA`OttDdt5QcPk&&9Mu(L zg8U1Y7zWmU3qVP z&mE>@1T9&qQ2!*W7M=iRIZQGK6U)k<%KYlGU_WjrK7HCujw21hO<#ZYXf+151ku2o&XOVlkTBBhzf3)8&J6G*3ji1 z%jmHT>!>fkeH%Y`wmJk6q8{PR2p0xXHx;WWiwi`0MEXQx5IDc=s1As6k%%CYfOtB@ zL%f1gre4g{3z>S!EAybGFOgaD{{%KMt{=k_e~T-jX{gzYSjdLN#)Uhlyg*1>yo%P# z!XtWeS)%34cC=hXEkmX_NhNkk$-as&5g~^_j1iPxFJYAYTk#x`GLd6MJ``2IjM@!c z2^D~UG8tmL^x9ADO zYTOjm`5h=+1CxHq5M%;{5$Xavb)c_7VJ^@Vp)NbQz}~=ygX>NNxG;_(0wo|b(F)Tl zkhq3tz`@ft!d{fN&Z(;w^AUHh;t56<8%pE=i45RW87Ne_2GsNIlyHxp07?SU zbbhW%{c+10T23+Ql+y}YP|IpizeXde-NBh+WSU-*u5;3UJ4H^XjBc*Vceiq=Bu&A^4H6oAhrx-8e{6!r7}vd&H%4fsv# z#uAE6OIrBT)X_4Lu}LRu{+7xCaKp;6!P#j}5&2N4 z_&RETge#%Z0xCFaHQpnLuwd{lLmGU|s@>Ec+71H_!m^s(cvny_cR;~3nDk5j9Apa= z5E$(G5HK)8o6Wm}T&LQpFesD)7ok=OaDdzyLO0Nk5mbUI41T4gVt1zuU;r0S*0{}7 zSk3S2VGY5U5l*45F>2+6b*c}i7(8mUkA$4*I82+*s)c}`racD|s@EdO^Z>Vo*9pG?aUUh~h z0K@_;*w^&es67H5e-_uSyw`K-T;Ah6LLCB%-J>3Nrwaj=W`)L3u(e9EAbxMS& zRF!1z1iMBvIKCI;WMx%spLDtc7$RD|3)xKC4Qcl+m;mCef;LE%%O+wkeJ^2ZR^g?P z-+z>^M-PySvf&mn4%ZjO)7=GATL_^Oguyn|2%bS5Uh#&sks0D*j$2V z&(?L@n*IXyMN5f1G-(zOS>oH9RK=E-4Ba7GManz54h2%0X5FT}7p&;*3ao35LMtGZ z#4q=K*eM`fO?8aMO~)p*b8jP8i*QW4jeUZ-o!37rNyzR7$(BSN`RJt1CvdSSF+3q5 zS0v(8@f}n(H95xLKzUnyk3czh~b1ZwN-nNbzX?7zHv=tE!mEJ=pTl+OT^B$DDNBK$>>L>D3sj)ntM5 zJZ7LSC>zLo1aQUSxl%|-xfF9#dq(-n9EF3FS3#YN%!h_BP)8VuHoK8-LuSa?G-Z%@ z@d19joxA`~7Qteme_z|gyyER84kpO~us6y>8&(1PAY_7nLoND)Qyat(<4TSO>|6yGEA+eAh$%+zjXvI=l`xov^PF*JUO z*7G40oD#+Qu4btuoQ~Qc!Q;OQQ};b5kILwDfySA==;T0ImxmzIjNp=n`xA_KD z`s$q=1ezGRb;1E9&x1p}V?;jGU$~0e`)FV%Q7HTcXxGh2k71m$t-U+WUI*KUBM_r=z6*Q1uJ~vcUSjh7Ce{ zMa6a@_DHc~qJ3ZY>{6Njf`bSlv1byN@|12VBu`-q_ng)Z%6afP9W*GVizn_2W;m5w zBzNIOtda*Y7fuKBc&>qRJ>)iFBM(67WQpm`27CC)z2PpjD9PurD@$ie3~w&DdrR#+ zLRTx;A1tD7H%^l*gnPoh)Iz4BI~t!VF}%AtfxHh=fB$$l_het3Xsso!=+tRWlr(h@ z$umk4de(*ff1Wh7xB;h2mBHNn$J;>NqqKzfwnzf|D_J$lMI830leS~ch zN-1#y3U-R3U|6t{_$tT1ln*wr#|ps5zN@iXc^rGJewm%6GFnhJ*c}?llj&3s@$_na zsKa%lex^~=5HG+^*{d_FaEex^eaiFVELC7~ygvsX_2CY2k0<8S-Rx>Dz(v}_rD=Y8 zFY5NPYh*2FCLcJ!T#e8U3Dj?4xa=>mhd^mhj#SCR#Hkl~fMf-2@cZ}AH18UF8kyV0 zVGm$54i||ipF>cv2kJ5%a{h6L}g(Ivl?aRuN_G=Go}v!_!?TlrL$g5k z+NG2*qRjCg`{YP+=wRH+rg{F!1s$b)N!N;>fr5#PJpo!r9!DN(-_Wtuh|_EmdNlES zsAnrs>@qSk{sFj|C0BnNBxW@_=C+oG@NelL*WZl>fRK|1F!FgO2e9=a@rQUpd;3Sz zhx13*Q79(=2JM-SK3EmMkGkSpsTV=t>Xm!&!3#f__I^({;ZVB!irH z0gslty-R*agcsL$O)%U2L~d;aTiCFW1>0PrJY-hga$v7n%8UR;TICljb zT>j2eFVT@yQBceycUh`bdYnj;$a6%<2FhKQZBFssRPjedb`zmsNswU>Wc{U+{TEbv zp2&Dz7{fSp{Q)Z0X-#@4IEtegx1AClEV4d8q78G8(k<-1;Q!VXoOGy>f_uccStQ{m zUFnMLzN=NJhK?jThI(crBTkBcU5_+yfWA1#};^gX==SXvjZGY7Tt3&{(*Ew$hr6K<2ng+ z#qrBLzp{igIo(Tqu9pw#cn3bBElxb(ATb~H;XPsA3HBcRs3#bKCfpN3JMYj(Ji+Iq z@{`6aIUm{o*doc3!h*a4@1Bq7zwnI6Ph#ovKJgK)rPoGAfX`fy^g@u+CVe={^>~MO z>O)RA+Z?f!+n2Sp8zq8_7~8a+m*x)dL6AatRenhlURWYjvDmtf;eUMW>yU cI!blygmPKitvYVQY3yz6Z@k)ganBe37uQb=K>z>% literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/nvlm_d.cpython-310.pyc b/vllm/model_executor/models/__pycache__/nvlm_d.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d71113bfae422591c4da23bc07133f4f033f39e GIT binary patch literal 2741 zcmb7GOK;mo5auo^ijpY#9Vd3urmqwN8oOx^MFYb~V7smXTM1$(Xcs65M!S-z@|7j! zB!UdIuz{pNbLhD!U>|enALyY!gx8+@7jm#WvyyE&@u4NL!`a!{+1Z(IXBm~tMGc-? zfBxP26}W$}GyGW4xdScv3kcUZ@wI?3LXf9E4Rod}UiXb4$8w4{d^5Vy)-qW}(>|LFWi7`-76%De&=Dx$Qj?~?w@W;of@sw0QszgDY~Z1lbqsdT+i z@TDkhcukO1)@t`3ZaEKDR@XMS9()gyiH(QrTWcHlmhY~giVq2xfU&W)FcmU<{?T#PT)RsJY{Cvcf7!D3MY+z6k*aG zUFdBq%TAKIGVLvPE2k0>@^k&zg?=4(HvE{-xdScv5Qxwi;Tofy2%S@Cx-c3hH+XK( zV2~e(O+Fi|JzC#5@1@-kZrYK;NxZJeN?|8(VmGa~M3R{hz>W;-I+#Cs3}A3wlqJhR zIN@|pOEs?d=q_E>b_q9{WLn$Nd!$Eav|kiD_Q?zF1!+*8=hmLp(^2mLw0L1!1J8=J zI>!9up}^#zZ9+>f18HhK?FVZIHrOM-!;*jKztJZYCep6Ckb1sw-jK!D(9P)F=Na+j z2uxDs&A0Pu5YNY6d=sJ!8M*mY&}jv?f~WJo_jo>jp0=WJe%tqh`2aKQh-aeSNh8_s zCiCI8A2@t2ex4Nyjc89O#{d&J5p2v0nod0eG(JoFD6CRB3X|D{ieVfjj>koqdJRv=*Ob{O z$_g7sBK$@_vg0r*D=aicD4xY~5gkadzR;fxjZ*u#c-;N^e~0A6TGv{b?{6CNnUikV z6q(VEys&D>OR$J6A+dq5$qW2W`YukPlJax%qEsNN60~L#ix|fSHI6KOoKk8XQQ*5b z{+s)?s+n1i6S{$LoUG_L0HO}S(aCJbdD3zH!H5nuAk&jn${FzO=35`GzmqF;Zhc2hHr!G|x#-Yg@Be1?U8T z@PMR7yKvD+4=NNlUgU3qT35q%-6)q3{|5#Rmx`GG37+tLP{tNYy`)6CkUQ9E#dM63J8`#`4LXl&!Qtg zMd=yL)lSFz{toh|%I$4XOYq;rG;p8hQTP1|b~rqvRxQX`G=3GyH6(}-`7x4Dkla9m z>6BF@XHF;%J(1UepqeULmNsdc6-p|{N?x617S#WGYc33xRc6C)M3^)HB@kA%8Y|(A zG3ViZpGXWYzyT#e$4@=jnd?hDsI1t(`yIfF3KX+Weg-5)TjJJb0W}M28>`Od*4+nN zERXC;ZJCvjS-Mx-TKR5ERRdO0OFD3!secSMD@7A-5+MioOn*3V<5);$pX*&YRaO5a zL%$+y;&iB=TJEMUD-Q9OPZb91%cw4rB$908ErAN*MKRB;g??>Z#G5|34#YNXVpD9B U#Lx|5kWt+(7j=si;k`=!1(fX2&;S4c literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/olmo.cpython-310.pyc b/vllm/model_executor/models/__pycache__/olmo.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d0aa1f73108c4792a9364c124a440414ae899df GIT binary patch literal 9558 zcmcIqOKcohcCCL^e@Qke>SM`j+p?{;EYkLj?Tlw&j7PS{9#gc&vYn)2r&R1$B#W%- zZoaD4r_%%mLnC$;US}}~AZ3w2ivZbWm+Z63x_|(KWMu@%GRWAHm2+=Z|EP}>Bq_39 zy?5WYUfp}{=XJMMD=WBO{+qww{Q>{LTjR1F)#EvLj`L15A1}BI zal>tJxfCtNOYRcq%h8c|*S3o%?j)QW%p!@)o6XFR_ zo+v0k1m&bS16h=S(zFx_sx+uzy>J6jk1HV8quwVfs_U(@$&nVIum8b}{aGh;1G z(#-jwmxkSrA7$o8{T|5X#~qZ{TWY3tI!MbOL~-{*x3d{;wY04CR+X9vsvzLO@ z7k=uab>)KJ-VVmit6M?p^?Kfx?Dl)v87iTB(v=>s-0S($kD?&*u6Jc5yd)e1H06~r z>c7*4+xzAYikkX(f=llPzQ~q8 z>9+li5%HaPBM<@~0J9ts!KUAjQZN4wUextPK-16jm}oS$d5iCL@Nwcmgnk-)#QT(d z4T)3GzI>(M>vbjG;WGB*XqeEp#$l3R2;RmxK;P{7*xwDjAnv6%y>M!@V%rmM({HC; zc{8hU!^poG$fQlYF?}%_fRJn;2^3d_%xgk*^^#W7#GKFrLl}V>SjQD%;%OgOHmxs| zO;uRJ{=DWo!U;+{Wl`d^%xNX4h$`ha%c3UgpBrxV3&l{}8n}Bts}Yxt8EWv?O7mRva1(08z0dZde$w(Role) z-l()k%vF@y#?;;tl-js;+{u3+NtFZTmU>%Jm7zLRJZ-4%=y>X>aiHuQD48f(Lxre2 z)-vFAS4&R)LisgX?^r{lpgcFBoKVoi9`)dogK^zxtz)))Tox+KoNqh-cM=qK_F3N76&%Wmq+o1O?`=m-CX*Xt%B!;P`wC&3n4;ri`AKnBdD z?L#`BK{@Ze3{t-bA#gnKewccmd>WNmWjhoi=rFp-8Yu2!ryqM`j`Fr4!-+eWV(Q4F zs+>@oWF4aLqK-~43W8RMdO|Dj1s-@M^)sg(g*|U?_nMri>FSfI=v^}hHC|74cjOYS zZ}iR>%(dyIAkwl#H3m9%^DH~{OfUlJAOI7z8ZxatG%1)cDJ*7EsEmR9I1mFLAwv+$cxEp7xKE& zyD4A5;(m-f`4CCjvQ$SkH0{2lEolwPsn-3v=78#`nr^81pQbwBYNb)hxm!g)Drxw0 z?rQd^RrT&*c?K9(Mqf57jTj@zYNPL)0yeR`*jyIItVY}C0T@|3WtI%mzU;I~1V>jD z84||}xRYy0hRQ8vsNGgS)%MMO5<6gDUzN*4b4R061uZ~RztSME<_?5K=)(9)`&8ez z_nm!+$t?)&&>lLsb<TC?=JMrJ+7FZkziMz+2X^%yU#QM+M?wu0yTpao9oP-}su) z;p_XjK(*T2{myQZ>5z$*EzhG$zKF!N@HOL}mzmr^u297)B`+aqRprZ+e~psYDfvB0 zeuTu;!GD?RW$Iq0?q=#Wx568h11UI-0agBx+H>+Ma+$r^mDhbK_E1QOt?PDKUBX`j ze|3Dk!I9ZdRkXjOt*^K0B$FLK4m>X_d!+pP5#_6%2c(RO8r$1P8XaGb>k)lJu1PIaSmO-fjm`gaIaNe}$UR3&^LC5Iv#Ia};n}NmGrxnx=hc z>DoQZsNdCI1IK=6pkx@@UCq?KHLS)r+_Q~NGnZAAl6R4r)sdnytsyrJ){LgGpc!pp z!?G(uKO)^UL`jsP3CpYri4UroO}hyFH2BQ}IuO+Vk@g#Dx&H^fm*WqCcAI#0bWxr* zhMvhEBYBi|gGL}-V5HDAtRsvq2-T7zv>^nFwTzY82c%(~lynbTldz3z%dAC=^{+L| za#22uv{jKUJeX7R^_@e}U6`?h$i$R$pl8)Bz=hZEq39&z8lghZE?L)ENQVFT{VF=uU*}!xaboGjIM_q5}k|0@Lq2wbZEhB%6kCAun z;^pPfsCptUEGZ;We}y|CIjkGyT)gK-i!KCPeu7%S%zJpqUr_QsC2N#S1b8gF@)A|7 zQ$pj*4=9;TJdxAOsQ5MR_LM znT9n<^`(cWeyWfJ?}b92?J$q zXdYGQ$*u4#TIog39BO&}6(tu~Lw<@;gWY2*us0nx z0Bqr~YkbHlCYKoPt3g%N(6@$sJ*X2ks4=)y;c(BxNSI@AnFqKuvJzu41%K8v<&gMh zR!;_-;f)zPm$qsw#$@MC#P~~gJ5&8jsASwGi!hWg@>uv6>KEFkPGbDIGcmZK z*t`0tEYuKeawp+N?$oHSpg)TR9`2*Cx76{7^Q^F)^)|VEql@x{Fr+0Cvc*&7OZ7|T zOYHzbubf|1{t|ao>7)9No=YdoDOn#mPZB$3&QD+tZwUX*e@7|NkW`hynej%A<#6RE z2q5@T@+LBQhk?DqNRY@4^tvej04dwdt%buD#W7T62S7Ce+=q*XdD>(_TDKT-A;a#I~EtMIU!-OXmQ?e~J_)w3IZ z+TLb1c(IuTdtMZDT37SH5Dl`P^H5n5W4kqk>^4IQ{R4HsD>p&QOk}qZ-9%n)Qxa0k zDq6x%FB(2U^c^IvQvRhQBCuN*Q;s}3*+GICXt_xqlt_5;Obs7~+W(2>NfU`vDeIPA zg@fME9XRW-PH|gz9lNTQ;jRxBCLB-zcS~Pimsimj!g0v-n^=8g+|KkQmAOSevHx`R z&V4t2Ibw_NfpHH_ev3OHgV9kmgO&>4Z4BeuJ&pXg)u+KYR5oD;DB#GzPcr2zB_0KE!(kpR7R2+-?d?sL-xtW#UD5X_^6 znfKeYL_;hB?ia+;Ol^Z=tm24RM%^OL8J2=0!7{ZN*`a*iV7NzFO&kM5pUl`~zu1)* z{C?s`YnKMA%;E3cNF|Id93pJ^GT7*Lf~5KG#o{pGy7YSpq80JYL%QVw(~!O$ea=K8 zndlUlCF%`wj#nO)Q5TFdu z++FN80mUX}};G{aN1{_TVT%&h$$bz>@>o z0&giDdW_#Bs)mmkYaa4O~VH+^sEmJtRRrwA$R7d?gd*@P&IaCfb z`99i$b72;ghNT@F(bX~{t>(}<&{Z6&bWV?Zi=wyKtB-r{Ur|xTHpzs$s$OQ27|@oWCR$u5_Yb6n=;rN9GN1vyGb7R7BgfV5RS<80@qgZ zZ#^(q_&B0Adp23l3=y_dcb=lMVsf^a3D8YYcfEXh0F2e$D?~VpdycntmqMTLO+)8l3(oSLgVqUk#C#9k|H_gi2flvRkzInJoRsNl}Ks6dd<&IIli^H6v zWTAA%ku-Yibq)q*76rO+q{YkAX;Fkm`9j(DW6O|ql3SgMg(bW{Zk-OHR=`jB&kp!t zjEzUP!ZrDcD<2fN^Gp%=U4sla=p>j;@BGZ~S+1Ks%qrZz6@-1mYy3k1NglrY)MMJ> zbPm5f;1DvO?(y-Qd+u>P4%-F|n&bJ$^x)Iu;-?9G`tjKDCm4|b?0`=d-KQTj{>UsL zBZc|L{lQ~e=VlK{7yndaY-C9`g2;q$&p)htzSsFL9tz0u`e_&?3=Hl=Kga{xT@s_a zH69;6R-b+VBgt)W%j{_4_Y6TIPl1q``QQvh`apu!#?GIAF-}1oj7*OQ*1c z^(R>ZepWRVh6tm5L>j67!vNdjf868yD#TE%k5sJQq=G}_0xY@`NuPw_H#;1{%j;(z?1w~*EFH|n$U$2>9OG(vFV$! z{u=+XEw z|1plwMknG$f05&J(aHF6|8d0kiTUVM{Dl7m#}}d}=y?< z)Wm_Xxcc=i(|;C}gW?bCtaz(I#^!r zbeGRGTOfp;<&`v9{FFAl;b!KG-7A^(Qm2(>-fL+no2ZppXFCbfZ}ihnuiK0=`<+1_ zWczzvq?a0cW^}s3)7}(rcKl-oo=>P z!g2BPa+n7Fey}WigMN05QmCHvWWZAi`c2u4qA&_B^<*T1q_YvC5%0_(2?ZvMxck<* zx0iY{ZkXBho8SHZ`MknejH)TK+FM~i>a?0^D00g5TfIv=3!#yzv%Td`nw*!tR+yl7 zkY?T-G%=42EM%{nj~6}D2xYGyVyf=j%^1rfvq~O$W03X-DRQdg4$i$AHbpl7{a&ki zVMKg6z7Pt54M8vaMA&W)qBKzZ5=6bG2&wxTw~0a%Yd8B^7dsS(qJ#Oo!^@P=Nc*zW zMGH;YMfsqIGw~@HS3_3u?LoialbG3gtj|Ixp|y@XNkRqtSgJTgi`ntGxfTXt+)u9r zXlmR?z86W*Znn~%ypol;VAQ-4%A`dnX!19mCz3uQq2}v=4nyd^Supr-iWy;rmay?S zcw9VQXot?Kdq5Kf#5}xS(1do87f!vB>0?w~HN5MvPT&%22fvin}V-n_Up=Z92U) z9ByVUlwFLjlhm0^LrPad8X9&~)cegOd5kNzJK#mI`lYZgLoom|Q@I`=x@2;o`5 zPG5%Y&gE8N;uKJvq>G6sc>zJHZE4r_8=9^S&7l?;sfo7^DWM7DngxoL+CzP4fNBX- z=tF1d3hSDO97kx_SbOQrCx&lb=rj}g7#e)1;mPB8%ZDjwQ1A!>zp54`kYNIbO8gSg z5xCHdFdwccPoNke&rSx|+RO`Z);nnss5WMinOV!=ELo$do3B`-kXcC>wfE{%K8ljt zZOcU@Khd*75VRs}TM&Gr-F)R_8uw53JN*-BQvy9sd_5kl#HZp*C!@}Vll?2{O0RqJ zVid(E72<`L!`1-Irjp6Y9v$k%{uOx&jX#AasUs*B9NjVvo&MezmaSr=u>ASP^oSW2 zN7KwIqdmvr50lIRcn@T^WuUpy&qc(-a~e2F6OBsN#_8_m^QOA$CDN?pu3zc#Lde=zn64KqC?u{QWy=iNGHkR zO|kPgu^KyYjWEr_!?<=H@?`?ozzKwFTiRv;I0i%mFIdCEJa}f!NETk#K0pbg9Mq6& zGY&eSIK#r4El;XgUV4#AcMHP;QFGd}L`z+PwyDlRZHs#Gz-?%i+oaOOfrUBhp@0H$H+7XJx;yqi_8igytYr*9iB<*F3rJKlEb@~w@9oxHs*p$a4JOq8tfut3b4 zS%qdAKwD)yqD2So_A2oS@ecHlRdX3S2Fa**%}saH z1IKFDt)Y9}zMTgmIP(WMKD68## zp9myDWs`yn2z(PKNPZi!OkdCRwM@V0mw2I+QkXgFJt0e2jJ$vve~TxnBB;7`{eJBA z40hMjH|BTk%}5m5^1CR!B+pW~)R>kpA|gpekhJjfWdxbshW(-hpCq7^uTWr6P@#at zpJFz+9W?nW1+;zg1qxoHU@x)vIx=Xf5~|>7hC4c0)-NVSo&k|pg~&tx=ZobDa)C;DXG;Ko*h3L4gGf5Fc612>Np%XLPOl) zIP5dT-IPQEDSlASA#`j~%UMNtST%Q3h#x?`zFNdXX6F<2{{qVoj>|5pUFhP|g+{us6SHU;lcbG2P4pSjS(!6%Z>fImw>kEC*2 zC^bS6*TO5o#bz`JbKRmqh5!}qvNFYetCG2~5Wuhq(%u?dB$*jUeTD5)oi02B0dF9K z{C*%^I??Gk2Xkk=H=@zU6l^2+E=b=4!~X#l5~^QxJzzDVHIVv_VQ%AZj?tePB}F5g z_HNbOIep_G^Zw-F8$+{vpN6s%elicHR!2bnZIP-w`ca%7f`xcjj|R zZQKQkBqZpU3=vFN2(S^vGyu^gr%q@yAX*HJPUs4E42pn52E|=)m_uPD1Pr2n1@+yq zO4P7ImIdH(O4N2(1v6QZ*Y_NGIhA21nNPM^ge>s8F_G|*9>&DqMr0ckuM&D`fr&S% zuy96D43EP~HgFE{!bjRi`bS2aRfi9~q4n3AFn?meO<}%EP7SC40n}_lKB_5f;jrQi z6&nlTEgR$ZYK(n_r535WqaVeIW5^au4kJjlt9nY>h#_EK3#`VLepS0_Y+0;ZWcs2~ zFHGJsb$|rweWEKDgsDJ-*M_lhtm$Rmb5S$CAeztH=ry79=V)cvjsSx@8xL(S@0jx! zpNrtUjFRUOQ8hQ*MTVb4bk!VOJ=0&76E})6|qjU@IS_ z=#gLI=P3wQ~Fe(tzdeDdoO`R_d%Yq)|88f$QwqZgezT@55 znB8Mpvw{){9g)cQ(NADH)cG=txr=0R-COd$(glc2O_pgwkQ@yGKXmiKRCLCy+ zZnw$+tS2*Uh}!JPfkdZc1vyVjG0A#+%Xmq9Z=O_M$a{EVp&FAp>pYo%uJ$792k4y+CP(bqDIPgF|aAO`Hzp31Yy7YE1aThFZHL zArr}YDSwMr#71;Elmt6+lLF#gzYLQO&b7SY{Y3vA1dWo~zTYLn`xJaY!37GoC}7z| zhcnYVnaziURp-4Rze>D8jN^Glh|n^i-P32_FH*u2{DOb+mgnWd^vjwJ)p=%On@4K1 zIdUM~r7$z}6W33mKC8=gy5t!OSU)D*Qc5z{gAXbw}8aTV80V&J251BwRLQIFXT#a^ctuPBYF)ilC3bx9%!x_jChi>bTKPwi( zxo}@N&&oIZuG$9Q(^>ukd!F{=pdQYY8}Lj|o^B43W_0%4nWf!QhLvvela78A9}5_= z^|l2J`B(z-ZbQgGsb>9Vi&A##Kuv~q310{V%Qd1EQc|bgEvi#W%Uk5LsStCnRuQYR z@PNjnEAPeBHp;3IGS<_^Wi~sLavzncDHkj`7}Two+)rdTQP*1X zhe$z<+6xt*MwVh6R+H$+;@VS7sDYwGE`ll^RxQ6Qo1X5NfI+1kO)TtK#>lVY+8uaD zSU1KyG9`gDa6kd0V2DGFBx4^~^NCIibQSp{AM#hVt}tN_mG3p03ma18xVU2(8~d5J zC&!+1IJc2a^O})Z8@nh-*BJdq$Kx~z$s7V6Ik1{Wf5U`Z9hbWD50L>c`S0T;e~;q% z+B0%p0|By)o#wkBE349lFP!SRPyKu2(UY5oMt|h4(eJDwe}vXsyGKl;<`LHr8Lc%O z{}ZFtEP3=zfHTmf>psJD3G2)>Ig^spC4V+ZmeZ^^wi;u>7vY&Zr7xuIhVr{k_Q^ z$E{xkeXllc$m$B5tfxAHqmRm?r_J;hu^4G?9H=Cx%|8HT9orG^b>D*>Sow)uC%(>aB&Z9O`rxHLA=)U{vp5V-c;N$XidiK7Vc? z7PHFE&krhm1&Xh1__F{tkNiS7Zua}YIVEq7V}&zI;Nua$M*WKIZwivkmH6}mpPXbK zl=Fl>M3H1g%fF!D2nAgV4kKtxDJk|1N2wvr(4(chOZ=+1#;FevaD18%D6 z6Bdg`;aAkO@^31#B9~BGT*@a(<>a5CyUgZX9#!U!W=DQAe_n=^#z;n+rq0ApfsvJ| zx4f@}>^ik49(lQ zZP_!p@})XOHaDz09^&*T7mxD=LMV6C7Ew^{ht@xD+;wVPcA5n0CfirVuC>JlA0)CAt0-s>X4j4TYY$8b#dj2uaW4^{G9qLPg+!|lx&q{DL ztv+D5XHhnP*fyS~Rv)-WtGGJFhg5uHC1+_BNPqDky;luZQV4@{wUV?=|M)#>@aHA@ zr&fGPaL@L)(O7*(B^RjuqxWckq|uUk%4g1h@*bs?Q-WZDzdRdX6f13-{1pC)y{fBS zVeFBMInE5yPLwdl`48GlI7f`1FhHA;70uVIm&@ zA+uE`qwIqquzu+ k)@uN%8sMs|&*^28-Ze^t%70@{$9>`BGw;lt-v7e?0dQb&`Tzg` literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/opt.cpython-310.pyc b/vllm/model_executor/models/__pycache__/opt.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d13c4ccb58deb45b33d8d02f8681182ebcaae67d GIT binary patch literal 10458 zcmb_i-ILo!atA;F1VNC?<*s&D-)qT|%-CA_>+DP7b99ldJD;VsZ0no^=Q|vi1MZRw zklX<%Sq%4bxm6A;K)B#8gp2NCxa2N{2i=2AF9wIg!|q|uJHe6gsC$(2 zrQle2+&vyHyUScI2hW5j+!NtR_av8Rf@i~1?kUb!g45v{_YCJ}gXhAUTSLAo=7L`d zpLd_<{Cx02c-B1|uDC1VIrp5Zd`A%n#KKKQEcjOY#hsk{5-5ve36v#JUIyhAP!5Vi zpd895uYz(|90BD>M)?LPN5wHvj%AeBKshd!L0Kk>|NM^bzAkbfDz#?@-+HIh^rhcv z_-DJF;KuTMSFSFHUEv4Ijc(Yh$E~%%U%uXoHQMPOSHOajV;@2Z?dD-vinBeh1~%nwn^x4$=aibD`U5w$^J}QuuZp z`yFbK%#O2{{kSgbaUIOX3-!i^KW2B<{n+dEymi^__mXF+gyvCKdOUEiSC{o5@B{C9 zR|dk1S_2=Q*u+L&v>5&!g6HF76H}_5lYZLmS zg?G5h^Avpw%~QZ{f7tK!x)S5QQp+Wat%#;IY(){e;Asqfv`x;0^-bUN!(M#DYfbc) zO?%`uF|S>DBPlXtP`}~Js6nDJdFfdp`ZZ*H#Z{sEno#|m(EMCezoj&x>q5WjxVkWW zqir@drkNrSTK<-zE3O5ZwUVO{o=aG-jxVlsqgE7cT=X9uLn73n;wiDZqukYo$hVb6WvJcLps@;`t*w5lfeVvmcAL$}j}s&6_b^44 zhnZ=`o`)(u@`I*40X}&W$$dz>=QVk>kF_Vuv85xx{& zKNqyt&h>7@8{N*iYe5j6OU2p$#BcO5y=gHz*X_kCy&Lijx_%jVMB_L0C3SH6iPN$& z8evlCWs?>)=w(J1<;QbTMIlx&3o6fowy4}wKg6r7)^hSR>XLl3E3en3kf%_pY4SPz z$+MKyD0!Ze7brQ0o|le^2gXC>(LIAfy-1@okyy%tb|f!}*1)~zDrKdp ziOfiP3u-|tXrdx=kYXLT;p@Wm4c|Pdh&=xD__u`CbXbxL!oFE@Em8CfZCf~;mi(fy zDA%+^SYSe+Gzdk0FSTtE4sG^eT7Bh%Eg8t2jA zz@)J3s(WdpC4}Gcq8J7sN-VrEOG;9rxAOq;Y-bxIe8v!zB5sn63%H|;NQTN?NX0$% zV{KdCHimjzC9e%Dt9_i?Hn;QJ7C0a!SaA!|lDlVY7eL7k3*5#=8yga>uYOA6_UU6> zFnMNUqu<$#k{lMMrpxoFN(`pS-$Y)18%eF0PCpGrzD3E~NZcG2E>XQi-AdHWM7`z~ z`E9U%u%tnToAYW_Eo1%$^Lu7)1VOp0tE;uLBzTZ7Q9^JlUqJ$m z>vjW4GbU-ODnXH&>otm_CU**KhsV@F22yS8r`Tql+;55_nwP1o37}$kxXEfDoSK)fmXV20GV|B z1F4!Oqmv;_pi>^WWwi^yBuWduEo{mGor=N%Iu#k69IRzInZdNZiofxHT0=_K=_>#>vNb-FV2)_h2;FKK3zLb?zjCr2k(zC65Se%#+gUvh>9lW#UV zY-4iG#!IryyCQdaA@U1-rtr9}Gs| zh)B(C(^M@|2mX;_SP!kdu2I>7dW`40BpbX4vdqdxCK>c{lS9=|iu8r~SZQFvo}U!<0oA7QZacPRNjB|oIZrQ`>ce2k=K z?hC1w^rM!5b0yy;&RxJ7W9MIi9+9V~=S|I4O-#0?nqOEsD(Re-VX?|sIHstB#pwme z!1cZcY%NV=D*@O49aY~)FIzOy(@21-288GlVX6t_H1+Z$?aY^Ol-e|K@405PQuK3V zCUF!#-+botrAP|Dk9|JV&o?bb)4Z_Q*DJ7>XS0W=k1({LBGON5N?c#|*&75V&-f)# zL2qT`XYmiu5INhQp`2fCYQXCJKwTzeil>h%D5An0i!<6DRq zL0!6~+){68P59)(xM>abzgC3#H=4-b$$d=te-QT}+=m(4t<{T+)$&8kL4zF5(M5T( zS#!7}2T82lR&Oh}wH@r8$Q2N3swBy|ugW$0%ca=HlTS`+szanciSz2i`G}OEl^^Y@vv>9lOPB6`V8(;U`8MEy=6I zJd@H#kBcPfPf{dz+Q+^NT(ZO-66En=1-xe?!dHyr^rwL0iKR_7!lk~ zxSBMgbc-N-5#w!wBC$_ni1qZOP?a;?&(`!RA!UZi25JWh1B2FIVWX)J$4 ztW%qX1iwdMcG=q6W*KfIfcCNVcrZJ9w=sN5#yoc1zY;h}8KJkC!#;veP5v%Dn?z^6n21z z9-<+>hRl9W`hxRRRi!?@q^cCb(iQzHl98&Wn^i*pISv@t6=D2B#UTR@7S=V_C;r^Ym;!*3;bs;;6siFZxTwL4Bf5 z6UW4Hv^&Vke+Xi8A}M2CFLvdHdOxZMmo8&V{yZjtH&93UBOr|O&r%-87+dVuCFW-K zpYDd%%z?F%R;?hHR#GmlUU^`zjCM*_2F41u305A^-^(lV_vxuWL-O@Ohz#K_2tg^Z z`3y^~cz~c(MHrB1Q%Gx@@+kg|b{=*e8$lf_wgFK^+-dkxh)Y!7pc$$ z+$@6_yYAHq5Zm?GNv0BmPhOI_xaHF!V&(ypR~@C$u|ZtxR6u= zJ`ki?O-}^n0V3z4^!QH>iSw%<@Lf`jL*SrI5R-k#6bj*c!ZMp00Fff=iS?d>#r=?i zp^|Lr=psy_djgZl$E5uX!X)hBnnPgHDAL&h>r>%~`q~eUJbEPpK-|_xpCx0)${3G<;m>w^Una5i}hQ~`| z)h5TP%3G)#4U>A2G)%YH@jt;KH-&i7^jOVQqGB2&J;E{IpwUslYA? zq7g|2kL}-(0YIw1P`9<1XH?KGKb&HWQ-w?2OiuuWKGd%%@-VApP9EMW5B0liiXkcoFkFLwdMXmuVEmqY#et8+F=efB@`N4j*oMQO^|6B!-K{dm z{{`%#VIS>0-dz5Kl1oTNgx@2=~Voh}`S@0_GjRSupFeqo3u#(R7z{(t> zm4V5aLTI^0r*);hXZOWK7p+F@R_I_;?4G?xsV8?64@_*1awGZ}BgspAmVhrY5)08T zljJcP*&Slc!%e|Pv6Kf<8l9i~De~?Dx@c^U9uRq9z1M@1AxOuE5MhMyz7Z!njlnJP zR|O*b#sHB!aqx+(Tg=AJd{(r9V_*caiNP#9glQplsI#oAXh@>Ue!5yQ9kqf8!$yo< z2C`|0H>x=8vXKMMp`@$T+-{lF&(^e`>i;r0JSF0jPnd9N4o|r>?#VT1biE_iZs0VG z4#oDF5BKEg3yVF>*?tKY5@*qod%VHlkW74Ul71DF9acw@;O;5gnFJF1MmmK4@{cJY zXHotMCFG7uf<1YIlE0*c+^31zW`FtxqHQC2gJ#-AW)u;bk1P{W&zE;Z59tr~i;A4Q z{h8)a4V_0>y8TEij7la-&szMP+BcKD=ZS6u2lQrqgG4GFEWaM1QaWscne>E=%#Q2C z2!tcxeu(cyc(&X!efY34J_zAsSe_rZ$Tivd5@D8GVT!=_MzXaAv=$tD^P#VMitAYA zywL{~*)H13?x{t3w(-mpcyUsU4^8-3gb$2)Pm51F+~=RvY})vfPvD+;LJK~w&psC7 z<3#tw6SzlS1o;#!Anq$q;7)h-NIUtnsPSe{>bTHGpL=${=IQjMUl(P7=Jn%N5HYaq zKgVCP+ma;NEph*mztyKt%_N2OuFV@s_;$#TmMI|^6o2#>~a8J7RCu};JA*o0o$*s*1idiP@={TR16DSs5-X`vRX%P>wo1xPha7WF{)MD+>r-lzi%zLrvTI4U@_XF_5Tq#WR)#eX zuU~&m&vf_i{rWX~m5Q(7I`xnLy!t<{Y1)5MV)C+)cn5bj)HRK1p~iG(BzkIuMrwv; zYK2y6hj!|OPU?nk>V=+8Wz3`$mhfvOrPL37X|t1Z8iaw=ourah!)iJePRX>J)Y9p2 zTIya>PiMlJbT*uo=~8kioeSrr?kDr9#M7k6%N&QfAGJQFGIXxAgN?!?I(X|gWHpk|#YHU9C*S@!7hOff1zz)N5 zxUhU5mLu#jSRN}Zufg&-I||Fu!tw)Hjv}1*@GxLJhSll}OQMVoQxZ8?P z^t#FB;>Wzx>z+9E>W>%G9*dL3Rxj;0^UlR2UcB7NR~J7VtgLjpD`%Q57~;;#YMw3K zronBv!hE;8DXjN8tz38?a7PA7=Y!U8I*Ax-hz3$l7@x z=e*hJ#_V*j+wQD13{iSF&*LuD6Vqe$T%0#qGjF0!`E;|j8jp(yD{&t6`_T&T4f^6~ zN}+nzUGLd9NSi zUEK@K6yJr5N}+r_$oqpFIo0t4&V3j+nV9>m*J@rI8Q)JY#*AS^@RUO=ZZ`)>9;tze zl3tU=^!S==6NM%|-pof`3{e`hPBV``mES45jrFLZynkWP@Ao*~;XJ;{d?%yNns%}b zO~}uf#;7ZvOPlL)6sP@sGwMvVR($p>YByVXk8g^yESNMmW1h8WvP@p(B9Z+VO00!C z5Wy(vtjvtqWF~GawwN8;v2$2s4%6HI*IHX=F7vKdLYI}8e^m>;uQf{xOPJFECL2xS zb7#*li}GrRvA8P-TTJiiB#poDi+prc1RFF+QAL&7X#mKSSbK64cvUx+q0aD`{p~6y4J9d~h;P z`zQOI{)xPaHI|?FK{{AXPo^k^b1jb64zUv+d@P-rMm-O{&e#kuK-x;Ry ztzjAW9NV~SIMq9~L$YKGCHGu3W9Z7RLF}&Ll2I~o8|)BPv58e|G6%Pd746ZAj@{U6 z8;3Ql>=NugY$fPr{QCF}n9;7w)m~v$toAaSii5QZt4TR6Wj(I4fV8$x{`9I9PDw1N zVHwSds{E*9?{T|^skwYxTf`gFoE4?+AdOamtCi4n^fjQu>0AMuGB7al>h+G5wc|4<4f*4)IjA+zN+!l5J9H8+U1zp5R0C#;G!C%$%W|U`XIr8RF?tHJ_3#P+jV=!>pml z?CX?Q%clz~wFu@WbEs}TpP}~T?VvTAIoI*zXf{6to9qLA1Ki%wMJ-phHz&c-Tju1b z_%VnT+_?VZU(Rq^!3`_KzyV-3Bi2d7U~;I0b+MVn{0tQ{`Fr@`ACe%jYRvN&Nta7T zg4q<3;Z%;cl4v|f_-V4%cdg^;V&X}%&CsLSL{AsahVnZuN|Cn--C_#y_vu+5kf=il zZv!QEf!FDn>~UI+7xldBX~GtavXUt*2IUFwv2AK5OwplW>ldZC;5Wf!GyW1X?nRdg%GI%UFes>vV7^DWP!p_zZ1YMF`KacE8H1pZXb46($;bge+mobRGD|3o;P)XttRofw5$M)##O?rLLZuA z4exh_D6l7qyTt7FkVrG`XPBsa;aRv~kXRvH4t+R72#uuLEh-PLjO+1cbcx7o;nqt; zSE*=M@|a!DD$F$Lb8^+9q=u=-Q+WO~i@MMv^g>K^y4aINaw9`Vu>kbSCmuY`0VWU^ z|$-)H;i4_ElBQf41Z+v?sb9 zB7cQDqp1Kmv@zKC2nZcOoFgHwde?YzKj?XY#;RNYhEx5A1K$w2lfVZaK0Vq|OX#x; z=mQVOetNXC_E-t<=Yfrv)_ibrk^!I&zW}{T9Iz_tRiL|Zm8@~4fWIkL+rz)>!k6Fn z9L18B#s2jmoIa1+ohu6dZ1i??kxsXSziUu-;qRKH$=7u9=#hnAqK$U$kcQh;q&6=> zN1)(q{cEEwz5Z2iXkoW){?cI9b@L0V4N40t3nH6W*QTg0?i#2o!QFITCqJ-ajN6yC=xU2FCS$^TGijz5 zS@W&`qQ+>J8Jb$`*0)>F?0(4cYF>IXK`bcA-h!fPZn;YmgEHQu2Q*1Yu0boLF*|{x zU&0UueNffBw~h!&hlORND3k>KitsN{m`RbwBh(C`vk2sNM^(xAlhnqvWXCGZqMj|v z85nqhlcE%NnSxB=6n!I<uYHVoc~hClVD_5CLp=I9wfXcGr0Zo3Np-#5{-22PBAPg#k(; z;!_knO7_bTqwUHj86@Iwk@hKxKO!-nqeAZpTTYXM@ch3~Hlu;{u<^i%8ie2VA^>E- zThemz?|KER^uF^k&Rg|~aUBt1b9tGkXy`!-d=R=z0!+`HfOvu@h4bnF| zko#}AGh+G*>@go@m}M94RhY$id<6Mk!E+rzen5~k))xCi$*pya4%fT$Y?Skc_t?8E-JbckQr>ung6##Z%zK1v_p#a5i*qKxF8)CQBgV5lu^10e+*a=t6xT@G9h|krcL!cJjl}IVT};)=gAYDxAqX zF&$X2i9nNDfR2jt^I2-8mdG;)iW{gy2R=_`H&dqy{0dS<@!0(vdE@g zSg}w1ka~aof!^<}!T$`cxAym#dM$fggEIQsH!0LT?yb(RB5%}DYK2or!*Vyig7Y+@ z3fcbNI%<~vh=RaWkkW)W#!m=X+ z@9=&X;n}Qhc*dc1389FMfMEl2Nz0BRxJZ^`8^Msg1ys>p?!r1C%D%{5Qm~rGcGTTcR zaS|medrucVVKJQehBZ2fV7sS-nQ(dBPjF5wJTSnFPLKH_y2Ag0M2o~@5RI~8bbk&# zoC9%et&f_?qO{rXQ^+06lsB;xrze3f;z?meo^G<@tW)^1gghq+%S8`mIWJqqX$vT= zuw@bT_`MJz0&1*iDmzW{m^?l<;2xnJkdGv=kMuyVBMM4!Pv9U@mR>h^)3iQ)!}x{u ztF5CCtkKD*@OAnEuhAD+_g>~Vu1whoj5H@PEe~Nd4EmXLnCy5jNBT1 z0z#>WZb^?=cMfJK{x?QIIdpL3S^hnvG)g&0J?(L7zwtxiMiJ|^aE|5Vn>5BMzDS6r zx+`ChAVGCp(kr>Yrp-eb;C}_W84x?gkuoRB7FOx(aA_O_lEHGx&%(0IDPnFFDqE8l zhW||9or}O(CV4I=78O1t^Y)%W7MrT2-P2Zi3>rT1h_W~mJ4S8Z+{9Azx%RaLG-PNxCWq^~@4WE2O`y={5WJ1oT@OzIat#*vG%H+RU zj8DcCxuH#Dc;cYy>RZe8%$JI_JIFgpCIRB$0eQeZbDH>JMYf;C8w2{67Ea7O^yLl> z|BDc3{5TB4R*f%0m%C26j};Dr6gXK`VZnDOFCsxCP0rZ~uQ%ov*s`PZsPz^huMFZ? l_S4vxP~@%#iDPJeRu4>>rj*PHj9DE=E63{R>+dYQ{ePp8f95dY>DE!9nm4C0f_$q#&(HNwy?Q*4ox;Z7sbLt+ga?d2+x*bdv-%1He54 z9SE50F70*Y(B>hJE?1FT`{IY>A%8&fKcw>1RjEoPPku>K>N^*W z#pA(vJP}O9dxAZx&W$GHy}@2p&PV&=sbET#z37E_f3RPb3(v%y(i<9qqJH9a_gLt_(c&qo^D!*gq9@Djc!*@%IXvTVU1|Iy=gxo6^D=dUHfAdmI=@pz1B%vzvPXj9%Tn8i*5R-JqV)?Po$NCx1>c|n*Tf}-CC^~#=KUOUabbqIuVtN zwc3ZBFv?nl1vKPHtrj+$tyG`h{tEzj|CCo0NyWM z08cCGo^I)?_Ky-y979(S<={@Dyg4BGfH2vatCL8?NsJNG0Q2G$6^Gl&N@uC=Jit*Z zoEv;evz|Vx6aNh!Iz4)?<4yJf^t5#?)i?B2y{E4jt43<}j5Ym^_PIU}jw}%`EicyU zN}NRj17zue;?F^ykGiTUUZ$q}R@Y1T-i9S1C=U%db4bD+%t?Dn8~{&ubk`7=|yv1+KTQ~Jt$`fpaiGZU{yz4#Po^=LhbG_KIQ0+UQNayi9@Kbx{4y= z3JrOcz$*l3T;Mf3aklX!a0e0$Ga4&V-c7W9nWPV;d`LUOCYc2A43JAV3`3`%r90nv zIae32p-Fn+<6481XHqv%F?dE$u<{DD4JYJrR}DytyjinEeib$a%?i*g>VxqjJ0+3kV26WVH?+Lm-x_S9MyS_`UV zqg3n_u&OQs^e;TnM35HNiwwuIVdz?~uwLrr7|Iw?TrX$qpg#Yp)^qU|KGm4}nYCK# zdA;IZjbWv|+G;r+>6LpW?8kdz;&2OL38LWnc7fVPG40g{MxjC+Rx5@k4xvz))z0#CZfJ`xKdc#G}J8iR_95AwQ57Yf!aED^TGv% zpR6dQPr4OaU#VR|LL{AxDzcLOXGRWEkaT}bv{Qrtm0HjPv z+gl*pjMSv=eGvgyD4(C;Ph2DL231s~2~8u5nf~tUt}WpuwI}mKvn>ncRP@zZP#Fqm zw&z4TjMrfZ#E&s7DB`S##E?$rcF5erP&7egnG?Jn0X^wtI~AjZ&q$^VUPoCWC~ii9? zQF);2lHE$^pi&{!m)Hqz2)QCdj-u@(WB|N;h4hc%>7L>0Mf?bQ_^uchX(z*aTC|G# zm{Ha(leCrdOuU1Eb-X(svg-6C#Ek>2W}6%(k;*HwrH(@SzAJm_08FGgE8JzLM_YKC^f?ktFcDB!{Ctd zAv)qg02$X_-%8xk#06AUb#agWOz|H6#QOlUG$1A% zGX5=Q%7J+xFzhk9_R+TV?(y$MyFa9w7gbv2Gy#OJnlu~DR1guwiv(r}{3$h*`qZlP zC>l^qc}>5hF3MrLO(nd_>2S44Rr5ud^!gBQxRcTHNN& zlt$dhbNF|dzUV1FC5|iwF3+<(^FGoLJbJ9a3oEd11i`aKRszqKSeciJkNF4!LW_;C z3ThFi@=-Rnt-r&@*#ug~5PFUb=b+%z3dVucX@rn8`;>w&@Co_{M*I05gq${?L^)e& zFW<)}Rrpy};inx;vBPSV&tFiOsF4UxkFcYdeLq4_nt9N=PX(?NY+Atqc9I|DhqxL~ zbE3^))9e)5d?k%XAPZ;Yk>NSNh3Ko*WQ{Dmyuq6YJgGb{olIxT(QP3~zp7ZL$g8id zWf6fubD+CLfFr7tYz`iB3`s7Chm$VAmaIZfwHkaKKoj1KPTma4PMV|gimPCr=})MK z2S-t>71APVOFcM}Yb4#|UqI5Cxn{x-(pJhV6-!$g?b(+#L#vIdnlbbnQ(C~9uNqt8 zEDJClh+C6-81W@whUfoYua1j5C}jNn0hK}m2MLhtD2sh%m9`#op?Fmm2C_yWC}|40 zXS+@BYclvc#MNRCfjR0`j>6_rhr&u_CS?vLGfZJ$|5X-!*O_@BQ$KM z2uQcZxitj%6zIZ3Rx02FT027vUKbLQYx@||q1Br!d?&~m$7H}@`W(V5K5X-1Tn2{w2pv}}C6qZ}4 z-bbI$@FwI#yGFPAd}R6)s-SI$Qq5tCVZ=go$q-sGEN#_TMHmgHG#5=c*)S#)t1#_j zh?8=*;cnkI#aU)9>8YJMU_|5h`nwupnH;lv4%}dhb1-U6N7?xdO#K@Kb`ZSW{1+qz z;st=fTL{zova)sEljP6wNA1jYOP0en_gjm;3d{X0EcY9U-yNCp-*0t%Sg&@}fIlYn z83Z{W(41LRlQ|{i6eIdpnIc=wCe1i(OA|hf^xmT||4kuUqMFMHJ&g&ZzAW{JL1};m z+1w;3g!L3*e)h4|hE8I)dooG+g9hAJUfZ5d*udN>LMkIHA45nv29LwV&)Kv#Evx&= z^H1@+Czj5A7%(J9h_MC_DH>8fKgXN=Ge8znF%4mhp~^T(lo3`jt)q#_o|*lVdQf$` zxRm1b$&FYul4^<+3Sv}*Vz zn1P~PbK4ANe+DznGw@k9HuYGJ=C8W*;sh`kF9CE<&ou`z5~XE!bH$8*qm}y9a>Z9{ z1^srC2GfytXTCjHF^IpFTG~EA7a>S?AkKKACZ~dXP&#>~IcUc$}&_1kMA<^0g%4g6vDie^$nS52(X%&$}7xFq~YG za3q)57AItkt~}y@!svviFB(PV)PFGKY;tbe|5nJ z$nWq_i3y1ruj5U80Ajuk=scuDlz$2@6>8=QQo=}&P=?N+6t2zaERpn?pfpY9@NXTK zbRMdlq4*0-D2=pLwKD<^aR`c8KCPzaQI6nla!&R0JxWL^j;amZ&wnFaNh=l|gb+se z)gM5d>>R#57gEA1U8=UQL-i6>hLoY85%wF&A!gh+1hNv$oFY+#eSl!<$>`e}DNm|} z0K>c*2S%e?=8!+N*t5`xRB3AV5y?ZrV%?3~rxB z3j6;+$7BqkXuEnDF_ncMay-vGyL<5mkf?9Nw@)ea+0h~Zl=*fmQSKLt_aD@hYn8|n z+<;~+?x*YJ#CO^HI-S+vp|l`bR;Bs=28EtY1QfUnT19$zHP@U`L^T^)&-sJ?J4y?= zs>cOIRu>vA-^-&9G#}H)HI*>ZR;(3!-BIC2j@&Y&hO$2Hnzfpvxtq|WNH?jT6nK$q zh74a0;g_@O^*nVvbOd<*^IyT^CAIe?%Wr3a2|^0;48^Yrd_Z84fbt4#7`spe!EjdsJm4x{!9{bT!jRlRq>ql$Ty3jH-rK%^2y0#gL?1SSZOoQO#R#6W^hNze(2 z9DxrB5EaCq6WEfmtt3DV4S$I@A$fK+!vp7i0}!wH^x4sS4^Ws zZI1438po*z(pXO|%lpPCWMAG>W77H>N_rFa{~1K|se$p1ZG3Gw##f%>ePx`Y76UCV zjdKTI4!(}(1mRQtiF9C$A&1#s)b2^}giSyZTX?EuU1!Ze zt!Fo}ijYA;Rc2vzc6@Z2{pw6liV>Q>LV!{r{kAOWDab4Z(`u3|(K@&&n~B25NM_Hd z*t~xet5_pA`aQh|@e-YC;D1*~wMsIh?%q@&8=UyAZr@U2iammntncuSQ|zUM_Ux*i z>7PHNCKSM zkSdkPRAwIPc`E$_fy)H`jDTWWy1!A$QR&=LSDz~05B3YbgqwOUwp$h4_qHK!!PKyk zE`T#PNF{ZXV0_rLb)gjO8&+mc!7%1mY`9Hcl;lvX$0UK4rNoX2r4c@O?MSmX||@cynxY?2J>Sc+Ax+uv(SP{|B1;AW8rL literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/persimmon.cpython-310.pyc b/vllm/model_executor/models/__pycache__/persimmon.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a1f8330cd908346230785055b41c191d1ec45c4 GIT binary patch literal 9429 zcmbVSTaVmEb|#CLX0zEnGd)+`Ecx#CW#o&z&TbrIz4ChPwKSGjUMCc27o(TrRev=;?VpZc_FvYupJ;MPE??K=a_DTnvTyjWLb4){Kysvzyavfpc?^YA$d%mgybYi!WZ{V|9jH7s5MUQo_)XD4prD~g)jBG(X~_kP$iu> z?sZSaJsC!)TD`d6OgopO@YIz~x^e2g!TNfqyZ&yo1wq(Z-$;|yFX*E-tju{o4OJ6O zGxJ<0Ni*lee%k4En^9(cJm^Dg{k)6vT0_r7w~KV0Xb8kINgML3)SG_?$dx}cvp7d0}(+~Ph)r_Jr3a<22B!i^0 z8)6Wb+BI8g(C#*jY~gG#8pPcTMbmSgZrD`W+(#dL@}THvy(zy>Ypn9z^ z!Gj^He>7;~6?U<1y>2m8J#*-xs@D(kmZgi$7z?7ZYLWSHkoE^DYHH&LoPRHD%53SA zUaNU|B>ZuFIg}FXkEblku-zO)X^?-0AnG+`NROZ8G0|vh+2-Hx;@iZb>@?HxV_u=; z3uv8&_U^?&zu!}MhYMJZ*RxEfDw6UT+ zbuFuK!>D-;AGbwYWBO+oh2$+{Le1Bq_oA%JvJ{~qjnE9OBU; zSnA9`f)8dUVboSnpq8g_N6*SZ(2AN#5(Ia&uRb`P#{JX%PXDE}sjvlKdLtfe#IM9x zPDh=~r~B8^jb8WkP87wbbBzhFhOGg3Zb+ibXt-^Gmx-EfzqaS?(y3XQCo<}m&ZfQgPwth+M>3dk+TRN5YbmZ`D z#&Z3lc1K@p80tCnQPfd2kU;PPZdrA-)G4-llIek3r6!!5rUs>UPhDv$`8EoPgQTXr zdJV6#yKrFgMKkpZn&89Yi*}oF7zCLc5V{N^%6mb8%@*Y~ibhi=CG(Wfa%C14v$c`g zxz?#OB&!x*s1-&Lw#a0HFH$#}rFtC6Yq*kGB#tIb^PbSfeOuJ-t$1A9LdAg}TU$ty z&yY-X3)(e5tzFWDmf6rUo3+f6HnglHORx=P=}_7>p^q+f&3!;xkshQKS>?7hPG>ls z4ZWhTE9Yb#t*Wf2HK@lzRv$0y7}M8y%Vu7S@Vrn}-oPzdztN9$wSP}{VJ=27gZpoF|hWiI8qs|97WbV+}4ui##hG@6HY4z*oC z&rNenBtQ76_AAuTXiJ(Dkenq+TBUki+!}oWU7ypFYv~NPMom6OQHQp{ES@!odmSz6 zDoN*WEzk(J>B)-+N=xZ7)uby(F*_P-pJq|Un-O5`QMev!Z~iSdf(`S%)M% zi{qSTX(F@c3m|u2yr9G$Tbilg(@oLcH8ocrDmz zM#wX&T8y;5x4jjxiR$)LoXw!?v_R3G%gX1-hOJS9%Jx+2%2ZcHNSGNw^lY@6n6+P` z<%0cGWvVu@7~O`&_>?c!cpd$t1z*8CJ;+oPqrn)sc&&em@sdxFxTdY!x+5Ij19&<7 zXX{I%t_xuT%m|Tz&A%5Vca*>D*nkhTrWOQvrtaMJY(nw8wRd;-*f%wGg5vsB^)ep) z4HPFWWfVV-40bul`VkEXt^>h=6M@a7E zAg<0K@4HEeWe>aS>Bba9GKPCW(vM(Xc`N-AjT1r*7jM6N@~}6UYW?upKZYS5Hv0wb zwz#ON^Jt>hDEWwz32+y)ibL4gd{+BtP|RxKA+wrlop~eO3pz4UucCA2Z0%q}10J&~ zZJ+==jA?L-)_e5xkfGhm>;E^%uoV?-Ev-F!Xl?4`Rso;HC1bC&=j;)R2=yx38N(9n zD{uoj;Lsk>eyn;6^Z`$>*6!mT4 zI`E@s(l7ZG$pQLM`@hi#S!PP$hOVqouI)-sR*5(;CGfBlX0i!Ud?#$d)l@myJ^+sg z$N;H0ZGGB9$t5JyWPm`1=m54QnZKOCo2iAeA;srZYp3udpx{LG%cKyHxrfLqY&1=x zm`tn3V6C3QLqV{Ou%BS-J&jUs0Lkp}qGn}c<7E80%+WFjgV-#2AjRsaL=O~=M_Y@_a&hnbJVOvK;eLA-li(Z{xNr+5Doi>OOx-W#dY?=w6%%eE=Kp{z zAv0i>Tmh|N8!B`gmeVu<0JFtrym zhpIgF`-aLBBcfmO8zj>zUly_sZ`@#C0Uo)ua-V^G@wUT$19{_B-(l~f9F{>$T}~@u zh3@F%!6&GqUlsWp?zwa(tWhqkwgr5Unw*t$*DZgBp<$N2h1z&^&r=L)Eh`x2SPwJ1 zuflfc>H$MYJCiO(hpWg;(BUTOJfj2UaGC79!@WnobVIwL-w1b}ISyFf@$+4QI{uBx;cJwW%Ry{?Z(oGs{fi`e$)M|pxSLi~ixb*kOeZ)!Ki zJ{VE%UFwQH4}u60-He=Ok6rqFr!vEE-i(^@W!Ze|KWQ*hK3fCs-`{Nn(Jo4T4C+EN?nsxq|qO5ItY zD~CdmcZU|$B+KlgT1>f1uz$H@YVC`6v_G9AVg~2cIRZ=uJp;|7`V|djbKl%6Z9l=! zDJ5^E4qVkzN8d+yg{fOk4NHu|`37+n?OwkMXQ|zR0~@GjcS~(UmRZQIhe{!@5=v5P z>7gY8QANXJBp)DYlyjWELIToMb&V2YZEBa2Kcj^9WTtlxgX#Z96@N{qjIXJC!h;)B zhYLi2?cpN8_5jy_>ZJ$1W`WTSV{J`s(Q{rwg1!IkY%MdAROR-5LfHsy6vH;Z9%p}^V(SoKFLM7gt|E#Zq&p1kXw*RK%L-24%1&`qp*Z6cr2U5 z=HBh8v+U-dJD&%UCQ*rp(?#O66N3O+0&@TwNC0o7h#O6j#ysjvk>C>hO>6{GiWnIb zg&{+*Pf-_&7FnB!1O?R*VHA_8yxY*#AK{^m+5}c}I@yPmzg}i>$W<+qfC07?<`8RYif|2ut<$*h$^FJ^@8ypZAHp~&FVLy6ktlnQ>jE3`Fq2e%{ zI6-SbOKFxzRF3ZUSEjRje2fdz?_Hon4nA|d&~yqi^1O50CY>Ms$gmUZ;J02thWT&7 zd~X|j#&B%ph58lh)n6h3e|>@SgmLOR5*SDR)(^ZzR-^N`K^o$8hfbMrCYXn;U5l)~ zDR!Sc^feClQoqI%4jhBftm)!eQ?QI@jc;{ZOwU+X`=}dDmPX;p`ju{Y6$foO4d87z z-g}vy@6PYjOz9CGjO$cXX?n5qmRYGdM?4UiAxItD!+%2|rTxUAjL%`CD7Xas79jXw z?3og)yaShd8^KY^CC8-2lH=Phj&-2G`$9dj?b2REU+nv$iF{U}AZMIl zDmcC}=_!7dk1b_!h?!a3Lj4tu`!nYnnndpLeOK6m^vVhLYBfm}~T zkg0w{$r2?uC=rwpyr^GN^4FBmYfZg8LqDVbpQEBoFHRriP|t7$FZU2q;eILRiEapl ziX79uE6SsijnY$&qR|^~WThaGy%r9>?DSfn0^B@BLd5*sizmx0h&d9euNw27VREKXeuZoDqgz#>?z1eB_>ThBxeP3nOhXl&H~3F;-NBQ~!o69= z@n~m<&*1a3)5jmt7O{F9`tw;WA0F^of&apzdOT>G$K6Mx$wd8DMgo>scE&$2za0LE6!l|KmbBVguSS-C*1L)0*pb(3XC+#;WGCBkf()lIBq(5J z*Z}pz3^&eFS?S`c+_R~&oXSCmRF1hNhg9y#F*v0vxhU5hQY%YlzW0C`4rN+iu<)k) zb)y^Te*L|#U$bAUl{H)!{{ElV{_!sKzx>3wJcR5P0h^@0j8hXb#IoXci02tI!;g zN1-`dXnp|AF?k%CDtn}o2#o~Z}pva2Z|tET}zYJLz>*Coq6x2 zfoh{_W?c%CH1pmcq+!3;jxzh&<^XE@qaO0hO+7PuJ;)OJy4de^!_}sdmEKCzphpj6 z)8pjjAZ^Qb+D3WhV!N{zl4OJOf)t8Ds%pMLzb>cjrc{VKFI zb*aA^rpXo6?*s`thHB=6%{D$~3!~TX6|bsi9(Ahvg8*OTUTw!17M0bC#QU4+U^7KY zecZw2cZ0Uf=0EOt+AAaFkK>g_F zEexHZH*{tBGkWJnIt`u6dV5IySGfNfsbTP{%ZjY!+5;oAR>F3o&La0f=bu$1asB5% zKQ!EwS#EPu47D^fPc%_RH={TwCXj+uU_0%-- znYVHMhCfjKbu~j-8xJNyX2PDup+Q@vA(}2;IydLc#UjQ#D0PS)7=2L?uPCyP%tBeO zS<0*=h`Qx)#=D+-E#6#Ud-#dRJisJJ*J_fgf&L-Ar zo=wgV*231{wmOgg-^QJM2vN2?-P7y3Wg5FPhNT<2vujwEPJageye|zmPkT?wj-!`R z-Y_h?in6>0>O6LwQM&qg>(Jy%Y>lQktB!2O4`q_oMpKCq50cEoT=7cHYBb6|aEKj5 z=BOauRK1RkcO6|-BxtQ(#GU*h#8CTO8yffZPmFCF>%(5xiF#m-8K0Qj?zXpGLJec6 z58a`6-$X6wpfM>wEDfA;^y`^^Lsa;S`|Kn# zC;wHyMfF+yHd2|}?W>z@C2t~=(D&EfIei9ym^poGZr_L(RUa73%MWU_#(M2I@cpdp zli%EoC|&h^Kw*@ZC@f)35gMuri8&HvC7BKD>8xc=?o<>Zy80;<)C#-o1X1Ms6L0%N zszR@QH0BiaQCWKeG_hioAn#xCRyqY`Y z4MrD@7u^?e^KA2a)OjW8+wh&A>qF$W;S*^Uj`jLgc$wvDs-Kad)eHqM9$q)^ zkaquKU<>>e^a;`zUPpU&d<_M^ z#GTOKlO0=MX z12Z z4fPDbg0YORH~^?&hJT8dRhz^LMAQ3bQ1?`jgc67D-16AH!X-a0J5>Vu#(2C&R(=VU)nyU~+3p{q;BRq%GX|Fr#9(@^gL~ga zEQ_w>7Q`L~gV7su8f+c}wqW#j&gi8RxLuDi*_9qJ83Z*bt(Rq)WhJQ4FLnUc0DKj2s?P8ZR?I{|TIRh1R!(01|g&=(l>qBJr z*3~^gl5gCjRRaLOht-AK;Z+2%M!IxYyQ|+dx+W2dJKoUxYfYMeV-QvUgdrSHg0TD5 zw6xmoh&HxUXR-Gifq5}@5m*D&{b{g6u}~m)_94vUXQTtjFLBy$m?&p zEne)2>Qb|xkRbGJ&Q5rty&e})l$CGL-ZkgnFOs6l#MMvx*Fm(^^H7OtrsO$i&pXH} z3Ak&4Em2`D;A+7sWhKLs~q55lj!2TL@5IX10Q zipGpx;J+e~oQ5ddj!u6TkzhE&I;io?6AyIQ+jUG(++A;%F>C*TW~Cf3iPx4!EEmZU zr%jmnWpf8@rE_c}EE`>;ohW15r_hqQuIq~UDSOXtr&k`@#6gkAL=9nsZm1FjJlehMY8|@FMshV!*O68ciI{3uQ4_nDqT(^q_aNYv zd3pw<*dU=u5Ia^039`LR4>OyMVglKC$_Oa`19!3n;dxcPYG9f`18Bcu=(IC&9(!(8 zFN1S3WaiUVKxWfiURDy1HmARt98^%wJUBBGYzT=As}D%fo=KS`$jK}ICYrr(ArSYGminqD zjW6_27ba5X7rL+{NHGYp@hel<(7Dh#6famcgd1wYTgCplRHi?93PBbGSOCsdz;O+~ zIV@L#IsiF`a{}d>m_qH-ayn?NPsTM8NY4j{s5F=>Abmj|zGI7d z1Uu=;Bf$b{R47eBAbC_C1K=M8p$F&7>V&y3G4Egesg34hv+?_2M@V6MzL=z) zP%E{xzC*z+3M9Gf4Wt^pRWX;Zsqdmwtem~w$r&ibY#Ig>hLscH3-xo9XPU1 zT-kMuvH^TI>V`Ak=2XE)9Rs+&u-{t7?oxdtOudOlJCo2&&WrAgxS7T=B`t{P&_;&Kc}Jm@uWfzlOZd%? zAi-}wqS5$WH||@s^&OECQ~d>MsZU8XAx1j^>P+oGz@zh2KK1TdosJ$h(*PlD3a!66 z{tO%Yf?WXi@uxmxe|>caZGY|6i9zwJH;@<&`h8>ag?04{l#O1KTH)73Ff+J?bA_C> z>~Ux&R8DSo@6kI_7gZY^oWffZsM5;oWNvY6vTtWhZe%>H|Aq{(SN{S#l$72Q(PhMQ zv9W`<{u!cD*0zm!f^|bvhc>E1^5HGSuN;n6xzOwC({2480(jY#r+U0oPr zp`-l5#scCHSok}JdJT24|AF@}4NL13-zp>4We>d_Q%6+0cVhhXmE!3u8`bgCAL9** zrvV-%X{9b=A4MBGhOTW?84_&l-?39-T)a(#`V)weVv7{-PALA26z@T7%@(0okj7Tn zyWw|LumPN;@N-t%f9T<5mJBmt`3Af_}@oPj-Eb2+r_|QcbM=)8LYj7}4REjs` zdS0@I$Q+`Ynax%5?w>NHDm7L#m3%-3MNsAGRfpJ%q1Rbf^%=upCIe6cld<#}6M5W} z2V>YR8^)*Be{UUq$_OW0CxWw_W6aW13`fil@~;OJBM>Q!7}b z?Ven@jVP^u@Ysn{qr<9wmGc;N5pqn#i4GscO&r?g$Bg3&)8oMRShcvP8npRnY*aK6 z#~D*KMd5|tn5rG6%6~~>fy6IK(3VM&i&ZyC{1u6-B=#ng9Sr-HTPXNH+zAn2PxB0G z*DJxn)YmX`Hkk9ZdDgzcy-AVjwQ+S>?w@X_Rx2 zJMAgzxA~*Y^?ljzAe8H*x5;+O=dv>ff^vmB0giWi>s#M88a*n~Lyj3@XcusFtLrvJ?L_s}XoqU9qZ z@%%H|A2Al{ z8WT{o7DSEPPlAn2`lkWj*@!Z4S#dlTB#J1MIsuW{c@ImZI7xyw`K}S5N+e2s+sntON6ymmx22hNy10jUa}vZ?SL?dbeafJRl>;Y#)H35HW{g;Yb`} z2c042GVcj7kK;!-bXnsX^_mZd=joIio<_W{QnB){;^e@WigDpbfPRw3+$6a_HA?yu zK_wmN)d%RKo>OjD2V-SUX^~Mexrl6IJzO9o$ZQXQqtF3cm^m{r2I)RyFyeucM!98Q zUXvoRG3j2pBJftviG)3M%SOxT|S2 z=j^wg5lOX9(%B0#n^wxg=Qzlv4s@dXIW;q$_m5-|%b~mrL~;}8RUWG}SPCJ7YCDAA z^Qg#`sc9Byun(&-cH$Of)~O2Y;cuJmuc~7mZxF&%L`djcxxs(Anq~uGS8Xt8RQLXj Y)|LGTW$U<}_XIuz%9;Qk?%*Bw2Zm;njQ{`u literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/phi3_small.cpython-310.pyc b/vllm/model_executor/models/__pycache__/phi3_small.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e9d52a427d6c6f58dab6ce10f1658763ed6203c GIT binary patch literal 13156 zcmb7KX>c6Jb>5ksnVmhb02UxXf)^-K5?3NY>#$|Zv<~ZVB$E(r%1$zoGhSi_z%F-o z7d^8)kVULA*H`ju}{vr6qMTP6l<-^-N$pw$O}vHpn?S$(!|jP;d&th%+M3R=pUE zI-|juGbYcO`gl-sO2LFP5llLhLbL1pf+=Szn0BUv{my=&W$Oom8D~cFx%$E2kaI}# z`TF7Dh;u~ph5FH8)|m~CImhIAq<&9u+&Lbca83mGI`;~#Sidj0-??A%qxF+P*(pnY zto}gopz|Q|<6fzLDmd+&mi$D0E;!?y2|nO_Ab7}m2((E*Q?qLid;4&IWYO@Zyy@2r z=h083y#3yR*HYdAe|kObd=Sqw-a$Mc^r!IrA@7iP__dVtnD?M}#5ep8A4o0g-cfJ% zHPiVB>K^m%LEU?NyY@J0e-xDC-U(1nbSY;+x!1c7l>54rkAZT(cM_D7UCI-nl)VQ) zd4LrE^t$0Z>1h{JE_j~Bz)-Tx#m(e3NNVUf**n*kV>Cw zSMZsu6@F^>6SUYSQ>xkW@uiuIl>mdL;*oCVrFPV6M`Y>Q=bnH1Gv{7*Kl{=%Pk-FK z@bt6KzA*m@_u>oAb8)f%^7B98SG;)Yv(1Idr5^FQ;F9lo)y5Jy*ys6+m3BRHulm)c z<;bl!E1pl*#^imd)E{;&o_L{wxea`;T8aFZWmLnrAjv7z&s}V{T1|!Ty8z~fBkUs2YcalPPv9b^~)%7?p73-Djz6xWrg&_~(5#i43 z^17gN$#s7g?gp-BaD@?)D7Bti(<2ROx|7<_J?*B}(I->!fPwj#f*f?ZeVGRWmkF$;0t$64zHYw1zT(beTx3uDc{l~NCa`0lwgQP4Wm zsinT)?&E$Kieo*W2XXPdS>G zaxlz(x@Oc&PsiPY@=ALKlvl*}MZp3IB!mG%1fLlC4EniA6LNCa38WcTw*7Y?VItO*yn)(pg#ie(2>Yt8YLrs(! zH}son=%-$38l~w@n))R_C{ITQmbwPLM9q|*#z*6W&H1^v$aJD+^f(<<8%QeG%BFmN zY+haV71T`?%6Bm~7wWJf@_jJRX0-wPp+1iGj#ZISaO{d(7hC4o&2}WT?Sp15`5Z9L zQn(+stGR*v|=7ed4{Jg zTCb-THP7^{*G3%6%Xl^rP3Co|cE`q-=iaJIQPE2=%5otsl!hA-dJ9ph595~}lfD(7KX+ASDYXs2qU z8o91I2fCA6u6mx|aA8s7d^b-!ou=7 z@P82v zWh&Im$d!%cgPua($t(fexPXpY_fz`c6K60tS1Za}M!k?Kp(sLNXseT3cf zTnUQAP!~w!(-ag6dcMG(qkS}A34JEDP9CT5%=RT|KaF1ZgSsF01Gpc+eFpa#DjfHq zC#bY`NOjO+3}uH8g3_>}BO6Caxv8z0c!PP69L`3^KqKd4xP(XaV!-}Tk=+v=@0N~q zsT`vd(Y>Iip@mcHuvV~QKjd@mzBQ|3J)8R6_*y2qzhiZ{K$oYc|Fv+p)i*g`5NSJB^E@i=Jy#%z0cHFaKs{EgPLi zp5^3W7S|Lku+hnSg&W}1ofDNkGA~wEVExPLC9JWyK-bI-7b>usur-Op;#i3Vh0A6{ z*P)Dm7FElJlT%H=_Uo=!4G^QKTytB^uquwNaj63R{yFw|1X?=u-Ad2tb^R+o+BDq6 z{V8Xh9GiyzO5Jb7*+x5Xmw}_hxWMe?E8VBurFs*wPB@N(ifgf?Y!txn)f#ss8eugpub2QPNU!=&*L>F{vX z{>#xlzE?G1rh5E@&Y|I|{J7!IP(E3l9}&L*jq}n7BR2zxfd$!uu@0#3%0wBDeSIN zguXz)w>I1W&VIF3uL3R@K_yy-l2Ik8*AZM98WdEW&)Fse*1GAbCw1 zyf{UEP+w9SPWBRF40NwTB^I)8sAZ|H;8o$@Bgthf&4Qs${sv%&y=kXyy+|0M7Xd;h z0X+(uwF!zq6H7DT00Z6X04S)R`~|AKYh@FlV&5s5?6C;2;-3+#KZjf zVQfP$i2001l8Ntz&$bxP0tSfST#(cckP##WR|QDQgt|93U`U{7;qov9B8ICQBB>dR za7buuP=o3fK;BC)!QTfG)?Z1lW!7vc@#`dX%A57BI)*`4eT8r1a$ z%yS3zZ>GTy`a>T%K6kQI)t+_3T!U%Y$aiwWaULA!fiN=jZ*AfySXtEfTkqo$&R`lc zUDnhpKD1m=4cubmO4Yxr?n71)*Qq5E%@7h2IT__k#75OvQe_e`X)|deK_`zgf0fBK zCfAv;eU8_$I`oseeA%i zWPrbr+W5!AtBbZ^Re%h^s?(t;q;5kaH_r{h_F0dZn=j)BJnkgBcf zX^Z@Dl=@yJHP{gN)M^4hBS23${#>t`c!)+MK}7+6@FW3pSW~OP8NR_5xm{@aZg>#^ zj=r9R+m-fPA=co)KGU$tn8sPg!$jqaPk{J9z159LCEo@e8p-3LRNEQ`xdzvMFe#vz9mIWcPwGjuhV?emjN1&CW4Y+_ z08YDzJg7JDF7=B{ewE2DAwh(9PaR=H!Bjnj4u|CH6dTeZoIvnP#>>d$AoIz1>6-PP z4cW@UZgCV>5BD{}mhtNOps$gG;%=HieTgp;NA&KRfV}Lj1bzXfcW{MsNOo5O#oJI{ z+Z2FdLHOBrhwv9c zh_O;wCdeM;GfKUoA%@c3H1RMP>)IQsH}rJ_gOHw&wK?^3sD?NSCoow@UqBAQ_+R5K zo#-f03-l@mA)(kli&*3(ukyrME^dZGjMNxG(EF?9tz+Al__?#<1B6c?leAsU%!v*P z)mPd3>r7~T%M(MIP~JWRYMi^m&94FQj|+mP-KcpPZdbgY(0G8a2l!asgVxTtAoe7X zmb^921MtF~lBYQ9H@t4v>T*RurGA+SmvB6~SjFz=K%uH*q#ped8(~|NYkz&o04K`z z*ZG1|Oz4@-aXe2TlfYd7QNT^uW55MFUAk>&2zxC6RDILlT%F!ytaq#n^eRg9Z7vX~ zO@yB%tc9um4N!0s-8Mi6WGJ5G24-SL`U4w&0)5Gzi>0j|-}e69>Tiko>FAah6}s#K zNl-^N-I864Z=v9pF5JYXxM$l`YCIY4!qkgBrgA%(+O2+^O+KsYg<=8XY!79Jg8Jgg zEy2`e=Q&Ia6o;_)qa)PE-M@tgs)U-R5bjB0mO!#Z)C|5bjKR3Hft*mWEon9Eju{@7 zKo$1dusLaAs~`z;SnE21G!wwtK->({05{t(b`~%QY36>!<~qi@v1YBDl=s@<^AXTT z$Es@Uh?0VbT*rdhLl8$Ko|ssEiJ0Zp2%ek~_?wGW1#G~rbXgDTH&GCqcwX|=*O6EJ zHuViwFQ7U?#NFD5Nq-Xw0;IBp{*VMQs<)V1W%5T%DA}=A#nucV*|u2~LwTNS|0Jr} zHq9dcbjfV&UnUV|x_61;EPw)|IS4?Wryif{T1zOmdG!@^RG!?`Vs15-aT)=YBrI~1 zjQ&0oqG+B-Xe@QN`>6DXV6}WVSHh3L)?Y@IQ~a(EBg059&3}Tf8SN#y*3IH=`+%K> z+X>@}`yIQBt}RP{w~#5lqu9$113L;Le<9At^bM@1Eon|VtMiXTAJ~%F84*To*M?zC zkal8fv)IC+sm*mwZC;|im~2Fr$3P$9VHezsVoX!cC=RuZ)yBOkzvNA$KjhJ)B?ns! zX9Dj&>CJ#Ti9;iPhj{F8teQ0tjdmxZN$fyk3thd!+O=YBAJK7b z%G0Ci4T(D6)HY^1=1q;;_czkX=?3guA{H$O!kjOsol;XRxwMW09q$xc6**awSX&zZ zc*1K3!S%%2!YFuEoPK#8zIYB1vk(y@obxFk+-|r9HgNUErfhwwuQ5MKQ+>7SMax5B zEOh`y646upSt-+M{ zH7N3rqcAb!++gDFY7!Ts3P40D!+lqYDQ)JA9AaB6)i)7a(sFll=8#7)w0`?=YEKg{ zXbh_WW~gW6b@i9zla^j>FxhJ771iEX$uUi{tqnoA;Wz)kjC)u>SCord%Eb%O6!v0aH&ednK&ap5dmcgpc=OlHZ!p>V;&>lFD+%EYt*@_x74>aY zUwxntakh`ejz)VT9rZhWF*#A+WwOfv#QXSdJGy+6eEh-QeC*oj$KV4a!X<$)1dVo_ z@seoC_QA=s|8Cj0`mD_EIwV6?e~c+n-{U}RFxfUCc}__8o!{D*#(k_o0xm{D-&DJ+#L)$liiiRCDgrb5CmC$hhJU-M~E&#F(!ZOwgEqA_0X>C za5353SO1De3)|loEIKf#p0|+2;#&}rH zOZIPgFckY0aHPYl)o5AGqM~XsImjm6HSr(F$N6N#2YYik(_*j>IXRh}aN`TM4=eUT zv6a^yC7wE$2uDGu1=Zmoz-HotHAQM{7Ifo(SC8#F;KTjUPnZ|ITt2qVQE*BwU`PLr z+(Zk9j(p_F;v^H=x$VxwKa%!OO#Ye4510t9_!x7PFZFLs1ZHrFCZYOmKE*T({+$WU zauUe@Q|A7Q$)3^t@1XQD2j~(qb_xTsi_c-v=eQT!XWf6|dobQ_8@c!NapT?HeUaQd zdWj{;&Oq+H6FJ5AC1d!Mx{Q2mVc#4YIN@5pj#8KKPokA)o~_NKhA@9Yq1@95|BFg-=&2dbL*ZLmvwa4{s+RC zocESpS?8X+^xp%$RC9_8$$8{1wUfXaMN&@G_x%6?;t4LFVIQY=>mwONInk|p$3>I5 zqtrY1?$&0fm8MWQBT^T~BicSTq7~)A$w?pvhco<89R(palg?+6mk6AM_taTFig=x7 z?hKPBnLLdoHW$HPBqzmU3!wrW_m$lg*HWS<3@FBDquSz&?x5M5`wtSgG(S zBzEc$ww()r@Z8$Nrk-9%%MV+Z-#5Wo&!WEEP+~Q(q{Jmd@;d^8Wys CiH}PF literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/phi3v.cpython-310.pyc b/vllm/model_executor/models/__pycache__/phi3v.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..359db8a5b9d3981b044ae1f7fffa05889cd35845 GIT binary patch literal 17530 zcmb7rd5|2}d0$`CbL{Lrd*K|C;4uWaAV7$eD3An*bE(A=B0x!OQ5p^A_3qAMj^*oK z0Gyh&9W7^u=AJ2!x>z`A=@yNq0Fk` z#dp)Q@N8A>n&UaOgqJXxZ&%$~(o5D-UaFS%(zT42sSS7owXBz|4SIvMA#bQQ>37^4mwT=@;Y~=HQ=P0$c~g>3RHth*-b`(uw@;qk>i*gR?|`I})q}M| z-l5uI?{F>e7@Sc!#rut;xwi>A~u3?YMVb(nHl}YbU%Dk{+&}tUc#FC+U&usoF=pkJO&`o|os* z>I=1xdLNbaSoLGI7rhse9#^^Q$7`p()3ukpm*jb(`f}|R@0Hpa?~FW8R?pU6^afbIN7SS0F?Hl_ z#`}bN+~2oosVCHv?>XN5B||->3hx=J;16%u-W5DQt&ZaPs5~$DBmNbCVbPi~yw_27 zOg)3LXZ(qktEl}+%knDO8H zf-cUOHStrKt7QB*La#9Xbk z%})NT$;Of{f)Px>|A|yEkuLRzY|93`S~l7 z5n!}|O1%;mtBs{4U*mD?lWQf6-7InZ_LQtBd975#+v#Yqop^mMM2Dz9+X*V|sg8Xsskj6kkI&(qbj??lQs`gz>p?^FtumFMSjCnV8;kc$Y&-i^Yqi?D z#R~VLxY*Fea<#G=O~tu^|4xxB-7P?av1@Cqs|_6ny#3Trfn*(+_C_l^*03m9L-IOJR9A?qLY0YsD{1b5*HD>E*>@S&+WW4e9(6NZ_@MbGU+60YYQlY#A%& zrUi;%ZCUG9%To4DLr=8K6}x3k8OoV5*6q53d`X2)%WT;yao1XRS`JRSyMPnT85Jw) z&b|BF+uNE$>LUOJ)7$sJ#EXqvzSb4x7ZTAUMCjtD=B^j#&%SZ#Lh*p??I8}V^ zx#x@LUSGI4cPSdI)Rss*%6)3ex3sy-~b8z0&d;_(X0K&`lBqBDj!GceM7&5 z+Yc@s4{NK(S1YT}ge71`_{A?};jd5jS318~eCGi5ubWm?;o>29ZN%W`mcZ2by~%eeY} z+>JB1f|mh&!!woPS#KJ?t;`w2ca(M7@Dskfk_7Jn1UCTpK)!w2NE?#xDCe}H65NSs z;wmQjDItJYD|h^AOn0K}Dj$lsO4T(#h@22aGjezUL65qK87#b?j#9;9y;Spy#VAuO zVjb41OlOP5ch*YPxJ0Kg(`dK|;!tmdLb-xsQ4caVf%PgcM{Z0SbP{>8_&WgKQ=b8N z6IVcyDr02LX)IVK<(N|P@I~Tj{r_7eMHEd$Grfhm&>;hz#fg_ds6(K6C^V19g1*2c zQD+B`+0P?y8CSrSh-Z(PGk4FKBi{&40d!`rQeQR}O=TRY`m3eDk6bXbHCf-xmJ%9>5tGW zOCM_4*No8mqFHbjqJ&UJp+H``Tn)g(KqL%d;P7e7m61zoW1_$8y7qrn59k!-e63DJN@-=g)0?}kf-jb9<-1f zkdL@l#z|Q@E**%2V>O?A$nx#(pn zGuYn8vR|!s_EP4`BdBFp7mx~Z1>|cfW5}{x(6DXC9NX3`djPRrWbzj@0lJ870-Y^V zC*>?Ug5(M10?899Nm`{y!&1s-8q#uFWq|5wHK4NZf%mFGHH7gJ4XrW7D@?Q?1hfab-4_Osi5&bFViSW-TS12Le_0%n7AuNhx7!=cULaAb27%a=xO zN@a4!lD&n)g5Jh)K6?!=Z((|uP0mX z<|OJ&Z5dJ%tRA(jEz>mK3B#e4>6ZI*e5V<{AKoUb24%MUNLUI(3z1vrX;}<%zXquZ;tokKgr!r6s{MC{!fw4qmJ1Z6&8WNzF zT&mtE25UFuu-_v$y+`t*-y--7Kw-cBB-1<~QF6HyK+e+o3(O)-7OpImK;PhF3Bb#6 z>O~Q1^l9c2{k0zqShH#^ro2;r%fu+&y<5+D1u73B2GE zXf`wgzpgyz(uMgqA>aD%tWo)ZMl-}!=}B?&0yySEhl(6oW-HSh;%fG)MhWGwhRdEC z%Y~6y)N-)?r zZ~9TXJN?KkN9J;5-uALP*^)ONpV4-|;Fi!OcF8=LR72k`Y4!Wuxg5YqDhqVT04d0LMTTrwj4 zkFW;%AC>+`(f?RDu$gU*Z4N4sW6X1?ol3USGeW}G$5Gd6jc*PU+qXb&*K@%AkycI! z@8wnsn4W50?qXm6#_fTHg0@Fb9Lt|P_4LV8$MPqhJ$b5!f?kqT3KXkJyz0{6n6&1< z&>a0j9wI)@QCBaw|fPT`q-~J*cM- zdqoxFL{`{{T-pKZsxTo3f;3gk53#%#2tD!!WA(4to=~>)$a#4aOV!2=*b4d^deeJm zWAa}9VI{@()V-M-qNexvC}%7(D_)8xk%U20I@IfY6L(q2P2}E|yQYZcjd82`JMktS z%;TI6?wQ@4&`8`%S1JnPN$hdz^&O1-_izPdL)kUn6Q@;T z9~A6Xs(p92+)W!IE;Kcem6S^38EsTXp-s-%vhkGUh;fT{yUd=gSTB`B@B0kak5lb_ zoHlD4X#V<-bKsxu()ov+_B~lujA~DxI0>#M*e1zS&wlLr9+|kk>}x;Yt{KDx$oNob_CwGhlQ zW^6yrs$U`a8CEnGB6B`6&*^vN>2fsMe=cwacCj!0Q>eL(D|i$j>mD()_zzjrK;w8z)0LB zs<-xC3&M@1fR$h$$rh~uPV;VWmHUVqUUnDV|8cf-Ph5Da+p>^kZ_{wsxf1pJxWQu1E5`Bt#@C3Eo(nZ0@ku zdNU7fZ-$4w}Ks`^e?i+WeF6Be#3$OC6vKS zX9T2pDIjR`kv-?!*YDUai$Iht7FDAReV-PGo_#>S&Bsp=`~(3>yc`eB#nVJvJpd5d z0rVtJMSqTmC{}~dBlV|n1=9d2*ghw%)KjtDAhVqsO`S?j9U0QBRWMBTA1d}V zB**=HcH*a&Gi|Ix%7bhNLO*4+60l)IrES6YhQnkn25B!q@PIZZyKM0Yxl+j^7V{uK2}Z8iFf8!In>d#g5hE!bPL- ze^txU6m`h{pXPqk8afG*1U`!^W$H~{I9z_tfgow1D%2C9yW=VVOi@i z?JH0m=I)2FOxakHL`cJ0qUCTAq5E4mP5lhGpZx+CY>(355KBLbQrJAeC1u%Nq+Rxb zB~OMn7nTkI2+Aq&EL+dF(k(~_@|0?&WzpS*i`RF^jaF}|LI1ysH_%@r;Eu>^gZB3} zV+*-nMg-K=M&K)|)9`gbW5PN@It0;ZuX~2c_M-nP`}zw6J>-?;pNf~iPv+$&oZ==N zK?B+}soKyyH;MG*cFM{^Q>HKj&Do~#qkjz*qilt~zV?W8@9{}cK*TmND|HRDL~I_g z?#|IDLBr;np`o~JVi!gAwIPuZ)gfivwXv#Dj=A=b5wQNeDjZy^VAo&?{t?5ZN!GnrIPbceUpTi zcI*&(AdPQ#Y1p?G@Ou3d1vatc=pd&BFIdbw+9m<|A!AqY!7FD!Q=GeU7Cxu8X}5T7 z&bv^&d|~d=%3Sgmhhh1BssK$ac;gbGNwZ|j!TR9H z-i19SbdckOiO^NfU1tXcOu$Ai3#0!EI)fLWttrVw;m7m#_pw~r%AVWK@^W#p;CVOS zn}~Dc)7Q-|B;_6T6BtRq2ms$vyztjw|HOrb;@o-tIE#Lh;F|y-!?L`;&5XZBAP6YJ z-q?fl@=_glGD2-?;#b~{qHSk%*X1nOy29p{!$8g|b0f4gdgeWWY?j;vQsE*9iMYvVRz+Q^Ms$Yifux(6jrc0+=@%l8a*2_2< zdCVh9_uTrfUoXSAi4*>NDAB)9@HYXxOeyYzTR+fC7dv>cS}m3R zo73Szm3z+yKg|^%!FX9aV~s!~gat{oK>clGMj7~C;e-MK?YW4E{4=;{hnJ=OS!fv0 z(LgC-9Dz}gv@;ll!N|U4Y8Yd#$x=fQfFWgzJPn5f2L%##3&%kWNGDtFN&-|h31g27 zUYMjI5eLSB6&!{j+QRD>LWm3P`d48VLOx7FV>AbK+P2c0yV{G9=5+l@87sY#fwAff z7;AvWsvUh#N#AjK9U7$jp26OM9_e~Ign20(Y`GgiiLb#B1x=N2o5I@;zyw83@w4HO zN{7R&qta8xhP9qmFih2tg(Kl;I2Mj?=2WINuw`waJq=fz6KY_~RL)IDe?6SsoKjiN zv_6JDvKyvjtfN1aeXBJT_X@Q87L0I%GWq~aUZxtFF*c{Mo@jw}IJG_!&Ol2ZQNtiS zqaoW3Zx>8PQy`IH{bck z?|cU1lgW;jp#0GE!eu{0h zo>b2IRBIeHUqb0&8s65YThrmbn-=mu(wanze7Fzt$`pdK#_o!Tqcx?{kcXz~36!T= z)9j6X1pC8BT9YvA*=qc*jlc;8I^`Ck5eYJ(yBuecL`MI0pexRwa8F@RLcxg4T_%UL zFgX&9Yq&&ZSpxQcF3Fx@3Z~3s~GcXC2^4-z$F?*eD zK6@SG2=~l47URk9Bt+5=21mC-q@INPDV|l2reTdq75jFH+s1{IT_PuLK^VD-eTlU^ zQ-2G2P;ksUySF2C8nIb7RO#jKb~YnrG>`sRn-RAIq0hEMvE;pgQ-248>Ay|z9S-|w zEVv0h|EtXXNdkIt;l`&~k;0IEjNnTIRHb6=kGK>E(237Gj(n&y)21ZiBG?Ao64|xV zYFkgz^i5Riz;XoO$$Y56E+LdHO2Ri#$F4w;YOKC1^jwxDN;8HpY%~zUlw8t{wbdK% z>IMgKh;Na-+E~?(a86SIg_)T1|1D<7k@*f&@^X|(J8@KU#@1QXBKR8wJaV!wUOHw$ z;Lx6E1MwbvZPz2YQ&?IG;)Ud9d&6;qj_?!cFEe*UjM44bH!#M)8$y`d4#wuioK=bj zsdjWb@P>C(L-`#q)hXMJA~O%ghCfT3;g)8c5s1!k(n0B<&xJ|||IUZbcFLt#9V;ub zc+Grw+q$2Hz?}kz9WlqO4BC#tPCE(PEplwuOd&jp*4Zqae|&bg55NEhIS6jR9PA|4 z2i8wHN%oyZ4q90b>d^&A#fam>x_jT*&Lm*d{Xfp2PZ9)K=47(fWts1P~imb2y}vd)m+R-5CMGQ?+tn>A0BUn41l*FSYW^(Sjj4v z5K%MmKO-z)5V=FlLC^qT94v)859Fc6kQ!znfg15gQ6u4xNuceh8bj{58u!P#edp8! z@^YB#Q)*I1of0_R8AZ+b6KWs&n8d8n-(r>l$?qk+DRoe49`dJUjCf8i!<5nEjC!Q6 zbRTBmGI$wf`_&QufPc_G#2Ve;hsV_uC_5~{4|(u~0wP)&@O7~l{kfovSmtPSN8Dpu zWX+Zt)7sI0vIyv^hnDj~vdfs@Msg5v$kP7`fOnn|88jl^du?;XjR)IdqlA$Wx z=xaQ~B90@-y>!z@J-FT>I`(2yi}12baB4e}hl>&q^)HzEmjq(R67`K5l~~9~=f%63 zNx1Y>m>Ju}X@hIrNo_adAhI2c-F0_Az%RIN=Ww+{uqggZya?Ts<_dU6Xm3zcw-%uL z#nHY{T$eeI?-KlLf_((!I=hI5l;${9Y8^PhAP&vx-Iy@j?pLDO`g9d;Y)E2E?Xd1K?3UTzYLcHFfNHruPPhtUVatZAsQ}w+$-cy=e`tM$k}i~Z*tadHJs38wBko>=k$({(kXRXKq4_cLbC{A5*9!SSyx?8No+on) z@^BQir4**b(?+r42g9@Zw;F4C#I~(gD`*^-}|=y z9#4|G})nc$!-%0v%wduSl0sVF7%lJpi^ zwGFC!zxm1o+tuqJW=HaZJ^8FxEd9d<(Pibpoa-1#Hd_BjroKlYsQn+9k{M3)de||* ze$JEDM90iosWiDuY$4xCRA%>V+J{pe{Z|ME$0IgDwSWPV8{M)OZNV{X#qPr~xIF~N zV3I`e5pis?jbmC|Mc+eL567{287MwoO0)~$RtMLPD8x0{-XH?CKED*gjgqr z6JjRPiLdpr&W(NB-T<9xZO3%;#l7)Rf@DIQU4pvQ2$#1#1#G^(`y8OpMV&s z{~f`4hBz{wDiAsbe3B7~V0>Ntk9kMs{pwkQN6q6-t%5i`Y~d-PEh-4_3Jq8>20wuQd9?7dhR& z29%Aq$WfWQWD{96K$2LC=%u_=nW!#C8v-wL1-H3I!X5qoQ zLIYwdy0hwaC^8I*M0^F5jaM;HOjgcw$k~T=(~Bk8*%xS3w}WQ>`Z5SJgE>IS8*5dS zzv1WC>e`1?!Z^R#!l3@A9PTFxy11zS2_NN@NMC38c#q&H0dYVgucM)}K>$qy-%kiR z?}YIFXI3~%z&DVU+r>?z!#L;~VTkiEC8a)6a2;(%0NDd*I!n8|a z8WD69#U(=S!&ofuV8bU7dK(L79}kHR zsQLJV6nT%npcRXs2ufv?hC{1Dv_(u$3U;l77dIP*g=^#!ISZn zWH>X7qs(TT~ZY;saauWOi{{{2X z|Bc|g08t{A;d5d(mIdk25q*r+eg~j15Obsd&W!&-@IMK@Pw>A8d7*LIK_tKOpQ{?JM8 zA$7yR#W+d*N9ZgX6&W1J+uh2pTL;CWUGXUM7=M2VfLp{k0tU8r>`PX>SVOexAtr_& z4_3ke#93rM4x|5=pdX3#50Lk(d@Eu}D#162ztZTVK>eMHIR~9Tbdvi2awNJpK-*;x z-B`=$Oy`Fz-)lM_py1ay9bAaaXihzFP}=IsFQWY6w2oL2LSKn}C|~7wXjCS^b0PS_ z9k4>f2V%f$(#-8dh_*I4+nnmxBxm7!;CFYk&52zFf~2zrySXn;?b%zZ+uUu} zhl{46D-3jJunH)OzgW2gd!7q>8gvyb*wc6yi4c(}iw!+_VMS>Hv;X$U71mw?=+ql6m65A$iq=7 znO;CjEHCqwA0CkM&r$jVa3J|AC1qG?p#29}7#v~?2iW@1bzs+WhCZ-z8FSLLMj=#h z!#;N3nqV#H``gwL)`g4zL+haPLs+^P!FV45@*g5x@dGzu{m^nVKd>HQE|~s@DJS!R zmFhgXcsi2QoJXOg{}FBoRq&VmJNh3monBn4m&F>YWf^T|x)9PpA~Y$1CRdN&SyqxH zT~p$Xw}Mn&Q>1QxZU-#|*4D2wgPTD!Bfh_zt=A>~*0Zn9&5JGA%XWi%@#Ux-PfAa6BD&vG z&c>f9%!&c3{pquym-pz8sNFGUh)smp;T_(qwhR@_@@r7}KGe(qh+0*CIW2Z2@A18= zJwUt#LA~L)Zui4teVEgk*p)lG7FMc(&ZA5W+}@y6fUg=~d>~4iSnp*dFRu9*GaQ-u z<81VVR1n6$2!}|ZpJeV+08t`tQDBBpk)uor$GphYKP31T!8ZwhlYr)!_+ytdOwAMA z2Z-D&@|mZ6<|XF0UJQ1vOg0v-s97*GbD=a#;pJewrWb^53^y%u;)z#H~{J!7!-MgJa!B+6Q`*UAf`nUHe$~Wj@^k?DXK^)=NR7DYr zs|ZzSEj7?wEy%c;KzH@PaE-upO_lCtT3I)Xv);-EmTR%jXypRiwK;9J@`2+zLBTEX zb+%Ou#@w->oG!NJf?e*eV7I%Qug6+92YcK-!7c7B!L9DCtShx{3vPFB z4=Qejug6<=1b4c3a=P5w8|-uUAw3}`Tl2wwcYkn~dlz3%weAiMxCerJ+N#<$lp?rU+qP0ZqY*3ULy*Sr_H*Nc5(huC>mb>ARvfcA~zMwU1EZxVC< zeRWIh61&gp?wj%K&0-Ip-Q!!$`|;!h(A*+!h33{&^B^?0iQA#MJ=Gk9rXucu=8jbJ z5Hxp+z0m9>jlbW&Yt3*EiOf?vp|G+39uT zJ#>Yl!jAN~o?f>qtF4yb@=kSROL$>p#YZKq!@bZK_?k*49)J7Wo<5r9I)Z9dWn6mF z@3tDXYUGPlGyY`f)W%FGW$Z|2u@QwwWvAwccqLTjxA&^}kd-QZOj=32j;bX)T_4|) zeX1IuU1VHH6HoM_ZZASbwmcV)M}p(N5REpq<&^O2)m|&|PWg?+rO0b_sseAD4XS63`=0cJYS&vn z>4mjw%lF#VfSRMkRl)PaHdo_`$J*%Gz!wcP!!d5s@Fh%j1@Ca4>UFyv8HLmpu|2pt zdK6zV(+H_Qf<_pkKyLrQM@jK*L3P>p{Gc11@fss#r~MUr^=d8Z$TM-CGq$Q{d>Phg zn2rAQ^dfYT@D*1D^k_nLGeTQbT|Jk{={X~3;xC)Ca=DzH%jcY2Ay>?eK5 zAU%mxDOVCR&nb0POo{2UIoEhj(G}Ok@ZT0Y$#_pX(KWkEjiBSZW8G>PdCUG8?_{;r z^TU^vRYiG8DUo{^S|@*f8Bp< zLEc0eHHgFCgixM6JAaT9bnz03Q@(lS;;{bGMis4wXK?VnLU zq5=>Vv}tCcqQ_3RD!iz(?6#r41cY0A$UH^2~$3*lXsU72S`i2(Tr{$%$;z?u% z!T^AJn1{GYpcPN1m(|{BsKU^zb^Llfk!qU_G=oH~8X+J&?4kJB@-S~3k>^!*U~v1b zx_kr1pL`RE`$)W*#Qh{5B5{bs8%aD&;t>*uNxX){Ye~G0#JwaQCGh}>w~%;@#9K+c zjl>ZWkCS+i#6gJITs~Emi{V8z&Us#~RmCTG5>rsQ^p^cm(B0o{boWKb;Ya)K4SGw# z-NC8-t;X^F-80crr@jAVs}<}|Y=nQ>uk`>u$z`}dndc&4y z(^XU3pc`IUs+RugS2V-EyfQV4C-cL$oRzntY=QvPA7@bqdJd*Z3CUs>bC$Yd2i^JfMqFH!fGl)tf(zXu=5NbKnD~ z*YE>7$cRP|8$eS3besugt&(vI^tmtz3B`yrXmHm7Jz)T<3gJ~u`CimQ9w)K!>)q+W zcO>0K9a9Cf+}t46pd~+uGQyn@j$vs7())+}{IYydfGOR<@7Bw3|wOC;7ig;P8y-i;gY!x8R=uoO+xR9(}q>KW}7 zUDvMYhIUyuol9Dt^rfqk@~8CS>;oHmslWxKL+Uh$S)`m-FDM_q8Lh8SZwO6f&S}qR z)NdGWnF~r&?`uLor=k}@6t*dSeTO29K6-FjU)A1;-qP{6hW=|?cPrr?r(n40@7*|1 zg3;F^QBRA za&<#3a~o=Dqn3HDrJdH&?%Q0;JlE2}JsZ@?sZ)E=SD?<&nEm`vWA?4{REBdwT?K98 zbOCAT`(|Ck=*}%jf~R8@O$wad?6U87G0s7(GeM)xsJx=@1AtysIY*jN`p8`=$|gV5 z?!bouv(JFg#F^^xFg9S+g02fX%otN|wBS_L!RI448+PguGP)*^XUUgPBoih!Yn?Wd zu@-gQEN8ywfQtzi-E*K)U;uIL)VPCfxKj*S2_*p20OBomTEfi|tHLae&4iREP(z>t z4}A!TUWccnTS#k|%t3cNy<|{?5$PUqr#2&Jq8=M5i}H<38OE`E*saRY4;rU^`66B( zlGe_YwUS!W%xgQZlq_8>W{Q|Am_a<3O;x{QnpI!$xyrL3G&ua+kFS-h|b{~<{IG~NF4cW zzBs%x_vG*+*ioh1?M}3w1wNh;E>*$$s9^xWr*M*|Nt_|E0^vH;AgP-fx)xtGM7X&V zXA(Na^pl#i-fP#0tp6*@2#XNYCcut5OLa6$m3^paK5h7sLqxxjCdj{ z20EDAjv}8bMq#cEB;)5!?$3_Qct!%Pv`seB{B8(hd%qKjC4BNLgcR2D?Mf ztQKXMTd2${v~%Rt>JcngLZP4q@dIjSVQ2JnY;$0-TcUY|ZKE%BL2M6~Pd><9Y2R)>}4104GefA1GA-ILwWy{nq znOW_-rd53T#@&*hyAJ#&prl{;EW`l(aw>o?Lw|@}>F{O@9OTtxalo@-xHAEJHeion zFrUJpBQo_eLtsG^83MEIwz!tUuy zCAhE)+`n;-;2{a0IFv-ZF})g%*09eXJ2i@Rr0~E49{~}Hdr*WD_Co$RKDw5J82qSn z2{9Ng?n8HvpaliPfEH`YYK}NA0${{?bJMWIu(vaBRbXpbvmN#sfiXfCguj|*Keuei z{Yfg#Jx{sYISPq2%~bb#(nVPWx%ipw${M}Jq99yT=Q1fb@Tr-bq&EPInh4FMR|u#z zb0RZAcN`AQ6*zELjK;VGdX{Po6zgY&agG9Wr3k@0)>4UBKjAwR+$%>2$2IAhTwjk6 zlxr5&=)Te{3X`9miV&VlYJ; zg;2^F53E*x40>Q{HT2yAV!ds@bJ`1|GYDz_6z)_q@}uk98h##sG#iJlpc>0eN=P2M zoll}fWz|6p$yO&bWu4N1DAoYT$ra>3hVo0AMI@EeS2UCE0N@EiR15c#oQeDDTw2+d z0Nij!4skxt4Pv8V0{Ir&gJ)b^pbz!n#PTXjjlvUyAKBt84z>XT2_QUz6pZ&2FgBVr z-f0eC{xma-u)gs7HN*(u^p}(=0y=061eh(%B?@^MtC$z(&A!bg=20RA zUNVWjkZruMl9ivp2`m($mq>mbY56lGK1t%|Nc=2B#Yy_<=SjgF^;4930m99o>*e!E z#p;PzU5?e0Zk|6m<@0889)WN2E*RI}OWj6|P|VJ$W&FXzP*-L)H*#uDVBq%Bljk7F zu#9aoEIiPpq%3SyD0IcvB-TZv79}=?c&%i^Lh$x|tjmXy1*@4#_!cA(MOcqrd}p(L z(T4U#d)drs-_h*ix3z<#F$c0jx>>#?snqLi548gQ!WP7c?NP-TqHGzW-M#_q0_yFP zwLv(bZn1TtNLv9*z801ibFfAvZI*f8hIP>?O;6Cd?g+9*qtv?;my-s6*so#9Q68ZM zp#^quK_u4!CNU7yvV$KDWokt>5Hx#74FnBaCX@(X!5IU6i3l=5Z~&Al41tQE1_ek! za1biUoTHVMTx8=akHf)1c?)pabS9zz?`A30j^q2yb7WM?TWTj5a^c7^OwoV=6%tJXo8{L17e{Ef2+iDM#2?(igD zPZ8QAXiv}?=P0^QDiSGIOhl_`;x#S&Luqr@0er#@mQlx z%MQG_LOyf64OkRWvB_HR`W<{4!|iA273*KkjqoJtm|gMod;*szXy}v5HZAy!1Kdo| zkP-4_&0a@H>#~()^t@_i=U~I8bYSr0!(5H4nad8Sbkktm-sT{Bz-}j;$SIO zf-@_?$<#*?>N&*NFszIjV$(@JoDoSvKCtN|HN2nt{2G?bwyF@czlQh8UnTMDBrZY> z==2s}AU{W$xE1rHzrpfk|BR|w)vP-v4Wbn&vbdo!F7sn)aQ?@D^UhGWC8m|lg% zEJq=YD77d9(lE7CSVP2sVGR)lCec~n5{Pov9fmksIG`8;01X712{R%IdrpGme~tZRrsIEYt_1MEsns>fmbCgOb3!jIGLiiKE_8|OltZ*^pK;6$5U>DL>l z6Xc@-IQY>#B=pPR2tN<8j(pD(Zt=RXoapNylvoexjtc~T+rYt96Zf$aJVVqpNyCYb zh`Q2Mj_v{r3EAu@k1I65bLw+i9Ze;SvsT~u6hc?Opn+Ouo}pX?97P;sQK>nemQY61 z<+zNAXi^{mv#8M>rqEN2j83vH&T%e;Ps<&-$P{sX0l0?IF#J)PfS(*Bw%3vJy!yQI zJYfAPMS9RyR2~BeynX>!7c&XEaNE8gigjR#t@{vdJua&Ee}%>yHL9s#v|U@db);j4 zATfU*ud#;rBau9|lATYMk>wZh>@oSvR6J7>nt7G!5fpw66mA~t;JmKf5U1lQ?p^;h zHpY1Ci`#CF+$Fjd@=~i?r1n_9FSpY>$b7^Ow-BuWV;M#6A^n?4pNFF?Qw)xCep@7< z8=GleX~4+Ol6Zu&>x9$tW=h`leYR_Wp>2iPze#0LTe~)Y6W2DO#=lLsh9Le$YGwv; z3$zx|Tnmj2^tnbsKn*K1^k==I8Hn%#gUVMdq(FhMVlfrDtZOTiTXjsFORNqBA_)fL z+<@?gObg@54O;=yRi_Q8PSktvA_&p%_3rzcx}d-_OSAvY2v_`V2DG4ACqrtf?$wN-=-i&46km@09Q6wvnO`( z`?BF7Oh5A0GA|%z{1kEUT;JrC2il;bFN};XZ6s~W0_x1ZS-l})a4B*pe-FCY;0*`z z_mP%=K;jQ6rvonz%japv8%h615Cb3RPe{QQCwGkUUptdqFU#E`V ziw7)tG$zM*Ew!u-yc>kz@n>JQ($JN4)ka6JOpJE!z|!cT*YY<|RHeL`W8XqGA}o+Y zdM$~&VwDk0hgbs#a}tV8@FnV$UPWXx4#IIqDa=s0du$XcMF1PKs^(%&b3!9JpUM#dfg+Mr~T`#FI%m$=#ek z1EXh(8^n#czfIia&-&Z_9sCp~3+`!RPVB2C;61Da?Z%S`@y?!0#9#htMSc2kp_) zt1XjgDs{&PZoX#c_LslNW{fjCxD~qr#sUQ535g}9aTSFP8)6gjgQEG zK=HNBRi|DB8%?I|jrDva2t`6~smZuyHJBaZI9#s6O0ALJ9=XE+}iX zbwdT6_!y`HhKvB+IH}0F6El4x8S}#c9QgQ2Rqj2pyKf-OX7#h_(hNes7gTv4LeDzx z-*;j+CZ7){*eBL^ zV25#5Vqbf;=EoL1f$&rV8$NbZ_J1L`Ti|WzBHb+z+ngh5$9(#oluv#guZs=7%e9KJ6D9e$f2Nvk znu-pUI@p15j3#6Yx!_2o0a3);`}4 zWyBG%#7g&UE&&#S?8P-`p4Gp)a^v;TI$A0DL8>-;qBm}$tnQ9FR^C0@X9lG4Fu+7l z5(d-r?sPY4@no~+u#eFdgKZq_7^a&!Lw9`0WQMBQb=APqrloBeogudT0~6&NDDyv( zm?7~s5@>%#lAVx$O5)#1yovn*`3NcJeFL56wW?!mEFhJ{_P zk`;y>|J1_kVR9Vh@=}<4=c_%zcA79OC1AwM+hem49_={ zb`ObLAY7}8&FZ{Y5dX%M?4{y;qP$ltad1CGMd-&nomR5(#C)9hd?(TT4^v5>CP6fw z=f%hr<;hA@6cMomF7C%PRYI~W`t(l5O9$KUumunJIjK4s2R?lXsggz;>Pi|?`s5?_ cZQ8vE*nqzcHayr@%46l%mY*y?wEfZl1$dSTQ2+n{ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/pixtral.cpython-310.pyc b/vllm/model_executor/models/__pycache__/pixtral.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b4d3c74afd220198fb8305cfce763efebe025e6 GIT binary patch literal 19005 zcmbt+d2k$8dS74D({o@5fZzdAB1aNM5n2*l>fEKclqg=JOhR0elDusxQ$utE7;p~o zb%O^o$d;E>M8(nC>vdu$juT*QS6MD6_S$i-a$HGjZRe$?(n>e+_pdySl*YvlcWqu>`BMZee>@CPJ6Q!n`? z)X&s6GzR^_G;OFcXc?x_81^?xid!FPZ1OirJX_z~*y3-IxL4oW81+Xbo~v(bZ1=ZI zJYT=3aj$=`#0&Kujh+5Zi5KhT#(n;Mjr;xk<$Iw1Kx3D`tMQ=!U}LwxThdDPhZ=kQ zJrdtgA8S19KP>UV`X?HD{k@HS{yzC0s*gAJ`}-w6T>oU_fPX;Z8|#lW9`zrU_(=V+ z#^e6u65mvRqVc5vq{KJZKh^lO|7nSDsUK`S$nZ{xNaN}wJY5Cq( zf2MK7KZ5x7@WCl7xF@*xrsF@$uV6=bG}syD!(-vmDQnc2a)WYkA99aN?)~A3;DPW- zr0xnHMCwUN-5u_pa)O6~JxF;je0=Ftcyigq`}y!$ymu{qUwB;Jr^Dx#&67qj7Cd~@ z2p$dxXU~L(!>4a#ZkYPdp9q=%iC{1C_ujB>nEngFzOXcVHW-)i0fy*(fAC54_1WM6 zM*9e!NAWxsp2OIfpAQ}noEIcpJd>upjFcn6vq*V1P5J)dXqaQ`qeeO!CpZ=yM-QJv z*%QG@ls%c2^^x*ia0)4>(v%+vGUtu4=ilZz8MBgHP>rjT^=cF)?o=(T2ho_BY?!Q0 z&V)f_PPOJj6<G0^xeDe~%GV+DU;8_*Ms#Q8sy2q-K3Fu~%!87yqxOS!$ zRO@L+c2YVuarVUvm2<~Wo;rWw+~<I$8xHS zS5lNAR3-(Yl3cf>8e^_kYeun}pNx}yqfx=EFTz$y1!n# zxPR_?Jkx6K=gRHZV-2r`lk;&)>CXr?ui;$O$LFpm_izh2q%Y6-3tA6uWJ6Q)XRkgEO0* z^%gUD=VtSZu54XlG27107MBJBE3lWXMK5p`b8%^QLy&3b+BqP8H^{zeFXr31z}slF zGYq#Gi-mTf?Tr|V#dvUOsO`3kEZH{O#jVDag?7+h+iT~i%@Lz!E|%H@?b4Km?axi9 zJs4?{k!?{AApSN;u94X1PClyyc!6OiTlJPw4`Y|O=pjh#~iY`|`Y zSJnN9`9o94(>(#+xFKU3G&DWV?w(-Kg{vgL0(419gQ4s@2BJ7TKi`~hXn=?khtR5? zWb&{sI~7*rd8NVFAM6&@-ABGh=peTAvw(g^1bB7uDB4Lj2lI`_b;+tswJc zjR@e%ShQjj$DgP`BL_Qn+Gd=&VX5J^*|gf`w7KX^8-&JV#;ZFP-Pmorf%T>dc(iA; zOI|y>Y%Y3nZYkgPW(&*aY%#Fo-0VQ$0M%sMnWYj-n8xMNn6R3ufjt0xxz*1Dh(Ob# zay%23Ya|L~j=dbW%C$Huui!(CZOeOVQ{`&&`q+3rU&ilLzLV^;MWmor0SY9C(9M{FL+yE60gKlf= z8yVQKuwJ=Rt*deB95zp%c`mF5XNbQ$W0U62 z$(HrZY0V1sjFD>=ft|P%2IVG5X0;Be4D`^kqC#=XE!7?8>e=Y^kRv{33N$CqA#E(36S&Ud~R@nBa$)e@)9vtDEYg039zBm;CiIpgQ$ zK;mr7UDv%?am&w6VTN``!>Rrq}g^#wm1mOoK0w*dk=1p|~_|>`s42N@O%;Jia zzTTQw>4CjctCmv?4?Npww3-hA`Z|md08lyv{7=qQW1KN^Z{+C8imXr+kC#uN3>eXH zB~oinwf2^yFg#eEiQ~EG;Qsy77~uTHanu)Tz7Nxz4JTtw+G|mSruKj8u>%Lj#|r8h zu0okX7kGptQBN`L5CdT*KEv2y1WEoSp4;O}wbYoaf$uysFEHp*2}QiQ8n(7;*ur?Y zGCCfwMTdbQL&G-%!?$u~&I}$1 ztk4eZ&n+Bw=JZkO=3Sj}LxhUa4 zIKVgRmBLc60c~tRd@vjg+;E6F;RcZxOTkbud^6(@OA8xOZU{Bgx}!!X#u`)hsKIfd zOgA5L~9Du#o1xRh68f)*@-?8fjj{ROc;-b5zK%S zcr_zpf$^i(WpIB2<`6xcX0;K5Bv|90Ofn*~u08;{Ls1y>Tp#{nHpjMyjfJP~vgYFz zAX9L0mGMK6GpqIJ@OZbKzOizgT;Pptd%rUV z6%9sr(3gqeja$To453BiC9u^rx6o zw?Rl!c2H~Rk!?&e(L6{|;&oO*eLwmV@Kc;{GV~ItYO8r%3r%En`~ff!Nb104Hf~ke zI@t3#Y(z;R&4v^dsoiM9FOU=J6ff1Q&FOiHTEYxgir%S;sxq0Iq)1|4!xNFq&u3gP#=}*bmE6(F^L7YBs#P$syE^7kfpVmr@IZ1Y=8bObu z)FsqXF@sSAZ<*>c;}lv~Ku+?18c7lHND+_*ID6=BBThR@a+>r|R?5w_BEf5jUb<2d z3>GB=7)Z0yNL4k-$a))*xR=1DPDhg*LFZRM#KS{AD_TGRVbTOElMBp66Hm6e8;f8B zHV&gp9JT1RJt?uT4eZzU!2aY=0?f`5gb93QV=yx9ECf}9?9L5mG0V1^7F*LLd5DSO zzyM_-Ubqls^H^SI;qi0xO)}ABr!_Z$4>H^$xocT~T=<0-j=iw*MQzxCX4tgCC(?({ zOLkV~swxUA)n-73y#|yJroz6IPev$MlAQOZ%7!KxIX?afbp?9)SL6dlPEvqZI0bMb zvgKEqdX2$#23$_{XBo5^EFwq>n(vYZ{h`h|>}si!fzFwWF;qQ|6(CBo>Dv4}nu^HT zcqK}Oln!wWz<4;fEt~Qq(3k`M7R*QI&wby?-W}l5_7%aQkNG8x5N>!>qF+XE3!46U zL%F0nF}`9P6dalHTgI&Y3iM5j_M)?xS#)n1fD7#Qh#~v!E;}hjB%So)4DwwlIUp!% zYI&>EUKXvr3}G~T3zYKJd*3k8i<$Ov=5-U?kCV1w_O{S8Z<%iZ{+7I3fJTU-Pqwpf zV0?hD%v##mp*Qc0U`o@ZH|`=RP1_tbzO;(Uyk%)BW7Z%kH*Bm-hH9jaq2ct#3>070l5^YlUB@+9U&Vm_Og zM-%hq#C*;#P1Qx~oa$Ee5w-M6-eN<7^#2@Uzk^3~AzsD-13hSsnD>~@K>mM=#AX*i zDq@_LWqnixj62rC)8FGVs~<#%ebCM&CGRIhfx1B8IzAA{mhpEUNPc3>Qgg^r4Fre_ z0$@XWd1duwMxRI8xADBgMhC5h$L>Dbbx_h5P&pYAdH}Jvi|PARk35?M1djTh5&~jw zj=2rx9~iwaBMBqDgjZ4`u+9S;)>{y`qA^=ffZeFS#9$Tq`f;SagGV66%MjBnjNff0 z{e2{^?(2?YR)3it_m0xl1h6x@pJ4uaqx&n!c%P%=8kaH)yYDtK-ENZA7KDB|Uo(R# zsw;y(2!MKGGX{whLCK<4_9oESnsg_~ART10zx?Y4(3R+HZx-LMsB;Uv8`dkfMv`&n zbqmOh0-V8HcdRr|4!V+Z z&z)gpF@Cj-dJ7l6*U_)n{F*87oNUA+h7Ld&U}`Q4$iO>A6ngDB6`+hIywq11bn#j; zv@)ag6y-4q5v9m;IJaHXy1j7nd!0_&_S-~mcb*2%te!>@lwNR%3>CD`yasyx)BR)j z4%MrTi$V49Fo(>=$QuiX|8zs`G?kd+VmX2Fful)m=tqT$syEm<$q8-2bx24W1s8ka z5W0MbcybhAR)8c9GjUbnjpDFF%(iMxP~o|HEmqsv9HDJ2uaWj&Mcgk?&6h5gzj^&r z;&1N};#Py8zj0M18PE@~?E$S_Pn%CNGBUqcB1B^p21QQQ3vc%TjjaBo95xwO;2|TT zRc5QT)!t!Sj$`I;4?9H|ItVr;t7vW&gPZ=gD3wD!D`}?l!NT7EpM8SNHlaz%4$9zM z*-9nJS126M*BLKXDp35?b&ew0R6H4qCKNTyU@HSkql(m3OL}CF^u}fxqZ&`$hahoV z7pbmP1*Y6!K$b?43sCnmASG24G!?0qB2z5@qkfJtE>JR%io=s(y z%ZRbzh~kE4Sa~en2RL`a7rD0eq3c*5SWf8!Ye(L+iq3lu_$4gphaS^2*88q&y=P^u z?-afKyVgGE!>zP0-6l_!|G?S~S>iUg1)GCrs;03dC^tEVGCDWMMHs53xonXw|FNdBQd_N%&Lh2`e+O+Mq2uYE)o;0bx3dwkDu5 z{zdltOANls)@`(&Y^XrOY_=e>p>712%80R}XXCnPQ^pOaAtR#1kvC{s?)@v8QXKbJnkA;Wf zmLVdLDE}`ia1xVMm^M~an_WR|%u)ZC9sC7$kP87{R6@gYcBK|x1xNc2naRDC5*k+i z5kGmplg;aQbg%cHUNBWA%X5-5GI>NitV!K|BM)9AS8p=&SQ1 zjUwtgiYNM42m-Th#}F4_!-BeJ*;tXUcPwz}UgYEqp8 z4j6y-YcLQjTWkU9Am+HpaaZMdXoCtM)OgL3dU(^c0CC=hMa^Bdp=x4#X?`cSZK5sY zPQy+Xcoq%Fd=W&ppEpnUQAsw!jUj$W=D(@`su&WWV(`=h5dqx&%8U~Ny~HK8H#*AZ2d z2q1Qoq`*3=MeAQ<1`8$`SnHsSwI{1FP5pClK=3njt$J1YxmGibU?;s4f)(JttKVSo zn+$#nfuE7tC+19IUWEl#P|8dva@F^A$|FG!rV{AeGN;0h@TSWE5mg=uP80k{4tEp= ztwD2}wGAM+!`fkjDmd=@!&&upWG2?N%XlvJ33`OJF1-CMd|q~&cFPZ0zFH({bxP-N}Qx+~wcPtBK(=4IW1tNHis4#{kB@&b+5CSwdf;N|O zR5}rk0X3-8lLom7{tYMtnF_|3Wyo%TCuc&*W^oVI&maH|;*R+nc%Ca&;A)NBlx*D( zu^5lOdY*-HY337;$szTNok}W%zYJ{SLVElxt?+1TEUW%C>;4-CL=?V#?Q#XdrHbz& z29Ut}65p%0ZUm<9`t^zeHKNB~<02%cc!v8{LOXjidtr{un2en-zn4RFn;bvlM6%w8}^YSiDKQm>cU*I?d{F%q1OD;0wF(L`EXp!G zxhP6&4mYFz6Urv$2_Jq-^?CIl@umJVg3b(nmI*TWl^NVX+H0HvJ1v_F8+*r@s)Mm! z{cMlszBu_F35PAIUtzG)h2kesilS2Q3i}~7^^9R_$AF}*0Y7S*`FrPC1Rn$Zva}@w z5VHb^Uf=~e0HP;=m;)f@liZPxFRS7nu373X6#gi3d!Z0;M)<;#&pMk8H7J}F;z%ci zN%}Ty4xkZEd})eUfCWZ#>!#+{O)adO3S|y9_xCWYv4I|t6-XqQs=tRI$#(cvufuey z|G>)|AjfSEwW3pH{8e{;@%7+lBYoda9lDsTbs?z<~DO-xVwP; zkefIScnfPzMu513I?#+k0f>8j6i@UE2(Uf|lwM*L0Qib76>qFCP;Cm>wr$yq*+maF zr7QrPECrbapz~$ek=ovxG?4r?X;^pIpS(79;!PWzA=-noizzv<71o;$?M?l)aE$0q zX)NX1`BXOrW|5e8%=B|W&drJ-j=aX{-$I7^?+pHsfgt;DF!oIb#Hwloga3h`V;qvh z_cl}6Y?8T1=T|=~?ucQOn3cr5oS2sq^Gaf`#p@zU{&;7PithbTit}$6WOy?W`ofCC zcE=y>t^CJ1wSyedcR#v+fFf_O>&=Mp=t7=MsR0X z=3!6(JDiKaE`?}Y695i9;T~&`c)?0ZaSFtXOg>GKY6}YsT_|2 zsnow@k#92)l(K?X2v)zs=@ZCGF4cl`rydq6xv+K3B6ODfkNVqE{1im>$B6xWA0%ei zKqAZB5fY;l$o-f^#Unw1n^Qkw$q8Yh4kDivdSqj{)}j{PQw5R3yWwo)L&%o-(_%?0hgp> zLlN5XKbb1ay0Td$0$*E!Rr*B^)5e`|rR9!4y->(fv`|{{>9V%b=p5 zao<4^VB8P+%B%}$8JXL9AokBWYIc@);c5pKrgy|07N+pAkzo!k-323lfFl2-4-TPj z>*&c+jL0&{koGng&jCtmFcPC$_h%u-!+jKw$W;`Ih|O+dy=V`|aRr?Ou-R*E*Bs}0 zRwPV|8f51pEKjq1DC-D&62Mh%87b)()IqQjfrEdOYzid#B{I*MWh+&XxNm<1jf+S+ zzEztA@x$E?h`e;Q<)PB<|E`u*Gvbtt3w=ncb&zkwiQO4>gcpYNsvPfVZ< zq`=V^Y4p=%TCOLJ20y0d)3gH06@sFq6$5%+!o?owhiw!fV8M42d~He4&jvumprYz; zeY8U&A!z3cA}eSI=MEfgkBlVLvr9oiDnLQ$UkHy30tdeHaUrF;MVw&VA(C(amrdyO zoY@R7Pzc&jJP$NU{b4|qhC7@XpEq%0EaMky@MBALvwjhL&B|BdQmCM^756gVg|(!H zTSNbzbNXKl-eK@C1D=?L?dM)Ndt6+JYf!YJ|y~0HmewPp-e{bEyC43 zNGB)lbpId~zCV}+0|3qq{Mxpc#L2H0RdrniPoi=kp5O`)R|uXUF(o!7(MM=)ql_Q~ z!q$`jRTNO1s^a#qI(*59Tzn$)3EdIB6sb{8I#w%{>nL>rChu3{{R9f1eqr z7`(t>y}45$u>fD%CIP20UWuS%nfE~{3k|ciuzl@%bPvV)&SG88=O@s(=#5C$AS8?1 zGw2hH61EUJiz#sCf};52PR2b~FW;EW+0F69H$Z(akosvQna_+ZHkro3vp$Z1yeoq(Y@v8_Lg7P77^yJ zxe3HX?If7qHH!{*@A%-Z6YpD;J2`UUXib;6U2>GRF(KCz*UKhb|kF5-(544GF zNJKW`iFP3n{-Q+RxFrbkU{b(GELn8+TLzQTHLqYXp(<5eR6jT0jBxQI#0@%uJRUtz zXz~Pd7o!IhA7$F>2zC$(d8S-rEA1`&)`2@MldZ=KtUHDCXe}xukSP?&Pr(C)!zZUj zUlcnXzP&RlMMV5+jA)%ugDrkz{X42kDrogoo% zh^Y=b@KIx=W3aLm?_ml*2JV;~$Q=i<4}(x8mVOV#Qth-NTcq&s7ht<)AR8gbfM;Hj zl=wxtt_4#ZZnZ`#i(-B@-MC(VK4Go=VlP9h87*n)Z8rC920{!4%gC9MK=9Q48F=>ngPju;=zFz`x9SR|A1+N z@$}@v*|@ zQBeYqbt_e$ z>P4*-ajwEc+ILPKKmB6jOo#P(*d1w`=wp{SgZ0>@CkPzF$fWw5WQS@3@56j%v$?QC zgT$(f39F>glc>8!r^}r_ibr%-U`8-mZEn))^A)AJt>^=!h8W;qju`6qQET-fzk#$P z948(FV1t9I~l4q>Ifak~em=p)jDo^vO9^Z|H?{Z{TBqsaX8}^E{>0PZiJ6RmWyyXC6c?NzJ2!8X}Qns7kl{b zNZfGzp=(%BSk0sH)?73`)tIS@R@L9WCVjlCq17m@#Qy`p#8jm>#IY45F}?pqfD=ET zG+AFCP;|;rn;2|nm0eOrE&^0K|4(3C>sa)QIzSO3{D;0vQMp)_{(lScCi3t9E~V2; zA99&Q9EJRS>y_4{63;9(t{uNB^KlJ3PX{r-e8)P`xGqM7>_50(u{CbS;<*G%Cs%dl zn!ci5#~)dhI)1Slg>t*s-?TbQcU)$*0m%c^k^hn*zEkRcX5__RXR3-@8EO|2)Po3; zjIO$y36C+5)AVzU9cS>%49+niBkphS{P&FBdmZVuV1Klmx$h8{j$qIz;?i;%{~DkO1s&t*A1nbQ5+CfOq+f$5S+es*sGo1U_llDk zJ??gTo3!T$z?8=trK!tPT90@kiy!%{L;YgwY`&%bZoBl$uBtq)6Z0~;<4ZydgF NcCvi{Hrb*hQag*kcIdSUNO>@{b&BLuD zOBC7P@4K@v5Tbj!ICJN|=FXja@Av-hMY~YQCh*(!v%kDt{^3O88%zxUa!5RlEBKjY zBH<<6glBkW!*C6`n{E>KWW#Ev+?2sGt6?|OZo28XPBY_Xnprp7%(=N{-pw})ZlPIp zi_I~2tXXnP&2e{Jo-rF`cS72mbSH67HMTXU+$kxu8#jQv>*SNiThkHl!PWMhp=Nor5?{@EQ-s9dQ=|bb) z=6&vc%~^N0dB1zVuoiA89`7 zKH7ZDeXRLW_oI?uZam)nnENq_Pc%N>-0kj`_+;aW<{o!X^AqkTB)zTiWb-NaDTz-t zo^I}S_cot#pON%*V_$Q>yTAFY`>dpA8qYPKcb}K|_Qofh2iyY^-_dxX`J(%x#BXXG zY#wqCN&M!^N-o3FaBO8WN3 zr<$L3Kiza)xA__OGtCq32_xY@;JxLEy03e8xi#-@_oR1^yWrjHde0}k z`@GpV6W**}x{!2zr0@4Wg7iluz37kozQ4F=Zcn&ND0{$r5M>Yg(<{rUT}RHGw-Y%# z{&aOD;6mlN+9zo6{aZUp{k9vdGbJO(N}(J!GEybYMigU{HouoG~1rvs4TRbtF^Fxvf)?G z*2Cq>C)bvi>aC^cY75Bl>r2aFFn5`FV1Cw!(yKKU)@zN~T$J3~Iv+XvYK=w>&qmh% z`a&2v2k<>QeXt&cQTkw=8I34?sJ6O_W+UgN)v(@fAuDxkZ50n@j-6liJ$X9yN()7J zP3l0iw&WvXwwTO@s@4h?+o~BLHGZIlHdosX^t#9F!qtKRaxeeKp_eQDN= zGJC_&Z?RG|-j5#g!9B#tAK%VJ85IxAAAaRn^~kf&A2@pK$fu+Hf!69;h(<&I zTo`2z9y}x&{FGe1RS&C;_R^BC(4p+ndSh*G%R^V9EZN9XbVvsN^@ zuiaQ{whqVdK8R1Qsb~k*(-OaZi?Z>TUp&64`uLId+0iV#Wb9yjsU8N0ReQk?@Ri6a zzr0q%2zF$s`{R;5;s>=RCQm&a<)oh{gX(IVU*s^U+N(Z3Cw){`iVAV$rL_!?Xg5Z&9Xo${-Gb2U6)t#g^jJ&D&{P+i25Xsh#4PAWEP=Y17)QhVynr6)V~ zo}F@cm|U%SPhL)-ceBZ*NA~>Yqxb&yA3U{mcIl6QA$RC&Pc5DQ>~HdEhjaYI|!n1&RQqkngXZ@Cd^WpkAzo91}%C9m}JyUC}`9WlbKmn0O zu$+G<9XZu%tJd_Z)hJsfq^&g=&sVFb*J=%2q8#)f8mm@otyVjf)fu2p3S+ad9NBu} z-bttfs3yUC&+QDGt2_KvfLIOn)f)E4()xb>chB zI~IPRo^7C^nEe=R$|fcGG6x0f}2L|aWCgPD;Y13a@2aIJfhKz;7%*D9fMdxFYMeve2jn!Cbb2-_fngL%>2%sS|r@(nJJX z`AqOY<>UjE<=5oFTFa{d^RKB^P-zQ#sk9d>;j&)=hC+ZR^c#-NM;RI4s#kADnbumf zx(w8V#doq^LwJhW))|BgwhWsN7*FBi(hq(CL72FZ*f1^uvbsh$Q8l}kmsm4Yv1_#q z$w_uok~7^+#W_|tv6Ac>r_A=9h*^x`$yAr2jc3d$u*;{+;Gq{1Utk;Njs!~5Qtn8b z89bK?o_QhRnHP*N7-4oJ*9EiPH6|06*h*gKbPY)r!s5mlo=9I}4@+JGlVZ&)Vyey_ zJvEj7+*ImBpunWHc}9;HwYWCb`C`c+l35|5xz9Yx?yR8OBe zQ{@EcCCwpMhfxyUx?J}>;6Tnx6>U`eYeHrtx}Ebfng>jYdaL9+gp_aJ+(f@fjq&An zOcB%dR&`aiS0V@NLdUFTEnJSQ;Is;(TnpqhMyInjGO+^ncd(=CWd{8(ddG+|)vB!5 zYIO(-kE7N}Tmcbc*0GH;{<4N+Sl8@iDVa5F{5eSz>9ScwS!ZSwXw3D-Ffz_X8GL!G z+N=eqB3m!&1&(;{S3-nXXCJQMbxdd1*hpMToJg)G*RAzb7wZx$z*x7tHuF8Ks_y|S ziIre_SS2gTNwAjFiS=~X#wtvIF~OQEcy2{Rw~HoPa@I5J*{-u;b(3ALTXT}nCod&8 zY!ArdlDY15GdC#BbRE=B-;`L-bxnbTYUpfau$FUp&h~5%YxGibJ>SYAKRc3ZuIIb? zl^n&$iw5G^8@xw9jXLO2p<6&r=`DKA-dd8E_tNaMXI`>43YQX+r^m&P$ECPvU_6T5 z0>-0=@hD&n$GQcK;aGPJP~*&BzJ{MVh7rdaNOBz$jMQTYE?-5_WfQU464nnH2$yA_ zV(=;gpp&|P7SOWVsE2Mwm{@R0zWOfKcpib9K3Vg;lg*}^WRMQpi(#{N4s13k|61rr z*#)qT^`*7;TA)77r#^#Vwxp_zbLFW81d$n>j*OF$QH59y!s=DeRRzVmrIvrT$Fl~@ zwN>BEfs;|^ROZ^4>rqcpX8}&Kkv}1(-3fS=X{jD`^|~YnDZ_YF>3r^XhOsYdNS1 zxVko(%o~8BO2f@AUiTj z+r6xp1WnhpJ>}bwC2gVQwwER?_Z%uAH|d!d0oc`K2%ZwaZC(I?S5u)4gaW-&zdRiixnu8`8g+Nko@*`njkQ@vy@G_E zyb5BGwNPDLP`5H_YY9u;DTxIk6SI~EuMYCmXBn_D^*RHYS(#L>g@1u7*nuExV^ZbM z`Jia&7{mOP&UkNZ`F?i_KvzuT;t+B;3 z46+RFW*}g}X4N?cgZI78q`dSzV!wndm_i^RU|Uyu%ip$&S4tC#brx`q{Q6An+sGa) zbU(QQabq!wrJll4Pl1gEZSm7!V;xbUX!ey$z2IguCfL`kpIgb3iRB&aYo2$ouLZ9N z_O)=sI%8f5b&3KqqzmO}auZg0rqNzF^+&qvrKjhkw5N3hk=@dqO{Upctt#?ZQC3(F zLC~_aMi|aC+UPT!9J>KjfW$$sgY^%lGmZf+04X5s7eMyI6rtvVu$5sNaT~0tagh+? z2z;3HST3W>dNwnFj>u}Xs=Fk7SkEzWPh^oRk=coo_=wrG29GyUroM;4Ut;hU1A#jE z(zA6hT;2+1ejoLMNd#F32-?WwuLm#r4?4GOHLQKe8Bjdsp@k9Vh(JV)B7k%cA~E}s z?V1Fr6@X3)@+v`UCB`lnp&NS#V`2yqh33W#+fmA+Fsi-}ExeQ5L}!PVHfKioHLYO! zQzQevekN5kI=6256@5y2UB8J1rQakFn770V(SR^>#DW*ew25#56p+{e+kXjvIuB^z zFXJKLtRKWp5lYneGx%NxKfquBj{=GW)OT?OeCteVlF)zimK`3!1f|WGfXp_uS=2yK zGbIZqN-aV`6Dp3xW(4s(^Mr0nPj8GbzQD$D34L=Crr_66f@$^$=JYVZ%w92b!)Xvu zwsjp|DI$}<;5*`#6z~d2Cvb}s*i?P zkySF>PxWPX<8LCku1o4~Avdx;|4etz#oHmfzUBD!Q@7h6~s zU6sZ|f)@H94kaQq(!Q!ggPqmVuILtvaTCxpht;~LSv(rgdq09JF(>nan=$x9EFQ@G zqG=_-{pYET5HxMZIYnqCAPRN!-6AxEh3?o6=yQ-`QRhOTXeCXN2#6i}9_Ub@*{M1Z zk*tZtdMUJ_g(*=_=%u>JOVH~GYR`uSJXeHXCXK~D-W~5Ig~c2TOK8dJrocpwce9>z z(MD_IVL6-#CpWfrp<6^frwa}neMxr5@kZ0jUQDk{0gUo1(-O{jxr^-KcC_J0yE~*G z4*MiJOSI7M%S~PAIjc8!p=6#0Fjj97rnN9H>X+wcQ!dP9)rDoK{QXAY=Bd`}r*fx! zg>0ED*A7BehneVQ4DcV}!td)}k*Qv~_{TrL@fW}P51ynp$6pQ9Uq{*HHxSIGqs$VX z($oDCaztb4+R$WD?L*S#Ixro=e0j!q%U+unFRfVg7pR$>H8to7BtFI_ZyoT~>K(TD z(+vI*gP&m_6vp1yZk>79i&7Ag&+Ljaw1SB-NNb2lbi7tHiI<7~v)0f$RocfC)@?#C z1Q-KqJAHx0C>IWx2t{64Bajn}y~f}o1HlvQY^--&g`OiyvmJa_WKnmjewM}g*gDbj zu$*VhnqCaPXDjOW_xJ+&Mz$az5ZR}o_*ufl>E@Qjwv0Iy(}%D(ZeBxNuSlmf=aUkJ zl;$8_XPZJF`V0 z9RZeyEUtgTj7toJ68&Su+%y;`>L8&JWL*#{%@YO#4-GOzQ9gfgZWEAy0j$xwGAO%@_fubcg}#q#xJ2zw4iJIpY2?{xXV z>?A3WYO`tQ*PaKKX^9$68^8|ihusT$R|Sn#Pzgv}b_QX4wX)iV(k+0NPGNXg+btr7 zO4t_b7)@&eUof-vGYEvc`>7+^cs9u3Gd&*-v;%u;MLRuQ_ll8T)anWrxW)eK*bILI zeYsl1K+VtEg4d#Ct#w|bm(N0qp!2qisNAiccP;W*gn>K+pDQ!Ej zPiU#z`M|MCM&5)LS+9ogL>GxWIpF(nZPt355Wk2`i6=>`23l_|h=q6tYRrFw+mO~f z*4a}ebJOuSh+bh6wh+PWr|>cAyHT|>*6VPL@IhQSFn6>@Jr;%44uQjP6C#uw;^#6t;E1v_e>-e^?bh8euKUOPstNAUn!0 z&?_r87HL>@_`UHjgL@j6z$$rx0j!ePR?RSZAprxm7K-R<;l!>L)8bkoIpSPFO*v8$ zw?$W;Sb=~;qe2%fkrtgox?Rw$Y{Li+e`YD6?kvkZFEv5f&V_lVW$G^pDruz4| z$?(7w{0id)QH}9t5Oc>uG}EZ|om{ji<4|cR!pt406xj>(v}g?w1?^uycd1xiQ=D;l7ev;hBF(;2;IfW`&10%OKt0&9CK6 z;u=d*nrGKo2=Oa<+XP4784kBt2BS zS+E!2QY3M>w;`S)j+B~kNL$H>V#oy7le(BDKD@3SVntoQ;Mr6V0r$}kS?uAypF=-1 z*5LBcc+J`hjR?;KB{dn7*|8yPcN?E0_Y#%GC{h)xDs(#aM!O}3uqX}w9t&E@9J&)* z_?BtxCD8J{Y>!k|+x)*DF~}+cJYQpOALkDB@^uynguK8Q7wVsJ*%7~H?J`)RJk$t1 zWF$<}du7Ll7z*EjtDSOx4*O_HZ2_o}1Ie!L!J|@RAWmrS$2c#+Kqn9p@56-3cTMpK z6Ku3@wuogA@4|x%SVziMpg}+ma1JKl5?=!vg@$SXFOdtA+(p`OU@Wz^>LGI9#=AAz zk%1f@zAZjU{byE}ksAyjVLA(yiMUia84IT2(vYUczEX1NTL+hl-qN3ou?(!Z^Rvax z_@&HuWEtETm*iOG#t7y2QRPoKvvY_5p`<{H#DBpCx}g7pxqUP8z%;*COqai{`0RBE zq|eE{fzn|F0zR%B^Klx5z(JQNgx(FYYz9jt&(-TN`lORx$x zeFLoWw(w$a6aK|>hcR5Ko@WB$5He|Vgae1z8@m={rsS7R&$^gA!KbJQ1{oxG#V2tU zfNO}Mis2ZQ)k|i-#Zox_w&6J!;Q14ly=)z($CZhnfba6#X5IJ+)k$6SxaygALPw4HNE0JwF>m<`IbK4^ao60fG)>oQRnQNFZ|dG-}P0UhT<` zaVg_U*EqBfQK`4k1EzBFdiO z4hpTjet=B~W=x*N7J?^`bjO7k)XulD4LK)DB<;rs-)Lxer64kvmEhL@#CGWtvZ!`5 zdfRu{n-Ha!Yk{EKpJWFvBj^d4qw7hAP&0pj#F~BFc`rxoVPs{Yss)Y|rp~@^bH z7_p>}BcspcVXGY05nX7-xphByd!*q2z8^VIvSRVfyvBy4$Le0?=klzPQQY0DYS~(Wj$z%l$7!yB7#v( zd8of$2y^H)Wt_qV6D>}`T{f{&>;kRONfrcyY%BBfGY~Ps85Az0@W#w~;q>RI|5zCd zHegK3bm1ED%GY9R5QL_A($)c4p?+uqi@Ov$)#8mbB$| z1BRV|8j_0F#`XYg|ER#N8;T^X@VKT{c~&B0P8@YL-%Ch>=+uWQXsWqH)vqAnjvlC> z!W#DYV9(jAn{IhErE2FDmG0O3ZjGh5-a{s z27k=pe<6Um1Dl0tNS7zLy49Z`06Sb|yuU`(pEF6uTz!)P7o(fTP9Dn8dRW+ZIHh{$ zbUqZLIs(P05Y))1M=1f~4^@3ULl8THcC1Vp%t^^`%sl+lMY6g&2^oukW&^;S6liU7 z(wG7RlLx~B@Qd|6U|mojTA=rxn6GxOmH>{LE^83%asSErdG%lMHVig7l!242Dl@r> zIR85HHgWreJS8kYhb|hUpW*?UjP%wSRbcsziT5{9K9^%D4cuIA?XAxhS<191*P6{qYVI+$^niURG7={wT8zI)zKSxscZWIHM;2h zuE8_$!0xWR3KbD-0~OJDhU~8xHwFzgYb|iaP?Ms>_hK8zbry|d9HAlXQ5kep8(7#s z^ou=#Bn4I*_6W4b{&$?i*w?zI{7T2%4UCe-PQJ{6L(f-_9@~55Sd?b;+4=q0dbsyf z)t6s=cD~A-V=ukWCWgi6f`b!CgXP-On1RE3Yb+MCK!OZ- z&yLZoo##U3jWh|%yvZHKP3|bxYe}|-N?LDf3p9kvjw@8hs^q>~z(rl}U_({x&jwQk z0J}r8fOp8Er2m`t$gj65*so2P$m>&Dnb3vzO=Z)kvVe1o^lRBct$+CA7P; zPw(sKIr+;*TQ7F=m$woe2J38h1sj?B-yxP9e)@9ljF0Zhr&IF>EhM*f0>Xj4PGU>T zhXLt<;cte)qi;fz;fXQ8U_|?G2B>o6&H!k=2m1ZgyQk1fA1*3uEp(^8SjQMtUh0j| z1N~9gQ#yD-zbS#ipt3VYe6oj-Xb7`{y@xIX`79A3{hz^K-Em(J^PTYS z-(8U{#> z*vA~KVT-q|clAeUIK8yc)yT#Ma!%S!s1IEQT|iZ;*X<}X=m9L3PcMG6rCqPq{c&Y0}d>g3?mU*7a-$(HtaQO3$(c@|)+?w5jReZa&_-cK7(5|raZkG@9KYf6O{)}Lz!A%Y<8a+6uS|fj z{WwUPu`+qV@Cv6)wFYv#&4X(loTz*Gpa(yO9!y9NCP`Ag zZLEv?)R!Rvayv0daAX;#x9GUS3A1dI^2pt25+j-vjS&%3{k*}Ab21fQldlsQ#3SxUbzp%46YutG2Z|G zB|Ln`ZvCcJ;6H4N6uuAa8l4Boxd|KqGZFC&A!&^Sn>c;M_xd=UqznbahY*0ERC?;f z`)0+;UW0!D_p!@qDF@}GibFz@1&@3fA-iq_hWbd9Cmt?9kyKX$>vnqB5+f?dO@N8$ z_jB{tXg)${MW=m0v(;yZDPRR87E~YfQ~D^(L#y5U4)g>J!V^P`NuP{TREXJ@Saw6d zFhuOJ{#=#C%s&^It31G?47yhhfUz+R8l41gQ4`(XQ5D+p#|P*Nsp(8S*Iom`j6Vsn z1N$qTG3@0>kNe2KV^uy<*(HRnbNdiY>a*|ibzcnKep-V z5t>MWh3B}C*%4o~&)lgy2IAN;aZSIbTUP#p4@WGBD)d;ZeC9plwGsDrihUIIL~%$d zRW?D`>rftYhnC*@Fu=0VKl~^vxhX8?wI+zV-V{uNHqaodzkwcOR zcq2@dLJarv?K6DKfbPY<)T{6=<1!P6G4T`=e_FbdF440<>pewdQtrPA+n{B!1$t<2 zWy@4GM%%U=&##iiDU>?}*;Nc8W+!1-7V{0f2!(*b^Ii7lI-&z9p-u@kNCqwHOt5Pi zTiVQ`m4)3Jqz&px_#HBaKPW9(%e1eeWy^s_DbEWiq9t`F`^E)4vbc~bF zXY}{!hG&haAbR57SyMoKR@1MwT(|&v@PwcdCra<5G#2=-eFRoIfXp~@-SjgD_Pv73 zb&j=<39fN7^F53oW%eCBaJYKt*}d~PL5tC&$M$1Ol%TK!czg&MR4|2QA_Fp?R+dW> z%ZJ?66~(eB9r&lKz==_&*#vL924~O()bIbOAR`kctB8;sGf_z}Q82%N5rE83CW<=` zDCv7J_tL;iiyzo5sHMRm!4YOXjSZ%0xU;0O&j7}lb;q+e9Iy$_n~4yTy=I=MB)&im z15Scifs{}0BC-)U%^`<#C9+}u5==hZvHJigM8GX3EKHJ-`gBLLSe;ou_ULzHxSqKCJg@l-E*CKGO|Fi$)e04lniN7Z5fUr< zH4#r^5-(~nS&qVd^Fh+j_z0^&n2O^Ro7Y+mNcscf8VmU`yP?VSmO%6WE~&?>Vs~Y} zFPw?2!vl5X#X?I_OAfZu3!ZFIe-Z5-W)pK6^m;rc?rEwERL$t;t*pQ@_x~!wPWE;7V+BEkYLN}GTVD|PIJ_krjqMex7s(10g zfCOu66G`lIgetZ=mFV0n^De528xz>a_#j@uZ9APikM)$T!|NtiPik?m(%CM0)(ZUH zHEtQOES;<_;zH{@qD$`!W@Uotzcp9c3$zRIx3b1#!YZ{2MKPe}__(sCq9icZtaCtd2HAQ-C^6U^&ZB3JD?a13x z*bh*$b|6S<^d=}YNOgms^XKYBy%tzs0E{jBOPJQ0K!5Pm>nI=M2dIScxXf=wgm!ZNk(H#wqXR7r z8A>purgN<`vuUpSz6V2=ZX&?mkMX|YY6S@(Rs;6rM=*BSw4~$o^@5hb2*qyV5!HfS(7ngFAUnbdKQ%xb1~3Io5NryzpjLc<;_ z!7*&2+Z`3wYe0-WRu0^%2=WzunqXge23FO)@NdhxZy-UT%j$E?hxu zeYdR6-PbMd!ET;G%O7fAA93*@f86Jhso;e&kB$@>xJU5N5W5At*G8gigTn+b%%eH9 z=N$LPUb3M$O!xeROhr4Jj2F%osB#Ft!r*rp{5gZ)WAIG|GC;q}*eGoid-jiVOlT#i zCmePz@-!<_EO@APrZ;~(SO~DHo_IGzwI2oyZxa1@j`82>cKMETJ zvHN}SF*wE=`WpnWVamrycR!8{(QpH?ZR&eeq;(309v*9pVjdPAaBQ>mnS=P-xpi~r zM;}lWTCeMHA1zVlk`A-8+$+-Wtf3ws>L*)K?5CS+K`f1xRGT|%Yb^Zp>?B`-Q^XAI zF+N1%=5!^f^7V1ZBRXn25B41}e;WyUv@OBK5^SlMSb-~nDd9QuX&4@HOdkxb)*0wz zPQyhLxpQy~#{zAUrh?SUhuz0qIFyU>BI5UhfnAGPaj7Q;-{f6&v@|hg{N=a#szSE zr}IK7vFTH9w{gw04~tfBklq#R_6A!8=IX68@EG$?cT#gFd9r3<^yFR#=e)wAj+5!I z%h?+oeH%pB9x4W-$bmULps#}ZIL1>w$(j5*V-F&j&FOWBbJG*=M{=tHJ+ zI1{qg3}7X}nba2FCw#Q2&Kr ze$$pdG*cXzEQ4(f?q)zHVPM8m?=vZAY1Er$$a!XtRi5BT6Ai09jB!~jagP0U#-3x| zImQN~@j8<)Fn9!|-^GzQi4>e{viP?Cz9t(EH8x#rrg&(CZDy~UccJ{-TRGgosp#8z zC;P5B$xkdc&+ru9iiC`!SehNFMK#f;|>xXHr@4vNsDxA5m= zn=HsN%&Vdla;JuJ=K2IW(7K^~KLeg@;t1sydGhcajGFp5DZSr@`eZjJC0Zq*XP6W2 zmL%JQDpi}+J@HRdGlNHj1_o+7a?}n6G(5)5Y1^c_nK^e$OQPei_6|Ls(=MWOgKd?1 z;~;rX_pTc}AkUF86BknV_8V1|gUeL~P0IE~bq8a2@(mxk(PNvr>AU#Ey*GG54j_z= zFcHr~V8upFXo(_D;EX+@L?b_Gj5Xq0N2d2DPr>Z7doID#mpYX}!A_c+IlN_`u$K1)$RC1Dr?5ftALVkly z{yu|mFgU~DECa!f)HjI{8|R*$T!ABlR_Ws(izGx#D>?=DCZH%lERHiCW`MO_$KBp9 zuc tp7K(;QhvJp(W&G1gy!tC*QWMO?lsGj0+P=H$-`!H#o0|iQ^&Z&_`gP$C2#-$ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/qwen2.cpython-310.pyc b/vllm/model_executor/models/__pycache__/qwen2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c494900d315ea245caffe7c579d10c1e940f993 GIT binary patch literal 10919 zcmai4S$7*pcJ93!Km!CpiWDi?f^2z#7s~Q($M$&NMxtfO_RQ(yNgG5rNs!$DSl!UV zfbqmsB0r3>o|A{fPAq3K89(GDuQ?}qn5X3A7u3@v^Ai69TavAO_jUsWDOv$eUsT<$ zuBxu(yWhQqZn<1k@Hz3%|F-(yuPMs!C^P=Cka-bT^4F@O1d68wYM@1Gta)0jdwOhm zMr?X!Ym2Z2TIL9Xp=GIx{N7MX$(dD=NjV=f-8P%y~Pi#1q~`T=lA)ccRI- z=GEdUZ;JDUXgWUN9pH2^nu%w)hyYe1~@jr_0fu@e%I` zrz=rCKI$Fi^h9)5e7ARZe9Sw>`D%1ee6M$Je4lq8=O?54 z(^JvI@q)L2^mK3_IvzjbJrbYrPH=uEdNh8_dn|t3dpv%^dm?_)doq5?drDQ_P=eXu z;3XwE7&>cDZ|mMO(98w%(9GwWXQ4S19ERp_uK7J^?g;LL=FVL695hFQIy80CgpY(L zwoUK(KtHQAj&A;}7<#&UDZ3R-w9yliihenw%EhR+ykC zP}Sb;H}QU(7|?Df@4G4;>aFPZLcF4Vwi#ojge>QYclv3spW;qs)WE4X!)737KIpcZ zOGD*r@lqHBm>85i5QNLkew6yz{PC>m$NrL)MHZP(tv+aatE6#L0J8_7{Vb~S2#vrD&Cr@t0xM9LiLe{ISbF8nFCY021B|n8om2Kq<^{S#OsWwmt+KL*e(@LNXj6lC)3>05ajREeA zY0Lp$Xk8cgrdELT-{O@%*SFPA)YRTq@Prd6cnN#)YYp|k{*~q#OYLSN9z@PF5^RCZ zgYS2eYzo7mj*hp_@e+NuzIk+^{tjUR6|1+C`cARsuBdmq=NB3UX+n$1b9~HMJN11r ziK?Z$f_3Bfdwx%J*TfX53PJaLhx4-dK{M)yuL;o=VuJ2jXxCrvU;!CP7%hwYp%e`g z50IeVex=HS@3*36lKB2t%C$F+r*ZFiuibk%ZDPNs4?h+6SK|}$`QuT0>3HvAy4vj= zKNm&u@eBpR3t)2Nv^BM;YWO+6dHZLNc z#qCCsriymH@5`c35ZMP5_|o-#EZ8W&qhoDLJxPV2w-N_P&_IdjA!Iq9h*lUynDL!1 zJxN8VU*bN9r*QG8I*Mi+*ELQ1UemSjG()>#nc8*Ds$8oc;F7oOVp=iuNq!Eovx^H_ zL8otU0J~VnF4hAJmyOH8t}f)ex`&CT%utMS^-iU4>nfZi_5hjt{)VrOUSooPn0&A-sC405SX!-=ZmIya1UP!waB= z;%lkCt!x#55Q-19SThEN88EALEt!2=`2;0MGf$`hq=2-bu?B^8QykAydFcfz-6;$T zq|J3tlP%JrMlF>;yZ+YlWC1=;)|Xt51hchLiJ6h)5vrC_;qnP1A5Nk8V@iE z78V7WFmVK81s6WQ{rC680)Ag_7#=1EP;$`^+Oc>GI%zC5lTe&Q5}Wf){D>cuAT(;s zh~t#zJ;7*ILDri{QAOmB7K3<$w6z`WXkF-77hI?3&_ba`rL~b67MmyMaGc~Q>j1MtI!(?u6OiO{HmUv_PBMy zn|XvZ{Hi~}Lu{#v1xuxGO?4rQ>a<=nC|$vqG}!CFAo? zb6Gj}`sVEJjW$ABSr~4jM3#qEIE9f(q=OORy(G)@zCPFlv&LPv)6m`;98yxy3xPvyCDIiNc{19!V;bc?r3nX@^<3OclOvV7viAB7vZyB7{0|jXl6p|LY3b0fG z|LU4OAX2Fh%)q#UW$imD0W6g$;wph)noK_3w2VCID}i-|+L*XSQ_blnBTM_mmnW1A zKbtFB`ds76w$YzdYW`VugaSIIh zB6pUOtn_+#(LYCQG`H6U;-6GUhl$A!36**r^+Xljf|-sJzXNa4N1xnVJPws+nFmA4DV62>*01mcQnrhuJ9MTjuN8Oyebp&&W{7M5Z zR`%keLv7Md6+3{)B%w*VB@zsS43Y%PKa7QhSYP5w$Qu9_nwa_73T-T25 zMLY+XSg~v03VNQdT_5t!=#%^>#1QoW699@(UxuFv8V%gb5HF|ERtT0A7Kj`J)g0g; z5DYj-u5#IPL9@$%1cxMQ0LlU5CJHK??t}Tg3YVA&tMrRHs^MfXiP~#O*YKNZop6d$ zVU5wU!Zh6U5HSl6gfn4QKX4RCss%H_ELxdmBt3{#Gbf8YQ>Q3$wI~&_dy`44C&K0S z1z8I+sLl4PpI(IX`VfX<*McXaKnq^SmHZdP4!VBIMEg^fXo7Fxm&m7?THDH+og6~m zAy*u^0)C5hA6My9(1|PbgxfLFGCSsOTEU%(%gSZ-vUZt6pdRy3-ciTF(3=11u)x2LWh42q1=bA-rwo1AWx7Qa%N z#I}eH4?mQqMM<7lLbmZ!)c&cG^*q~_`kq1My@k3qQ2Hw)Lo^n_w3Bv2%Z+6ixA_Ak{6K3p4+q+vXGBRse0-!cohf#z*sr> zm$4<@_95M#A2S!!G0*LRYNy?GgmBt{S3-{BW=LO<#d8!>%b0H&Wn~$0s^zu-l#6C( zT~4L#kRph|m;&1$lvaMfN->Ou05Mh3Uh1b|AkI^D_fU05{-1~$n=7$KMGZ1dAfQjs!@Kte&!6Bq z7 zHC0RvESOJKlm^z8-5~?Y(V#6mP_~>-g^jD4%(2T3(mAJW+0X&_wjA)d%w0COwO5s& z&5&b4EW_Y#zDX&c%8FtX70+y{oJ~pyq!AawgmcT+BOCitrys>A(L%y zl?FEUy0N&klQ7ckgV3&mD@j{JTqHr$AWiIzl~DX4DLy9ga}s|*;!YCV5JP{4=Zfw0 zpU^GxXT)U^pOP3gBGtAudGWKOvuAeMf0(Dqq;=^O9CAOg7+n}^2Tsk<%_O3PP~LS1 z0X_OU-&YL;#cE?ypWAV1=Vuwa)*)H%5$E||WL6MK`BL4|QViJ|tR*6{>Nbb(FcB>b zp0Nc2JuuEGVs@i4py^qF_lPNmuLz2LP8D}=%nb}owmq<}qCC=|)>l>W7$!=`{l_-u zK+ZwWw>9w$%7QgvViAW~Bk!_^i8V2yx{CO4r!ji^O#bwljY?`!dYkKJhV~(J!y1>+ ztGve49uxvAMP0*sw>4GSsO03mVItnao+=^|9BUxbH`1+Vx}S~f@|eW$XSx&kkp&J~ z^5cxvS$tF4!=NxlVP0>jUKOVj-E*NI;Fzf0f=9nOl?P(5dl%ZBbN;djH#W_MB?PWz zdDp=P;?G9VZlzw8;+DbAVL~Ep5%waSR7nSzn$Y=`IEEI*Uy?XXVui#Zh(;*`lV2e1 z%>a*E>%+PdA15X~*a!(@A)4U*)Ev+h&3R=$T?z8D7FpyH99#8Dc{jP7ldK{_3o9c{ zE|T?OH!+niRhCzkyg&<_JZ?v=IxZHr25+6DtJc6%X_3>y2X|F#I`UZjJ6h00a98c; z#_#fI3Ui``@%lxFUVCmYd2K-}$TaU|urEU=4JJ#3XwpKJ2pebNuO>aPMBG^ur>45eo>3DVG8(KU^YC45nPLf_gCI!eF}K!4o<6 zoeZmB!cI6z&xJMS!IOE^fWpzHHw7L{&&-6=D8XzPOxWTxV#7NS%)yWK+PKgcg5Hsk*0R5vfJ27~@D_&{#lV)`C6t?^o42$?v5`RYG&q@3R z1k3^<1fN;JGvoA`p@PpZ7WdE$S&F)%IgEER=cdIl{7{-MT0RLakb$;_zYV}nPN#4p z0~Me+ooWa`6ZclvjAd;`;e#%78-X`EG^2pBPtgntOcQr5q=;aU`?*cI5}tC!qk!)c z0>_G9=@iom=&HCT5#`XCp;r)MhHjhcLQt8x`T>4s2AUa$NMlh>QdHr5yMsfmjIbq} z|8X{I6P?6iIjQTgFZHAwhjq}lI!>VL8Ka||D%}+D@{=aq$Uqh)(XuW4T?%5ysXL zd;OTubOFy59pEH9eZxfbDmB&17bATlnS#X}G!p}4cP!)vdi#E_x?Q2iNA+HE=*U4KRINYGtG?83l1ZXRT+ zGv0eFNIELyI^-k|D0x4lcsXso;m1INwj=kQCWiZ-$8)agIJU(9M>0mPiGRR-@g)S1 zfG38@+&@AAH-@Kk4CrZEWQG1)0&fO?P@q3b{L85K6y%`ZXB>UJ_q(4Ezd|*wy*;L0 zbC0V?3`ZMWYpl1b_$BTQJ4&?(>Zn)hgctC~5%LX_y}h-v#rLn&OKQT!`pIqUcn|X> zN;F~@m!w{fdPXaqI3%&iHBA%(bw%yqPYQI}fcTh(co@7L{4JiFy$t#JWLs&uLw;8=H}AM* z*^Cnh^k{K0oAx{Qum^9Zmk}pycW{Fda{{9p{=ERR4GWOp_a;Z)*+@0FqZ<0=dFg#= ziA%4~xYMN65(ygYed)&?bmN~$93=5;5*i6w=Hg`%G=4wKOFqZlRT{NbBpd}EFP%<& zZ(VcB27;~>pTpwiWA0&uZg|YCJ#1XtkDa)7@mtNMJ9IGa7{wb1<>omHxnqu?RvRx! z+xLTR3+MC}SQvI~)^&baT3XplFxTRtAlUKT6y8~aT_59*GCY7@g`)|h5IG+c@@RP_ zzLTFkaO{#t_m8M@cKT?oc=vEA26TV;kBU4jv#|5u5&lz$ckDK0*%VSpfF1^(OGXisGJa-JG{c)!Q9Y00+yLhgX{({_}wm%~oa{u6kMDG;<- zyfQbEgd2VOe;_;bUXi09_$OAFh&sK^Q3z>f%`hC@P1<85?jdn6iO)#fN8(iyEfT~H zcy*7V)5va1=eca-lP9DkWYmq>!>R)_?W$F+ick}!rd3yG4f0iujz_B6VHHQwch*kT KUQj-sFZ>@sGH8(i literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/qwen2_5_vl.cpython-310.pyc b/vllm/model_executor/models/__pycache__/qwen2_5_vl.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e96226c8211f858a6748c3fe830555fdca5faf32 GIT binary patch literal 32068 zcmbV#d6XQ-dEazTPft(p?CfH8af9H1;(;ZJ1qn!`L{St;93(_w2>~D_HAM|(w|jSI zF~_317sq;5A_+=_4$8Ld$cJoaq1cZ3hy5JeaSq3duQ<=n@toLsi92~OmhHr06316! zOB4v?_xq}Qj>Q6^HT$Nvy1Kf$j_>}ys%Z}oyBT~Q{OPaF|Jm0vnLpx1|7QdjkK-Hu zrIpF}8871-zS%IEre`*@Ue@4swqZ4MUao0-b~Eqgn~vu+3tpk=dTw*b8)_E4VsqFV zZkD`KbHp2wXROAkH;QMi#)f9uD<@@R&9a#>GR<*sqm<+to0^-w&CM;|7P+5WqJEhEN+}^BsmF6yQms}SbcQoJSy-V^|)qSD&$~~`ijDi5d%Qi(NpDiFhZ}pF4|oqWAM_qM z-ciYKZ5(S(c~g?#)_Asg+&kX*Fe zk4k=L<6}+F^O`SsFUa-njgL29^j>U!!uy0=R~pskY45b;cQtCwX>S_&JN$PwW}3d| zHv=z_>pL6!o3q|*bIzM<&U^DlCU_*MUoyOfr!xLs{@t%;{JVqFrL1=b*Z26lalKov z8^K6$CTPr>TQgo0_a^*%aqnKa*TVIE{{6VVKPWG>@${nK_9uhUg|q(NK=}_aH*5P3 z`ukqZd0}Bs_wIhF^X@e@UH?7)L#P?~56hkRN}9IvKL3$dEpG|$e82xF-uYC;TT-@?=tS5hVxwgD5$elq{p_OtKlZxmy(IZ5$-i89ws5p? zhrj3_4_@&e-Lu_t%m1kVF_gb5@7C{p1#{{7 zFW{Y@#(O@Uyz?`X*YA8yYJVK>crkE;PojOS8+uy|UI>PQ4{y!z8a@AnUqvgI{nNo` z`Nmnxulds``J6w45+Bb8fh~QR4RZcm-~_KoJMtfWp7-mh{dsxYLhxR^;l1eF8NY$j zF9gR|Uk_foV&MN5{i)#PD<*Q^9ehSgz6T|*q2!^}?+q@a_vd||Z=K9csO5u4TC;%)S~J0(cB^r*^5OGAYv1I1_B?;I(ro)dqcYQO zF4m&@bR(#ouSfHhraDrMwHXuy^||>doP3isb;67b4YgFAsm;s>aelF;qI#_{ zF%)N?XkCn*1GPq@hE`(dNE9f^Sx4((6z7lDSrRkIewrKnwPEj1S}R@vx9v_ICYoe8SbwFraq ztLN%|&<^8b*u?0nDqy2{-(V$qUx?xjPef7B;!utSQO&PKHFU|TK3SVN6SVwzd|GF! zty(jv;>`hG9q(k9=Nj$lT7$is!CPxl)It~JDRgP*0B86>yER*%i${+wHKO{lwqI-L z61<|wCKeZ~bE>_x7?-dTn0pL2jA~I3SJ0=ht*WxT)$UDHsm|4+`9N_3m!f(j zI{w_L>WM>79XWaG#K+>|k=Ei;gf~UOg(xl@J$g(EING6ls~%Mw?YTK=V)SIKx!AyR zcE@hS`4h)ZPGP1oi17pM#!|C&Jn7rfdJF3qZ#dCDKUjo@M~=4V>QQ)HwP%75Jx5Xb z!%H@one-Acx7#(8;VpyJ^q^XyWD-lBe~+vu^UgPM<$jeam&!_-wZ45QyQU;6C| z24F&i2F5>j?75?-jvRaT;1frClThx>`9UnIKxE%KbZ4YASPyUACdlDedtkZPH>%N@ zB?VN(+BIqsA(HTS1XHb=fnUekosyLU{GANWE)i$>dQ=*fK3CsUeg0?y7Mpq{&({gz z$s`%^umDHjTkU#SompIZ)6h6vKDo5G*j7>4>wN5{Wyg;bCXOFJRu4lg?a|sr%=Tt9 z0BCFKGJ&XOfKJEv2<`;U#pq(S-ZzY@hQP2oTbqg6>f#K^NB<{_i^uT|A3zdhJOiSc z`CKN*`o`8wVEN{wnIN}dBbR+NQ%LjHqZvO3qGre2dZQOgaJ+sYXy|2$i;KLdo~t#M zf-trsQWlWws1}zrp_o&3zZ%V-pL!z?WYx%7jom5{W~mWWt8uYfJ-bwE2-;SagFeP1 z)hb9z8wiHE4pAqEfz8avwqCL~GU{1WljNH_p$!zl+tiW*hxRf`jBfvO( z1HwZVBjf0D5E|?tA2%M~_b(Ce5zSf@dy)mBl0Tb%-i`1=e)E7RCi2X|*AkUIf^K zJk1E}nXzRQI$@sQEWRP9F3McWtQuFK?nGwC=$M^s2dnkE)yY(|HH>ch()yL4LdI8CV6X(LEMUWRDr?8x=+KuwK0PYo3K~ERr{L-Ri)P&sd+vi*Q zMiyfMeXv4E;OZl6@}o?oagRB^;SD1$RI4+MS_nk$1KLmE-gn^}-hsq*EZ`acoVP9O z#++*{_oz@hGdIM}tCbcI<)?ia?9LDt58%uE( zEFre0+wF$Ru{S+5BND0mm~hI~0+M(*S&^9la!Rr)f{8w>2vK<+xr_M9`ZyUgXQFTT zzGa%`H8X3zZCd8l(VY1$+b+GebF*Uo!!ilP12C*cP&*g+Y7XTCtHKrG%E+p;km(rV z#!hBItOb7dbyKfSWGn#8hcX>=*7Qx5pTzq4mJiElb|_MZxTq}yp9imGYheGQhfj=%UdQVCo6s>I^VABd91>V-EuVnStE+3T{IQpT? zD@J6H6oi}nYzr2YOXdnoFtg$9-4Zi07k~|Z&Ih6RH0B#KYm8?iv^7wcy<~Rqmd%{^ zT*vZlU3bouxyAZSILj})6?`r{QrRt1z9sG{y)p5XUY?R%wHAW7eG1p= zB^H`c(z^9xyDj>_+F@@W81`a2ta9k*glK;atq8SqOviA9+Nnz@BmErs3?*0L_aKup z^@5@Q{4$w)U;*Z{$$u;PpTqwg%7N_(L(lsT7`p8jg8YI5_OJiDVrj7aA-@R5?iLJx z%paB-B}qq;+IR-`h*5t7Y#l?w{)>R}@p$tfiSBBqQ*lA$Ho`+(fC5?-fR9JWNP`)Q zvw{)A88vAnLZ_bsGQ7^>8=6od`uYB9_6nA`3husYV>!($U|ZF^Tp_n$$+Z&|I))E~ zAI|_m&DVj}RW}+!eWQC_jD{~|2n2p+DWg6Rl@>-;?2e5q16QLR`;vf&+zp})t7Sh6 zpviag>NLo2t})geUvZ2~$GMbI&x2%Ssv9X;M4MJO`}r#d40=vl*2{Hr(Uyg+XH3=Z zJt1Y=P?BIsBlaL7H|Q=rJH*Z5VJWOJPvCKxH#fVI?s#@PP;fZ?f{S_EKf0 z)~cwbR^>EQ7P$jcL8aGxC2C6}-4bo`NZXn`yu3qTZ;u$2{B{slpeR)!!%@wwEDxtO zQG9-KVuyN>^CF!8cI3P}sj#kZvpC;Y4PRz7&PR(nLslbuR_l<6VUN!7`|{s0ZsvR3{E-5i@)C%sDMnT0{@E9sP@~pmi?vHa!npNcD+R zrw&b>I`ZsP^~om=JacI3pxVa4Jj8Kq#4=U=;9PwsfQ@XiHeGMjql;cSg7qp)7GP5P ztv<*4BULO-A5g@$wynf@h+!>gUur9>xrdKFaq?-tdCGG>didEB#}1u185d71!9LOq z4k-nLNkPkdpJrb^!{jw4FYxhEh{8Z~SecTw_sXeRYWgB9QsWzq?Q=T~s1_ zi4-QV%r*Nhd_yWrZr(L52w?Vg%e0whZ3@d^7$P)ZvvZDNgQW70HH(IAf}xlY*si-d z*Tf$f3^Xa0USHP0PZNjGeoQ~6hMHScitt3qjiBNK7 z<0{|1$_p>cEP%v7fwMlF5w*?s^RJ?Xj@`-6W)EgA8QnJ~Wg_>9l&3DE>mnX#hHrQ! z!fO{c!x<8h^g~IJ1Ky8sNa%YRH26hFhL^dV`C{gI=wxS2(m%~mjo3O@56-_~#^&S$ zA}+jPc!gGQzN*EF?)t`sN9Tf8aA8qBHikEboLcjxEt%za--NeJhA;p|RUpmeqmA}V ztr0#p*==INRV!#feV$2$$vGzMx}ucmZP!mHm|ttOTXRrDf!p9z;P#7E;LREJ-K_sb zBq@*+4B|Y*vMS`a*qjbkoh8QE*f(Jl}GVXLhS*3!#_sS zr~es*S?GV%`cml3EkKe5{5gc%S2OFvQ&!mIUdU9r4&>pL9CBdC$l1ai;oV#*fCWN2 zZotF2;x?U?AuvnI$eB-5vIemDdFHlcRtu2Nik+baw^Kx^156n16c&c`n>)j3C4=kY z!tiS8O2#j|o?j_NBb}1(UNTol@oWiW*w86q4CcxRMl=FAADR+4zX6OXSkQtZU_3`K zEB=f5P;fvk8EBg)`@pEG$_Ca3$FO`K+-BK_5sI#AbVOd7Q(Xhwg_FbMU*7 zpO`2M6;xL^PQk8A$a!1q34@>HT&Vi=WXkZ>PxAiu^l}oNo}6vY*CxBKRezst{S=be zj@lfiSDaPtW(v;U9nGK#8*Vk2ovqK*X}!j?33E;6$+Os|2e};rj%OV{|Lmfyjrwu6 z{k=#gMtc?s^;MSsJto9l^`lIFjLF|c;<-I74F(HzxUYvqdtiO2-eApPjuvhTZM;&i zun8e%U5X~Ag-u-beYjc5k7iwPA;&0QcaU?9qB&~p5WYn&wn=yyxYu=<6|R>+1(t9B zHV`slP2r>I?C=^(h!QhLlfCNMI=V&>7?A1u(o0DTh-we^-{&O>mwFY+zr;7Bm;t*- zVvVxCWy9`a!|q|j?qS33VbbpLkh$Hu3N}kdOv}|(D_8!Ov@wIn2Ruzk=^V$0@;FNgbyssn%v3 zdEW6${s{D(5z$~qf#aj`*x+0XCQYdbT@#rmE;SpA)kM&va_iwv?6x4;CICQrM{6)N zN2DL-`OV@RQiADeFp0h~8kvz;W}%&ce1gfC`ZvG%vyai16M#%uY8}b!6m<8SStuul`h;Y5JT_Y`c;jfK zrGG8Xao7*)fBR%{V~Z#)Hm^8~nVQIJjO)&y`Uhx7CQbbub23{3WIc4@^dQVbGe@`J zfEf}^v`E))2{n!4)kN&?ifpoAJ#lT8FX^KC02j1yi)8o!zJi&UT-bOe~Hj#Aeu16yfN1`1pCCI0g zV#kU7+t-1}uHkdTT690dS2dUj&Xc+I**xhKP``*Iw&8n(ex}Jok22hv2K+3ZL@Aw@4DjCoQZ+YP;wQXKqke0*fY#RwqU`+ z0Luk%JPYj3`VPLp>BNFT8^y3^!9D?r8Fmb$aB66b&I?96SfLH^mcjWuPAB^zI+5Ni z;gwqArR@9WOMr;jMHs@&yjJqYB6t*$3LKbNhYG3lr;nuimeBEB_1T)5U$iC`xwT_1Md8u`A4%2jsi4rKZn za6kIN9ibWUm}31K&}|rn{0I*kKsN)Z1G6DKf>|IP{DKR%uvelUe1i#NE%q^1`T@au zftE;tdfFR^8WyA?px#Kl#$zM1bu554T209W#d%)`h{Qvw6h)p&&JdSFARR(`sxTQP zfi->BK_XZBVcx|zi*LAyqz}>HuJVns46J@cI4UU_wHb~OYavHv2Anoao1t)8JF-^u z5fx~Hl?kp2-{|WWN}-NKu9U-7piAdq`$L;U3Afej{quOa`UNDX6#dQ<#Xf`-G-MYA z7u7#PF18x2>e<-Uq?ln!VgT~2rw$!`PAoveMN&uFZ}Z`Q#Y9-=zh>@c{PU9plsi$w zb)ZTL2Zd@0I2MWxSuI6e>+RxO%Xh3pBK11m0&^X-PV5!TeHt0yC9oGJQDEaLtR9>& z!A;<8*B&BP5NKtvdtl;W+kmKpDTQUiT*;#>4}-yPqA}pnzrjBxNGx+lh6)l#kQQEw zAX$iFvojCs41BI>$v)4$AORwbN79OU7Ix9~u>1!&PvisiG8WAu=BkKUT9=e1Gm;{D zR>SbrL}^{z?y_3-%j_`$bUg`Nl!Kc~-M@v$I_X*h>1|oRuCLL7WKEQ}!Ew!e>6<&Z zDn7wE8u*aH@-x6?{bBkGfbf%}wsncf&8dH&AgJyF)l2bh zgCKN5996<*8N%@=^7hNW{I#aqwnLq!asM3PI@Mh+G ziPQm)KUbw^=`hM%m}o=bCbUghlU_upwAol>+JDc>V@P5fF=mJlR4?Pg8&Sa=A5qom zrCG!RB626O73si5AD(9MTqTz&OX{`xAK0+ZWKb~vaRNgb<6zTMxv!*ST5gB$I1H=DYzT{k!UXh35v0!7)ElMj3 zmR<+Uwk;l1QUctS)B#C_Z8S1h3&Oo^PzaLXL~$7)r@xDCXiJBweh+zA+QmSOK#l** zTP%Z;@}HRRE`<7zyp)OLQYe|oZuK|^LIA7W@AJ__FprUkLpeCBUg85X-`z=%%S(%h z+k%Oljz+y)Kf^WHiMq7gapn=80L|Tn8Z2_rwG#At>#vs|xb=G6*zmxLn0Nv4KvrZ6 zGAVt9)r|IGtPrhbse#p0{}pxgz)VdgIj}Xu5pwfO>_1VBL~r@7TlX?0cZT|RcmmVC zo**MLEz*p1f(W5wnQkr8{0{1Um7O8ZJDDA3+1)X$SUQ96z=yEOPx!+~`UMw%V7VxeUqUmy(>or6M-2uH*{+!@0E zp{OXDNB%PKTucnZ5iP4CDt*?toVjdn%d8aO3@!M^Huw|p%!a7EI`&%THRCn&wd`xw zEP~4X+^bGB4o9d#pMwZW8>3Bl$7ZxZYG7RkZ5GXDi(q_S^HN~suVR2&X#aKQeuK#` zAi2S8OMn=|H>@BLvu%-PTXqpIGAZcE-$uJ45nHq_1zm z==;zh()9sJmrSo`_r)k~CIwDx?c+_oS)`OG0eDF?dMNz|qYD!EvcS=8gY`#B&%=Akij1%l?ST zaV8{6zs99_92pmKlGa?)l=PcIA9N@YjbU}UTC^!JZ!+z?`%SZuEE+Z3Z><%S*ITv$ zK*fvU2h%kCuP6}$_CdQBz&%sCZK*d=E;@N^>o9R)Y;PEP1_s%Y-1QDLFc_hHPZWq-5VX3goKVxpq+j--!@pkp+$gEX{2i`4=I(E{4*8$E( zn-3}40enLn&CN-4&cV!!FG5d%*A!L|`k*Z6_@+-yA6DQF^?X~(b5afk4E8t#bD%um zv0)RSw>7tob4;UYH-E)ig~1e52|iWi^Y9Nr&-h!_hj(?=iQrkKClzsBCa9OW>e51Z z1(A|)by~PPL^{bkj5UJa3Vto^-J+#>wTSmZxm+FYIM9&_9mqc@&2xM!ZiiAdsNe@M zC1S+riKW_#5aI?j4dJYmaN@#JD@1sHK%9ZMw1%x7dBoRBjFupM9&&3F!VY38iM()A z+6Y0ymJoG;FBxTWo-JjYwF`o)VD5r+e}PgjPtRc;k=CDL9b-N=&I8MmLeDv$=7#aK zj%yZ!)Yzcku}adR3Be#J#f=n00ZHB@g56^w*`L|P*Q(&ETsjM9K7Blj?yg< zzDl^WR*2~b#lH6pKp|-}y_)X~LDLz=m(mDe>x#89+8LErOK4*RIFXYQ_&msUsqb9LCHF_r3*do% z#?PL~t`=}VHwSoKg?vFsVZ_&+EyJNPN(&x5w8RM*U}ot_b*BWnV+TebM2Nz1 z`Xh)rT@!`yCS@Rv3Ih8Z)PH9IL0bz6T~tv2mt|*=#M#A~ul}Fp!lkDA8|Lhpr6w<^ z@ex&k3BLMiURc-?))X9=81F%TTuk_(@VR&coa!lq)P6s2q`v}i#M^WZ0SKCLb?XW< z)8sF<E$qr>b(AU-Wz5ZhJIs+(t*qKUQr%_wF5!* zzWNyq%PZ(2#8!BvUPM3@+ddO^Wkvk@KEonBdFC_u*aI>+lP<^tR$G77SwLWBrKi0M#)$o4>{va4ef zv6zKuo>|yP;rzvo7@G^?MC<0Ja~bv1ox)171$BYp7Z@AV0U$5NdG-qCakHLV^lS?V zi-DkwEXb1{H+sNvYxMpvSSF3T!qCJRjs>HrEmHXJ>Ac)zxw^{apRsgnY$6_5BoNlp zqRF?IL->fcwRkpg7+dtb>;j}y7eS4Ys?EU6x_p}#}Gr>L(TnQg~}+4$yZpf&g82|Qr!Y?@v_s6GXp4#P(_-O&cOf`Q0zuf zU$_Q`MWKI)XnO*!m2IGDmO%s5HpUJ6ZP%hb6Flglrk; z4OE4XgDCA>LvWn&)7j=BgB*Fq84amp=HUZ zSh=^_w>BUA0V@9rrc}JQ$ySW3sW*h-2xhMIE%O0{lVfTm@>Rv>v7I2g zAREczVeZeXN`Uv7;Nk%3(e{ai^icOBS3`jArU*nRGA=O(nU8WRI3@ps2>I5Jz_GyH z8PLqL*nE^lWM7t+NuVcC-M8V4n1~!7QGBl$zP$qe4U(f3!@s@)t=ZqOX!W|1J zv6o97>hO>&XvugzyWpF{PA`q#kkN{JAd2BL{)DnW?6rylo3!A~d)v4ttrrHDhw6Ugo z>ujZtNZ~ZHn1Q>=oeIpph$_Z=D(z{8eaAW378jd~5gePg+zMk`3K_yCTbI&(oXa~7 zBUm}W&XZn)y0P7Ng%dDAgAg#HxP~38*lURm+-kopFgQzG)SDbZ5U`zDy@%yFDN+nF zRDXvB?jlHn9)kvm!l4T@P?H&$`+f2R42Otwb<`-IIgBE;fw>(>;t}Z!H-PC$P@8yb zGmw9+j>>F8M}J6yY;e7#HL_ehT&hLSg!%Dk+ZLJtAGVW7Y&wZ z3k?<}*G<#9=H!c4i$vsW>JIjzjAX*logGL1RBX@7rusN{zV1i!vD<`KTlOu6$U~q{ z@&&)0>_3kk^mL|;c;6W!L;4v(rmM<*2^XE5DibNhxGS8QLLlS+ z={kid?Vw(1mi#EUJo{7-A*LL2i|zMV&dP4Rsouwi8=ftI%``F(6=+cvP~vHzM|(Dn zWRKd|NFVAwIVhFoO2Bw8>R{(*WmaKx{Uoe2@8Wddi6pl5wqu3EQZnjpCR_!@=@&^C zl{aC&M3%)FFFhycH(2^>NU*DI7i)Lh>}w0ht|Y*>8#xcCLiPr!QO4s9r`Iwpm1Aux z+A7c5sR2?CEZ#_IltN}=m(;~;A$3%?lP6U;mp@F{+>L%(!&(ZY1V}51>o~BoFshBw z)wHfqf^@U$KD>N@tWvt89Zw{>yO9CpWS}l^D;8xs^AcjBki%Y8Y_S5=z`+0q;Y4n# z{;4njV5UBb2I`sPBiw7a-Yf}ra)L0~e7Taj<%&S*t+!-ZCbD{#yLHzNd`t#Tq|g}7 zvHvQ^UPgzx{f~bxZMC~?30q0E}YS4Md!G8`q1KyZF?d^8p?(1T7=F@A6x z(gfU#6p!HR==KbT8d5HyH|jQyhwsAJg&s*Y0!&4&I}3U^?K?6tI!@l`t#A~?!yM357Ay&g;zA)^&P$?jU zLkAy%unf^zdR_o&Q=MYnpw`abZe7BobV#ef4{RAX?pnWUPQ=_vnQTt40#w>?P%+Dqc z6ZJkktjH9-4H6?Pu&8-c=xt0W6yyM$ouY5m{q)p@^x7Lss=*e}t)wB$9$Gq1wyR2betAy`|^P>*AGMC#Vt^x&!cr{iSB}B0a(s3E*~;OKEs5 z@!6{HiDWt`8ho9&cMv@;W>JhlC9gX$bZdM}OWd3#dkbmWwyqWtxo2NDv%uTy z6ebj-`UG8tyA7f%DKzDGL01~pHI?29*AB#5?5V?l3nHi(V`ZB%^=#5Ln!ikJIi`cy zgz#%1f)!68?#2PF%ZnL}L3-BmFOfsvg!7UWH+lXH0K3-E{5PC*-3mUZ<{@k19FY08 zPpK1;aM1O9@Z)x~avEA;Z_krpML4O+7!I|N$2H|ka107$LgjF~Yp6JSgaYgFRIHnX zgqKQ*r*$J@%$k7vFmbWa4S~Sd^%&A1+lC}Lj_FimOvp_%lj=zftR829$%r3&KVhv) z#i@8?;1HwPmLN?$s)CvwtLo{QmNL%6GZDz1Rr zr5;r@SGU0p;jqsbFs4#^8LE8(xGV5?NDTKCSnr_KP|6goOn`k)KeShN2a|J?_oplX zbY0V$i3AOvfGOe;X^}{wSEjO?`zR(J0giw#5P#ljlWc>ZbY&~MVOB{RlCsK?gWQZM zsu-~$2oW43+T9KB(I~>!)<8#PcY0PAT7IH3yVR0xz|`(!?}b(t3E#kVKF^Opda9vg zTYRL_Zed$C=>#>*Tl8T7-?@1nhlV9B$gPy!;#NAoggC6*owcA{+07un7cg5dCarP_ zq+eWJOq;N8c2Yl@EZ+%Hbo*AY+l*el8(X3BzV29)$(jL*c0WWeTGL5q1iQsu?gCYK z#)o-119X58s$JzhYVk%^yx%xN!@t0jhA;v+B|q4uDB@qGtNUS64Zgt6Y(f-LRlB2T zlTMi5dai7(lqS;cE;h<8-(-&#@FhcKUL5K#1KD3h9Ps6}n+&DvFkJi~YHADHp~22F zcI26~v*t_X%<^5g>}U;Y>#E!BMAL<)^N?8#x@Xp^%Q$8Zi+-8;0mOlQDms} z6RyJNQ4R4L6TcF?o4;AB9Xr!m~{aM`> z_Vhj|wrPNRLQpE`FaCK_JW40Cftx0C5F8lJsh&gv3h+MWMW*OdfC~gP%7}J` z@o)sD<$byX=$|%-tubw_OA}6@2`x@#6^EyAJZnZBK{ByXZO6ZOe1^vo>a*>-P4{WU zyZFFEOsFQr1;HX({?%3%Ze!9z<03DyfEjTW3e@YNZsxU zh~*m&Mo^XuMo}l-HMF5yT9#6tZ(@04i2EB&p0|>Czj1$KQ0|s(@;9U2Mt=+6;cu06 zo21)=O-U`+-{IeeTAOjE%HE{rPN{Qy+E>34Z1H!YPg^lE-9J5+t#t3rmUuoe&Rf~$ z-zm@C6>Lx1w*9;Pd(id{oc_xm=sUUKHaum^*}s;z)4wmcJ*Wi3tikqh&-Cy2_u#o* z632Z9MAHZ2-d2F4hui8v8%`%$Olys&uqhvIFyyD;QrE|VZV>9EiBnQk6#Pt8^%t<7 zHy-CgV^~+{r^~t*(Fp&Dsfr&0H=$1iwifP?V)Ql-Z!;ET3nVy7_7bQ*jDd_BPBuaE z93NbP#$Q6D0(r*5u-Xqpo4bJB;MI}V2%;8^g$?*3wt-Q|QfftGmmp76M?imZS0Wk_ zt0-lgqb-c6n1gJzP=*0AQ{9?Gk1y{N{ZQ;Q`q&g4YJ0<>BM^*d>la|jnVF|^vZ$J2 zwHxgD^B6Z|l|SSRahgD6WwuIye$S!x&d2GG6We-Gecr~PRm0h~)Iia0HECC!BC0>f zhbK1S^0_)mRe zj}}AXJnV#wu~U#SJ0^HMV#%=YU-ky%4xQfdAnIBv4DCtS-RFI}1~G zI1Za5jM%VOnuO*{##b^wY7x%UdXQco2CRu2MC}p@idWW|ta<=SCZ7~6z;kj4P-I}k z685BCLVOMm8p0L#-vE#8spayK)=V1*z0N=}OO&JK;$-D1*c5sFNJU#6FQc2wla+A3 zz0~k4(?Mn5L7ELk1%raO|6p>EuYMBLp!rAzb_-#q5`jaHEjBVi&xzgVdEDtCI?3dG zPD~N%3$?(Ja4@vXn&JfXjN(Q%y@=hFjBn?0sdWfBIA9AhFHnHNY+`K|cmMBjmABz_ zqX;&^p)LqS(cFk!%0ZBcYc4bn3DlAABb=^C#+`h6=2bk z0$3321(Ik{1V(#W{GE~ZAQGbuo_v{Q8+iUjJYT@`0GICb1qh^BIFDhJ7IHQ;r4C|5 zJ%8_VS!9ihH~}mA4BKUWZfzwE=v;o_M)0)H2AVT_%}uz7{Aw>MpHMXjn(8#mYE0zq#7#K@HYG6uZw-$BM!>`=IEcU?RCZw} zmChhreVu^$I2zWGQKKL;j)+0r6#^0H`e>H%RCJDrj|c=v1S1GcNjZoX&VKmjI|JWa zcV31uMxovuAdb98c#Adld%)x`NnI=!|8+B3u1=n&*7evA{*AyxL2 zhF;N+x(Eba>@V;R*l7f`5%`hpiApv$)dxXaC~c42j;AcT>(>M*k8=)7XtZdSjj|4w z2M{km{7&dL@xB4{JS8%T{5$u}^63k+ygo$Fa`@ z^X&$F7+r<%h64q17?=PTjr0(Ytus6;M0_o29NwegG0Elv8FadRn0--;OwKZ)njKqB z9uBkyFm@zcuGj77-JBu{?@pfE;+H@8P5?tkH>}$pwBvRtDhAGg#+gK-K7{cw&k;dC z^bZ=u;{#iBboAe#BY`kWpR!Oo7h{82-+-+J`;=CIgRsc7fm^Wp)M3*9apQ{yWAU+z zu6K<%Zae`6A3>AyU!FPzW{UmGzFE&)qVvFAMKA*5q2cQVPCrf8mbeo>2>TXTHKM-q zup&JT4F<7WtM-*l%68ET{mPKfUt&Sl8ni**;nTh1NT`z~F6$#Q9Vo~{gsL#SO1vU*Ts z5<_$apFv*&OSAz2mZV22hxJwhNQS$i@Ik7BN`hnkCVF-7Qy!2y!$oS2tvQOZiR6rKiPn@ zd83FUps78brW;0rhY^Xogdfn*KV%Y4+%I-{5nT~fgflE5Q=i_wU(WjFVQ~|u@4vAX znDW*6Ivx2@-$Tik;rrp`VZYN)|M~KdRF3n91`;7noF_qLR(ob@x=Vu(vyszN?N)&E zI%jbfX6jO^DflD+6@fokUvs*Yi^CluqBus>N<88BSN2bwri(*Me^3}$8SqfSQI&&j z{cYdfKhWeL{WasoYlYl~-^s|6CvP@#O~=|0){10yT_K-j~#`*EaeRGEWX z2nz(;2EPziu|zIGTbw%2$r#xGM2t;N0#z^eb)Bs!cHJAo zp`S^_&JF#ywe{!{`^{Gr@v|4p6YrqU`Z*kCD{+y72Zb|mts8wv@SV%}##b4WCo>O? zKKnXMD-glqK6>$o;3C1sM4q(<7trRjFe-8zr>Zan4JDPcCBzXR*S^n(qvu>^@ji$E z_z&lntqy=GXU`dGKb`}FrsE0*oji2&9K;G9^O{Gr146ZN-^AM{5g$`Xc4k<3?%A`4 zus6g;#A^Yw$$cVu-@Rg}YSM1?>>)V7u-649qyTl3cXaX#g|zj~(4{N`!(028w{)L> zDS7^-kKoPS7GC33+QvsRtq}y4yBEKb1eOmWtQ2!gE1pJ09Y&i~FA;>Q zEI=lMBRjlN#;KnRsy+@ktW!u?PPZjO;GRqm_gF3^HqFUo+pQ>2(h!@EhX;PNX*oAJ zjRP{5$DZTiv~rkn^2=l}h!w@LDYAa-z_lCF z|Er!rxq2DZV&|FQqPC#baEXRzc#FsRc$*N@hF|AO+7BgS90Y}1un~TYQ=b{~PQ#iv zEWb15C%-y{FrkpKy)Xk;lSz!R#SG8rSK`Psw9Ej&Yih)fSp070;LXqt! zt%si{Qc~f9E2toQsl~mlH*qp1UH@1R_kSTe7)m9N^p_ZRT!GMaRB}^!?_~5!wKbNPvpwg|uX7hA7-lcwCcZ!uPf?_b?MW+tl4mI#?{Vhq)Ez zs8*>VCYO2HX0A8$J}<@NdJ=bk1QLDtlPJQzml1KVVdl8^1+yndahg`?s=1qIUt>=I zLj7>+||vMnZp`fhXpmw;bfuS;hj8U$g_Cncz`3v&`LlhBsmX-D4>Sfj*V?u83 zJv2~pZ7iaGFfpG{MmQ@wK+ZL^%bEKGyvg-i2%1a#CvSL~rHUSGufRk4CwX+d`Vy~8 zaxJ^TWm^k@(%U>(IH}KShZeULclFWo+gWc&>S;fp#vUS*H!8(E4Tn0x_A(tOuB* zACvuTW_->4$!Ub8t9SF(#b{)|I&_@75)-)SZ%0qYWH(Miy`S}5>4V1UN2NeaQP|vrt(cFYKz)$O z<4m4lLjR`bJm1G0g<&GkKoL6q6Q{WhS3wa*4^SOn#Qh*O>ekliy+T z2TZ2e)aRIdmI?QGdAl?j(6*X&HFhp;x zd%qV}?``Q8>bdIy0C`p)0~S_%9P=MLj{=!+dM<+SA8EmTdsA8W1#d@>P#-(&098Pd8&XJ77p9)93ts?&m+>e>taHE*CWX9{uS*uRQaXru`QsCVx33UceLoOxHA_d799L5$ciQ z8IkFkk>y!Y&dWu%XGf0bM6T!Rl*bJ7UY_5DsAy=q7L~j*f3d=RRP>4*&xMs}%A4Z2 z9Zp9x-b_^Ws+@Mh*=Ww2o-h&*kgrAF!ct@-x z_`FyMzZ5;~J$FB!~swaWRw>{xnI;ksb!RxKshK5fpRFLoB-tkaTt`tLjtc}}Z8xbf0! zomL=&PBS>#?Sz+WZ(IsGkDczmQj5AG2y4x5)N3T|3t>>Z)J|4vulARh+nwc;jV1^| zdwC^^7e69isXM9pV&`&dz1(glsryqV2kE7{ zo*JDF!s)XzNMxhk3B-wRr`2Ar8)^Q0wHtc-)mf55?~_WFriN(7)7`q44AOMv&Ha&VhWs3`D;fpqawCcG!Qh zBTn~2q}-GJI1rd1#GN-!pIhq6sBWcuPX}_D`X2V9&e^P`Qy5M|rZaE6{?=`z={JL3 z*lsqGKnydz*}b$q6B?L0)m?5U@mbkz1~EngQuU2~19P&0Md^03v7=WsTG{Odm^SBJ zBf=ERw46oG^pjpc!I#RYfzz)B4Ux{j)onH|42iEq7Xl%$xM*dU2wIJPnD}aq{jl2* z0X09(bt2QmI?TS-!D>f=Xk$F*dCFtzX-~F0sG%V{$nST-VjlrNG<+(Y>-TzHiIJVH zo9RM3rge+jaZDL|n7t@Kjp>o7u@?A2)Jrb=sA|+kwyv??YBZCsyqp#}W7xPH$hb+| zIQc7QiTG7S0?pHbU`Afg8=@@Cz!VmqTwn=1$OZNuO*nxgT%os0?`thxgov zl&)xA{(a5TyaIMkB`s!KB{+TR?9%zPxY8CP=qSFL+16cVwLL2(CbocZP2#=*tf8A24wR%^$#akf@h z);irwi}ie(0~IUn`dHw0;`?$ARbwlIll*?q@5%0}Tp&_jbT4%{ofh6|g#F+ZDZ5h6 z&^H_H`mIjgPOUfyTe1$K{5%B@Q$XG4E`O+}dEak_jX3uG549VwA4{U%v0l4(G-+V> zCP$x&`YX|+(WPTy`@*r_|}?tTyQ9%f%m zY=_j06Df(0hX$An_$*w(3MW}!BAp{hO4@zYlPb*U$Nmgg4)Ir+VjgYz)qI6WTnwT5Y#$-NEun{wrasU*!W3YOZCN* zP#%i-;%783mvnpn+QdF)z$rpafwXOes9$aBFJR&%%aq(i=IxtaK7dShozbg4FCU{K zv{&U6g49WRjM(yZB0Wh#6@fQ9(H$Qh^^>mZkGBi^eY-TG@p={N8glj42&68gPBs=f zPt91g>Ad^nl?Jdr8|IyrFvButrR8l|Q%s0#u1Mk-&(1W4>b5LCwSaw3+g!dz0mB|| zp3x5Hf&Vr>GV>R5F2IO@%F^vy)-Btz5VHVP^td-{m)-Lps74GB8e1x4w%2e5P#@m&1hEqY=NE8ELH*W(TK2GNeZAy=ST zXms*C6@H!y%(mK4uKn#(y&c^dWxs{YF)e{@6!a3-w7`E$cXi_y*7#P%va#a!E!S}I zg;w9XRWLUW++~$BQ6Qg1mGz>05ij`?1+j^Up4 zwYTxaG(Uj*ob@Szy=_)LJ#a+Q*Cw8Azu7Dme}Kps-1A1>#AAqEnYL#F(oFz5ppLY@ zzzN)zu}6dUmj}H7S|0Ht-UYl%!e~`lAe2P~0--3Tg3@YPOmjHHVKu0T62)2tF)QXE zAf{MXoCe#>!vs)fRWPz+KAJwFX-gkzH4L8gl{8O?vjVj|PD>PByO^a4q}TfZ&!mx- zI372Fft}@aA!@wi_quVLYys%Wnf{Innq(Pe`lSK_-gkr_!yhdU?L8Au{1`xVEP=2) zG~YIj)MFcx(1jGFZ_~+4Mx`2cXutG@%2t!vtV|xYkgn{o^qWg|fzExvyM)IDM!1lT zRNp*Hzx)zQAKv-PYGQx>FX!bK5ItG9JggiP%ga7(-O}V*LHTLK01o5>WPbxM*`R&h;1kc-8Ylkt!wX&`@x zf?WtwcO5y#TDRR*iM{k~I2 zEt^8o*6CjbK-mDHIkRd}yn;Vb08kKGV{zo(F!BnjAgXQ}juK%kxIVEtrNmh2la0N1 zY{Rjf$`3bEtiavHMDR9B%QQ?MCMi1}nk1}-zY-#3ma*^>h(Ppcq?kiZCsC7IC59w6 zXLWAbB+uDZ<85=(*>thFv};h-h^?-hC;=KMIZ9)@m;>v2ZWF8z;Z@)oi>R>(JTaH# zIkW{s+Qj149oa$(atAJjL{cwgK*0(HZ3>7t>m{{Lt3)6vBiASh5qKt6odrp%uc!K2 zs$cYqJZL3oQackkW0WQ}oZI}f%-1FPTNEzUXKoi7 zq?V_-7Ca7#k{tw4@)ZimK$2w6vjs7hC-OImMxFQ-66Nd2@!;7n5d$ z0W+e)W`yyv?Ha$d^TscXLgjyr7c9sh3&sX)x|@{Cva2^F)p~~{qTbLi?je|vM7o$B z8XAyA%Hxp3+ku?1;dLmmoPh^|y$<;xzgmDiqOizeDJY6E$){D4Og6n+B~cMmkT~#I zz&lX}hs>lS6Z7Su2~VCpMb42W)(q1;drJJacxMbdN->+{Z4wqFBv?TFCnA#qf_R!l z1fUcgy=89ZI1VKYahu~Xo)CB7hM*mw14?xWh#WxHA>M`|0&e%61h6lhEhBzxS?f@~ zJa&OGa4+ie3{nN*LK!;)vn0Pjs$mf##9cGw88T~7OW`W@q}(YbWWG>&GH0gorKk)Y z0KcMFMTzsy-a(mTFfT7DFeLV5XjaHYdb%mFG<&GK)2xyJ3QPq!7 zh1O2?3j7mh(2MK4Zj%711Lp+xqhwZ7wh5BTH>$J*MO+Im`xnVMlBo`Pa)c>gYW3?(ZF$iUhe4cVYOaZA@&_ANsXG;hDOsCo%sG(}0 zL`-)BLlbH*>l#l1eW~Xs43j=pHx>xDNeb@Y4)WoDP%1u!py0r4VM##BsbP#If!fRN zFDqprK2+EO;E1rlaLX`QM zxfWOwkQjs_q4j}E14x3QRM<2+gz%6MgWPq4T{)y+=-_07_k@1cg6TspoozW_&XBW* zsVxKh+gJhz?a(<;e-~}b4GMmbf&qfMeOKt0?^326a2;nSnV%((iP9}hmwOBF$8ZS? z;id)a(7jcFov^Wg$4Sfp|3d>PU%4IS30NdSy~6na52z=o<-hU7Pa*)Q+hB`NhH!3= zyh4>5#?jm1+Xcv5PW59+b=(2rM%gl;+zBc~4a#JQ0Me(#^p)I%5iy&VhtqtTj%Jq9%+4R}q}1-o zpw)ioPADf>r-z}OMl*);4+tFLEykO)VJpc|43Et^Ip7&_xV_)k-#1#UzFct!%KrpU z(N*(p(*2>N_|VGzQc|Xpcq-7Ysotq%8e+{^HbynhWIpC57;N;TIPD$+8Frv1+6Q_9 z-RYVkV7LoR$(H_s_JKjpZScRfB)^KRN)56Uu5O1F=+(5SipM&LeW=ST^fDFnjG%bJ z|LGnN8_@;Pcy5W9zk=MBW`Ox&8iVoen;Q>pTY(WwEFOoquMx)2A)+eSa27N8{}y%P z5(TVty^C1AI>vXe)Psr+=Mqj1+QKU$MMp3so72KYI-XJPsRL94$)L2vcAhE{OBj)w zH^WweI(|^iro|YRM+VJlKIjOA(y5(wizZrbQhgTTv-})I@BbC&E%n?4ijUZir&Tgy z{{hv?s^Q~^;Yd?K2q88c42;zFX9nC%72VBMKd$BtU^I~W*2c(#%Dg?A*)*?2Ok2bZ zcWHj8mB*~PrgPg4S|E3DN)~SeX0lX9D*AVkAl_fq0nC}_eA9+k$>~%Wu=Tjg)-C+4 z@I66YA=ZNuL+U#zGPy0|W$hdDq^Lugaq*2TTLVgs38fu*cEgDPE7IA@0p9IRXZ;|z z=fo$V=s@)m=J$;)cASS(ASV zTAD+2IgsB*T>cRSe@r<`$cb}~EaO3(1t6KR=Em`4RLZUedd~(ME-0VYre5-`|cSU1m@C`+@iMGiFAiO zz>l2i0?T;qf2%Iuss4_mm%MFC8@Op#K{!cT)d{pxFnH6lwoM z8-T3<@<~B-kR3px!@f;)kTn3&0sj=0oCyHILyMthLm17%R+tfUL)s`u6@tfxk8B>_ zXT?G=7wih=sSf2EYZ+p<*n@8iY-j8S!|qQgAsxm7UP34m7AKvJ(`nvU)ax)o&8Z z?#IW1P0o-C9#V2$Y&UX{xVmisP)9o1#JY@NRo-p?Z$nN(%@K0WBLU2U38qw4D6r(< z0BWgytINMc4)`K|4Sd-_-TV#>iuP;)YNZ85b7S8SEhYz$VTSV1%<%k?v>f?nNWmHY zhAeghgm?}CcR$m%ubVhnzzrDHc|-m+zRSNtFkA-)!ta3qk*VDK+k2f>=xPky|C#sy zd!qpWMtA^+b{fF;Lh}1)pm}?zspH(~X+(x|j?<5c4(pO6*|4uvi=@7K#ZK@JuDy^t z5#I*csjb-U$J9}3qLc$Bk33?Nv#c_vPIkd(+i?vk#d$p3!GA?v%Ww++2)0RrnHA8_ z;B=)eiHioXW#|$DJX+VGz2T5Xhv@t_h($1tfM~s_$=dqDfCTEZBafI1I`-xIA>nY2 zN7oMxa`1q``HCsXAr9Zjbv=HOWBKd4JPDtIg)%4C4?)Djp4u|xDJsR(HxPqU7lOJl zD8RQ+925r-wuAhZspIlQXLDep{iioegVNg;d*#Xy>8cmq^$U}2k81xW(^WsdHR{K= zvlhRNuRFDvgQ8G|zaq!&s{YeR{%%ymPqG?*vc52C>mM*fH0B~k`tQ^moJN!}6O}=Q z$M>_OAfUS7KzM3UK{!pv!n%f170g!Fj|5{DGf4^lm~8x82IXN@yt-7+L0fJS|G-bo z&fhK)BO0G*u1Z4b5iN5PuPfRY@k;Z2v4D$%EhsCKo#8c(2SAk9D0qT`>j>Z@pKY%7 z5v)0<`R2zpa z=v)`NXzih3ae1+}@nrpx+J%0CYc5Hx#bxO9n_6q)&R31@MGOnGh#8jKTVF)&N3+^v zF3*>*#9~_Bei^H*POA4RrCp7K?@q;j)adnqi%OV~uP3zxuH|{t)UFucu!>Vh;vyO@ z^QA6qgqW`Q$p>k5eoVn33NBM{KZ5#{!uG#G+?$7e(OetWHEPk@l}+gOj-;?_aH%WC zSp;rBS=0-!tcI1{FiQ)ZL#=c%8z|+IKSFb9j=yrRQfD~2zvYg|fYRv6aM2WN+=kXA zYAtIk{(FMJKSya-w~9(dg(SUMvPpUxWSqdi7$ z59BV(;jF%!R&LH5mMthJ6cEHEQz*6zr$qhZK+uk)$6e z=loA8_7w`oQ_n`>aiaYNf=e{*=Mix==+Y!6Z<3p4tZP?3G4|d~&n|pylt=@ot3Iw( zxM}1unVhnbI_z=_^;c5I_eHmf8%TC?nfHJ}&6^Z(hk1x>PH@-ROgM|L(B2c4FuvV9%K6)vM4oF#-$a`Q-C4wnn@nf649v(ad|( zOcA&jE8D<`K(KMg$9#MhzmN%2b&FN*rTP!uqyC}#y$>0guf50bky|-q2#5F*@krrU#sT?fyrVl6SIfv?Cl@nd z>nH6nW>DREt?y85WsT}DbNz9!-lzN8@*vR)Oj8vSUV8=hh~Q={F54UiAh~s$X}-*q2N6VXva=$W!}`Zl~eaPhJ;&w&tW@jF_El=!+=IQqb=#v lfW|q!WKw*$Ub2{`)2&B};cv|8IM1zBU#h-QJ+=Gz{{d+X*jOwV(7_Wc4i2qcefb_0iP`60=|gt2ifE5sr@qQVhPGgUL&OTV_N znq@bJiLf08O>DE`8m&QX$F;7+}$$qb#*ay_0+=IC59Pv^DVh!>JY zx~TbPyp$}{WzDzZm1LE!CTnyJ+6O)oNQ+vv?7{{DJ8;DV=pekn`i1OJPCp8o98ts8!l zF&_KOIZR*9YX6Yarg4n&H6i;!;4X#2Ks<21O=GCyj!>c}J zV>7#TW!#^Xb62ytNYb032;-Q?n^DR`QFd=-`=?bfi+eN6VlK)i*0BHE2RVloXTB&x zSnv=m%+iy6OpQCewu{;8DcF{97KMu6=BdntEL+v2pBE}G)ORL2!7LlwMV@Cu$(uJ( z(iYx8S84+bmHoBy+GZ9qF8b)Y*FS44$P2)5gPL$D$ydg}#EbdJqZYH6{m7v9SB7m+ z2aeDw?OS~F%Ra6@`H&}2a5lLFL>WiM6Z6!4`&hdgE zN+T5nq6R|*jw>5sUq$z|f4~4+@_6vnEbBqgk3%Vg;HmNK&l@VqH}WXIq(U*|>e5F^ zv6Ea*_BZ0_?nZu~cCvKiejFzo`bq`-0q=t!L?z0NJ-l&&NY?WMu?lORfg)`n4YOk& zw$Zrt$AXMl{%zmbRLLoza+0Do+58)6yMJyWb9>?HVA9NKtiyW?`{fu@?7l>!H$h56>eGpH5Q)5Z~3z*5>7y>MP19yX2L&EH@ z3AI4ud}C5Og8MNvs9V=Rw#I6_#ysxv`m(|5P&c4%K;4A8$=pFh-?J8LKWfuf-DPK3 zhqre-Y)aEmWCOJ&a!zuW`WN^50B47o68p2(j%KL zuw}l;m-sUFz){df*a};PwiSJ=SK(HlEoXJ;T+75Qj)zjkkVW0SAQ#y#b?=Sq`&C^Z zLS$BKt;jV4aqsQjtJ=$!ZlAQ+;>X}a>Sns5_z4PimhGon-YVU`2BosqNAw9M5*%@= zCQ>ubL^u&#!BrvBJ|>jOhkP9h-ck8AkP+F1BvY7TPMIT!=pzCgVPs_PT2Sl(Sul>w zznjW|Xmv4KM?YxZU_VO15kVXta3K%>xgAFg029z3 z)jq_5FSCUEcMIhM?D|zdKazf$DIb6Y!H=_b|Mm{(bj16GfcF=B-n`<>Z3+}7TXOZj|dN@op;31q0 zUk4Oehcgg!K(6%&ZRJ%e2o&UkvYuul3FGLH_m*g9yw|ZW%GypSL#0I7_)8cU{0~B8 zLR^M)9)gl%XD5N$n1 z_A|jn2WLVIpw^Ym&9WAt0bJ1clJFi6c#^9F!0%USDu85)d8+4V+^e>82`ACp(Hqk! zbpEMPAdYPUQSPL;cb@?EA zz-5n!MVPE?#qc}|#tTmj`V3pZ_4sgl!zOVB+76)TyxqZvL-mq9oj=x3PrJ3XRXq^a z#6=h@K0xvz5W0|u{d=&lD!Drkv7} zN@$0OAEC}aA+O#XJ-IAn^fu@{&_mznu1lV^YK{MrHntEwqVeKFP2f7c4*;tImi6J8X%d{JC z+m&T_yDM~TDG{a;0wjSvq+Lck_=;Qyh^YQ0Xs^(wR#q9h@+Z2Ip3`cLwkI#n^(tAd z!#@D{r=T%cBkT`$9tX=S^!@M$X4Vgc@&5^ZuJppYdcPUy`YdT%ca=IR^m%pMc)qlu zVQ>Kdxulp!g|mlmW>IC7LPfEx!p6O!-=kITK;OT1_&$n=P^I^Ab!|U2L6YZz*v^{i Vk(LQRGHt?7^V1-qZpK<_{vWZ3{2l-R literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/qwen2_vl.cpython-310.pyc b/vllm/model_executor/models/__pycache__/qwen2_vl.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21a2a47e857a46b537d7a039fa8b20a2d9d3a638 GIT binary patch literal 30618 zcmb__36LDud0ux<&$+X+i`m7o00;~zki?R}f&?i_0wIzh4iY4=gg}szTBHWE)4jX1 zm}Bv}7sq(kqPUbu$%bqzj^o63W#On2v0^*2%2&mSkH~SHIF6lCj&did_=xQ=iR~n+ zvLzA(@_qm7o@229B~@l?-n@SO`t>{h`~TnD*6?u7z~{lA{+0Rq&l|=c^P>MVf{Q2c z_5aj14A*cB*L1DA*{~d|k#G_wuM>5*yZe!e71gf;~wW8 z$>-|#Hg-F^839TjrTk6Z#?2WB4s1>4>UgLd{FYE^$#@;I0q!Zq5f#&pmR|2#rlUEk2#Mu9(Nv> z>#_P1jYG~M$&c5cY#eqDOMYYhNaLt;v~kQiCfA$lPc@!)o|gRP`timy&NGtVQh&B_ z!Z{)NiTcULv@_j!&UvnJ$~h%vTk9WboOVu2ep~(d#u?{~xQ&d23?N8M?B!udqwCFdo%-dTUS@k!^Ck}uWEjkC^KcS<2F3UYs@;crr~|in_DrR`KJtbmwWeXhI_YHSV=fFT;Jo~i|c#kdchm738* z>E7Khbskz%Gv_|+z8^J}`-t56fTVFNA9O$Tn(g>_=K=Royz|lModHS?x*tZ#hoh1u zlsx7>j*`ctk_#w#!aanNLs7{^lsxGkM#7ixM{W0*_kQ;& z@3i+l6NdY=*TKk6&1T%=?lb7ovX}6d-DlksS1jjyC4W-#pUO^WpUvLsw%q5uSKL$h zegxms_&$%hVJm06S7rV6F6ugbgiJ72+Ee%y8N&K108C3@#; zlGpG2jMV-F-tm%`^EzlB>xSOuy^niC-g6TMuhH|D-A|&G&$?yrb@pV|cF(#Ml)T~2 zprnfDT`whl^t_}y>t(!Gq#gN1pXc0p)c%~jt>!&~H#~yAEx6}U`gw19_4~ZBWi)YL)urP?$K>yc}i@SL*c& zS_w1915ZiLK2h_7FnyxNf_j)fSy@~}(_!Yh#h}(|q9}Q0X%UauXD%;#VV0?Tq*e{W zyb?)L6V+W}}zU!dRnn&MVJU0u0L4P5EmIFVtMG<%fB{ zfgzWb$2Xx98x945*W^G>dO^jl1Qj%wDL+}Mp7WY+I6k8@aw&3Ou!qt90wi*+njcVuRm{m4?SdhV&CCtg4w#t*mZOO58K=-nr3O{`M5 z;dJZbU=i9HInkP{1^y}3s(QZPV)Mn1ELAXZZGc^?*&SC}Uh1iEILbV?6rjJTU+6Y^ z@@cQ)qGY4%%~miqWeuU|dBv5U=LEVjVP2h!d-8=7XO5qI?#Q7Ny=f@+=JW^_P2jC> zp1X5W7_5i4ZWYjX%RQi5>>Jf+bx8ruur~EdKzPGEjUC5itA^*+ur_A^i!B8}dfq#~ zM117yQE6EE{Njmd&Bvn47i)yvXnM?WSeCu-eW{vXt}ZUUWoq0kKEJfM*iwPt>voun z%TAplJe)dpvgZ3())SS>nCH!C0C3dMWn}_V!$Wt&dj%)F#$s@}TyhutLb#knK$;%N9F32xPU$okjDfKiup^h_ohRF#gCz(9UWWB+h#*KM=eQFCi zLnc3o;jsLs;fO}T$}b{`C*L&*(z6NIA}HgFp?Rr=G&p_|oIgv6;dNuybW?8nwP7dc zX1t+=yqjeT$sy+sf$I;yZrFxXzy##Of}Q}u_>(72O_Ly(XTj{2z;42k`5L5)zJe_9 zuUr=-&I71HbE?8Gs!4P*{xHukkz4`?T9|a(=-)# zBQ!5jI>2l-s}>%PKTz{z_=oZ9--o1QtQsBjs`-+&Y%OExtC)tX=CXyH#T?2_l$+0^ zlhcz4^)jAOpJY--GHI)`%o9k$+$EuyWWAvo4MU)f2ot8k04`RP`+c|&q$vQ&3#M(h zHw`ZJR8(XaDnku=a;9{$^_puT$ zcFWZySb74bfYoGmeyMp*CNfM^D}frpgMjt`Vi{JQ3$7Y0vMxR7SeN$q(G^+lAw@3Y zmFFw<*)O7=KZ?XQi{`j_k2NlUoVua- z&{s+w^*OaO#9Qh5ly-}#EYkL#lJA|bd(Azi z<~+nQe@}@&nD--V=*Dav~1Oq;j_w70L@iK4Z|v~Cnk;Fq0xN0H2hCiI+H z<(>Bj@Q64>$Ph@FLWVT!ssS2hl6aFkty=iCg9O*RW3MDuv7(@2$qu9_<>3(ycNQ{n zpn7BCE1fhYr*g#y-|XO8Ewj*qQq!#$rdlF)uO0TbfnhJU{4$4rK}hj0pcP@J87pIE z0QF#|ub_-*Gw>OTu6%_|%uKVU{&Ndpr(mXl`9$<PJX zN5EFo3mG>j>5!x;rcm1R?l24jSz)v}!0~7}-ba1A`rCAv6qz<;?P>yRl??3FB-YBh3Qki_$(7Yf%2hhZbW9hx zJZ=E@EMdsmAcxvy#QJ(D$gdd7sSX(OlA#_Ah8GIU=}sC~Cay+0=@qkkw;GJDZg3NT zhfF78qw`xS6=Dm+Pd0%?n%3S*p={ zLdqskcClA4*y?6GsUI`k+)pG{%va6Hq3O3c6_W`kSEI%$bLR|cxav*hTNEFaXfI|s z0bOJk62ZJyhxeB&7b-QeC!A^j%3-Zo^(+@PXr~@jdo8(_<|}@wQkq$s(-w@g6vOvQ zYkR3&pDop31aQ5QhF01=z(hftS}<^wmVB>tiRzlC{8FTb^ML@gY^wC})>5fjX_nMd zvvd~HzubYoUFtPo3R==gw?rES;Jyw5Ve31Qb9Pbf(|Q+m87M*2(?sFF*iv;@<}=K>u(Xs#@efmOjZ$*BBa#|vt|)`@ zab|Vb1iZsL3Fw1m&<9ZuF9>P``bfGvf9`@7>}=wL+JSn;T=^9 zc7~i5)?Q^_R+zlTgaW9O>rI_A3NaNp4ns_|_)aml+{|2t(dGiQP6hM69zz!_rMpTb zDv_;3tm3l$I=()Oa_Nj^gA}H2*!V@7NQpdagDFt1O(ipC3gnW1In>CTDGSWPf*^Jy zm&{rC1M`4(!qOYM#opdKNn>b>unB6)Fpic=j9m*pI3(=FS9{OB5= zVIQryw_A+W{8eLh7*uztlhbbppR_tdoguWEb#qs;%lV+t$-6_Hted}*SRQVEDi{G} z{CX#~3I`AK?qOJibr z17;=VZme%@Y+EjZuCG52+c5`Yo#MhBoiWVLrYqL+cxS`Hj_3{JXu)tdUr8+wV>X7} zEi2aQPCPLTSSWRdMHk8g7V_Ym6VqDy`V8PjWNLUwdLN42l5|o08cMx0UeMx zT?6exZuueWJS9o9T?CaCAsph{Ovz_<($eyF4q9wbn_Frv`EOdGHT8g&%LKo=NEYS} z)hGWy3&f~&WKgI#vY7}ri8tTVU0}GhaG&4-PlseB$&7oL6lcu7$+#Z`j+h{WG7GFf2 zthGp+^oA#~UgwU~DsyxdoUHigCM_p(J{q`_J$E6>!nBzspD)kUDt>4#Le$4Pdu>^U zsPhE6-N;W)-l?cw!8!r9sHmQ+&&&0Nns-sXfvmI5hwZz|bSw2%a}LH9@Kf#E+(Pr1vt7FF zmfaRsPeDk5k*CX_olGpGsGnqO+v3Y9%+}OwW4NT2BAh$h3G(kf6;%{Y^Zst{|D**fV&jzroTUX2OMZhML~RGQf~pRcM|M z&2yo7!O8X1LzxeF&AYIwKE>yTiR5rVX#16u1HUR`0_K8wr_k$41IrznHOIb$;mLTM zbi~=zm)M6j7(~OzH}Um9i=>b)z-9#QdOc?s6FKnJjG4cYL5}%+VyjgGzaY8%5BU6nL}#OI^z+^H$#6Zjq;xw}X3w&*Sb5w0wi_Z*RXH zHJY@i@zL7a&|1@v*mr6i=B=!rqs)&f;8H=(B`u0SU*Q;#zt+1MztrDRA$g;yYlBh zzxvm|`IX0Ls`5bZZ1pO7utrb?WBeFPDYQ=J`&0vVSuG%e8;@q-X^b^Ya>NhnpZ!X} zhf5jSBn)E619hq|PG&Sxcc)JMJX@0&{Q`3$A{RKFkp@A!Cji!E!lIxcWW(@Fcp{h0 zz_v;30?tG@k|8_^w&dS!Ppn<`Gog7_@*!S?AvNGSxT%HM+QJrHVX#bsK{LkT8gwklQ8!$f2dAGk6 z8*@Pcu4R(eJDHUAb}DUs%gPkKS(qS|f`Rge;?p-_d)H8N8h2xChndF8CbG6W3LH-W zyAy62-weLM=EyKJ5Y1;z6nI>^`Lf^WjjTsj3c zG6fZqtSCacYwaED&hH@J4cHxtZJeXS`1%Ks&;lL#NuUox#~}8J9!OpAY7$sy!14%e z<6V@&o<&TA@5Ivj0<4T8E`1rT>G8`5DOt!&^dnwqo(Rpu>ml8zW2DQOqh!f9#=*P? z)ZrfdbiFCI`gOjg3rXsq;c_y&79Rx_#n?g-P5TXr=(lFCzlOVY_UJBT`q^tgy1}0! z>~+VOV*M(<1E1mO%BO2-0KH703ap=SC?hCB9w(WqlO8{SEIl;v=D=+Zf;>S$B5)RZ2M`bE54{SuNhV4nYi7i*bkIMintQ|z%O z!T%V!(5^Sj=fj-#voU;0Y+#PfxEC>k3EKc*pXGD^n#u1l5yts%nB(i#L)h;l(Cos^ zT&4iol>8D_A$)~eLn@2W*M2wuc6;Yq^wn#28~n|nY2xr7s z6$}OT!V&}jC$xJ^C+sqiZD2o6bPzxWDF@RlWoeY9;ZgW4GzL`tJ^ZLIGx=R6WS_Ap zK>$arEF=baf^`Po;J9SJn&ks?NTAy-1?wUCmpD5l{`8VkvWCQtxw)B)Me`7I*o0p8 z>B+*nsQv$k!TvV-^|^TYfb%I?vbST5r?-?Cp8|um(uHsBm{5FzlQQrjb>+wJXR5F? z!Rw)e2;AgssxNFHDXbM%G+;HwW!p*V67g=>E?KSJ=iMlT+R&^6hfa&HO}HvUKmJT! z{l?dx5Rn;?F7pUl6JN%3l{gT82touUmPyP*A0`vqB321-vNQ$*ABoFaz%{oSW5zZ* zsSKC_;qriy3GP{#tH@w8E@?nk7-(i%_sIC)kBbpTh!G3}SMlBu4E;ixuCZgV1LRX}xdlT=? zqRqqj`ed0}(miLZM=`zThaw5LEYrAP(yt2}q(RLDc}OJ-w7S}qPZn&w1eoZQ__3VE|T_v-g~ zDRU=ct<2pj%Vaf^zA66`_;ydsmEb`-j5lO#%;*Leq38)f7o+z8RRut@{ zeX@{Jpc-Tsa)|E`s)cayTvRrMGND>18-j|EMNky$GAK%`2*d8El=k0C(H1^n3K$=d zY0mU~Ny7f8!(r{|i=v{!jF*`WCxOQ`fWXV;XVS^deGH^GPa?Rr&vEVLnF2?0zsG(Ia&d&&$7ZFGWjDWG9$O_&Yz%)lVM1L zSfDF>;=eKJGNs?;^c_G^4l;;V2m4aUDe;76t`*XhJvUjYbnaVLHZIR!8<05Ho8$q& z!TSu-%ftA7gWM1l5>P$?I=KMmw1AxlR03HIAVW^C%`Ii*!jul$7Ulr2$XVVMaOlCr zRg_)F)8Y^*S_7uk{?3VK;g&s9Z?WnBf#jx6sQ<}Zt4#hElQr+>4GRNQe~!$6t~u~_ z@d<~SI2?6=oy0_iQ{<;GmN|*xfoJe#NZ*jXVPBvn%ZC1JxzwT&PTHXkos#mTltaaV zO$%`#C{M#bsUyLXTRGvhLL$}qtM)3ajvyAMn?ybhPadTA@2GydW36Ta_+RNEMD&*h z(qgS55(T|OY@}(Nw{dq!yp6cSh$MIr;m^c7kutuFzE_Y%_+L92Xdc-Pnoiv2Q z2E+>lTq5coGGdrQpc3|9s3kT%$|PX&CX*jV5@X+AAdklp$W%iRbO%?VIUkxAfoV~p zlerk@hUvwSWfoJ<*r0D6zQTH4WBwo+|=OGM9*V!R9Fy61XTQkak-O${Q?1-h?o6U z-YSF^0`pdRn^o1HGWjbe1TSZU-b|$TH%Lq?{H03`f2rXl5$F2Qe)Th`7A6)euKG(p zB}DAMBNr?GRhANp^vcsc<|Jn7=ho7adED5^wWs)6OuFk6>)-7!p8lFAslrpqAidR`Ab4_o*V*Yr1`Cx;3yN4~waHQI%>4d5c5j z7=aeuDM7C<$$Ltz8OAP#N!c|+=5=@)uyD3Erygc`Qi{~O95Gc1=<+~2Jq8UB z*`t@Lkh2(U_Ji^S6h*{tWz^ez<|vbQn2>>nBhnZ8#`Pp9tO1AulGyJ|@LGd^;r2!Y zSk5>^y()Bbm(NGrah`~YauG%yiN(4On5K?-gTbsekdpdl9wui>Q7E}#S@!izI)5!s z%iVRw7!-94eT4Bsx_TY?Ghu38_D+V$i#0cx4|5IpC}c;Ak35)3l+U`YXh&g~K|iZ4 zM6gwf2JvSE8R{m27M#OQQWc33qQH}wB6OQh3Xt+740+H*n$T1_=q>XAVhc#Lqmy3F zH4Ai10@Qa&0C@tANubSE1m7dPDA%FY)r3BRu&5}PM*!6jTF)Z{eYk_bpan~W4W!Ej z)X#PbvkbjYwr8L60)$3lX0gi>i&@&Ow@JDP3Fj74gnP?8l)%ADh&!ACaMYlLwv5=W?x$ycth5+h@4Z1#)HsFj>j2}uVq+HN{fKe?|k%o(TNpN zJ;HD!0#mvFG>V%>R^Y7#`8HoFN3)}-i zf;5`aKaxu*oH$?%P}c;Wrlu!gbPqBBAqecS9|;J#)(Sk`1P7P90RTR*Cc*YQnVhbD z>PLRmsEwk5nsI7`+;6?<>dq8qoiEq&7MBJ-=UP+N2m5Jcy1RAP4tz`oPVCT#&ar=y zV=tn^WIX&Mx8!f?uhnLb68F0w$~j4VWt6*+;V7w+z^={lQDUVZkM?hoz!@NLL>RxJzf5EQIE%RUdR*%-`&LH!vSLh| z${+~3>X$ePd31;|qKpsCAmT)AXAc+^!}uA*KSboiA6R(0n!0TD{|-(GCWJbhY$bq@ zPc=iP2F?IlI6CPq2*t| zyU%8#XN}clkfKTuq*pVY+(OpP&RY1*b<*-X#B2C@x_y|nAVeYz9P+&kI|Q*Y zkFZYmJPRtPI{BJ8i!j4Zehy{hhQ6B2PYZ!o46)PN1~S+eBxVJMWh26yLD?}6ZO>@x z4OY`>od+wV*U}xwLn$WcM~2 zdm$6-)vZklox+PWBmVAQq)Fp@y@(QWm^l5^ldz>HtF1cNYhgX;)*M>48PCr~rZYu- zMSTE?vq8eh1Rgb;@|}$lje^vTjq&@GDLmudZ{YyhjHTuQ@h z$Cv@R`?*$lj}+rLCseG3*$p4{Mx0|4tvT{B{TDwiwab7meQC-#TW zw_FOJ*lu6LK0q+$R-<$_=$T;sDZvSUDkfu7^0=ma5r)AZ)unP%f`u=hIL=cXPQ{|# z6r{XZQar0$(Apmi$9<d6gK{iksQ0Rs0B%(?_# z=!UaFi;pp2K*Z^LD-E4$oKLOM{kIA?TbJYi7edr_JebdAp!#XX1}Kt0s)c#PBC2}{ zDJcymVJ1eWaAe@birJ>%OE{|ON*BTWDw$#YAs65^xVYi}5fX7Tf)&07gAd)kv@BCG zBG4=w0JYSLhz2WA!Tbrzi!fiM368A$@{?N9Bq&m%RQGqXb7m7%UqI6aU4}gACUNHp z04Bl~AXRc3;t)M3%Xvtm1X!qL42yTu*m@|n^j(;(5}iB^PRvP9Gj0|=wL7`k1`cD? zPN_#Vjgf_DgBurNh>+(1>O+-JVwwf`j*9811YI7Q4Mi@|%mmW+_Jew9w>LF6wI`{ z)G=a%=fejo8s1Is(I~>!)<8#TcYH7lT0T@_RJ#c1(C{QwS1XKEkAUcWo*#kqSWCzD z%|ON7LLfJZ1hvfD^kD$mxychJ#6&H~t(eo|Ry@9lGOXL3wV++v&B%;TV76Y4TICW* zzqq=XHsRdtq<%D7zSFWpyl(}&&FIy;xfLq#>y9;=tO3(Ak;}$cyeY(P@uuW}D%@cX zcMs435~%j2^r*#~3GtXK1DT-M6=_JYE1>#0b8Jo=ZJvd^522ljU zMQqhy&kv$i(@3-(Vx#!ts0T^~1HA{Xh)Mq;{E92)(?Ir@5#{xU@gnr>^RV_su20?d zKSn*ZZ=jc9_TZe?M7Pvq+vC1kuM~~;-M8&){EV1{v2``elX7rk5S+#&$`Ktz$T-cq zKZO#A47>SanIqY8&?DIQ<3^aSHDR3ad}r%g&?)nbqq2wsi7HHmv@mFaLK6ljO(_K{ z;fyq~>r*2_pOz*A=8A;Ho5%Fmm_sp>3#jJt#V9uWdk4$&U@ zgye0l<1J8nfX_gjn2`WZ=OyZ4_r3TRxTDlik3mbqZd=&#lZYFDHy2i51aY_|$2V=kv+gVe*y%_{)!A>UZBnJm52sTK60`C~QYOdy8kadXc5ZX5x zZXT>P0EOT(o`5BXswi0hAkk?V0ltI*tm2w=X~GU%Xb)?TCM?`c6<|oh zX@7>=g9Mad7xN-pbSc4cViyI)l)+dC0#o||-2wDZ8^;FBTF+QSkXuyhtQy*Z0 z!~eMNBZ;4Iyvnnv^npj+hWnIap(l6pDe4JfR&a>>YE*)SHj@XL2p<0f)KU9+`3>d> z^Ln5g`EYc!i!=cQRGV1meMn&J7OngJtk2##dCgxm|8??O^w(Emw%1STU3Xy4HobH` z2}#oeY0DXoo78k8t1n3*UPTw20=1DIp;dd?mnkN)FgO@ddt<}xw_Alfxoit)FHh5= zgt_G83qVSl+lfhJqj8^QIYjvhP;tS*ozILq<#jqQ?c-f z=dUqBO*W>yt8gx(QN=3yiHN9oV(q<1GP5ebbx(P%^gx_XWUn}w*&7K$)G8s!OzX30#q_xEfx`VasdV0hCe zdPv5zwM($#ROg{tI(gOb%iWNlU&OfKY561k=r{`pxo|j+0p_tSzLTL1&&BazlUsWM zBF;vy$#V>;ilTi9MB@(hL&Wwcnfw78{X@2whVc&mXGPD7k9O2P-V)0psXj=SulEo_ zQ9c)r1vRgXJ?JhE(eQ+e;8-}x$0m$4tc48uP<#1!?gF;1&giofewQsj&SVlvIEoGO zIA5>ND{?=J(>{;z10_C$D`)3>oahFt?OD|88`@!xPr$0e8$JH>5-u;3kkLSzOJ>CS z!6-cVMaYZn%-e;mn5AK>BzekR&)6h*+{bRga@pP;Y5cc>`)*v1xI~1YhBnV_8!&95 z3q8N~C0sx`h03}BJMS#4+5R|;j<93HU}+JgSIi$Xe!?cO$Mqn)d;kC^E)jL7$WOeo z&n46YP&N6apaPzgb1ecBq3HIc} zj1w{H(00Pyi#!bPDBWf1h!nHFkPIAP1eSglh3W*C@gx&j!R|7KJ9(I14I++*Gw@v6 zuo+Y#CTHowRW9*XG%;$E2Z7`87y>LX`BVl`2|26%_QgZElb1=B4ik$0NI(l`vMq~ z0WTj8QZC6Jd<9r>q<|D`R|HYCDgB~7E%{cZJxIf7BS8ZQ+raZL}MQ>bZT}MG-Vg;tee6;|<&TNW)SbRn&gqX5hN{seWL?;u9jY`}7!c zZ7ngCgu?jksarMJqecVGS-s{aa|nrOEvnN5sX@3@(=2&=dPFXQ zycCs#Tw$Z`x4sS7y?sxr7)Uo8bmM% zZBfsoJ#_|2yC{x8MXsIo4Om|A0CiPc|XRCv?-P2QiL*?h)j1_{eJH_9J;- zTLP6CE}q4zbWK7^ix z!&>pvl9V5A>=TuH-2;LBO?Q`7qO~II_5lW2X zLGYD+J6IeI#=a;VDvy&QnVeq?(M>fJMt?}odNg6UNqOL87p)c=V zz*tkrV3vBCzASMeJPunHcr&8C(y$mk4W$J!TdS$7M$Bi?3f;(z9HhO9eVYN7##>P` zgu8kdK643ZjoB+U>ai!Lx`2B5PFAc^alO3M8`kwWMz#-V09Doj{ov?0HK)VfmHEn- zv1B-RiXH_Xo&hI=uiXo>bnq@m#%J2q2mJ>u(dGhJjSiHK=>S&9fxDsOL573&fn%vA zdv$PDJ7i3Mk;-CojzLSKW8t^mAv9-gn=D z(s6ha8F@%wp;n_XuISWmoEsSg90ASi@ig5u5yzsJ$vt9=7eN)_L^y;E z67<>K56L-{jDeUuyXWRsV7^xuYd$&>^gR?sP3(bZhy9K}{bvpzD4pV=(~(rh*~aT; zwKt}syEOPP8#z1OYI-5?OWSqPt?@TgoS9+O!n!aFEFAUfj(7SPhKjxssKga%oK z|AJyDv7?Gw*D`79kiBa=9Um7G zmA!3e+V{Q}&x2`DUfxjLf&K<`C3Bt5^1C@_kC%EJvv&OFdW z9Y+$2Ix_ru6v#1zu(GU~i?G0B$gur@|2hG;r#UTTl{vABK(L9U8*XXq7;He*@Uoi* zz(I&uLQjwkfSr$|HQjkrJ=@oBwxZZ?p3;P)mb7ncZRb4bK(!qAVZ?79p*S?ar(~c{PpZXA!!_cXc3B$glJvvyc_J!CJ^5 z`dj=}^iOq?tB@XWlHf4H6VP*9=Xsbrdci!7F+w@=&D~L}Uvh_~)h~T}tpXm`X%n(Z zvc07{%CAJD{K`j-^M&p>f00-5NIz;cM-Zo8xcu8uocf51n075KxyGy3MFS? zN@!hLTGWlCql|c-(Mpr8WTgsUBBU4Fy`518z+dpnF8(u2jZ#oM-faVMek$G=-Y!J; z(J2jw?c~%9j$KjI&)PZkJSrDruF3jA^h#XJZH`Q;x5r+fyIPL_jQ-ipfl@d;_)kTO z;>U7EX@R7-J^C*hz9!30;f36jB>F{|f%n*_i&;G(Z^KQo59@7&bS23N3Qkr6rXX~A z>i?iH%slH|)}}nMp9s~T!i|`Q*Ew06?F;7t7>Hkn^2_tFCEQ+)dl&`3poNil;m5k3$37Xv zHh0UwQ)ooSbF@VA$W+*X=>CM&0ddck^@{y_?LF^B!TLLh1EQis*xt8xueF3e-oYiK zr&VodavzhIn2^-zEj6^ssx%U8Xb+TFTX2ZB-h2`NV~gXoWgHjRzGm(;$7l{O>|2<2 zjb*~fw=nkz6Ea(MH7pbf3qL3;llwQg-gzu8d-zCruo%bVj@1iMe3U*Em|q ze=$s#aZF+r|FR`35luKv(N`AlW2oTPFuGH|BUe*o|9ZN(Nb;Xw6q%G>R+&PER;ytq z-DPzUaED{vrQGu8N!oYX%@M%?ptc(8( zL2sAZ$)v>O4J6US+D@bHWbs{0-q*XQ4|!I*STZJU%CNitMMqG;xqF>EQ}S<6{3)C_ zQ@bF0Pt-kZ<<8sGh3wCNDt0eBv743ezg^|E?lpSdoO^Hcg#1UA=)YSD#c=MrP3_oE zcRzbW{B|C?O>ON5B(atwz`GVM?Izs2-jjMJg(IklE9#TN)fC@Wy5%!b2(P#@oe^EF z8yu`2V1FNEvY*Y2uem=pgAjT35N~ZMO}N@ei51$mDyNyu#$uOn#2ZuQB;;CSPar z2TUGiQ~;$Q}lsINQ4C@Z8*MXJhYX-}XFbqFc-j76aYbF>tG=p+^diGuFM| z3*~pVbPM&A^cJ3crcWuF=9E*#4KoLUH25C@kD-F$GqpF5nh%>9gz*<4apduTUB(l6 zT#bwF%U{YOlt$>8Vh#SGRQsvoa%&uIj3!5ip&G;TJ!%&>Y@e_Sxp$!z!w#;wAbD>l zI-oIkqnI^5Rm>H474InCQ{0hR6b}oJqLu&DV}WDZcZ*L;JX^w*P5Ybi23;#Un+g#@ zIGc)xKjyw^O+p5ESJ%kByQ!%Ag14hbsE?lTOJ_a2yw|P!r3AUc*7zRZj>GD&r|#1w Xxf`}{b8dxzi=X}=`PISg<2U{vDBX%7 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/qwen3.cpython-310.pyc b/vllm/model_executor/models/__pycache__/qwen3.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58765bcf884f20df71c15c1bace36ffecac9c111 GIT binary patch literal 8751 zcmb7J+ix3JdY>ETLJld3k|oPm#j)csSxZTrO|oe&_S(*6S5_Q5-8QE|XK9|1OlgLs z&kXHIrP}Q(#Xh;fzO*Qs0(Q`sJQOGj^tF8}iv9`nRG?1*+NZS5#&%ZyecuczSyl{` z#C-Fe@7&M%?$fPS%Nl+!{@p*_n0`gm{(}m;e>Mu|@Ff4BYns%ACUt2_&V`7YHPBb1*hzZ8s zs2)#>$#_akaoLLw#M5Fro)I%#E=LFBSux9aB{~!z7Ka)4qa*QAaWp4FM zd`cdO=Hmsi5WgT^;PQ0zV*HXg%lHiNm&MDBAB=t&zan0VUlp&$uZh=mEu0O1w55yJ z-__)-JoK3+4~5?9?`)ak4NwltBcL29C~wN6;hSw;ycJI1B#+_I$#t>5aMf`YS8#&dNK@76@T`jf*S+wWy3se~z$t_pb1L9eNrQ4~hO?XHSskhC{wrb@fhPJ^hsvJxs3ovW9x zEp=6lsU|-B>CdhdeHPoDu&J_%tKHj=lF(&rvAfbvlPjv*3KL8RQvJhz6Pw+@5qCSq z0`<(LC8;hpXk^uE%^1g~KIo^tewsN&5q)c;2`;}EHYF(IT#4%$o)>77?{{!3aVXo( zH2jFSnS2FF)iAbyEkC@3jwdS*OINO-pG(*B(&a_M%Q;RAfg74Q-%q>0m*4$jw<$w~ zrChn(P7=(#*jz{1kDKe)!=R_a<@Tp7VvpUwqeUV4TOgq(bZ89&+-pcvTC#*U{q4|P zwU*1$2}{z&z`KHXJ@n-yo+)W8ANiHG ztjhy(`ZH)2ITO}a$K*kVvkVV~<8q2H%t?}FiHTonmL}?uk)v6Sxpc&RZA~+@rIkDX z^^bSI`Ioz(%aX-&(tta76*W!Pb!$-GWK4sPITlFTP`?bfgqH$M$}-K5QI?oyLL zw&ToKaT0XER6*EFGWRArTyHm%tW3NRq&FZ1ncpJ52BVX_Wd2R!pHYe1C9P(Jo5<|^ z_%pAWrkx;#ykxct)4uAoYP+su47i8K#FMOoGg56!yQ|;RbZwvyw7^KsEp4+lFo0VF zZPgmorlG^vjAZ5~+Aq<9D2FuEmIfxE4T?Ret(DY5juoxvsddL2)QDQpUL#uSNQ_N= z4ryC7i#sUw-le%$EgshcR>I>=(JWns7jNSkV%ct<^t%FMdnOhWY%(2deOU3b;RJvfOEGv9jCfczh+5yFj@a1*JLl8<08KLcX3qW&8j$W3PQj6C zfd-$VHq6_!Lv?_=wZpoyvpEB!KT_4wL_!Gr24Bg_tp&PoZyWbjK8CX$zV7OFEZBYBbb{=Vaf83qY zH)bAHSo6ah&b*=GNwVrtTcxlKP&l|S)-G9t27r&a8<3IR5kzS_nQHqwL; zHfriEoCu_<7qwIM25|Kz5v*J7@HWUdnvgc;%O8U*{N=8?-BdDY^<^_FL#LOkd0NTf(~;TwiBvBfPZ{TH$N$ z=AML|zty`m3WN86WlpiCKS3dx1E@K3`bopm-R*J1*H41ya;y&St)a9ry@%6^0UDE} zRyxf%41%m2kZ;wG2={{kRx8SDl+Tl1ymaA%iL1~AT{lyEbXy!BeR9^7h1{V;cN+3sfEX!ChEaTei(B56}A0GSy_ zz1*yiwL1uIf{20#@_(~~FjDERC5&nsZmtDG8fE94$Ti9>Jll0L>53(Nq}ccn{tEPj zv}oCJbsy@?w_V+aeCGxqV9Mf?0#B0 zb5)qMB~)LFwAk=RI6Z7o=|%=HjeSk|0~pkyaosdJWRPHJ2c>%k3>4SF)auOcmelW9 zn-H$M#-lCnLL6Y;n7V90!i^;`(LN>w)ChB{Hi4MH8bHJTUYnyj)QFFF=Zs19{u`d; z1c2+|lyr+t2b5uPruIV%&WpQUHe7vU_VE)OS`Tw6S3Hd|^)byP2>by7j{w<-U1s6u zR3sNmH3^W9l^X*(S+)p@)b6hJ?$`l|ioe8@kb4F@P@<(;4@m3V(BF2UzYV)~-*|Da zR(D|wYEJ!IMWYA`wQ?n(E~DJAGB4-swoLfcv#Cc!7mbkzfFs7Ol3c2ihAzhCI_zV+ z6~O5v_da>dXwWIpFF6M=G#b#V97camCc~6uE+|yC3$x*b&ayi)88rbM0j>wODCaT$ z*~=8;7YA?dm9Kqdu0`Ps{pdbQcpN;e`VRpi)xX1|sR1r6A@4JzJhC>NqnZ4}{zXV@!v2X~eu@ARFny1-U>@iGEvgrgK_3WSXLI}fdew}PN2CprCI9#R{*|L$|ukZ3WHeQ=fH zT7)ONw>#XSkQ8f2QnE_aRm~zHlgF2PxV(=cxMCzbBsZj|MzZr4DC~Bd$Y~movtn|F z#Y*A)kls%bZb^L$*)n!zoYaN9dFmB7Gi7y}LZ?*(J%Qhug!7}X7LE`1GSSQ8UVg_) z$G13t!j4aE3jtVAPbZm1xejOb04N7oZc9s9i6dPYj6EA88=N^&p7LrKJL!hT^e*}3 zPJ9NOjvxV0rwH64Fb*JmxK#+ikpgXY*0Ra89a1J%?)Eu+_CJvHU4sTyx2fx4>RP_l zZ8fj6OO$(CC#h;0AR9;O7TH(2w>1rWe^n^yp)K6A z>(O=Yr+M%NLQBX?DciPzfFHJy6afPI@7$72p?*ODeY^afYx}T@w&`vcmT?y+k93sk z(w5loHh>L(gp=aEfQOYGE)HqDfqqZFrhTa|;X>$yvnk3g=SFGAheIVRR33VdO{bvb z4cZ6I;OiTw_R84&ek11A&KqVF5{^?<4@0P%04?J6;g3QFdU)vvfv~4jcs6kK?Ry9T z2F^82{Shh~I&Wm?2=bEIq1i;`o@w$pdC!K(NqL(#M9HNxoD^7lL~0ucAKO^ri|{Df z1TzXe-e8m$q4OU8EFdL3ya~#2@g@rMchB5}=ngkvsCCpc*FuW;;5nZEV^A|YXVYDt z%6mkv(U*-r6jok+nbuEdR)$Hlwk>1h_+zVow1=XAzO{G7v}#^43EuG7?-{#StE&yv z4HrtIC|anfbiz;RvkOb$qeqlSC-oJ*gU<$(St92aZZ7jV@-Z83p@HV%MpcuwHmP>{3=t;S!_ycj*ISNPne8&>V>DUZ<> ze@@_a0!;$f36PvCI`!OfrjIm=q%`-Qj}Z0?0<#4ElmJ~9Cu9^cKA+9^d)Wz^m~?W) zO|Awni1IDJweP!C%Z3b^q!@9M4**9#Qto@#9!x)Nc;NGIhu*ha_~g*4_g!RSe7F1? zqeiuqr*WkW#hvLPCVZXZ=bj>-w_87d zByYk`VU-eShB#CSpRlM={+H72h3ijxB|k)RI>lkG&c96ZcfNUYaa9wkt3*|)MWtv# zpLFt|hHgoWGg;zOs%l?{>`OL}d3HYU0%d@&k*ZFuPjYLv&A~7WJ0=7?Il zNgr0#Q$$!LZ~`DJ_0x8gAUn7t7x_*?i;x}c?;sGC1iT6={en{bcj9$IF3KzfK1CvDCaHE ZvToL!)`!+MVr3(jT@aWlQUeY!ezMozZm9YIdh* zW>5F(GR#8AI@nwk2oMM)E>@`!qCgTJLh(Qqyz#^XmB)UfiU-O?<%I+i+mgli|EK4& ztF;_vt52Wv_qm_*|KI=r=WM4~bQS#W|NcLn`<|yL|3QiIpNYgVJaMWjN}za3paxn? zjWkb-bWe{A&xlOVj4aQJY|oAy&xvwgPNjT$%k^Bmjg}kby*$&*Rv{{SMUGpoQZ(UB zL}jncX}dKUO?guscUsfYj5ovaTx(Y}>&-^Hz1`8AH^(%$wI`bQ<~g2k?Tz+%`#4@` z-4@;M-Oll1>yD`6RXAR1?T-$42ckQ@JEMc%!RU~8i0Kopk41;Q!yGTS?ux2jmE)7G zyQ6!&dpJJTx;HxF9f=ma1x`=5jz;%+_eJ-6_j7ut_3`Kd?*WeQYJDR5r1wdV&$b?n z9`YVSe0MO{dN_K-dxYbAT8~Db@;((k<~<#nFPj2YmQ=se%ZUg1EjB*^5+k-nmxg(=I4N4{056b?GasreC!JVMonNgkr z9yD8hC&K-5VjvU-Fdnab%L-}sdu7o zEoq)@g_R4<_nlF>Ck~hFufsZP?B=B|eo?6k9~K(A=< zqSFmAy~Q)N2umSeSWCKVNor?Fd?^hZIQ?u`3)0!wI`!Jw0rBbRY#0RC320?k5H@OS zt;Clb0+Ue-crwad8#0Zp*YwFYwnh{NO^oLiUV-=>2pI(xxo6~B#xK5DHPh+vA~r)E zlUutK3cuM7!i(wdW=yLXHRBld@~TB4DohVYwUy8hqi%8uSw~G|s~r0cRMioe(mZG6 zI^(nu)h?Y4{jLZb&5L!~$K$_ZmWZE4Bvd>Vl2FU3IV~s#dZ-5m9y2roD>OrEj}q9S z9XNs7D7>jO)F2nQZxlTz$OnZt6fgItVkn*qo-3vKjLE{&r(RrqCC#5}20_@Cdv0<| z=Xo2t*>?;eEv(XtVw)3tq$Rlxu^VH|f1Fz2pEc30TVtc3xQ$206GPT3iL#+=s#g?M zNwmJ&*Ot^kolydC4fRWE zVs9w;?gR>k<}ALWp@kDywbVGE8*KDtU$7Lq4i=w1>Jp_OBJ;@c6pj;6Y*hCk=JVbGi7*BrB~7PQ{X*w$GX482a1 zKKL1k&{{u*C%zj&U)fap>J{~MZCzW}k;0l^QP)+(u*ytVL08WxSJg!<*25?xK1IPJ z2&#s746jtZ=-Juarm8PKL*F?#Mqh0#%!Qf=eg%oxK~TaVN~)purpG&+<*I6nuT=B2 zbnRLc`hM#AB!$;n6fgKb_INA%qJte$Jx+w6eJu7-W5(G|r^T}-j#IjrtxP?HU>@(< zr>V#>3hqVlIG*?b0!PtoO*P)rH0=XT*RE@ZcFi)i_ccpM&>#r(&vO2c60Hv$vP4_*oFWrqaAvePgi9CB|MW1=9R&JL#MOt&WD@N_49 zvel`tET)!Zao&P;TF=3c;yFC@juQmq{LH+FHD1PgE*mR`*q4|AR(hE@7OTB%;H|%H zGUr;9#$)4g@ZT|vhgIl->Iu+lqB z9J?e!@S&(A=W1;PVWqtWzz~&AqY_>O+a_@(RToZF?KF=sen#`mq|^0R#<-BhMIP!2 zfV0*@{gNd=fnirm;z7LRKA1!-wUaK(Q1JrMAEjUy0&jY(2_KC^Rgi6XyTE~4(SpX{ zm8d!7>aLL3IGfs8|HK|@#vnETpAcJ!(-g3n;LSGV2=3Mw@R3>2H61W7F}9&vH;fyW zVIXFJwdrx*vmAPBYTODIBET9d>$LVz z&i5$9>6?)Fef^5IMFtq_kn^-}fCJiPZQbfyc$8(J0`}42#OzxeDp95UfV4Lj-?@Qb zRTF2>Oj?9yg*7K#2x^Ep1RCV38K0|lp$49Bh8M)kpoy0#c#(oB%3{ljq$)hOBbrM- zm+%It7c?XBD&?jDdDD$1v`ByJ9BdUvEEIak9FzFBG*>OCCEddGV(O8ii8gMydhhlR znd(eRi6>CJnio&wB~DQA3?h!iUe}iOUEe zi%jD>Ng_)xUB7)^(APSiEx)~4D*iSiBgvD~aylL(XCl;c7K{ZOlE}f6!{b6V$&+dl z+M(0X_9#$M3ZNH3(-AM>`$U)v%1FCNPvV`&dkSjIbf7inS(42JyCBI5!E9JuE(N2Hd&&xmg`^}&K%ileI zB2}2((;PABIVhS?$|$;WK1;czsrry1Ni8jKJg!3&w3pJ|QSGAN?Zi#eZlU^R8flz5 zRF991g91_PGdG7_&{t6_=_P5QPR2o|Mlt`UmQ(^ILk%=Pn)ec(p}_q-RMnBd1+yr= zmgpPGdJ!s<;zNCcNG;BS)mF54_jAgZNce&>Pf8M$D1^jRR=>DnilZ`?{t%X&iIg(t8+M+(kk501J>MS^!1)_kR{7@=sI^58DAM*CkqA z58C@^13)dP#i2Nhd_8lamLzScE?+=OP$g1EpF<3)oQ};M%3ZH)X})CKR}k$T9x{{jSLJTcD7AL4W9+Ew3wreya`9zsnwCH>TUFGFgWaG zQ6Pdr;}EkHMFX^vQ)`9afIkKx?mBf+_JtdxIgl+<*F;D`gMzIyA%*Y`xb+p}jbBIL zPFO1aO+kcOK(85n%9v8i(CkV~u~b7FAQ^D%{Q~qVR#&e>sxh{4>V3-|%LTkEJ2Pr; z{w4u20=e9Ptz(HT4rY~EHZ9VO_^|od0Z=#5%fXKr)yBif_cKIT^M$Sp&DR8Snajj7 z#7e9oE$QUcx~#pfuiNX+daiFT8%qj86S|K2Wys5Kshk38qKq}Ke3i;uUsYdLzM`xX zo7tP#=Fp|xzPo8qUc~Lb!}T~@>Y>!m^)Q2?o;=Ft`+3xZyoiH^H(jtO>bs(21n46u zc}jy-^}cb%TnD3V7WxGqVG$!NLh|a1;uVY?=ClqTsREUe0t~~2kcbFz(WZbjb@3Vn zyhmchyi$gIP|IPBK})00s}u+dx(Ei_ZjA_p+2T9}7Z7+l#4Dq8s;;K$N~)gsa;yH? zODxE}+{!ST=QV`3KrRPMzQ>^Q7}ayNT=NMcY8#>S(EC)2py{!fAWiX!aB)Pr; zmDowZra^u~i%%>mZK^L1VFD&SuZlWSg`fZhy54t!;%-vLNjoU4Xri8!z}|XLVl13! z7m_l*Qu#?dQ}{B$2G;~ISJLgE%z0;SqU>VioLrP(LmQrfP=WyMe6IwalYUgzfKD*m z_uK_?d-%kU)q=FUJ`_tvFv1dJ>msESQLF2d2U)u7;t__!J~&Msx(@*$nE`8o2qPEE zGJTre2FR%m#|nBTcppnEZZrs16(w;cyyTxJ$Bxt(8Sm2EaA4vvYESKRwU{gWA~j7G zqBkAL{pZJQd;4(`c3J;Ubqt1m0As!&2H}Zj8!}4{kDdBT-EuRQzpr%-Ed&? zOfeR_H!NMEvU8m*Q0In)@`W3{+Xm8eoFRu`EOwCfsRNf|Gk~AwCSN0&MvTC;?chID z))Z0b1Azxn1PurP%j&>SIIlICC4V15V*+Hh6YKZEn++E z&^b`QjJCy>DEMs(-at^b^QI?2g2IAX25WF zZn&DG_V(>Ki31eU2bfdZjyWTHas(`lIRAfi`Au~BzwpH5!XwL)I%>QxtxFASF0nzg zO7Ce$wY8xe3t21wy zW7g+vS{$s#X*$M|;Br;pt^l)K+RB`DEII!Sgrr^4!$9*zL`J6HTO^X zhR0$P%}yM|;q-q~eN$_&rul}$0HOmxjLZ7#WC}uu^r8Rxg``L&@s!|uqk1QjasUWg z(uOroW*+uo2hkn)@ZsUWGmVG+35oKS`j+w*svtIECb??tgsN4Q7y7Cy*eJ@TW+r4gCL17 z^+)18Xkrf%fwsI?=)!o1iw-9u&A`heMdwV2UrgQebOt4zeEX>Yfk|3m(_a>e#obiL zBpYz@SVcCQ=4056ndnY)VLOmgJhifJk$jiXVUYY5RjT@qk2qDjYJRwsXE9F7o;XGU zjqOpI!2^iEAqbfv#kci=24P%M9b@XfDMup@)eYxHZ|E839Xy=bG$$db8SLQuBPEsg zD$8G5DVDSF@t;woH%k?N0P0mu_WDl|-zN6b89E%4O^iG$uTsCMa#!sg*t+67WoMzB z<_0!f%;rHlv*lz9fQd88I4yEpv2@?WbsLQgex#{Vnn&|pM2P#ARWaSSfGVoU!$WDe z30I&rk>wRA>rT7Gn4`vf*!jX%7i=;!fTVz`MwTC`eddOCT>0EA6g75z;2Uf>;EfG6 zk6j{j@Rq^B;cS@F_FBD*Tesuq61ZQW%<13MHZZSf9hM_(G(s{P7Mu;HGZx48wsem& z#y}}3V;o))rKQQtUX3? zUO`5*Kd0mc3jTtEZ&1MY8Sz)DHdB-LIooI3)|{d4v$Vv-J5J6aTVJ!_v367kx`T@} zz)y8QaCFJ|&Ie^|+1{?LTOq@`tJ>n?&8F2a3v)CVf@H2Vr+CEVeftQ*>Mdk>n1(>7 zJb*qvIkT>zyHIk-xk|@8rE9Z$VM3|Ky94lZGDx%levYlZTX3d+A*il8o-My20Fu2i z0zgj(00II710cBdEZ7WffR+Q-s{}zh4=#B*m`v_6&V(rh2o{49nMdTn#XS!BZGv+E z`sj^1Cc_e(vSv7e_%u0dnOX?T6bmOZ5F|&f=}p2>3o0D7Q)na0+pvPUU=P}v4(126 zVU8I)-15WofqOFT|eEAAl}AuPEkebNZ!i8O16IE63*bpun*rjeBl&#quD z>6bH1UDlTK;$GtH<-#(#Q(eDE#@4d4L3ILK`J9ZLoE8i%xHTDDjm6aD<67~%=m-Fk zEtcsKG>z;S-Xr23WD{2?s8H|~3K)^>mh@$N(@8U=8zsS*dzwXXYRStIf?WG)S)Q4Y zCs5wq5O)ySj^*VH@h4RC*C^P*Hdv=^PkiE<7PwZbr{G-H@!^#N*OIpmT&)3ChHC2A ziVeX<$`iJ?N!ky7TVRExGA53Wk;i~Vlk>pb`)A0pUBu9buTe*-u7Wz!8^*iwp;#hz zkZxO`!a=R{FjUMTB7sjD6lB?70aTD^9>5^2TGjcbh3F2p!PjXtgt0Dc1IQG(UwV57 z0&#p)4S6)q(m0*uB_(}k=)$C}Kllw;g2yofDvu*Gj;`o9yu+<6IR;I98{fsZ5WwJJ zG+=rE9T1?@$V0fT{Z31CJqm|(*%{q;Xb3cIE-VEDhRq1v#|N;rkoap9sBiByb(}j* z_%@h)oG^}cSQUSTZ-c&4ErR;$<=eREGtl0)b=Z;|@I&e-HBo?DdE_`A@1)F_+S#SF zEk~l%C&Cc!;Ez$)613xQ!IVrevr>9vM{CQF;4nSu`irt@UQyuNF%k=BJMa%dS@>4~ zJa$olBhd4TxN~*BPb$Qx;Xa1*80td~bX6Q(JrvlS;}ffga9C=C4&{IkPMyHHqQ=KT zg%jm4PAxe-~Nrh zj`knH$y4!lgZoe#_ClfTQ8+5$0~W%QA&t-Sm&LD- z)6UC$eMx&5uQbP(T(~$k3KeIvJ-AYI2t@HW6g)z~-%=jnYJH`f_`{-bphsaw9Ns>X z#7WMX)y5=tM)jt%b1f*{3(fX|vZ5yc-5YIsDY+0&-8}vnio`;``JclfTNY+k<)d0ryLUlt#w} z%PTk0Ho;k-E@T%n<9{IWNjik18Trg~SFLLWi}ZS!p=5l*sRacX=@~Liq5mU|w?lRS zbbkluE|f0t7nz6b1bJ_T>Vz}QoY4nmUFn&G_fh76uTr`dtccr{F3D1Ve&AQ;^;&{*i*`DHyE)D{04wwu#^p&3*$BM}cnkVTQ@vM;RvW zFSHVwEp%heG2Clf4r|6K3n_;kkvOTn!EIJgrnc_~ojNYmS;-}4BhtJC;kFzC+gRi~ zlH=eJ5lG6(A;rck;QnnphW>(W8S|8SCA!o|;zatp`IMS1HE%C}k@v6pBIg3R*F_Ci zV`4$4HC{f;L04D#6wNC$CBO}F(S)1{12&HZ*;;%#C@Ug)S4>kShqwq!8(bTcw<0xpw6iUgWhHQ(aARcw0-0V)Tb&_ M3se5o>A5HV599&3ga7~l literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/registry.cpython-310.pyc b/vllm/model_executor/models/__pycache__/registry.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..576a5383dd6e8127e6ff7152ae0e44aa8ae3ec46 GIT binary patch literal 14695 zcmb7r33wdGb!K->&4IxH7y!IQkwZ!pIg$uUq%KpGA`pOR$^ZxvASrdMG#gU|V94&7 zQFQ|#CZMf|ik(Zo6z8&>kZ}}8u^s1hyvfFKobTH&*<8ChoZZfsZ1(eJ6D4sj=UT@S z_kY!MV<1}t_Ur0;uU=iR-h1_`Snlgf8~EM(sn0F_>8N3Ri9T9?J@|MGkH3o>#yLY6 z!gLKQdYhK1znR7KJ7&f7n_C=j=El8*mGF{Q(j@wroAP?B9xrXB^*48Wy^NL7>A2hH z^;`YkfHmM{t*kd_4SGY?kT+}%dt0n6-d1a?rZe4b);3+oc5jEZL+2&jo!%~Mmrf_$ zTfE)YZlqJ9$Gz3tW9{+wT6?{mmGkyl`*eQV-S6FI-R9kH-EJCkmppLIwC*SxqE}>I zGDV+;{bB&=tQZ6w62pL7#8$vDD2LSI7 z2LTU>Jm8(;u9r;IbGJB*uMu$s@E&n5;8Af5@IG-r-~+n;2gO53zfBwmd{~?Sd_?Cz zs^K?@$B;9s(8E1a!naU_l5#DHZ@1#S&nVdishfF5&O8aD^vI_$$Ah ze8sd5ii&tvDB-^xvksvxfmlX~6|o9jD>(Q7eK9&5pTQhW-SPt`E@$&C0Z@#z;$@ze6K_!)Ur{Hz?5!*a)^`*#{UjfI5x zIq~x^aPWAbE9JeK<`=|gG|hvu2W|RA@mbXHS>48m>j$Tb7& z$(+?s{?}h4tudWvVQlo|SlI8(`+;%_fhU8dvOso^Z~~{`I=&yq7mCsqc{A*B)v{f1 z3QIDd3R7c_>pJtU4EcDm5QKcP=m%lyOeH9mOO6}H=awskoLj5NFiG&dqr&*P642n) zTPzg=+bu6HN`=ov(YKe1mDfzPD$m1#CC9gmrIJ*(A2@*w2Z48jK$eKRy`ZEFhyCSB zrK|$q_LkkC=#>R(?QLXMD(J-!ib-!?3Q;UA+8~i`xC42Cyb{=p68s&_P-Cp#(Px*7 zt}{hsC>%%sJSWUf((CcE8grI?$DN$A1|zzbtZ;Bt6@p@+B`eRYgrhUnE_5m!KIs(4 zmX@7T8%>y+cZvndR=3ot^k*BD+6|n#l?bvY-E!G$!&>oq%2Rs?KR8xil1jD{qzYxv zb4o&~y+kuGKCiB{Q*ePSmD3!?WuZ7Z)s9aHWV)W(M-;>3QdWE^FSpaAL@itGC#tP= zs`E$N>3ZvQ_R3Lpo33)=ic>CiG&&)5rn+5IJnp!Ka{Jg>i3Odh4iH85ltdrfYl<&Y zp1MPG(D;s~Bs9TMb&x0qPPyKF?F{%LvecmnJ2Q85a_a5U5j6p^_+lkEs`7;0dTM6w zWN~o}qSMZy7c};Kaj^i5y0gyciFTqm5nWPu5qb!mre*o5c8U~Hlw|pex;r9L4%Sac zUCG5LR~?R;6n#yVJIK;CGOsMv2q>-L)1^SFQrnz|Tk4q|L)KH&#IAQ#9ieIlo)|qn z+c7SeoT9JpA#C;u$D8k*l}nVT?gd`$r24aylg`v=ONYbkRF`6&E_RgI4oyMXD93Fr zgS;ypd02xnfozk8v)w0z?L^W{6^~&vh zco|vhZG_DdwqxGogqG?!;rq|Db%E?axe};{qp}K0ww-mnOnK@N!f%<8$}f6eCrwYK zp06I&b<8Xk+qi_W%2H8%6X6Dc>lg(NiD2&c*u z?NkX+QAfv!BKzd3EOm%*{8`FVd_>y7*b6%*mN{lvpJ?7Z?HMmoy{@L z<`l@RzOxtF+nR!TgAH{(Yk0PnY9~m8KxyT1G~5c3TDqMm?w4H{*X%64+EIxsE0`4+ z(5Ljf9m%l_gK+mrKthyDq8M54Qw`IYPg*;omQ7AgRLX^=77q0`Pqx}4^>%brohI-E zb-9PQYmKSifp3WWfVQhmyKD_>3p-J_({=TlisnFhwMjzFXXUD+8u~XIY3~+JTOuXj z!765DYWgB-$0W6@HzL(J;jJg#Vns_y*D+vZVnLv$P%_*%w&Zxym67~!QlO_$CsNZ? zT?U1MQ|9E9Pz$fFF53D~@_n_gE88!)c;AHz|I^EXw__!!I_d=Df%^yv^oOW)uk=rVMgFQ@0rr?O{?YVo6%JDP6E#m_!iyr zFfN@%S6cCj(NmKXGz*>-@V>3YlztYJLJnQeULV^&(1)eW+G zqhis=?l#I9CFhh=EXhc%ggflBXU3-NbJMfuW@gTuotqf9r_PK|OwLw6GwT*W( z6$R&-Bs>Q`^|E*8uzhlTnOnv)_Kwctx%su6FN1*EbMgEng!SC)ggtlm^xT#6f0Rz)A-BkMq*AOLhG(-VgToJc%S+z9-s%yzIDe$FhG0bL$cpQFM|8hz zc?Ihot3CSkk0`lF8-cZv%39cuY4V~yzFlzrPlCUn1z;>;w)}myy0bZ&Bkg6T^XV{U z+a;KH+s3F9w`G^o8JqS6ZVkid%Vjqlux+P=6|V){CnET22m(Yg6qP~UAuE-_1k?_= zs3m0RZnuC%%A;ZX7#^SIF>Rz+%B=q}w#U?XqG|had_9K8r=3Mz`eaO)k}oj9L`+Li zT<6rJ3;A5cYm!BinhdwtC!u4`>S09KAku+Kg5?I;A^91@^A)N&FrGJGHK$)S!{O<2 zaN2{e@(`SnVuIE=vbitag^rFQic=zu9W-a@mxv();H-K_x3-LID6(n8lmZ{+69@>b z5Fjy&01b86@0K0Wtp0O^T?44z*0u6>YKVs_Qi)3hE)$3mpt(_32+*9XGJy(#DuHGT zbV|?ovp{?nk5AG36ht&@ve+(DQU2yLJr8VsrV-mWEoc~M57IHU8O9fBnjR6W=d{du zS|qe=Bt;6b!CrhdverF|Yf{%Ijb52gLNT_KifSN{zs}LGl z%gsv(_3u$+E;4_?l7v}+9jvG_Y5ft+DoksCbiqMZ*b}X8tlABeOTspWN3RXiSl6Mg zZa{BAFR@53F|`f?brm2VQ{PHyJtMTvtT$#R-l*>F>~@EG z_&Vz%0bc9IC1W-c!n_%^tBV+FVfh1)Ytc`b)%_c4ehXs%ZZL|({#}&nRv!uC7YX?s zK$m`Y%6VUhoQFx<7UhC%MR#Yx~9U^%x$19AJ|LKnnq)FStxV!3vyHRVHV#MB8m>hr>R^e>1Nsp5X4`cSvg*{F&~ zPPpZQ9$?8uggcq7IB} zhT5|Z{kvkQfq)5&>PK*B{81c{J&$w{L;fC&HV(e7vVdR5YFszg*^QWKRO9RB^~4S1 z0yHBxg5=sU6Gu^|0jiXMzj=e1#?v@*o;6mDe0;jPWr}>*95oLi?gAWHnLAj$?O=U_ z)!bZR4;1Zfu#+m4S4XPHyA?g)>!KCJp^AtcHugC%Pc;U|a|itDA?>=*hK(kS_EmbA zI@$qd`5r}nrMd_Zrq4U>veZSDLm7$aOe;(xQdI&Mz4F4I3G8_E(OB3+n;)HowR#^h z5>zhc`HLMA77p9+ZE;A}>azSTi2eNliFnG)nkmdQhxy_re}nNfq%c8WHxrD#u09O1 zLhHOzx-qYJ<3rDl(U==^4F~snZrF9Wk{xryrmJ@~2B~JC-fWzBB*h;>J?aAlUIM_1 zYIM4@zuR7{jJSuq zCwNO~nbbx?`^VZhN@7=*k$tlN(g4nqllqi3CQ>-njeMo7=xvnBh(4H`L3mV2@EWR@ z0!J=zHvzce25yVa#a#vDZUv7F+ES|>`@eciINf6a#kMXKTfi0f8-U&31-qkOd(>vo z;uZvGcXpxO1uSky0DH?OSlpTbcK0S&+@=8b)=jXuWdZD-O|ZCq0qovQu(*{0Y)+(L z`fd;Vqa9mht<@TI*?^D>4#H1197ya3*TRHetbTXXO6p9jvdB`xAi#~hfz55*D8xeU z(nj(JK^XGOa?NisSq-;69ENFtS<}bPQrM)S{B0zhw66Nejnn$4-p-LON&-7=t6GvxDPEt-)wg^q=Oy~fnRbp|a~c))7yK#@ zlIys3@VptM)?@E8uEtC8pyzrT<;|<{b?oggfjh)Lj=@bhe~u1lwPev8_r+WR96&FN@c-5ECgtJWXzsZ7 zg^pju%oUbKN~IAD8+OPjL>CSa>_I>*fTC7%0h`iNfs~BiMv!5M_7BupVueW~EVMQe zAU@(@KT>PW2{Z?6A3c6T^L_MUZgmNV?e(VVt_0;qA96X@sTNUfa87Xyq!nb*Xp+T zZN`_Ddt-@HJ?qAGm|nO`aP-$ra;MhWM~si^r3;6@n@_d zvvuXvHr4*|9L(;^Gz2tw@H_NqeGF-97%ow#PU{?~Z8jq0zO+$|ype8T2_ODI#{hLN+cq*JIpjo>^83mjfSU3%Rg&C7gel31NA6Ce>BzH4XvoKTHS*G< z>DTI71HH{4{7GtP0*N%^W|nQx(_P%GP1~nl0iS3qQELG?A?n8obW}}<(~YX(qM_;y zqeC9v29IY$?Ib%iI@!%-SVVve0=lGJt2OhZM(3^91^ceH=rCq1_O6 zmTBuB&3t|K{fF>D>tY>!p`MK!&kS5e$Ol_GZR*x>XLZ?B+ty<1K3Wj`uN7C>9c|WjnQ$i46h#S9N|r_>TTK%^$DtlV!%*% zG?&p)Nw>PU0b51g>XVe%x3T7qi-T{WPJJ@drOt2|cMqGl6{87xuF-~0_puA{TWG{q z>V3CHYz5y6&Q9y5B5KBa8qK(U<4|lQe+v!Kr&C=U61j^VcRjzSArGwD*_5sY-a=C- zT-J3=wqr~h=ehcRX*4Pypi%j05|BGKj!L%z`F)+y9WwbL7q}xE87$g=NBWkc{ZYuL zhf>-;liKMFt^O$G(Rx>;X7#QQe-MiomA{{0xDn z-qWc?%KLtRLG;*PMK?LaXY37ihR#GchB1l3#U1aTyio%#u4^y!a4O0%;HcWj`Oe4$pU{PaZ~*Q(J^~2 zWDD)bvI!t);&jKphy;Z%Zy4`?2Ej|%bOh1NOE}SkHDp2jQX;@PF@=Q}VJJjQ@E2oO zV^wVbuJ_!4J-lF)wj;!rUVFeat|Dww?G1WGd_8t4bG`2d0(Kw8-S>6=JpTX3RsJr% zYOWeVKk*ldYwZ17uEv9b>)G}A4fBG*Fqj;F*AT#K|C%-KHT=0%cpCKgw63*DOa2=0 zq7_eh`Q&t!9rmiXy~$ieuC!X4qs!ODQe_!eEelSC0)TdTIly;ycNBlp?x@~}Hn%&+ z(;-vy`N1$1NUyR`#O3p$X+++|96rm+QEQJ=UT?uIFN;dCa2X$QvFug+kVg^vCa*At z77~SOrk-PO^dnT*e>3~#q2c{s)@2JQ*7%WvBBi7yu!IE3muq7--qSeKD^Y2tK4p5+Of>SNS@>-UHh7dhCYwMK82SQB5-L*&rEw zTA?cxq28iHozcB~*RHtBi*T_*wmP5hwK7_8YI*8+Q9%7B0B-4HKz;QEoyO1?G1l|T z3kV#9oK7UwZ)gl{VWpz;Ufc)KrD+D#ZxIr=C49Qd^gEQ(v9vjzW_-53s57`ru#F{H zh7FnfZz@`@bYXywH*FiLL?UB4|EPK|aiZp_&l8}=sb2;NW6pd*k>jLKj>F!@|FV$2 zA{D2!wdh$`E+MQ{c6~+aAxs2o6~qt}DL_S2rL@7&)<7$L^1f6bfqnwN0$`=;(G`6l zZT0H$i-OZ?fC$n$XE`XM8XeFkt>3&$(!5_{4K&Yc?Rnf6h@2{IUy4+Qg1gpsdv4;X zIs4Sa^u*cGxzo6Y6y2M$2JMNdlN00Pr>9TVvxe-cbCYwYK`=U5&)aR!&P*K z&S%U+W{Tlx2+t0dWht0b4)e%iPB~{^<8k&MJi)%olkETT9`iQ-^%`mZ)1PWx3kyy4)#}kC;Lmji~V1I z3;PScoBbbtEBg}P%l`(X|iI5*;|C1kL|AXh*AM-oe z7x`W6kNDl}5BXvC2YiJ6K0m_#JHLnhH-0bsJ${t^F29fcHou?!7Jq>KSNPOrd)Vvn?n<<~6^^whyjrjAjaaW8;oaZ1!@nK%%MlJon1AhX*eh{n{CzJA zM;}8;VIV4zm6etCGV2}RdoR(Qn({LEocYPWU;Ez|GMWFzlgZBvPTs^3{h67`1Q|aQ z7=hU`d;`B$%WPY|)wX@R?f6dH_1$*X&lI=eL8;HQ9%?`AKP>gx)+6ml{YTr6`H#u@ zTNL(_fLN((=o2d)!=Zr6fEF4 z5-v5|;An8{hV4JSr}cPn0}#X)`t6{m#`xZZ^v@BPg zB==q%s@h5`OziW`C{A+U=<#W_R^nXj4VssmL6|(e8pc)hin^*QbX9vbRcp1vR`p6(wSsEY z+~gDU=g+_YUZ>ZOBb<1=saB7xjZVc%=HKqN`t8mK>3!#$ov@~oxo>{`qrFWZcCYMi z!dRtm_G_SVQ&xC1iG`24_1em?^SkYpFbJ9*PVaCKHfsG=T)h%DSJ&cdt6K{~RWXzK zdab?|2346*cO&e8aFslu!d_SdwVePn!_lhM!?kV;_f%u{;(Ur1!b|?#HC=aA8244D&KJlJ6x`@|8}%rMVvrda$1@i)w~XamnOmkh zjy83I#i?QO)2}VX?cP$a**hKA6zD$vLc71#KGVLk)M~CQ^{&Nh-OkeGR;#_FUrl&5 ztoP%t(x=f|J*$eg7fz|StiEi~9Tgz$M( zVj(>+Zkk)>0N;5|!dzDO<66a#0V>vbI(}i|N#nVxGP)-V6J&MzZB1GjJ%wwsNVZ`b z?%kZJ9%AqOTBBMQHm(~uAAVRLQ8#`m4469;Eu>e;Tkq5Q(U@z#BeEPx%i$*%yk1q+=mma5ARTqu-Nn96MS&H zc6Iog^usj|zJUJv4H@b&87DchpSWuGa;WfH!z3Gpm-uoMkE<#@qhxww#A-88k7J;^ zg@PZ_%z0&T2<^|KGO&Y8Kcn(`4qHyg9N1ew%)obo z6_?2#MctoksBT+J6y9OK13lsD^{?QvdV+`FXm$qBBTLVs*kRm8@P+S<4aO zi6b*jTs<{K4&J+#3wkZnMISBXGP=q_df6P4+U z!SL7v{&7Vt;sVxy~L?0RlAlz9aF_ua(a1Q#CYTP>gRjSR#oQV=Q_Q#HrH-z zuIj-I$hdwsZMw(WNv8=Wtl}vV)uov=p~U!gznFGPvWkl9k`%Sl)*3DGpvJ3Bv7ozz zatd8XLixuy8JOExPp;j-p?F49fjMw48){|%;fqD@T{pJeRCr%QFQBQdY-|m(vAu3z zH`QaYGceF6*EuS!?s^uj&knNdxsR=_{8nzu>lEUAV8z}bd)@e9<}>rU@qKXbRxU0K zT%o|;!8;9{V;QJ|T;SexJ~p=sTgA9IC=LpEX7;9q_m^8vX4|mFO@fD7OBPk$-RAsz zoy)aWGbjt!mE&u@@a1yi_Mj$uaWboaQ-#qCqKfIy@3YO&Vtz3-;G$H(CvRbi)SE2c zKv9`WEa+9WfO?V_Wh!Rh1LW#NaSb+IadkygB@S$7eJyb>3f@VaXsy-@ll(={_O4P8 zKbvH#BfNnMrZS~oV_odX>#U7bm^@uk2>cW#P)jLVHeuzYu+|KMup@}U_aMGiJ!OB2 zb)gi~i}teHabXemt%ngo*;wyg!r0N%D7@S(MAb8MX3i)Y9*$|F1U-lk*v}Pa^nPaM zQ?|UVzK)yfICgy~$oxd#MGx=~XwF3Z`#3zV|7Vre6S?%q9yOm1^KT!gEpLrUrkgeo4_# z1zV)<_ff=|>zQq12W}H=QKo7RjCBjYP_OF_e!=YPE`Hq@432sp>e|MEHc*b4++;5g zXZT%rk7okw5R_K6fHNG`VmyUf5eI|^=hLXo1a>^TKF4_I?QteB!6KfjC5IRH}++!?_~0IILdgMa%tPjXf0m1bP6s`oLH`WqZB9}+TG zgJydSh!o$r?*e0Aeg<+ybJM8l2#u){EgIr;Si* z^rX+f_d)sbpdZq8PER}CxV%=o3>F-XCTvk2y{dA*(`s(O(T>+bRsN`XF{+f~wOU+; z*V?@jp`Lo!9F@+w>UCSSI2?_~4m}l~j#Uk=YET|Bav2_Am&_lPqker&588~&beTz7|jQ6SW zXF2GGQJHVCO*gd6Ac(w@i8nF14@PO~s`HBWPt{nA zbbx`3wn(?ApG|W4&?1uZ-r983aamzL~m+qn0?)AKLurAH3us<+W{V-A;JbHvd zQO{7pjWGQ>PN1qXAKP0NzVvyZ?AFb7cr*i;VO+BZFsnDI{D9)Aq;`yLH-Hk{F`*Dl z^&}KfW;=@#{?LwfB%{@YwBpKoe#aQLRqSO&4@)>u`vRVR;oo6v$O7TzOk&2}B-?-> z!}tN|k<+oJu&eAr&Qae)F?^j}Ht=gwcyM2Z-qz*&jTxtFF{k z@ZT{~#O7(s!&`v0xl6$1!Xm-yIPM(=3v6@q)IHuzI?OHOT%|~R-GN(GttMWTu(=O8 zt0u*2^-{mq(pMDUh*&={q3Rq@S6RrMZ0L)YigBJyrDn4pwpv5eDfH7pk%)&nkJ=A$ z$OLkkl36@X6A1))*L>Or=`@$-9osD3k-qzoB3Xl<|Md=vAyUj6fftw|EE+<(a1$D~ zphh@(H>P}-SaCfEJczPkr)V)Rc+i8rC?rSpN)01?qe$hl;DMxwwdepr>1YsD)bPVP z3F2?`Cn$anI3x%ZFnFAhFN-rC?mUgS4HH+1bMRi#^21DEeg^XgKL~wNh?obr12|R- z@O$()ayO0jO^jNdiU}4uzD;lGCSmDJJbRtbnu|+l=kwf%TKCeDPH_ehGn5n*v zw&W0p61f!oDwHSr(fCPDo=qsa!S0S&7JqJhN1(brJkZak6*b3xxl0?D=?AT=lAIDz z%PBqPBpdfMbw!>X89l8EJHQZjEMWSUrM0_I%y}vii;;lKeEu5k8elpaV8WXsxHP>x zMVrEoI=&G4FU)LOyPb3AjmN2N$g~e4CVuL+wSs7Od-?M>@oWLxNI-~cE$&sFOpW_qsu=*VlPMI<%aQ>@N=<9* zIYD#nFld68w5prMeTmhmpV1l-wP&E;zsu*16r5iW%ZYp%jLAXT?knV=GVbx9?GQNv zr_lbBe*kT6<0VP0czZKVFJjF{`b?4c_iHh(Ight-L{FfAj6;)&j2m!rQ!TcIwMS}U z+>SW7PMcWj@8K@>yDa`b3dH$jTJku`)dYW)dCC|39nOSse;Sk{17K{=AX{B)hLOiY4q0c6P2KaS^-fei#WGEN>tlaetgeBn^p*o9vnE zI5B{Zh=xQEU`Tj(k%k7>33A50XGH1m?lGG(zjiWz2jn24V}-q_GB?5eL?Qwi)UUBN zg(8{#l?(4yzkTi-AHM(YhZm~f_~3%*gCtLwLr~FC9{ch&BL^)j0{=FdZn0YJpkBpe zlEV1>jJ&tRYPWim&CgVvJ))w1og?31aR4e=RrO9*j3@DK+d3vn@-+O-xF z@|zqNl&aGL!mrUNKe;R3$l3D{+5Q?TW7+-z9Sf$~n75t%0iFstqlDU0rm_~fw7e`t)&^QurPF%gl`{jNcB&vVHLgfC(td&_1 z-TXG`r8)3icmv3=*ygP1n$veS(-cN3;Ul_}DV^2RL7)RL{_s_bnDaSWfY&qcaIGz< zWrSfN;>ajpLVkH@6Akr8=sTqTpQ0|Z>`zIU63S1ecTaKs@o?QCuC$Kk=HcDzK8#72 z%gaAkJXiVjUkdyEW88tQZo{C1>A;QpLZd(Pwrfv+VICSGK=$1yq$4{^0T>EtVE9(v zgqT{PB&I|>b>QI_70-wCD!k96B^}eZhI^zwKuiU}->;!<^W^Aqd33#;c37Mc>0Jnv z;+n-FV*3dyW3i3RZHod5(PaG0-LbS#B42At%8GbX`%%Q1UTngz(kQfuNW-KNMj{G5 zX3|)KxIY&whF{SBNTaEcUit@kxFTCuW+&uF+fX_Hh{6w(GnAD;L(as3d%Y6+rP1U^ za~Ff1cs&C3TE)juUi(1v)hVJfy{40$r8muGg^8xj1FD8I&}HZIsI2y{4tB4un>!zLXbaU!{22g zJ^vAFV|ZHTDJ||_aE3PK!*)uG%Jf#9+e@nTBdd1o*E|84K_@?@UXZVr2(5mZ%GCBko}q zt|LN+>v(6lI+p2GoY|i_3^k71f~raq%^9^RFs|+!g)OX`Ug3xnv$2^sTD2V(rGRPA5+QxJ?PhCXntE-x; zP3$h!=`N(uWXI}SEh=xGz#dh=>gcKRN-Niv8zYg?4a%Fj^x~qZmt-c*Ibo}- z1S>CUg>!g9;RrGi&`)1U2iU3WlWc0j5(|6S#LQ`Pyuof{kO}=VjVM2Md6F(FnIsEJ zp@)?Em+0r`hL0C!OKfWNp~)NrlWh)3EpU)d`p@Lx7jcmW-^I2Hy=?-}{&vX);KvuS z{ktXGg#!QpgTM=1So|3qT%_h=%`cV&mj#TS9j^x1C-PM916Y-(~bnyoucDmVQ@IZSkwLm87ifJMskdc z*kAMIZoxsg;~M8f^>}aJ;7@V&KaswNQK4JE4}3gk-zNMpE%UZ*m%dQEQnZax@2Oncz)Z1aa>G>oLbm#>J@(o+UVPC4< z$Y(<+8oi2>N$s`{@EKc(6gaWnEirAUBL&bfV=ZTDEth%{;_HLgwlUN$oc?QQ5a(X~ zL)4NY0=gY!WBi3IP1l_UK?G%%?jzzy*7&9Qy~eI$4oKpV^YZh}RC9wdEKw?h%$ zJhq?8RAiH^&dZNCgW|r-6b?dOS-$^*3Tac>RY0vD8y#a6aElujx6K{+p@>CMo7wBw zL54HfI5wdBXjAf@OyT-VqsCgk2(78B}qgj#-&lqM%uD5YSFo?#JIe9Vv^ez zcbl_47gJTdsLgOkx*=b|heWVZaG&Q|MfZ2n9eyNgnlz$q!puuh_ktr~?d zadqfI{abc;m&G|2V>?F`qNutP3(lST4?MkR(&f)_HP5wp3KfEK!t%ZLZ?P=0Hjmtx4h_7}L9^USb0Y_pJ}!JFoZKa%O&U zVnb3Su7>}_WYeM;fM_w&|6`FeJP_Fq{>e%rzkY%Lf5Ap6{<}#PIO?pl1c(u-CU9yP zs3HlVE=qgrfQ)S6^&gSmGIVwL|Cv<27RPB9HN(3f_>#Ve4PbkgVq{42(~^VnAN!I! zhKU$v#pT8OXTPNPW&Ytw#901(HL@bI+tNRM(4BfxB3x=Ql^*OvYekBp>mSzlFwlXX zwbr7(w!j%2VTUQ{fPYxXf1lJ*w&kuu;e%Tvb&RL>Rs0o7+wlVa4A)h$Y?Rh}Q6At( z53zWZ#bYc^vv`5Ut1Q0G;y^Oh)MHz-jL)c&j1RF5PKfj?qV%i3vS6RIi~0SlpA0ejUoV~ wvT$ARr{)pB*q;rrxVHI)>p+zpKCt!Z{xM1zqx3}S$O5Ls{Z$!|6KZ<)93%c|NqZXr(AXw{EmJ92WwyWoTB_IWyXINGAHpQKTs7VP&_42 z11(Zx&C_Dt(__OkV$(BY%d=wJvt!3|RH~y#1+Rd=MpTGh&t;w&72}du;J)@7*8Qy*lS7 zqX*&#y$9omyocf=-Vx^2qNVt#cQk(3dpJJk9b?{9^ho@u_b8{QqmRXpd5>{=PxSHl zaqn@YXM)-2iTJp8JU-!_;QUd zrF*Bq*&FNwXJ5|w6gc~X1K=FUIj6xn7%YObm~+m6b60Q(oI}J3kB29=Oz)XMzo^vj z-hBG`_DU$ib~8NEX-C%=FLt7aSd2SC7%etCakr7SE=A$uwN|>e_}s?oYOB3^w$TJ5 zY^|=P$p=h90W}Is!$iL7{Tb*_z%FLHGx?r0xw~=42 ztC`krgH|tyFcpnfI}A>D+AFQqx|S7ArD@ovcG=`Gc|J@VK_hLTym-3NTnlCO%stn6 z@l@Va&q}Ld>UX>Ts_1NVvjdbvS4l_sJT1T55RE7bBVRPym$SnkU4E@2qQFmDn;{12 zUTj4hr`o~!jR-mW#q$@JJ0h;@*~0lytkTG%jktXw@B17k(-7I@XFv1GPS%T^Yuj1q zZsJ^LwUs6pM5h@h7%5n_&u%m@|4n>=PCK8QnjNCi(svQ47Y6yn62D2S;U~>TL}PWR zmgsatER=n*5o6IsR?ZVIY^2?dlm>q0nX{)}KKGLU$_uAYeagRZ>dcwvm!I`7KJPt~ zRfa=6|6JGzvYA&p&Bmnx^O^Wk7zEe~=w(k3t~55H)R#NLk2;M2Z=1P%b$n3CfIlBM zu3rj$5yp+Kk2*YLzukyw+*KYT`W=2~HJf_AjqMbNL93C5FY$LwP}OQB{#vX14s?)$ zS(Ywtbh{mqCbSdqt@7N33v~l?xp2OfBBBfvE=b~h(o+1yDx5B4*f9h zrq}(}*jtCIywYf<9dSJ?a>a;xOPVAAt!54j==pOOmd|EN zR$OZZLD=R!noVvSGi}?<&C4 zdzkrUT`Z+mVE1&ia=xu^sb5u7drLt}Cr~iU!ty&B-gm3Kw2E);Z*={x=&aYx%x$%? zb$Pa4sl)r2Gl_T<^~EDZJ_fSsE(M)yZEp6qDV_k!D~{FQQQb2V$oA&k%k4z&n=n|! zinXq??44NLylZLk1yYVwZLyUsj;gJ6#A3U1ZK+u8VLUe?vQJIDJ`YUZHR3iza6>$_y@i38NM0kcgSDNPAYF6<5X4FU$ z-+x=V_3Y6!?jG&7x<}FmR8o56@pxk`J{Dg)8nrGR?OspUI_;xZqbNQq6=C>V*xbN| zmf7TJLb`vcdtH12oqrNfau~!dXsV_P_`kt*`ZU>TE+;Hwug$>q8(octV5P^qG;(%0CSFNaqx>+5LHm_3Gm(PkNRH+x~1Cl!O z{mk_VF*YJfmwX@c6Xhj3L@m>kR0y($#2g4Ds?&*h>BMuCFXx}88AcIy!)S1)s0ocw z908%VPVNVB6uGj-dzz+wpy}FAG(-EbWoqwhmUate?`clu-RgdB)kK4Bzxh0ud;^K? zS_hiPz+(mr&^kJ_jscBhNR1PkK{2#KJ9NUriZ-u68#rIldv8|Zr5_^tJJtIX zW%(UtQBmXv$SkR#c;{z@_D1Zl0f3UML`eXK%(;Yae!2#OAS-d2fCMT&n~fW<`Q1*^ zBHV;B&ygw_$xc?1Xqmr=BuP95(ky`teq|&Icyv6;IiSo)`$4xUzLx4+O25>@-t~>1 zvTpQBGq7?mYsu`Vm9JtS6K8>R8fihrx4^M_rOT!`DpPswC92&n^h(6dc~1~8wFA6| z+AJtr^hPHw^k7AUM|=ZZdbf)-bndzfO|1Auq?}=n`g8+hib~4R#)@C**`O0?mFBQc zZE5UQuh4UPMQCFan%cTSbtcnVUP?V{!x}bfIIPR3(&@Yp7rmN2D=093PdWoWj|qRv zc$}Wwa{~7U*1(@-4PRUq1l;0Ykj$Y^>IbcO6*>I=`EPF(magDav(~qHCwK*}#Gp2j zcG5=VkG4i_#29Rl9U6YqUg~gn;y7MVH$3bg=*a8zRoH#*g5?QI4Cn@zRk(Y>k5hT;m9XHHu=-@^QB|#b}v#v;(w#n)NID2-!WyLGh z_GgIfp^DC6?h-DneB=vMHW)YdwI~w9zzf>-GV4Qek@{fB;NkG-xpSJ1e8UJqcv`gi zGdu0_hy-gUR#dE>Xr82Kk^sc5SStOisU=WXoz`ncO|>*vwQg6m3Ue(LstXF1{^bDi zo>p+j;6if-WdIXEiL5JYmu_v&Zr2V&Fv$u7iB7Wez~;f)hDqjNIa!{wGA)G@Hw|eF}Ni4|($(O#WlTUj?dsXk-eWzbQZ3Fg<-E(g0 zXaOELCgmX)de69N_907eSv{9~ETYFEw4J{E&Taf)t?Jlkbz9J4!cOQ$Ex0>Ki%Ud8 zA}d5_o$Dp}Ey>~%Eh6hgE`xYFc7}KssZ70+sh2bLs#oMmOWP*1~H!MIK7nzormVvGn zv;iNj?2x&XBP3oa!CR;TJipobM0Pb$nc0}C~6 zPTPUArFS3Wbb-=+Cp95KBxCBj#eA1iz%U>OBuR%XX*~lNR0P75ZUD)AJ0&cm7C=6r zmCo(T)F0QZpymXlN--^=2GuMF)f+UE%7-`;j69ReQr*)2W-XsBc0>zd4Is)m4vbq4 zteQy_vI?Y4MkRdmpUyJOcvEbegxCG6WX($p1uh5~@II^O;{Q`Kdrlm6`AwU~5|VXG zYWIg}@Vh$%de}|s)txx?C2DYu2yJmW;L%j1NmUq*mo3bZ*8xO}x?&0qVQR&R-$vZS z$7`|)>5j1>2s>Zp2rhxDABdsz3CzpJgTNdERL3=|j-QHYaWjHQeL;MlD zXyeR545k&pDQJ=tF!)iL9@1h7nFc)*qN9jX4^T6Joxav42qDADys5DhL%w$*q`XE%wtD;iXSwhUirm%)Mib9lG-St37AgtQm-&q+$}(m;Hf3P!4T z2pm5IKRE>A6tGIFfwgjOLwn|+ZvEKcHFPyc-JJjM3JwrOU(QivVpkNQ6_WrlM=6F6 zHnUR7ZV+LVAQ1m4P;nJi{t8d>1PGwQWQcg5Ai{#ew{nHAS(RJbv7IpBz}2eQwf6+| z@*yZ7bB}(>pMh+H0s@0Q9{~nNXtR2E0tcW_Vo)dqEhq3}xu z6(QmxfB{^ztZrarv5b`fbj3DGqsCQp`Qza7$E|DQ-dIsD8W8LabQ=q5~ zSm|3Rw|mwsxg<8H*|FJzGm;!Zz9^2T1>{QjORfXesZc$HmZ;x}wvkpjR5X!Jf?s<> zc|(0eTLB0M=Ic(+1b|q81^b5nD%GFD12@IrllOWloymKgg&RXaF*odSE}akXT~=uh z6?VB62F}X8m??WKu;pqUvC7WJ$hT~&C?EVNO-n`!GpT~4%A4w&%A0r@iS z^{-N(SUNZb0#qtWGIxw!p#>b}1qDpAGJ>)zEdlu8sQgsg3Vp=igE1V=N~nWim#iWV zq8YkQgXOurK0IX!ZNET8tYS&Y;rvTUP9ott76sZg>$XEs1^_ET ztybV~5tK>ja(@CH1#qjVj#Yctai>*B({%V)x1HOYQ@ecltRP{!6(rkYa%7#87}Ksw z$dqS5LhS9LBA!7OyYW})k0vMgn@I1#hisb88D#GxVZaHOl2GTax_NMHJiFCpB+3c{ z7cOBpC!5|D$c8b26A}1*{Gvo|h-6!+n)LF}y}OF{lU7<+#Z+##7JF9TZWH=)aItR( zO5bT$7@5^%o&z>!pw26O1Tq2`-+ct4`vp8Mo+2I!rRd@jQ0TNkpI4|iP8+CQPAR;D zu*tf!rJYf}xYDomDqLQ;s)!eQ_FL{v^w)FvZ!s?`qU?)#83H6+R?5ptD7yydvW(!1 z9T>gxoHDJ<()fzZbH;d1-lmK;Uq`6r!);2tv{BYEMvQ%;Hvt=i3~(B;J%Y}WEffs6t?*g+$;Qe$x!zeKy~ zw?I6HPyV1uSbQ?)QgXT3_S37G2}Q6PieICezfLu6S#&Hjn^C8Y_=(IvGDti+NPJ9u zjT-$5k;6p34pN^S+sAAG{WcX{Ch|K(ewWB_pk6`tzUChrB_A0i#fU5XN5nVKG*erd z$y$Ux^c|wYHVe*rmE=pP>l7S{p}>G8`G&0LvTbOU+a$>~y+r0I%&h@MY zj>t#4@n1nHPX3kTED{6T4BG70xOFxPc~{<>c~EnW_n_g}S$A^V&4QKYz`ZXywRZ*e z*~GK$N0>st1ZVt1OKlYZ{2f>!fxfB*#KOr!qXGvUGbpU8o`s`09Kq4? zTQKo0)3Xaz`WHL|yUT&iRp8+G6#kNZHjTd&Am0;IbC(|n{1hBVizC4t2gT=^F2F`Z zU>tS#hLbqNFN8H}7fum3tmSS$9q60hba0T{EQWhzczwmG#3GN;%DTK2K7FfaI z;2zkHbHTmge7F$qr5ZAzzoG^A1^1(DUr-PC@+ghqf#5-K_H$_A03>$_XRl-(p6!U! zjg6!cojZ@SU4$$t2Af%z^S?4Cn-%!bg^TRL>72&VEk3A$n?WCh!&&m=h7N5QO)@)@ z0e!E!6F0oc{Df`5t3wlT<&i!)dN#ir*vt?-O~8 z$W0>O0a@M^x*tQywkRSA2f>VvC;2;&k?~8h4Il9fP&)k_x^aji0w{blwE#%A_#7e> zaQyrN%;WodIMqdD!&rA8S2XZair!PCgJKy@&qy9g%Wd=tMXl#2s}m@xiqoiPu1~I0 zgnq)WrBjI6!2IQy4aMZC517JyK7eREo6EB>bLXVt%&q4!ig`9~t>w(*^FS=(&}2dE zdkYg}qmRuzidb_HoV*0wRgeewtEhwDzdv1H*kLZpuD*go@drdmAf-lr6DeZQraWBD#s`DG%sMKViXKM-Fc&Ma}NJWd?-4=z=R zy~MT?c^^XBiN68Xz$@O(eEUP{?i7*JM9vT)lkJ<>T?yq}+kzUWsKel&^S*-!g`rj{ zGRglm!}eTVfONh~F@1ytHs^M#kfj`Z7*?}M zUU~8vhB0~_%?CdN2|^T}#558-i99HBQ^#pAZY)Xc(!?L3oYg{p6fA-JZE!P7e)k;! zoK@)7%m%I#$l`w-j-G=@7`?Pr$3MdM7UECPdvoWAY1BO8DiVWbze#7>!_lhZk5M)l zDD@&}pkA>Jk7?kwP#>c`mRWL_{u7OoUQr@cm!~7+Bb2o=JHL&!&Au<88EGsY-2>=I z!e3uYX`HynvBf9sFjs#Ork>G<;oCD{LhiXz!YLzC$$l7_#5r)Kgc~2mmTD+{OFRKn z5m9cA^xA=$8Wi1y*-ShWQCyhNTbim|LBI=rFX0TO(0BVq7^cQmRh+qUqG#aj#O)QZ zO6dPBK<`aeyae5;qx_{SCsLdA=$0lt)Wt~XT|IlD?8K%|)72}c_5^vZlMEXZ$(dG6@m zS7vf;ensYTnlE5@#e6>8-bvPQl>^a+opN2Gj`MDle}gWd z&{GahEH(0;K}%{4((mG-QVMR?pr$F5IpR_q)cCUT51acxq~FHhoz3zEtNg~ETw~C5 zZJSh*njT6eg=oo!fJ~~OI^U~8h!+U0^U1spoVT`2$WdL&D0VWOOtKp0l@#H=EiJi! z#91%(t+c!a{DeHJ$03ht1un1#SvpSVEU*@;SEf>&TG5>=-s#4&Afw5nV^sbon@?QC zxy9lzh`+ejNN~kz@iVV1HtCY#Vt#79C@;fNMbWwh#d=r0CMMC9AWbOlBEpDzgHoRY z@mvWh>GdvdrnRthv3+pyR2m3hqiO}J<~)xpZD}jLE~D4?fS1`A1-*&SNxj1G=3rKO zaRYT@d53BMYJW+6tx{ifv`qmeufVPuMXS6@J_BFQz?+aoqvMYqx_Fwu@*qYbS1CE9 zVW*Km(3$uE`YC!-dVktR#71*W7pLToX8bS@$Bpal-Fn1vF&wmK&}IGLju^Q&)0J|D z1;llvp`bwk2N%{j&X!+SO1#OT4LxMdk5~gcD(}s^`x5S+Hrm0Z>o`#Ncb92-*h@Qi zEKJYtA(=xWCBk8Pg796P!n>*9dqnmT`BNeUZGuEXI+=e#slO)j0+G>oI7QhrM7|Aj zjimf45{`nSA4J0t*}vtKHC$A)K0tgQGA#dDw{YN#w6QJvFa1kI{~t4M4dVafo@L2) z-*c$lZ9{c$Yv@p3$FdCf$NQXxr+5tYvzhJtL8pnkVHS2Y1u(EB6rmI0dGSi{it!A| zRFdn&yS#+BM>~Om6yp*pudY|2yLn4P$>R$iyp&#%OY#d#W$K8Q-m5Y@z$I%bf`K%% zSpZ2Cy?eNV6uZG~zdY!>G`iu)R}a0zcc{w}!U54K>h`WXRppz{eEE^Dv+yMr@4-8@ z;JeRbb~>5w-hFpyF|KI&f`NDP|5$tJ!*{2bHqurkf$)yQ9AUA%`z+`}j(6w|FCKUr zdr^@$z4z!HYD-6m@Q*L255<-JAn6uS$lE)zTOaZhlUfPIQjVK)UW@c{*2bi*cHa>n zGRy@Dt5@bXCE=9~`u_nOO7mRi$*xH=d$^N}yVFpmVIuCMcij(?nX*Qm7;HdLAYDe~ zt@EtJ6ZmOzc?!kl5qELn^DMAo}uT&zL!kTOf20T-B=TG~B8ZQ6N43 WF$eLoynIf@U8Q?#pRZk5i2oN=sGX(& literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/stablelm.cpython-310.pyc b/vllm/model_executor/models/__pycache__/stablelm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4d0eb05e4cca0f13c4a602444df11f447d3be02 GIT binary patch literal 9418 zcmb_iOKcohcCCL`S64U5=1-)4ZChizZHXL@?eWBRJU0A^<&kL3$m7HnC*@+lBH3hB zck@*>B@)eK#uQ)yfwN75U;rtrXpv>M$s));*%q=0W WW|i2MrJQ@Ky4jRCvJs>j zb?d$RKK1H-&OPtG?pCXgg6p}z`-hDm{YX)MLy_^zMC5hcNvbM}P&`GbLW|T`^R!s^ z^w{u>*!0ZU@+_6o^r+;O@HC=QYH76##OJ%@lsTaC%g#`+tFk^ zvuG#$@)GaPoJ*?7*Ii|4)h_=I_#W!Q{E{KSEJMM8Se~- zYth-b;WgsNyvH~`5j`G1;XM(b^UiU6GWuTpr1vC;r=q9gr@f~+T#uUZGu|@@Pm7u8 z+4#J7p2M@zg?PzZikH3R`1{`XRps{-F(>A4C}KXeH=jGuy&phxLM%YDP-vcqW>K7k z=47FH0h&|dG&H9R&5O{S5oe(}OPcWff#Lm7=vS3yV=ubcSqo*@X@wWMooKglH4Rq7 zOP3o@Z=`82d3kwxy`66KSC?Adc$w1LQ9Iqe@Ww@sM9A56;;smz1~T-5w7nXIjcX{< zc(=d4-tMfw6SSZR+v^)?vh)Rwe$&kCi)knWWX+6A?Ig|Y_j_r(+X!G(efkn?Y+M9A$Uc z!_@Ef{B_yw_p);oL-C|5eQv+s3uF*QVdP)y%1HQ0doM&K&ee9*f1@KV_oJqsEnW`g zIyE8c$DJ!hflFvrAhW5T{^Y|4RX^)qJE}sB6PLQ^(&UQlw!#FjhpPV5et-|zL-%z% zMT6DMrY6d6FT}T&t_CssMP}6^@P0q-^;4wOMlZPhZWxGc?!#^?SRE?gj#on=FbsIh ztO(bFew6z8X!ub#5Fx#OlIujKvCf;m*ufCRp=bwb_yPA*@@g8a_aey`V+_=YR! zllgW+-5R%(1XXZ1#v#gPPsPDj=!bDH-SyjJwH4i-_-jEc?aJM(!WpArHX2|$e zDk90d2!x8K0w=VxDjcDOy3j)-G#3-uCjNy(iU%IlASuLJbnY&FR;TJIrlddm2Eq$y(6t-~FGx-=IRSuL} z>TN|;2I@fZwSl^+b>PO?k)G)y$fl8^^sn9DeiB?&&Ai4qxR}@Z#UiOc9wUdC|=H&T6jHd^)d8$ zG+9oHp;+qe$|m0Y0`BA#gri%kji090aBHgljb#}0yyxnBGh-vTH0*&)z22f88(szi zG*oZmPM(1nD7P?Bx7CleeQh5deoGx_x7B?N-7O3q>s9F0tI8d9rK!tjQAS=MaUP;+ z$R!Ft58;)HF3eP4K1W&>KR_ret##$KK#I2zNofjp;b#HpS)kQ8zAB^F|ICg?K z^!?2736=U0gy$NtH?ZDmhJpx+^o$m#vM=%Gas!o)ai-!c`W zM#{$^D3HuR*otNt_t44rElsChyr5Qi%gYqEB9O=4|YSequ+ zrp;>;s}gIoBy3orvM953SXM$um=s!bge$68s})|WF4n+AHZ|?w#23qz zr=~oIMzyGah8HbX>NJ`K$zKA(e%pdgjRR%hz=AYsX%3W4YhWzWQlv#{<5`BTJTSIQ zc|H%#fs|}%$+Pb%pQ;_CkhajhNV*gtmFHYk4iNHD3xUI?#r0NDuM4f4RtF$AEDfO@ z=m+|~nbwf6GB9V91A>(aiV1U?+B1o8iQ+R9n?i`{&0}@01s#1rU@}d5+$H2RDEFqm zrOQ80X9Tz8uN6VxRKnBwm3g3|#j}(LHWHyNMM^7Dm#_&up@Mey^2!v%Ns3 z?I40J1ib7PW@8qNx?)SP$ zo5_f=8YCfdxKsn0#jldDp;(UIXXMiqrj?sRD#4PhLU_|uBpA|~yregg;=3b%)E=*V z7~}$5;jX_EME$TRuy9!5fuu}FOFf%fmNAD&x$YX@Z# zuavZc2&O)>w%NpFC43|q7SA4zLgsu(?Ri_uu9TDbvW$}sO^wyf?F=CgbtQ&6-M}le zVYi|!vP=aHG^5kBa@ZyFcG#0%Dedu#<&R0vz&3(l?%&JMClQ2WkRhSlRc*~tXY{(E z0g|ozHI4L+YTTdHTtGDa<^a_A-8VGn9vPhc2eb3FYmxaOLj<&h-l971TF}yuQrxfi z79WDEL-SM~PLm|74(ApJbIKG#0h_e=ZsJ{I#W0NkK7uy5-q0Ub0Z#RuhqJ}_>Z`wleL zz~LGzsIdYB)>po`j|cVQ6R(AJ`;cj+@1l$6?EkB2t&nkL1FJ14 zBO~jwk*%_k9nq<`!WK4k@)Gg!+|C{@xrfaw%}$z;Y+gS>;LyCLAXEhMY>M=hW}ePw z&`d2DJe4K5FdIB01yhGXgc$`BCS&P5q^T5~II}xl8T(-`$!ff9@Q3mL{I8AQ-2D6h z`H$B?QFWwa5*~srb$M1MDoYsKVRi#VjCzumlccZ6%K2xbdP%UHO|(1MXZidVdP#Ny z3pgFa=ut}3JPxDeM^yDO(DYZ(Gx2h)8bB$BpgO<^5ORO-^l(0p55@>Ze<2x%bSj1BkQIBAz+CC`=m$G6HN_Qst(4 zQ@IHuepSi0W-5;p7~mT6+|hG9WHRyu6#2xZIh)z9MnSwPg4h0)D$AE4X2box3lDeW zXgM#v8o_51C9ffnm)H@ylgW z=cnB*_;6$cdAup6kGYE=e@j+Lz^n_D%gSL##uZ>7Y*XiI1wXm2ThG`mlmA8SxTnhn0C4J9#{ZnYhW?_<`ck?L>as)d~j=R3Bx3V&X(MQE;A8a4`qa~ zj7fATr;D8M+Y}j3lD-SkbaLahO$tV7Ng*O@G9f|JEmPZ>$umL{WF3ds{|0^1fUs>> zbz#lwz-U5jST)=ZaN8xatqw`@IIb)ehz-8EB12Sy`Tgy@Ez^@!=DVpw@}{G7&fxeH zX+XaP#EDG*6YhjaV$S6l#tVlCvpIrtqjp!DA40iZFgnZ9zH&?TFD2DNeRJfRC}PRK zATSQ+V&S_vOyF8se86eKZW91_C9DV+VD6HmC9E>ba|Maq_$tlY^i>r~5R#4dl@^5#B&IZNtNk~{S!ef`k0_yABI!yLav zZstUeMcdCBZem9EJzFOxiS4WgSR8Spu|XMG9QoVU`4Q+N*c(CL93sphnbO18Wn#0) zU^Fz5yYv;M?|>u60S6B3a!09S1M#u*9^!fwkAFac5IG!8asSbw!)UKelWBMjC)k@% zFw^Gd7u5H*4kz!(zH5@0mfV9Fjsx#l{{#vcK_0yaTAkJCz^e}*dY-&6`T#Oo^Z}3O zhUw@B94#dO6fZb(yg-enn;Siez_8D6lec=*U{&rTZP-?-MN(V6N+-OIbFJLkKSyd7 z%uTF(x;;(}rB}G-Aq*dFWS%iA6~}f*9O|@RCOG8={u+UGI6?mgn}w8`igSwzX;M_$wln_3=}H_3b9z>{GP>;qlJiA?A0sPsog>5sPE zQE4A7C`w~5S{68u0gWu`IM7rDZKhM0(n{0BQNX+SS`SVyc4J*dWTe~5bsvoDc5>a% zbKSF4w#)|xT%1T}ak}7@hNm#6pxm1-oNXX(OYP2%zb3=&z1cBktDEGGB5_2G)3>48 zTv~-IG^-vx=(96JwBb*kq>}`3cu)s#pA35u&S^6n)GwjaTX_aekq0DbpydKYvy!jv zPZ9R!z!+Ow!)g*g4thQE30B~&4dMje#a0TQh&OpvK4=uhv18_N4nFAcDn$!9pPg*r z#Kdol%;YTj>mQiJE>%`kmEZ(jNoW%Sw;uQg?Vb6w2sp_5Xtl19laI6(kkL@<`eB?_ zsiSGXFn+zqhu(A+I1V-AFOsx}%yI6-cc3KSnT;$`wZ9-i@G5^vLL)(jQC=hQXCz)F zao7`V6lhJ$>k$8qJNX#|Sj+KS&qd>_;~9(drRGwKhK;IiICr)3FlHfk&gS}?FJ&d) z7u^=lqO5e6hB?Nz|6V5Xb9$-l>GN6!+A4bu|NEZaMLkS=K(?LGF0yzpY< z7F@E literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/starcoder2.cpython-310.pyc b/vllm/model_executor/models/__pycache__/starcoder2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9accf5d55fa77b999d28e2ed04683cb8359586e GIT binary patch literal 8867 zcma)BOLN>-cE-EW=m(okiZ4+TCCgUYwn$5UWhRaiN3uN{OSC+aofM)8(&`J6O#-k_JoOCC->;y-`DR(NIcBjJ`cZPX|U^bj{=eS%9j)wE@JeNzsvGBNioXh3l zM7ZECgeTpTTptTgg|E41dHL4yA;0Tz7w8v&#B61ikK49cNH=1+w14|boX6wX2dKwvl-_-aOT8O zaE@l2_raML$G|z3aXtX&xHtjM3F7!?_YC(#pO3p;ZQ*S2r8Wn*pOdR&){w(#Zo zg%G!3@mo5tnwi*_V_(*91%6^&X-08kf6AIGuZqH)Ripm^1f>!KdlF{W^_-dOVo-HWS!>~%Wcs%&>V$!V%#c+{32 zug&Y!WjzS|z`Na+f$*Z{j*m$k>Q`^X-b$;cCzBW3K{sq&%er1^w*0zG#=rdh>*rZN zYu`S|!dzol+N;etx+dEVKf;5-s(#t6;}v%5^cvYx)x@TU%67-cTjs9UL+pr5N?GBH zZrtg{Sj^m|Pd@$lt1CCWufMqX@gI5DKE8D6^73ci^~>%jNqO+_t3UDUBANQS-KgK{ zGd~G$`9d^Xt60Gi;jh%YLG0c3o2zTF7qsg*Ff>&7dFV8|f;TtxF*Z39);E054?FR;*BqHGJB-L%!EU$Z zc2eMuL4DhoQG?`W^v6g5#{c{oh_ARR^j*uVc~z8!=IcWD4d0wqgn_#?tE^b}loeH& z!n#|6&gUSrd0{inGj)7N6sWY~h@vRnHQd5I#ZcTLBzi2F7)swj(3TUc!IGF18zg65 zzqUyyn$;S{$f60KY+O37=q7e8R`!&K>LW!}dTLMcw4S=IKXKa}A{j`+ze~&P}RYTFad5J+FY<8|6mlOe^p zRgZkP1Tk*;?K@r+ZwEf)y4dQ5-o}=<2Ca(Z>zIb;s?8^6)a^jNZ4YAIj6F}DMq^S~ zYYO4F7*1pf9Cw0dsP~~OoyZ-Fv5vr_CZ5^N4~slt!{7F{>Ot4f+N;9>{naEoUO?99 z88%HqFO^uE{OBZyUIb}wE^5>Rs3g65$M@nj045C8$g4v+GkAy+yBql)?^uQ2#@@s< z9ob%&<5acUX=RcsIcrP>-UwW=0D9d%N#8dbH-f8x+z(+su$*HrtlW;=uxdGZ4o`;EcYFIQju7M1#Se#lgbv?CRnou7Qnp-hhDdOq4%wC^w$9dEpAU9g~$nq5!xwU0^ z2J=F6Xs3ANVe~pB|r()Y`yx;MaHTdX4PwC7)x7E94Po(mAld2OKTZT*6D*>F6fNP(&eW z|0y)8R4&kNZsxR-WEJ(0oy5ETo$SsL1 zYjBb$14wqi#V-zH9dsn2c#==ws4YWy7oZ*{X$FXD#+og7`X0N$)R4@;ejpkd_zr`X zePcwZ2b*%3WdAMg$sCAd*pO>7TlUkkPG$R<4YOB#wsT@AAESFd6z)UNT(YdwuJ?Zg z5_GYwlRWicomk#ssYupgrT7eb1_1>)fQD%W_p~4GX}^bhRvNSeuC(QvahO#Bm(09$ ziFic*6r^Uo+zi>$kD3DDKO({d<=MyJXdc914}i9d{uZgf?m(fiaoW!Dmuy&Hvo6y* zeoEvp!|*j4eueAhhGFbb!$9L+&>zA)`bBp@hK2#5ktW6w7Qq3}LL)N`jmRvl*lZ|r z!iJ%MVes?oj&PV3`~uw(N)>$sMi^5z7tE1Xo2dCMqx{V{A++IGt)THr$e{ z6#6eG%5&adVs)gy(!6uPwIKl-sP12*GF09B93$ONV68tF0YT#SX>ohvZ9a2Rbj2FgBd5-CgqkJ z6ixJhRHkJTBu2F5v2tI%uiV!l$&>_v3HxJkYpSF}0zS|=?dz%QJG49lqm$Sl1@-Wj zs9*Rubc{%CG{sV_XFG2`w~qrQT>L0NNFPKOP)LXFK ziycGH%@}0!lqC`O&uFTR5gkVXLLxOZjBu~;D;~b!*!A<(uq!7V@ z$nYx3`z?_|S7K$$BD9l)$MWYisX>?~7pQpR2a*Cc^H6-FT}?sGrxI%|`Dy=xD@cye$CM^qP?V0^-XXa%aoa9sZZ6eMiz)SLdA3b9KHJLo%iKMxt zD8)fK!2XZIpn#8Iln{E8A)pB~KP`YYV;qcxR>a4PB!A2=?o0N5PPrIc2 zd}5aZUaOZwn+;!G>sj#DiD%3xvr5m{Gj^@bqda>q`cZ5PwP!WeJ@^iIQod&aO|71l zN+-*OBa2uL$G9s^3Ckkutqr*bUSguS>dPj|a-GNq^(-P9Mv#_uBv_OING+e9PKy|9 z8`@MNCtr4m(9;sNnV2jHBOo}y+5bQf##!4gszr^WGe@-$p#f{3+IkT+xcAe~pH&91 zh)tH|Elg1xf4Tcopp}rqCVCXh)U6ykGCH-A^C9`Q=m3Afxv8VeKjVrh;z|8^_=-

x@9rOxv+{q2Cz5MaVr+gMit;*5fgo$==T_> z%$k@KN6=msQ~rcM=^vqSG%wmTF)e1$HpPZv8lrbJ8F4;8ZOe=GZd4DhT>U|Rjx>nQ zDEb?ap;3Sbafmkuzm1JUcDHcP$&;ihX|k+EX)5Yr#n3|Nl!^orIB^j^pvEbtd`w9% zYJv8lXE`3*ql8z5XW>*A8wNc?dflzIHCT{d__`BTN_43{X~cb57N>Asv{23$RQ7GmwYKw zuopRso^I!1F4A*aC`Yf2#yW|A$-0`BPGX1%=ZkA z-LWCyjJ`((J1MS>^Inqu>?dsw#LEUAMy|p|ToEP8_yscO_mRG<_uw$3%h%*K+9i2z zaAWx$c&UE@28<}5{{W-jei?l7LSude7 z;BBP5PeyrJI`&iWKCP*4MfzcT%l8DnqBR>xE$&QY8E&A`Vzag7tw?`!r<`HNV!E$M z>EOqod}4@ZBX%p44->tsxBv zpEWVLOS+h0d_}HBQl!bUsUm#+%#5{K$x)_PQ_Ju)khg_*0SIGQGt>%ztzRb{ay0E5 zeO6 z3UdU$AIs(~K%iiEHvP!*I=3DCI*>(2i%0@CxA=prEYmVhzM?O_l;8_*ntmh{xho9wlGxKA7Swrbpa@$ pMCU<_iFtTA2uA=3xNx6TiwH%ij1spLwMiA9k3O%?SKppF|NmG(Xu$vg literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/ultravox.cpython-310.pyc b/vllm/model_executor/models/__pycache__/ultravox.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..199e76a02b3fe0741d6b873a9f793eae8eb6c7f0 GIT binary patch literal 15123 zcmb7rYit}>mR`NO`oZqzQxYl3QdyR5wIwwz*_J&T*%qyr>5(W|lI zYt|O?opYPq5eB$N8K;9n82hoX`9Fg9GjX z&KLZH!E^3&oFDNI1&7_kf#W*-T=bt0j<`oSU-DlFj=D!VU-n-NUUFaJ{HXtOaLhdx zRNM+bkNK|z$KB(cANNlLC*6~rpYW@}tUDXL>b}a)lm4mTw0oNKQ~qng>+b8E-{Zd# zoN>=^ey{&#@Rs{l@E!L%{JhUU8+_OOZg9>$$IsLL+rc~TJHfl|yTN()Jkw_Ux!^tb zJ|Vd32}eBtxh9_XN)Ju<2A+?I7x4T7KY!?zy&K+#O=CuLZ+aPV)LRrUdNWN+yd+-! z+i3yS6ib{Er$PBa zzus$J!COQL-ii9Ji#Jf}a_JfGhp7Licng#tG37hnQBcl`?}G9P#(qw`jj_M&O|AYC zW>W*@9q}$G?|R4ZdWvjo)YfEY&6Cl3ocCq7#>7fK&do=jtS|XqY+Y@I zQJlTnB7z@huhrMqTJ7aHcYQ5tb=n|hZgtmCF@I})%@Y?}jVR9CZlfr+8uc*3JB@Z5 zX`$I|H&hcm?fZD872?I0y?WG@-lYc^ZX@bQ6qS>rOYH{6PL%PvDDv9W_L>*fMLnux z0wWh_0v9^%W@|YfyVmui*0qkP`-(#S%rD%yeXDl!(v|teTQ`3IO8)BAYxC{3ZWQ97 z$T`%Gyay5bD=fDB?p#}-r?{}}MKz44)})bVZ(dtm=*R%si4VJV3}mBDgGt_G#M#Aq zfF+P|Dal;#MtBR=zvwmV7#c?A`Byu2HJ>rAhE?oOM~^43-M)Hj{@V46b5{pLHg--- zjXW+bcwyv;TV6Zt$PfdcN_LXQ9Chw`?bZf{^1Srcu(q{&8+%;G*RMCcm5z@Qs9Isg zjHf@siqvjh|K6pA+Wf`Z+|8SFKd9aM!HrAt?Fn^^3wt#xD+4z>tuozQOFsARCc-Kg`i*IHhQLbXP5 zDS0Gr!(<9gzfLSrTWZJqTkTd~{fEg)t9rS8|fV965# zOC7e1|D$WmBnO%{bH1T6gA~?;?aI%OAmeGUCz@;A);v?_Gn!`!NhTz zJdLVYjDtgqaVcH(WI2jzIe0Td`39QEGe{QXo0RQOM&;fn+Pg?zr_4L)Q=C&1Xo=XC z9+XDAK^&HTC{KHM0pE~7LZof!J?*i+sC}U?R17%=nmgK*od7z|!=zhnPgHa{2l|d7 z(A(qbM5L1a!=feXiAEC|ND79$NHxty^x&y+SSaCf&^sfohml0aV|~?ph|x$YsaUFc ziG+5gS!=KwlgFsbgm==}Vu+VO-aW*b`$~bPJT@33zu86;LoevowtOGip&`=v`$L?@ z0}b(^zNtt0meIp7BWsEmP7O7~J;YW}k5-`5y_O#r8!O%R-J0*UmqDAZ&E38@e_iR3 z#oCQaH*1TRE?i%@h*hVd$Ky|}lvU~14>Vi ztRCRTv<7-6%Ji&WW*=HY{EY?4`puTKn({>?{dKr;b?(BY_pe{Qh=pL?s)lT0xbkIG ztk`NtuT#q#NZfHXi5)uH9o?}2wU!9o(N46I>TV+JUcv*S8wBe$S{G6?wPr{5R}?^6 z3rTet83Q71Jh2VyiIFMj#x|a}Ed#V|VwR#Q7g4Q2lS)6di2a2;fro@$3}P2rx}LFW z62l0iX>6JTiwYW+w?zwOEhIV@bA1Jkfj~$#LI7x2feqHw3VshyATq1dnn)V6n3_CIIs)yum42-!Fv1Q0WDN%+ld@)9ccFsB)?2F|Um4ua4g}<NGD7nwHnSgV?V_bclk2y2jsBuZVLyb0{mi(p^|Gr| zBv{l+WFKL*DTkG|SNDkAeof@}X`8udF9~8VcUP0Y9F1)4>scbxOQ_RY(|C+A^la)! z6vT)qKC(m!yO{48ThqP#){MwJ)*tJ-_UV4iz1%a@+$*CCv03+hV3`cSWaF+pk7eB0 zce@=bfa1YVfiO8_zRx-=@*5{__+CBqoK`!G(AFX2`a@>IPaJTe!&xWkpjt89Y)7`1 z!M|=^nP_CcWUWE+;AZ_!dl`@}Hr66QvQkJ`5`d*9fxO}@up7Y_x2$Yl4Je3kUtDex zkWICJD9Kn>_QwEVJT8E=*din*D=3IFu=tIY*j7kRKCftu+b0(~(LBM|01(>~m!#}S zWui-{C5gA>Z6t2CE!cUGMQ+{nJM}0wTkVJ?S$;y)EI^0sg!Om?T>|uRZ*ER{JS)%5 zv}?^~ByUn(6H|+A0(fl!eGX*lZFrS}QUS!{Jf~E_^K@C{Vgssxw1Xu1aHj$Q!k`-@ z7DL%2cMq0f8K^2z4hat1EqF6(5d%hL3{97NF2k=u4muO<1Aa_rfqdmwi=_D9!6J?M?=qRISMUckuaLNbq z$Xd-jG~^k`l`TxP$o4W@*TIl6_=4uXde1m>_`2*VKNR;0y z^hSCG()ODLek!1JM3~dsG<3o0$N?%T0`WjkAji5wm|U~iE3J-+%#=oTV1&nM{LraG zL?#uoI<>AlFD=K)XaATbB`+f2#b@Cx4dy!Pel{l>tfyS!Ls?A~%x)Vw~ zxOHM2dBxl@RnUf5DG25k6onSm9Z}NE8Y(r>iQE+u{U2D5wu_U7&oStnUB+L4$m8W0(iQGWCZ1qUA{I{<@;rRlr4PLr!SQOAKx+--PV3Fw1;c1P`yv$XEi9mP-F zU8m{X*~w&gs!aZpaEE+=WJpPo-J-9FZ#ab{(jLOSc?_ctu%y+Xs8Scl1mAbbdxy8n zK6{P3LrB01J-^8hUmz$`t2O+3NX{)gg6#hN3VQx8^a=u=v{@T7yH}&y@AW+D$oG&8 zjg;{ph!^k;$x^_x(SBrZ0-^#czyny-_d&fvZ6UXxQ%3p57NC)UQiZb3^q|<97Qh7* z(2mynmen&W)$;d9=&6ADw%W@SDv30NI-3MCyRVq186#cze zzZL5j+{`VC*6g-ZS>THMbt(QU>V^lAkZVphhj|zLIt@N9Gv`{y#`IH5l{AMd*+XO4 z*thT_zeCAcByqM@6P*TjfUvrx#UE%d`4g&9N_MH?`F^@nyj?%0%DiX4jNE_5S0Pr- z(x)#7gl5a!4R*I;L@0wt6xQicALPmFdq@T1?f`Tu2_x}P2 zv0jh$B{!Q4QvR4K4P=4V|9|7l9?gV40R^@(^VC9A(>>1Z+Fbc-)Lq{g@!RE}@4sT# zCjL4K{%88K95(TwA&K_QTS%-l*h0@tHV;dO&9j8%Wq>F$ltRndA`2~-drC zx_>uhXVs$+aBM59Hnc;uGqg**LtDt~)H~Q2t%j%tb`2qM9=SrG0j#V-E1+&kn0En?qVhv%gBk$~Yd>1IZMZfl z1m3_Dt}{tGu##E0T+v-?sZ|eMJG|do_Pfj?l_|vqBxUZ%N=pdDHABQ~fd$k9c!$MC zDUC9RZ`lJTdk`lsaS}1wk6vQiZ&OT5mZ?koDgs+|Xk9oF_^q9COE8maX(^`7+X=4+ zxfyCaLh$;UeoV=LJ!u|4$2WA4z)1yrmZ7kxU^5GB zd5eb4I5_uY!rbTwP5awD(BtI7JgvuR$LV_k-|##VfJ?|Hz$JDRA-*(+9Rv+~3;m5? z8|R>8J)I&B(LTuOHdaaeD6;DrS$3%DNA z@-wg?aDhJya{3f^_vJLdTTV$AK1=t0v#|apwD?neRjfmsFb+*gqBZax`jEvwk1yMf zfv^IY7!Xm?vl;+N23%{$d#`mwtJ(6z@G(sN7wCkML2@qRkbj3$5C;I_I()df`Ikc{ zX_K5f6}bK_1S*|Y>S1yun{;*E34IFr`|DJj-{{=)WSDgM{$=NuthYm`H3ap;m$|F7 z`(y|wlydL1(FZV%ci(BY9(dtdu6wHLEWu4zCj%rND08}N0I+cC0g>RkchCS1yXe4I zbnaAl#fI;2Hvn9sC83z`4osF{m!}SBt8TMQOe5y!NUSdJs=RE5oSSNt}3k z?UIM5FvJ*IZ8cEBbSHt>{Q3h>KdSp(kKJa6BN9BqI|}Zm#|d{*ET{UTSEpm8&Uh4< z7l2>gr+41*9;{&oI3`J=rij|!;rNAD=TRa|fUp#1$W8ivopcymk{+NPYh%5zXAQM} z2nXYwk=7#tmINe~TdPi8!aHjJuOs@=!a?C#XC;c(!n3op%P`N~rD~%S%&v5oQMe2| z>dnG>_xF2t$?q)91_;v6hO#kS#I8>`n*_ksK?eG$B?O#g@C+6`&%rCm6{*Yh|G&4l z!m#Uwv!~y9^Yz!cvyBqJwVLexkZsCAvgpei>VF^uivmQ56oF9+CQcE>O7m=MGUT-P zjJuXR7Qr3w89&9b64@)FBM*Ir)c#qPLvPvD+*Tf%jddI}ni-`OAlb*H_Yid5D!?9P zMGiQW&@DoB0xtPhN#q~ve~v>0z%LepX&Gw0S_Zn+kZP-=qJXd)J&hq;P};N+NJHQb ze@3!W2rKG+O#DhMZ^^McGG z0A5MBW@E|$s}c*JaF(bdLbSr!I9YYB)$e);qe-|i9g-mKQT0as?(8wvt87~tcYpbW zFqjeRYjKVNA$S9pc+{`2BYwUDmd61@c(*bk$)+KGgwuww)0GX67`;V0Ep#&w;CrWF zxFHz=aVvT}%}_or3`CSIt{kOqtYy#dx>@);K{xW;(Z1xU4rBX1A8^9=Ayc0fICh6K zEFV)_n}-&<1vSJroi*o_*`-jnJNXRxjY~ta=H~k^l)p=DN(radxKFpFx>o-8Gffz- zpJ?HGXlh$@E%|j3K~ARE}UgWdg}K(ve(|8KHNFMZ!^dKIz$Tode(!7E()I`X65F^hzB4kij)G? zmP8q-wkSru5@;E(jJB2-6XT$ciV3d_>KG`KVhWUTv4>ml<#eAnLBDwGv^Obc&{{ne zy(!9hdpOcw6#K;iwB0KXa;xXOeaYKxaY!6S$+U2Glw`aazQ^*sID)eMxL0$)JLo+} zHK;$78RDpT5oL!MiywwKzZ{oRw1VyP`~{Yecznlcx&oaz{-diGDP}Cf%(PPkDGy+c z8)+bf7M$`~d`*1A-$$|&xdUe-q=}R?mKlzHpnaw<6Z*Dr+NQ7KkO28CaybM*b$J}M ztSyD&nWhjpg)pOH562m(jgTYikCLu?Ncf(x`#87$V5l`{Kt9GJ=ugBhp!-%4daO+_ zcDELOk-+SzbKjHIiisOyv?eu#n%3wX6d~<<2({yu3!;exEjFNE+RNQKT|c0j)ym`m z=POM!u`@cUmeL(pPyq(2rW8b|c=)eiq0-Pdk3n7mMTHO#A?MnQtg;yI#d~)4+(-M8 zJX!hIsi*Sr;M@{}el?|*z_m2!T3GG1XdR)N!NrgCv?29Y*$Mbt)a&1-Tg0BRs(&I4BHOC{QE?n*(dx1vm#B%s`;r+P3Z7_Qu{JKI}UPLx@Y`&aTu$+$NFo@6pKrh?0XyzR;D% z-KA%!yhHP$cv2A#jcsh~86IEKwIp5rwEvQNt&9%HouH7m8erm^-~ctT-~&217k&(Q z6R|2>H%TrH#0LE1KSUou>vSX~|1R?B=J4wS8Us5mNeH(e9o((-gMZ{`39NILBvCx+ed&4ax>WD*Vh_C_M;J5?vL*#*n z=)gnn?V0MAq>^d6JcWY~BLcFD5KzT&$8kE$jBN7o`$v^PEabgE0t;{ty1a^3=8jgE z(JF&hI}bQ>i1@)5qE=C!4k$me1*s0Sa zDOmO1*C_WclDIqve}jEB_A7A8{;O2>k0>El;mIix?#pp}7_IA$P{k(B<7yo?w8zk}aQ$U-WH zUJCW`7z)}!AH0GKA7GD7^VcX~mk8BX27uc@Yzp?C;4#>aVJ*(h zz)S$bhc~n6Bb)T`ZqMGza(kSZ@tIkb(E&xYa=q+DI*Y@ZHBQt3ka5mj?b*;gxdpWu z=P_75w;vos4G9K(Yc`UT_~nZlWCQ`lb-a;;1ZU;+R@}G1?{6BPf<+O`#THPY;veD% zXDRLE2vzz-He#dts%xv`V)^$$8YtVvuKB;4t zo!ehM=XV-)KYRz7v@2lNJM^9Se(JipB>qqXSI+;pku!fteftoANn$TiAMaN2XRJ3yrzfDuGqL|cr4r~mR%V^iK za56-)3EHXiZ)C&x|AxMq>*crxP!r;0 zIoJ~TkmQp>2*_v562|IfdwE>>ur{GHH}$qbJ*iqKqlhD=q?0Cln4*VI}YnsNwHYvI~XL8OT2&aeWn;+&I~pF%B*h za>Q}1sV-XBriEa$a#~et|H8`d#%l@R@{8heMsE~8N@cM@<|X}5o)F6ge1A?pB*_>> zoq^e)gcU9`)S8q@p~CsE`5i>KaKBnjp6Bzg&~*2t{tamVohC(Mp2M)M?T!9D+H34Y z4EKmYKll8NBb5J|UPEIA`rv)C6*x`D@^7j9S!4MV6#Oq53t6IaW@9RqWi<}fOPo!7 zlp(83%abk&&RVd;j}V5Zdny9gE(ussn2L^N0M0Ux=>K#Go!bi=`xO@}XNx&2 z4YXC|-$C~X-%Ri&DnL|qM}S-|Q1`c%5iRj*-F81jqTDeLSNvUDUF^Z+Nz2QVNa7-{ zEs50EN}%qT%LGquPZ)y(4z+MY zZ!;DaD$ef6$1(IFrzpu&LI^;e0ubvd_hevD$(F`;XNit;;>mQ6=ID;VJpW^ZlG7v9 zl!cX8OHNQix3=U(`ZBn$!dFMLlHOTaQ74-7G&y+GLzMEfY^x`9B~iI?4O#jxbS$5fs-D6D4G&^yMC+ zjB!OpsnY!&z8U0B?If%Jq>$@!AGLUqi+J$;{}-sL7-bdbhOK*ih$E*_m(LL0quQ@J#T$AE~WMrCg=_ zkzGkde!uVaTmZNSQ2plhJHPk6?|s+%x;ZwMGw^rfx4ye{>a1b>PreNPP2lBaJkg46 z7+*31BQR@*&tI!%)-B(v+rDk`-L5(HjGw8yzFW`w*}CU@^$~xhp7V3{QGc|a_w)5J zf2=<4kJl&s3F%|jCjChnb60)JpOP}Cw!1#id!Rn!&q&^_?Wyne_ty9M`|A7s z{q+O>0V&Vc4%Q#^AC$aTd#HZMKP35)TA}{1|FGn9wZrvC{6{1|T6?tqnE#mM^R>t8 zNBkp_AFDl4FZxB(2y@|+;b{2eK4Zb!XZW8PF+%HABNz`RZW+PEy0vclNAW%x?85u5 zb;~~%nDRgM4LjH!&IHrwy?37xJb>Q^@jHXxefZsj-~IUAi{AtI-4{+R&n}Dy`-20w zT>p4D5gZIBm!Cp8(gi1YFnH*egV|0Js9n3!*;K2HOJZRP0w7<#+@iQw_@u^x;^f+ub{{xfOWlYprhJc*KLgU`rX zj!JqAJ!ZqtroD2(@!%=6`5dr#B6u2Dd^*gp+x~NSeNX z@LBc`Ow91P;5qa@89XnoW@S`j(g{8vynvdQf)~;DWcXb0Quril$3pa#wx z3ci4LU#J@X%b|-pue;mf>EPv)MsT_}vaWH0Gr?K(I1`-1i2d59ft6Q+^Qbu+ysAeD zUJE}9NL~-Vh|+Tc0sHI=pG4cA3NE1ZmEfY3u-&EL4V0Y6$}b0RV&!j=Ca~^T1M_X8 z_}2G$7B3a8WF#oZ(L;O)>FY4=wdt>dq>a_ zaeG76K7-$=5HE#=M!6mqnhS*xJ5$g&D^wR)wOXi`V-&L0LbFzD-T++`R?0Du)QDz* znTam07s{0wxJjm(s=8Q;o25p0qZGB~f$8D06%{U2Zw{A5tp%3ZSHsoB1nq1ssw>AZ%VHQGd#>JEs-LLeI9{vHA75FGmzs^^*K4)qM-5sY1kd|JcF7^9-J77RgsYFYiua*UUBRws{9s zv0*yK2iWJ9sjS;Z$6By{+Axi47I*%xd8ufs6X;j86KnRV#Nj#aQz!8n<&jwL<$xD=(>0yV_T$4V$0@9>&Yr9_!7QUwv%StT zV$ss0Ki!++Ip$=ZjP(3G=OB8f?e^{+M!rbyv{XuRC2~iwQsncc(zRB(#zo;R zTPg+3N~xrXH7pXML6hhBz@U@{6QxqQ(P+lPdLwiwpo6AS8;4v`DHNs->CmGOA zWhrqrCsl-5BaGD8N1b{Qulw*sGe~lVV@+98=9D>OS$ii56uMQ(96pbo>Xf`h(Ecwzql^tDm8~Kx;R6C^!WDY|r~RFaBPf zq(ZXmG!A^a^%q2NmVX<#%@y$XT3CbRD2Pr`h?@lp)>L3bg=zyO<-!Y6`@)>;{A{B! z3z-YLo0~1X9v7-nfnrkzV{1$W0aK;Auv!ShIIP5l$`WLGK~pQ5Hv-5T1zLs3i3&GR z85TrQi7ASxf9TO5p>AmTR%kXbTC-6JXLC|_rB^2qvtO;bSgk;$DS#bRuU7*ch~Au) za-%R0iHNaPH3-6nj51FM7pe_*FBj(KxcF4Vi00-B0C1@pq%*!iK2TJ`9De^XoxV1D$4sa79Gv-hD)K5jg(44QRpE|gDg9O5b{O_4;Mw25}iZx6=-RYDp(Gs%9<4$%Vq~WBygZ(XFjx`YK%jK*lSb@ zTxsdBWhy#$ak0hhc1gsEB0i7GB#=a7S`tA=fCQqBFd%;{inGL=RcsA81iXp3x7D*K zNJi6L7m3t;pmN(FPUpoKg(oT1xUt9&_{cU$yKz7DBL~MfhOrd zt85?IX&nQ760fV)3>|b585hCDe6$xy3?&i*U#pNf^0j5e#m6^#hD@-*{Wg>;zpG?EwQBLn3S0aD1C-a zc60+z?{MVI+-Cd5`@pkJRPBff6>uUES0QsUTOS28f`V;BT_+UOQ1hg$&Gr*NE)2=& zd^rkJHkptklY| zifnZRWfse#$7ATO9rx?-j&@9>&Nnmb=2y*YR_x%!W#Y`by_W4{Vbo^Ut**&ZbT7$? z!&!%EUSEN9@9Ad^adC9*`XlWFN4lz71XTvwR8R9L9QlEHBpDl0EQ+rBGM1tin2^0C zxt}iATHz~7H5C|I2zvm^{~pv7Jxz~GY+7X^*2^X4-az6{q$78asp2W$$_GY8qai0O z*W0L%4kO9Cc{68OrUyZf6py>a-QjbcPnfNWHP3}k~9 zo9mX!isf&?*0-=J2V-~@z$Xl#+yVOGSXBo*5dZ6@w)Epn$GYQgP+h>rbgX44)q(vX zxQT4C*ReOiUEX~WB#>S6u*D-_l39?z2=ckk2uR=y*N$T|Gw@toMA>L(q?74nA(y-j z8=K2I7j-|2x;*OKp*rKv7{`c~1E(_rQptSNUdzSfotzNNaZyOzOG$oEQ$1faeRv!y zEft0BBhZW$)aj_uXr|h40V;8yAgE>pg(Hg8T%1U9RgerE6i}^`@ynMlRwI(oh4LyC zJ(o6aT%l}YP|{@{-l2IO%f@eU3*jp`&0lyD{B^~Yq&)b54w)~_;e|4p1x-KJ?zEe`E^w77*h6`AQ@#S>8(%eju3 zbMhcZ3aGp_X--*PGR+H#dUz+P=4`gI?l~W|AG#lz>J>oYbKzNmrmR|+qQ{h}RoQr` zObXdPjtq#C4vY~HA*3%IO&}8pHj*#iK|n$RP|7HcA%VsI<{D(Tx*}OlDQDZd+9t~W zJ|1D0o|`kLt)1W>B(LH=^#VtFkqO-(>LnzJ)qvybq8NhOPmpERX(m+V`yK;zhOb;< zpDGe(9@^6Y2O7EDH3*eZhpdP$1VxCUNob3q64q*6t3hPL7uiJQ@HymO#uHJtd4^+J zlh~(^9m~3xbF5Et8S7)q9shwfHi?phcJ9-eT~pqaZJ_~n(Y8LFL@O(k``FsMt)%FF z!d_6v&|>Oi#kua<$!M*q%B709!INEe`ns{@^-UEx2mAR&9r{&+*oPQvizzYXv2~bubl?Ki4l@xOh|>&k&sfY;rM& z;d*SgO>UW(o_CRj3$oA z$uaCaPIv#`{k~e~TyG&6vP*b)`EUn5qQ<~usTC%f^Gy_OIeSTNp++xzqcq?6Z|o)t zm!-aj{C&q(w>Xd}V)q+Y2KXOr_Lq<(F6gmRj<58FB$%A43+|R7p>lF{>U}0MM-OQ- z^8Z4)SapsIK_kt!4NU|HEs}>q@*1(ZY(e(s!Mtp0-2#3!TXO0#w9WBhx&KBq9=2GG$qQq$ItrG>ps=3a$@rZdw zf5^KdV9?YDoHbbdzhqx?uVJNqG^WO;Z{IAE{H z^+{zwPwq8^vAF^;`kXb%rX3O+d#`S@a)-VAGr~?2ET4m_Ob?5dLa!=#T;C`+{aDm%o+Hl?ZHneK)SWgG0o;I*!4+JJoA~abz!;EDoaDvR&aLLHiC#TUxZ8q>& z+bt`Ow6#|kce1z)XjQ=6%kY=Zw-y)aFvT4gm@|03+#c{qlHoU6ASWePk3G7=!DO9m z;LtYfO>EnRn}wm32xr6sh}&0L4PrX*Lni0~nh^=R z6;u+|K7vXCPk)m~lX~`tj5h#qkI1$%2XWTG=}4ZZsOoN=_E%6g$!Uc(xDRQ&Y*Vou zzAkdrQZVe#03rG3)XOKuUpZsFB)GpX(@{d=HliDj>%32#@vY*UMrMlV;E_jtk~CS zJ+16cMhvJdO(&>BW(T+Na%tPd#Et@2+L<_~P}1k78HX=Oj+X&f zmSEtuR^YP)=iELQ0FcoKm}Gl;$c+Fc4Q`YIsQRTsduC9Grl0`8_ShhdQPE56s8vrK z-f;^bOKf0B_>lTl?4r<%Ht9U|Yb^L{On#lo*O4Sny&PTjcY{Un$mt6UJ)J+2^4~B> z93IClB$^zkbFWG}?i7N)v_$sudE6Oz79Qwu(_*K=Vd^-h+Q9r3JVSq^$(QknKcaur z7=lsqhH=@&4Il3c96uXmAkbU{)X*?K-w<RzaoyJ^R37`v1(Hp0JgGqk^$x(c(Zy|Y}6L8s{ zvEP!CZAh_E%fG0Aq>mKe77r^Wi03KdB0F9~Iy9;BB-ji{3QxVm;K=n^>| z*6sB@-m}7M#=g)fR4>xjv?42EjB@a*Bgf8q*Y=)@Vvb z>SZW`t2nenD@4;C*~kSK7+T@gjGB?WyKbz(4nVAi^nu3w z?3$gnK;UKLSZpt6JNB}7XJiAy0KYl?yFALb)S9|`^(q|-ame5 z{j%stMe7QjgZV}iMkk_Du0r%cL9yayYM3PE)x^9Gdifm;rs+jUC3liRBVv}^AZ<`u zyN2tGZ@QnhKVL7e&WAlJLR=W|)>lMu1CJvY^b7i9;h27#6a6L<#0_TCramL2;;QG0}eP$68u(7vhJ)n=xph>$QqBBxeZ8-(>Q)m>fjnkI{L*3dl)K zf%+UP{x*}p!{oP^{5BFlQ)#ZOmi!FFV_5U^VJ)OXr4)e{1$G#HT%-ECY&@NwVP5(7 zVVGx?$0Odq&7`+a+nUomE$&TXgm%t2j6vL<*pBI1j<`S_J8v`k%heeAIPc=Z8E=l| z!Y%4fIF|ZH=$P!HKwpxmijpw>DDmXCw4eoLr7Kg=S`>Y&61ese#4Bz@X?C*6m4I*<<>an!ru3IeM z!>}0L5+1Q`862zd(9`$6z+>jM7cZ8thJ|v0T0e~!y^aE7_Qb4Ng!$U&inKH+7J(pT zo+y!h+vHg_s%r(ZToEf{Ko`Ri%=2b+Wkp~8;(a`QXXr|0i+G^$BerM_eKdtctf368 zp><=x7J}_a$#YlRt%|taA_5d`p^AFJ(I2$l-xC%c*7|ik{Q5Df# zM*UN?^JliTl7LC|T~>~%a^ot3U-UNmxf@M|m}NP!X9xkcvLts=oa7fUQIbK2#Sn_z zKW24*-JD7fIV!=TdQ1Ef1d8fxl7)JNn5#%H@?WvNaO%;49sGc2nLH=&@g|ff*SqK1 z2cZ(X_u%UO)Y;5s)bFD~Qb@h6VBVm@R6)~INqiyV%S9g>A^@rYe9q#D-a6a7 zqCWjjvilatGCvH@y{nFW`uyU2L-q5kTl7ZKOT?E25 zyubF{_Zknv@eE$JL`j6<)Z`ju?)qYG2rB=Y~t>~_$1kiswP$`&~ z7Q#QZ1iU|VzsproouA5Bj`vCX@{a)SCteMB&tJ#z+Klf(t`2CB3AtXwe`HQD{vVhV z;E(*6ziV>Q7KHMCuj_NQ!6ih_>8;(}13{jsYPLUQHoS zUR|c^<#Z4fz)|GZoqnyd{`f?HZPG25c!b6=!80@=QJD&-VVL)QT4~c4;M{^&pg&v? zAkG+HvQUdY%BY64?nadt1W?)#&Wj*jy2a#=yj~B&oOs&cJHr=7knO8!=N<@e$%iR} zP;hpt1aQ(F)-%m=oWo?P-I>dbu1HsNw9tfnQa7qmSXgdBdps=7Q-DMi$+qD<9RofG zcw_JQBzdiw>*4>1o5d}d$*KQ@pCqT1X%0IijwvE&k;^vCAp?<%?P1V>2w;eXcc<6v zHiHRFxKk11LV^2^ac#tbwZ4kY5t{sudmmA_JAU)y_B32hLLjDN(E! z-S(dfo?p3%Prwk@O~EX{LDLPwn}r3{tV^Mmv_c3P5wF}V*Kxg(BKlb2aM-BezE;g@ zMSJ$C!-Zp~3Qx)C+iOqE^1GN4ZciK1?nJlU(_1Q@?pBB(pWQmiEWW9kJt0H%Bc7y- z-X|)x?NPlVA)a=jdu=EI=(wj>%B+XF@;)K{w?fk%b_5ro|qDRGVsSB;%1#jDZOyifv}ga;YYlGNlxpE+*53%j+skckLtg$ zm1rj76zuJ+^!&fvN$yP!`uU{U-rYxC&+RBucZet3=+J#6ZdXIBgFc{WNcaZ>p{a5J zdcuy}Al%7li2=C`?n4aCe=r!b9fd22O_&$NAb}1n>pD~Nd#4t#VSGae~6_VGIaVL%uHnt zds1F}O}&HCWRz|h`SbyX99g{nLd%7N5%NaKEY$yW@Dy!rMVu>gc^zam%3Pkw7!#^n zT50pTALoeC>F4w-BYeb}=I3$N8WFr=xJZfAB>MTIV%mbI;VWLXizCswpr}78DhvJ* zX`t!6``KWZVN1MH`q8d>;J&SOXL_5->*YtndHiELjXl&ljV#sX_8!i#|Gs@99ZEE< z8}tIPW*OeA_!XKzCRI^*tp%YFihAYeq)bmhZCgjLeSx_LnVe&CgULFRvrN9m`>a`^HB2!m6AcMV4L50K<=kNYf~czFniyt&uPBU-Wlj~F)7vM0^)$Bmz} zMyQ~>-$%NMkXyJ3tWRCU$hofd12=1Z-|=$)&)UD&w44GSiW{_}3&F{MVvV|=^4-Gw zl#}~pdiT_rH%4K^$T^leh_9PGFzq@JR#WzV*Lt4wbZaNkXYb5@d;io-Zf0^SKQsOR E0W<(;D*ylh literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/xverse.cpython-310.pyc b/vllm/model_executor/models/__pycache__/xverse.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e621a3cb8cf0cf7f8a4e567eff35b644afbbbc61 GIT binary patch literal 10169 zcmb7KTaz2db)NeS2A9QRZ@h@)QY0lo3YVrV$(Ah3k|oiSMXo4OQdm>6r$F@VE|xO@ zwr7@?4K{IknUG0^PU1Y;u8@9;9#WNGkW}So%uAd~<$*sSazs*MzH@qj#V#*SfSQBu z(>*;s-KW3v^=b5~RbRvB$;ft@&klemGKc!8%=9WyQkCHz`(De(iJb#`1%DnW(QPFziDK`ogGCOGfL^<*-b zOd3Ih^IklaOb64+OfbXwQaqc?1#_JC<2}jVU@xc3@xEk#us?Y!cq%y%9AI4~KA0Q| z4sp5~&nJh2!3!2V(=p8XX9@tF9k0pF9$CtuLQ5?+K)6bC-z*^ z#Gc4s{mzydd>5L%Vjnd73eBs~>=#c#^HiaE4VnYuAT$RH&G(==B<7)+CrxxRdU4AR zUKi#$t$BFk&G)-Yk&L?S=y+n=oW zp|(HmBEQ(wbEDgZte=%pCR?3uB;M|ImpaQ$BQL#`Wl@*z%O^+4Gf~zOt*nLe^4qQU zN~Egi{^{PwZ;fuPEJs<`?}y8>x8Bc>QVva}JsI+Ah5eRn#c>pemwPf6VcOY<&}S1p>%vg^ZgL?K0uzO&%!p{IwH{}ony)bKwM0aZuXCHIG#XPqpL)NG znM)$kX=TwTJY?w|tXvI`teji#_j@u+X{a$+MegibjL=*srSVQWX^IAUe3J+*<eQnwPm^+`0MMCknfaO1XBYGGN=JyO z%Vv;I?&u|1M(!8o=xJVAqd5!PY^1pZt6i7fwmsHaN)zgtxYF|wnYN|f(r;_JmKg(m zU@YrGpVoviu!MQj8fc-JSp$@eY3u>s=%Oi~$sB?7ukcR4F}L)e>zTWy;SNt|cnf#& zj)C@1ePQI*g-$D#&m$LDDXe&7&!^o~O;9A}(echDwzAjeHx4h%e@N?`s?B%O`R!^; zJvra)U0!IGavNIAo)==$I$0RXDKwo|mSGj)dOz&T-m07-RY~+NcR8Q?pS9xk=v^s$ zQchBtgXe}z-KL#eX%sKXXQ7nOk!X^j-hQFyr7&#Atuzh8FSNVwoyd~@iGHVlJZoXe zWyimhtgj>|lglUK&V>{GtJzAgd*V_YCnwZGj;=)Qb=Z%}rYEj2crEm=%5S0dS8%0I zLHMqr`?`Uz_qAowFQq)Iu2+nW#>f;FhOgVEaUFgvBxo|<#+6d12ih%dpx@R%H#Ut; z6FE%kZGBTm3e(7X9eVv7UT?8!%9FT9zC_|hh^8f9#!s$a2`n1VTo2`UsgT91RODhN zF1Muk0GZT-sG;{Y-O@KExBFVuk*}e=>C;qoTS*j#xgQeBt;dwEgdq$zE=o+;c5bGb zlw_cCjs%U9{60ipEheHJ#W7}lyH7NXk~$`zfuI-RLG?7lweA^)@r_{`UmKQj-?5E* zhEuy+pW&JhEn~76`lP>r*j~mZqh#W;N(S7Wjm7NXa&dWB)FoQfksEnS#$FAJ+lRgk zT?y$*hs<(loW)3>2l zNE-B8!YM#(w^j| z@wvI_ZhD)rzgrgU(j9oWP22$;G&bd7b>_gjZDX}*x3Gq|#WGqf13=8hI}h*yu`)5! zO;>(|C&Q8U<4z_&L|UFDagGEHt^63GSy7YrDJf{IB@tOsfWWgoD z)8D6>meaT^sqHqd9iLi}OD`cY=8$wTQ7}s@vf*67E}2={V3uH!V3v+>nOS;~2Lvp! z^jVhSlq#qXJ??~aq7+@^z?LeaDr#^%6?QyT*iAhz2*^888+$H!nkevM?kI}6jYrBe z5E&VM$i`_@X)@A(gc$b>WZ&c)z$;7QE!g?M z29}##6FeFkLzHe}b%ajl7I(l;0IyBqGmrJNGV;_?MU-#S^Q#YOYK+#EMTK`{c8tCX zoXGxyDfX<<)$kJ0N#$S}u}(Eh|1WR@(R!BJe~!dB<20jw+ML*h?zs!PiRC5PQdBRH zRU6R4&6kMAhO`Em-)y&>n@QZ44Lk_$KS{$b_KhK)l23HH*sq1Wts!N;2hf&L6Gg+L z5&#C#LK%@bO@g6%7ku+t*$LDCfV%0M5WeF9k_nWtTpkz(L9avbi?muu1kB!-Mr8=s z-hIchbnk(Kx|IhT`xQQn+u;b4k!$OktPB9Z zLnz)fx&*5lXp4Q@U?xotUI*|R1hPWEX>G!Uz^HfBxCJu=*JkaKf#qW?VxAw-IcQ@% zTXsk!Bvv5+bv(+CxIuQQ5|8cn027%08?N+e2(N@;(k%>@_W&5f{?5>y`<6!<-|+N} zeUBKzA<7zyi_O|@WcBHxG+GKcw<-Ar(h*=O;H0(w2hfu|-EVNEw95c`+AxFrgqjYp z&|xgB-7}8tMn?}gT5}s;NxJ{>2>BN%9Y@HL4!YhL?!6f$?E)no{JOwCkapn#F@beJ zOxlB25z_-cu9k_qLsldHCHfv!#RTrFA?-yIR2tO^#H@?S9aMcPFRNibLouqweD|Ws z%W+S(Mtk(WjFviA6nxT14?i@dHZF$Ge@0>(K5sCwzM&IA0I9a*?7#v6wN{<}G;Q9y zfYeO~=-ZM`Afo95?K ztA=X=7YK53e?xc-qf^;54d1dsdM4;I45Q^m=DW;%n|jdb$z>*{+uMvbk>cTp(&RsA zFVE~C({AWDv>Wga=QOpC)Y%HY(4|D3FH8kQ3`vYeqlJ-sug9(Af@r<*duoh^)zJuc zHl7)qnh|;|ydEPw6{l|?p>Er77gVfArnpxwkzoEqtFSpc2F~>Kpwb6{&UzOSrRj*E zj2uO`kiHVk@MK3<5IzhqQ4mZK>k{#NDoS&IOkJZf+RjUg4yestg-1<2B$dbLsgum| zRd7i4HZP~xNfdydm!hsv*q%GZ+mH{C7a?+s@LnFJ< zJ)i1dE~G2Dc0l7gMgpHk{Pi9aJTvhiH+064zvmSc+54t$Av`oSK9xhiV=XQ&%08NTnAbnb`w%Bq)SgUB zZu=p^O683Bbv??Ge|$Wj@1gYHaiz4KP#%fArgeARE2*FnarxR;#-X8i@(S+mz&lmQ zJ5gH?c_*4Q`lJ?yYUrJo5&ifr09u&KhOmBZ1ST~0Z}q?;)-F6~Y*AX)1E=KER~&&) za^aCY4k}gg>$6AlIIuJ!{34o6eh7gj+;MbwSc!s4xMwD+i&@-1iS$Iop{HoF@M3h5 zV+T`WFNdP`MboI^sr!~3KEtts8F64o=`3~(jyNa|!T-(S^kz@AH`+%vR4{DG5QoLn zDBI7wiKk%P$MPaXc&aDgZmp-S`1BdX=Mi_HU_$O(EPhpRAun-6my6sYu8Va+J6Dcc zf-Z6T!?K7Iza}2D=wfykjd*Hb)ox zXxgN0`)CRC9H_5nC0tNzoA~QX3SV{V)ZujMcJyo^WNgvRcKjk4;Jq zwY)8AZz8L6fSp2A&T0di276ktOkG8Vrgpg|$sK3!NG6)w9GI5-k(Sh9wFja14yLkB zL?=#SY+%$_I_hjj&Qc5Ym`{pmqSR55+)HXVRcAqRfLzml0uVRk?)?r4h0U8rMjgeO*cKm@z}dIp z$gNGA&cH6=ctKzb$iE=pmo#}`4Vy>QT0o2r?jFRzgd)VkSRru$V#s-ZiF7ave%!t|Y$gqpR=-c@ zAfW8Bg~$j!3Dg)*3aWe(EsAsL+~*p6h!vEJ9&$Y|UBOuxxPESP74`UC)>@&)il)*& z+3f4+4gw-Bfg61dV!n_SM6lIqAV81y^LoSF&eKx#4dZ9le{CFm7){0>f`Bi06`#Hx z*tM$`9%oERak{ziMc8>*%AtZ3igWK6s{q`pAC38gBl$wFO1@He6GS|Hz;#S5sl^O zHZkO&&T4^wx5&-~pt|54gd_|Oj&T_ifpB<2U2Jz3w$I}EkUTj0gu1*v;B6#rD}%#N zs>=UH;bVF}K;qN+;DsmMv72R(yAGax!X5nAhvM%l9P)o+``gH*{)!^^Q2S3mq5UDj zA-cns8GQQ*wbdq`5P|>6Gg>Z+deOcH)wD47$b|xD*0WBWGMGJj?jGU|l9o_V z<@VEPZJqu;C23m}_`EO1zgnVH9;O$e$YE})CmbQgQ4+^U>zv_pdKFB5htz@u@kloG zv9&j%K8U<19C(A=1n>H-nWNYoQs}pC)OGx)0pDr8VzP$vMcK68uo?#&A2fcr@6G=O DC2y-S literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/arctic.py b/vllm/model_executor/models/arctic.py new file mode 100644 index 00000000..30b1f1cc --- /dev/null +++ b/vllm/model_executor/models/arctic.py @@ -0,0 +1,562 @@ +"""Inference-only Snowflake Arctic model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.deepspeedfp import ( + DeepSpeedFPConfig, DeepSpeedFPParameter) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.arctic import ArcticConfig + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +logger = init_logger(__name__) + + +class ArcticMLP(nn.Module): + + def __init__(self, + config: ArcticConfig, + layer_id: int, + expert_id: int = -1, + is_residual_mlp: bool = False, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True): + super(ArcticMLP, self).__init__() + self.hidden_size = config.hidden_size + self.expert_id = expert_id + self.layer_id = layer_id + + self.ffn_dim = config.intermediate_size if not is_residual_mlp \ + else self.hidden_size + + self.w13 = MergedColumnParallelLinear(self.hidden_size, + [self.ffn_dim] * 2, + bias=False, + quant_config=quant_config) + self.w2 = RowParallelLinear(self.ffn_dim, + self.hidden_size, + bias=False, + reduce_results=reduce_results, + quant_config=quant_config) + if config.hidden_act != "silu": + raise ValueError(f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, hidden_states): + gate_up, _ = self.w13(hidden_states) + hidden_states = self.act_fn(gate_up) + hidden_states, _ = self.w2(hidden_states) + return hidden_states + + +class ArcticMoE(nn.Module): + """ + Model-parallel implementation of Arctic MoE Layer. + """ + + def __init__(self, + config: ArcticConfig, + layer_id: int, + tp_size: Optional[int] = None, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True): + super(ArcticMoE, self).__init__() + + self.tp_size = tp_size or get_tensor_model_parallel_world_size() + self.hidden_size = config.hidden_size + self.num_experts = config.num_local_experts + self.layer_id = layer_id + self.top_k = config.num_experts_per_tok + self.intermediate_size = config.intermediate_size // self.tp_size + + self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0 + self.is_quant = isinstance(quant_config, DeepSpeedFPConfig) + self.reduce_results = reduce_results + # Some other parameters + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + + if not self.is_moe_layer: + self.mlp = ArcticMLP(config, + layer_id=layer_id, + quant_config=quant_config, + reduce_results=reduce_results) + else: + self.gate = ReplicatedLinear(self.hidden_size, + self.num_experts, + bias=False, + params_dtype=self.params_dtype, + quant_config=quant_config) + if self.is_quant: + self.ws = DeepSpeedFPParameter( + torch.Size((self.num_experts, 2 * self.intermediate_size, + self.hidden_size)), + params_dtype=params_dtype, + quant_config=quant_config, + ) + self.w2s = DeepSpeedFPParameter( + torch.Size((self.num_experts, self.hidden_size, + self.intermediate_size)), + params_dtype=params_dtype, + quant_config=quant_config, + ) + else: + self.ws = nn.Parameter( + torch.empty(self.num_experts, + 2 * self.intermediate_size, + self.hidden_size, + device="cuda", + dtype=self.params_dtype)) + self.w2s = nn.Parameter( + torch.empty(self.num_experts, + self.hidden_size, + self.intermediate_size, + device="cuda", + dtype=self.params_dtype)) + set_weight_attrs(self.ws, { + "weight_loader": self.weight_loader, + }) + set_weight_attrs(self.w2s, { + "weight_loader": self.weight_loader, + }) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, + weight_name: str, expert_id: int): + tp_rank = get_tensor_model_parallel_rank() + param_data = param.ds_dequantize() if self.is_quant else param.data + shard_size = self.intermediate_size + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + if weight_name.endswith("w1.weight"): + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :] + if weight_name.endswith("w3.weight"): + param_data[expert_id, + shard_size:2 * shard_size, :] = loaded_weight[shard, :] + if weight_name.endswith("w2.weight"): + param_data[expert_id, :, :] = loaded_weight[:, shard] + if self.is_quant: + param.ds_quantize_(param_data) + + def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_size = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + do_normalize = self.top_k > 1 + topk_weights, topk_ids = fused_topk(hidden_states, + router_logits, + self.top_k, + renormalize=do_normalize) + # topk_ids: (num_tokens, k) + if self.is_quant: + if 2 * num_tokens <= self.num_experts: + # If much fewer tokens than experts, use selective dequantize. + ws_dequantized = self.ws.ds_selective_dequantize( + topk_ids.flatten()) + w2s_dequantized = self.w2s.ds_selective_dequantize( + topk_ids.flatten()) + # We gathered the experts to the tokens so update the mapping. + topk_ids = torch.arange( + 0, + topk_ids.numel(), + device=topk_ids.device, + ).reshape(topk_ids.shape) + else: + ws_dequantized = self.ws.ds_dequantize() + w2s_dequantized = self.w2s.ds_dequantize() + + final_hidden_states = fused_experts( + hidden_states, + ws_dequantized if self.is_quant else self.ws, + w2s_dequantized if self.is_quant else self.w2s, + topk_weights, + topk_ids, + inplace=True) + if self.reduce_results and self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + return final_hidden_states.view(num_tokens, hidden_size) + + def forward(self, hidden_states: torch.Tensor): + if self.is_moe_layer: + final_hidden_states = self.local_moe_fused(hidden_states) + else: + final_hidden_states = self.mlp(hidden_states) + return final_hidden_states + + +class ArcticAttention(nn.Module): + + def __init__( + self, + config: ArcticConfig, + layer_idx: Optional[int] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = config.num_key_value_heads + if self.total_num_kv_heads >= tp_size: + assert self.total_num_kv_heads % tp_size == 0 + else: + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = self.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear(self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + reduce_results=True, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=int(self.rope_theta), + is_neox_style=True, + ) + + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class ArcticDecoderLayer(nn.Module): + + def __init__( + self, + config: ArcticConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0 + self.use_residual = config.use_residual and is_moe_layer + self.self_attn = ArcticAttention(config, + layer_idx, + cache_config, + quant_config=quant_config) + self.block_sparse_moe = ArcticMoE( + config, + layer_id=layer_idx, + quant_config=quant_config, + reduce_results=(not self.use_residual)) + + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + if self.use_residual: + self.residual_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.residual_mlp = ArcticMLP(config, + layer_id=layer_idx, + is_residual_mlp=True, + reduce_results=False) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual_input = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual_input + hidden_states + + residual_attn = hidden_states + if self.use_residual: + hidden_states = self.residual_layernorm(hidden_states) + hidden_states = self.residual_mlp(hidden_states) + residual_mlp = hidden_states + hidden_states = self.post_attention_layernorm(residual_input) + hidden_states = self.block_sparse_moe(hidden_states) + hidden_states = residual_mlp + hidden_states + hidden_states = tensor_model_parallel_all_reduce(hidden_states) + hidden_states = residual_attn + hidden_states + else: + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.block_sparse_moe(hidden_states) + hidden_states = residual_attn + hidden_states + return hidden_states + + +class ArcticModel(nn.Module): + + def __init__( + self, + config: ArcticConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=self.vocab_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: ArcticDecoderLayer(config, int( + prefix.split(".")[-1]), cache_config, quant_config), + prefix=f"{prefix}.layers") + self._attn_implementation = config._attn_implementation + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class ArcticForCausalLM(nn.Module, SupportsPP): + + def __init__(self, + config: ArcticConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + **kwargs) -> None: + super().__init__() + self.config = config + self.model = ArcticModel(config, cache_config, quant_config) + self.vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + self.vocab_size, + config.hidden_size, + quant_config=quant_config, + ) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.num_experts = config.num_local_experts + self.num_experts_per_tok = config.num_experts_per_tok + self.unpadded_vocab_size = config.vocab_size + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + mlp_params_mapping: List[Tuple[str, str, int]] = [] + expert_params_mapping: List[Tuple[str, str, int]] = [] + num_layers = self.config.num_hidden_layers + + for layer in range(num_layers): + mlp_params_mapping.append( + (f"layers.{layer}.residual_mlp.w13.weight", + f"layers.{layer}.residual_mlp.w1.weight", 0)) + mlp_params_mapping.append( + (f"layers.{layer}.residual_mlp.w13.weight", + f"layers.{layer}.residual_mlp.w3.weight", 1)) + if layer % 2 == 0: + # MLP layers + mlp_params_mapping.append( + (f"layers.{layer}.block_sparse_moe.mlp.w13.weight", + f"layers.{layer}.block_sparse_moe.mlp.w1.weight", 0)) + mlp_params_mapping.append( + (f"layers.{layer}.block_sparse_moe.mlp.w13.weight", + f"layers.{layer}.block_sparse_moe.mlp.w3.weight", 1)) + else: + # MoE layers + for expert_id in range(self.config.num_local_experts): + expert_params_mapping.append( + ("ws", f"experts.{expert_id}.w1.weight", expert_id)) + expert_params_mapping.append( + ("w2s", f"experts.{expert_id}.w2.weight", expert_id)) + expert_params_mapping.append( + ("ws", f"experts.{expert_id}.w3.weight", expert_id)) + + params_dict = dict(self.named_parameters()) + + logger.info( + "It will take ~10 minutes loading from the 16-bit weights. " + "Alternatively, use the prequantized 8-bit weights of arctic " + "and set load-format to `sharded_state` will accelerate loading.") + for name, loaded_weight in weights: + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for param_name, weight_name, shard_id in mlp_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for param_name, weight_name, shard_id \ + in expert_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + weight_name, + expert_id=shard_id) + break + else: + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/baichuan.py b/vllm/model_executor/models/baichuan.py new file mode 100644 index 00000000..54ed548b --- /dev/null +++ b/vllm/model_executor/models/baichuan.py @@ -0,0 +1,463 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only BaiChuan model compatible with HuggingFace weights.""" +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor: + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads)) + base = torch.tensor( + 2**(-(2**-(math.log2(closest_power_of_2) - 3))), + dtype=torch.float32, + ) + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32) + slopes = torch.pow(base, powers) + + if closest_power_of_2 != total_num_heads: + extra_base = torch.tensor( + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))), + dtype=torch.float32, + ) + num_remaining_heads = min(closest_power_of_2, + total_num_heads - closest_power_of_2) + extra_powers = torch.arange(start=1, + end=1 + 2 * num_remaining_heads, + step=2, + dtype=torch.int32) + slopes = torch.cat( + [slopes, torch.pow(extra_base, extra_powers)], dim=0) + return slopes + + +class BaiChuanMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class BaiChuanAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + hidden_size: int, + num_heads: int, + position_embedding: str, + rope_theta: float = 10000, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = hidden_size + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size( + ) + self.total_num_heads = num_heads + assert self.total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = (self.total_num_heads // + tensor_model_parallel_world_size) + self.head_dim = hidden_size // self.total_num_heads + self.postion_embedding = position_embedding + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + # pylint: disable=invalid-name + self.W_pack = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + # Create the alibi slopes and slice them. + if self.postion_embedding == "ALIBI": + tp_rank = get_tensor_model_parallel_rank() + head_start = tp_rank * self.num_heads + head_end = (tp_rank + 1) * self.num_heads + alibi_slopes = _get_alibi_slopes(self.total_num_heads) + alibi_slopes = alibi_slopes[head_start:head_end].tolist() + + scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + scaling, + alibi_slopes=alibi_slopes, + quant_config=quant_config) + else: + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + ) + self.scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.W_pack(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + if self.postion_embedding != "ALIBI": + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class BaiChuanDecoderLayer(nn.Module): + + def __init__(self, + config: PretrainedConfig, + position_embedding: str, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = BaiChuanAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + position_embedding=position_embedding, + rope_theta=rope_theta, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + self.mlp = BaiChuanMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class BaiChuanModel(nn.Module): + + def __init__(self, + config: PretrainedConfig, + position_embedding: str, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: BaiChuanDecoderLayer(config, position_embedding, + cache_config, quant_config), + prefix=f"{prefix}.layers", + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual, + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "W_pack": ["W_pack"], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + # LoRA specific attributes + supported_lora_modules = [ + "W_pack", + "o_proj", + "gate_up_proj", + "down_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: PretrainedConfig, + position_embedding: str, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = BaiChuanModel(config, position_embedding, cache_config, + quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if name == "lm_head.weight": + # Unlike Baichuan, Baichuan2 normalizes the head weights. + # Refer to: + # https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat/blob/84603cde5ebffb6084e476cfaeceaf0b8b91fe54/modeling_baichuan.py#L508 + # Distinguish between Baichuan and Baichuan2 by checking the + # vocab size. This is suggested by + # https://github.com/vllm-project/vllm/pull/1022#discussion_r1325652704 + is_baichuan2 = self.config.vocab_size == 125696 + if is_baichuan2: + loaded_weight = torch.nn.functional.normalize( + loaded_weight) + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + +class BaichuanForCausalLM(BaiChuanBaseForCausalLM): + """Baichuan 13B and Baichuan2 7B/13B.""" + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + if config.hidden_size == 4096: # baichuan2 7b + super().__init__(config, "ROPE", cache_config, quant_config, + lora_config) + else: # baichuan 13b, baichuan2 13b + super().__init__(config, "ALIBI", cache_config, quant_config, + lora_config) + + +class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): + """Baichuan 7B.""" + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__(config, "ROPE", cache_config, quant_config, + lora_config) diff --git a/vllm/model_executor/models/bart.py b/vllm/model_executor/models/bart.py new file mode 100644 index 00000000..cbdacf77 --- /dev/null +++ b/vllm/model_executor/models/bart.py @@ -0,0 +1,1003 @@ +# Derived from BART implementation posted on HuggingFace; license below: +# +# coding=utf-8 +# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BART model.""" +import math +from typing import Iterable, List, Optional, Tuple + +import torch +from torch import nn +from transformers import BartConfig +from transformers.utils import logging + +from vllm.attention import Attention, AttentionMetadata, AttentionType +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +logger = logging.get_logger(__name__) + + +def get_bsz_seq_len(input_ids): + shp = input_ids.shape + ndim = len(shp) + if ndim == 1: + return 1, input_ids.numel() + else: + return shp[:2] + + +class BartLearnedPositionalEmbedding(VocabParallelEmbedding): + """ + This module learns positional embeddings up to a fixed maximum size. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int): + # Bart is set up so that if padding_idx is + # specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. + # Other models don't have this hack + self.offset = 2 + super().__init__(num_embeddings + self.offset, embedding_dim) + + def forward( + self, + positions: torch.Tensor, + attn_type: AttentionType, + ) -> torch.Tensor: + """`input_ids' shape is expected to be [bsz x seqlen].""" + + assert attn_type != AttentionType.ENCODER_DECODER + + return super().forward(positions + self.offset) + + +class BartScaledWordEmbedding(VocabParallelEmbedding): + """ + This module overrides VocabParallelEmbedding's + forward by multiplying with embeddings scale. + """ + + def __init__(self, + num_embeddings: int, + embedding_dim: int, + embed_scale: float = 1.0): + super().__init__(num_embeddings, embedding_dim) + self.embed_scale = embed_scale + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return super().forward(input_ids) * self.embed_scale + + +class BartParallelLMHead(ParallelLMHead): + """ + This module overrides ParallelLMHead's + forward by dividing by embeddings scale, + yielding effectively the inverse of + BartScaledWordEmbedding + """ + + def __init__(self, + num_embeddings: int, + embedding_dim: int, + embed_scale: float = 1.0): + super().__init__(num_embeddings, embedding_dim) + self.embed_scale = embed_scale + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return super().forward(input_ids) / self.embed_scale + + +class BartEncoderAttention(nn.Module): + + def __init__( + self, + embed_dim: int, + num_heads: int, + bias: bool = True, + config: Optional[BartConfig] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.embed_dim = embed_dim + self.total_num_heads = num_heads + self.total_num_kv_heads = self.total_num_heads + self.head_dim = embed_dim // num_heads + self.config = config + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError(f"embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads}).") + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + ) + + self.out_proj = RowParallelLinear( + embed_dim, + embed_dim, + bias=bias, + quant_config=quant_config, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + if self.total_num_kv_heads >= tp_world_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_world_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_world_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata) -> torch.Tensor: + """Input shape: Batch x Time x Channel""" + + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + attn_output = self.attn(q, + k, + v, + kv_cache, + attn_metadata, + attn_type=AttentionType.ENCODER) + + output, _ = self.out_proj(attn_output) + return output + + +class BartDecoderSelfAttention(nn.Module): + + def __init__( + self, + embed_dim: int, + num_heads: int, + bias: bool = True, + config: Optional[BartConfig] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.embed_dim = embed_dim + self.total_num_heads = num_heads + self.total_num_kv_heads = self.total_num_heads + self.head_dim = embed_dim // num_heads + self.config = config + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError(f"embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads}).") + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + ) + + self.out_proj = RowParallelLinear( + embed_dim, + embed_dim, + bias=bias, + quant_config=quant_config, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + if self.total_num_kv_heads >= tp_world_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_world_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_world_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata) -> torch.Tensor: + """Input shape: Batch x Time x Channel""" + + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + attn_output = self.attn(q, + k, + v, + kv_cache, + attn_metadata, + attn_type=AttentionType.DECODER) + + output, _ = self.out_proj(attn_output) + return output + + +class BartCrossAttention(nn.Module): + + def __init__( + self, + embed_dim: int, + num_heads: int, + bias: bool = True, + config: Optional[BartConfig] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.embed_dim = embed_dim + self.total_num_heads = num_heads + self.total_num_kv_heads = self.total_num_heads + self.head_dim = embed_dim // num_heads + self.config = config + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError(f"embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads}).") + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + ) + + self.out_proj = RowParallelLinear( + embed_dim, + embed_dim, + bias=bias, + quant_config=quant_config, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + if self.total_num_kv_heads >= tp_world_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_world_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_world_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + decoder_hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + encoder_hidden_states: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Input shape: Batch x Time x Channel""" + + # (afeldman-nm 2024/07/22) TODO: + # Need a more efficient solution for q/k/v + qkv_dec, _ = self.qkv_proj(decoder_hidden_states) + q, _, _ = qkv_dec.split([self.q_size, self.kv_size, self.kv_size], + dim=-1) + if encoder_hidden_states is None: + k = None + v = None + else: + qkv_enc, _ = self.qkv_proj(encoder_hidden_states) + _, k, v = qkv_enc.split([self.q_size, self.kv_size, self.kv_size], + dim=-1) + + attn_output = self.attn(q, + k, + v, + kv_cache, + attn_metadata, + attn_type=AttentionType.ENCODER_DECODER) + + output, _ = self.out_proj(attn_output) + return output + + +class BartEncoderLayer(nn.Module): + + def __init__( + self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = BartEncoderAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + config=config, + cache_config=cache_config, + quant_config=quant_config) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.activation_fn = get_act_fn(config.activation_function, + quant_config) + + ffn_hidden_size = self.embed_dim + ffn_intermediate_size = config.encoder_ffn_dim + ffn_has_bias = True + self.fc1 = ColumnParallelLinear( + ffn_hidden_size, + ffn_intermediate_size, + bias=ffn_has_bias, + quant_config=quant_config, + ) + self.act = get_act_fn("gelu", quant_config, ffn_intermediate_size) + self.fc2 = RowParallelLinear( + ffn_intermediate_size, + ffn_hidden_size, + bias=ffn_has_bias, + quant_config=quant_config, + ) + + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata) -> torch.Tensor: + r""" + Args: + hidden_states + torch.Tensor of *encoder* input embeddings. + kv_cache: + Layer-wise list of KV cache tensors + attn_metadata: + vLLM Attention metadata structure + Returns: + Encoder layer output torch.Tensor + """ + residual = hidden_states + hidden_states = self.self_attn(hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata) + + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + residual = hidden_states + fc1_out, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(fc1_out) + + hidden_states, _ = self.fc2(hidden_states) + + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() + or torch.isnan(hidden_states).any()): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, + min=-clamp_value, + max=clamp_value) + + return hidden_states + + +class BartDecoderLayer(nn.Module): + + def __init__( + self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = BartDecoderSelfAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + config=config, + cache_config=cache_config, + quant_config=quant_config) + self.activation_fn = get_act_fn(config.activation_function, + quant_config) + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + ''' + afeldman-nm: personally I would call this "cross-attention", + however I left the name as "encoder_attn" to maintain consistency + with the name of the pretrained weights. + ''' + self.encoder_attn = BartCrossAttention( + self.embed_dim, + config.decoder_attention_heads, + config=config, + ) + self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) + + ffn_hidden_size = self.embed_dim + ffn_intermediate_size = config.encoder_ffn_dim + ffn_has_bias = True + self.fc1 = ColumnParallelLinear( + ffn_hidden_size, + ffn_intermediate_size, + bias=ffn_has_bias, + quant_config=quant_config, + ) + self.fc2 = RowParallelLinear( + ffn_intermediate_size, + ffn_hidden_size, + bias=ffn_has_bias, + quant_config=quant_config, + ) + + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + decoder_hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + encoder_hidden_states: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + r""" + Args: + decoder_hidden_states + torch.Tensor of *decoder* input embeddings. + kv_cache: + KV cache tensor + attn_metadata: + vLLM Attention metadata structure + encoder_hidden_states + torch.Tensor of *encoder* input embeddings. + Returns: + Decoder layer output torch.Tensor + """ + residual = decoder_hidden_states + + # Self Attention + hidden_states = self.self_attn(hidden_states=decoder_hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata) + + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Cross-Attention Block + + residual = hidden_states + + hidden_states = self.encoder_attn( + decoder_hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + encoder_hidden_states=encoder_hidden_states, + ) + + hidden_states = residual + hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + + # Fully Connected + residual = hidden_states + fc1_out, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(fc1_out) + + hidden_states, _ = self.fc2(hidden_states) + + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + return hidden_states + + +class BartEncoder(nn.Module): + """ + Transformer encoder consisting of *config.encoder_layers* + self attention layers. Each layer is a [`BartEncoderLayer`]. + Args: + config: BartConfig + embed_tokens (nn.Embedding): output embedding + """ + + def __init__(self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + embed_tokens: Optional[nn.Embedding] = None): + super().__init__() + + self.cache_config = cache_config + self.quant_config = quant_config + self.lora_config = lora_config + embed_dim = config.d_model + self.max_source_positions = config.max_position_embeddings + embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 + + self.embed_tokens = BartScaledWordEmbedding(config.vocab_size, + embed_dim, + embed_scale=embed_scale) + + if embed_tokens is not None: + self.embed_tokens.weight = embed_tokens.weight + + self.embed_positions = BartLearnedPositionalEmbedding( + config.max_position_embeddings, + embed_dim, + ) + self.layers = nn.ModuleList( + [BartEncoderLayer(config,cache_config,quant_config) \ + for _ in range(config.encoder_layers)]) + + self.layernorm_embedding = nn.LayerNorm(embed_dim) + + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata) -> torch.Tensor: + r""" + Args: + input_ids + Indices of *encoder* input sequence tokens in the vocabulary. + Padding will be ignored by default should you + provide it. + positions + Positions of *encoder* input sequence tokens. + kv_caches: + Layer-wise list of KV cache tensors + attn_metadata: + vLLM Attention metadata structure + Returns: + Decoder output torch.Tensor + """ + # retrieve input_ids and inputs_embeds + + input_ids = input_ids.view(-1, input_ids.shape[-1]) + inputs_embeds = self.embed_tokens(input_ids) + + embed_pos = self.embed_positions( + positions, + AttentionType.ENCODER, + ) + embed_pos = embed_pos.to(inputs_embeds.device) + + hidden_states = inputs_embeds + embed_pos + hidden_states = self.layernorm_embedding(hidden_states) + + for idx, encoder_layer in enumerate(self.layers): + hidden_states = encoder_layer( + hidden_states=hidden_states, + kv_cache=kv_caches[idx], + attn_metadata=attn_metadata, + ) + + return hidden_states + + +class BartDecoder(nn.Module): + """ + Transformer decoder consisting of *config.decoder_layers* layers. + Each layer is a [`BartDecoderLayer`] + Args: + config: BartConfig + embed_tokens (nn.Embedding): output embedding + """ + + def __init__( + self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + embed_tokens: Optional[nn.Embedding] = None, + ): + super().__init__() + self.cache_config = cache_config + self.quant_config = quant_config + self.lora_config = lora_config + self.max_target_positions = config.max_position_embeddings + embed_scale = math.sqrt( + config.d_model) if config.scale_embedding else 1.0 + + self.embed_tokens = BartScaledWordEmbedding(config.vocab_size, + config.d_model, + embed_scale=embed_scale) + + if embed_tokens is not None: + self.embed_tokens.weight = embed_tokens.weight + + self.embed_positions = BartLearnedPositionalEmbedding( + config.max_position_embeddings, + config.d_model, + ) + + self.layers = nn.ModuleList( + [BartDecoderLayer(config,cache_config,quant_config) \ + for _ in range(config.decoder_layers)]) + + self.layernorm_embedding = nn.LayerNorm(config.d_model) + + def forward(self, decoder_input_ids: torch.Tensor, + decoder_positions: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata) -> torch.Tensor: + r""" + Args: + decoder_input_ids + Indices of *decoder* input sequence tokens in the vocabulary. + Padding will be ignored by default should you + provide it. + decoder_positions + Positions of *decoder* input sequence tokens. + encoder_hidden_states: + Tensor of encoder output embeddings + kv_caches: + Layer-wise list of KV cache tensors + attn_metadata: + vLLM Attention metadata structure + Returns: + Decoder output torch.Tensor + """ + + inputs_embeds = self.embed_tokens(decoder_input_ids) + + # embed positions + embed_pos = self.embed_positions( + decoder_positions, + AttentionType.DECODER, + ) + embed_pos = embed_pos.to(inputs_embeds.device) + + hidden_states = inputs_embeds + embed_pos + hidden_states = self.layernorm_embedding(hidden_states) + + # decoder layers + + for idx, decoder_layer in enumerate(self.layers): + hidden_states = decoder_layer( + decoder_hidden_states=hidden_states, + kv_cache=kv_caches[idx], + attn_metadata=attn_metadata, + encoder_hidden_states=encoder_hidden_states, + ) + + return hidden_states + + +class BartModel(nn.Module): + _tied_weights_keys = [ + "encoder.embed_tokens.weight", "decoder.embed_tokens.weight" + ] + + def __init__(self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None): + super().__init__() + + self.config = config + + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.encoder = BartEncoder(config, + cache_config, + quant_config=quant_config) + self.decoder = BartDecoder(config, + cache_config, + quant_config=quant_config) + + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor, + encoder_input_ids: torch.Tensor, + encoder_positions: torch.Tensor, kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata) -> torch.Tensor: + r""" + Args: + input_ids + Indices of *decoder* input sequence tokens in the vocabulary. + Padding will be ignored by default should you + provide it. + positions + Positions of *decoder* input sequence tokens. + encoder_input_ids + Indices of *encoder* input sequence tokens in the vocabulary. + encoder_positions: + Positions of *encoder* input sequence tokens. + kv_caches: + Layer-wise list of KV cache tensors + attn_metadata: + vLLM Attention metadata structure + Returns: + Model output torch.Tensor + """ + + encoder_hidden_states = None + + if encoder_input_ids.numel() > 0: + # Run encoder attention if a non-zero number of encoder tokens + # are provided as input + encoder_hidden_states = self.encoder(input_ids=encoder_input_ids, + positions=encoder_positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata) + + # decoder outputs consists of + # (dec_features, past_key_value, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + decoder_input_ids=input_ids, + decoder_positions=positions, + encoder_hidden_states=encoder_hidden_states, + kv_caches=kv_caches, + attn_metadata=attn_metadata) + + return decoder_outputs + + +class BartForConditionalGeneration(nn.Module): + base_model_prefix = "model" + + def __init__(self, + config: BartConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None): + + super().__init__() + # currently all existing BART models have `tie_word_embeddings` enabled + assert config.tie_word_embeddings + self.config = config + self.model = BartModel(config, + cache_config, + quant_config, + lora_config=lora_config) + + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + + embed_scale = math.sqrt( + config.d_model) if config.scale_embedding else 1.0 + + self.lm_head = BartParallelLMHead(config.vocab_size, + config.d_model, + embed_scale=embed_scale) + + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + *, + encoder_input_ids: torch.Tensor, + encoder_positions: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + r""" + Args: + input_ids + torch.Tensor of *decoder* input token ids. + positions + torch.Tensor of *decoder* position indices. + encoder_input_ids + torch.Tensor of *encoder* input token ids. + encoder_positions + torch.Tensor of *encoder* position indices + kv_caches: + Layer-wise list of KV cache tensors + attn_metadata: + vLLM Attention metadata structure + Returns: + Output torch.Tensor + """ + return self.model(input_ids, positions, encoder_input_ids, + encoder_positions, kv_caches, attn_metadata) + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + stacked_params_mapping = { + "q_proj": { + "param_name": "qkv_proj", + "shard_id": "q", + }, + "k_proj": { + "param_name": "qkv_proj", + "shard_id": "k", + }, + "v_proj": { + "param_name": "qkv_proj", + "shard_id": "v", + }, + } + + params_mapping = { + "beta": "bias", + "gamma": "weight", + "LayerNorm": "layernorm", + } + + def _rename_key(self, key: str): + prefix = f"{self.base_model_prefix}." + key = key[len(prefix):] if key.startswith(prefix) else key + + for src, dst in self.params_mapping.items(): + key = key.replace(src, dst) + + return key + + def _rename_stacked_param( + self, + name: str, + ) -> Tuple[str, Optional[str]]: + for key, mapping in self.stacked_params_mapping.items(): + if key in name: + name = name.replace(key, mapping["param_name"]) + return name, mapping["shard_id"] + return name, None + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + model_params_dict = dict(self.model.named_parameters()) + top_params_dict = dict(self.named_parameters()) + + weights_tuple_list = list(weights) + + shared_embedding_weight = None + shared_embedding_shard_id = None + + for name, loaded_weight in weights_tuple_list: + + name = self._rename_key(name) + name, shard_id = self._rename_stacked_param(name) + + if ('shared.weight' in name + or 'encoder.embed_tokens.weight' in name + or 'decoder.embed_tokens.weight' in name + or 'lm_head.weight' in name): + assert shared_embedding_weight is None, ( + "Conflicting embedding weights.") + shared_embedding_weight = loaded_weight + shared_embedding_shard_id = shard_id + else: + # Skip the specific downstream task weight. + if name.startswith('cls.'): + continue + # use Pooler instead. + if name.startswith('pooler.'): + continue + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in model_params_dict: + continue + + param = model_params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + if shard_id: + weight_loader(param, loaded_weight, shard_id) + else: + weight_loader(param, loaded_weight) + + # Assign shared weight values + encoder_in_param = model_params_dict['encoder.embed_tokens.weight'] + encoder_in_weight_loader = getattr(encoder_in_param, "weight_loader", + default_weight_loader) + + decoder_in_param = model_params_dict['decoder.embed_tokens.weight'] + decoder_in_weight_loader = getattr(decoder_in_param, "weight_loader", + default_weight_loader) + + lm_head_in_param = top_params_dict['lm_head.weight'] + lm_head_in_weight_loader = getattr(lm_head_in_param, "weight_loader", + default_weight_loader) + + assert shared_embedding_weight is not None + + if shared_embedding_shard_id: + encoder_in_weight_loader(encoder_in_param, shared_embedding_weight, + shared_embedding_shard_id) + decoder_in_weight_loader(decoder_in_param, shared_embedding_weight, + shared_embedding_shard_id) + lm_head_in_weight_loader(lm_head_in_param, shared_embedding_weight, + shared_embedding_shard_id) + else: + encoder_in_weight_loader(encoder_in_param, shared_embedding_weight) + decoder_in_weight_loader(decoder_in_param, shared_embedding_weight) + lm_head_in_weight_loader(lm_head_in_param, shared_embedding_weight) diff --git a/vllm/model_executor/models/blip.py b/vllm/model_executor/models/blip.py new file mode 100644 index 00000000..228908d4 --- /dev/null +++ b/vllm/model_executor/models/blip.py @@ -0,0 +1,423 @@ +"""Minimal implementation of BlipVisionModel intended to be only used +within a vision language model.""" +from typing import Iterable, Optional, Tuple, Union + +import torch +import torch.nn as nn +from PIL import Image +from transformers import Blip2VisionConfig, BlipVisionConfig +from transformers.models.blip.modeling_blip import BlipAttention + +from vllm.config import ModelConfig +from vllm.distributed import divide, get_tensor_model_parallel_world_size +from vllm.inputs import LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import SequenceData + +try: + from xformers import ops as xops + USE_XFORMERS_OPS = True +except ImportError: + USE_XFORMERS_OPS = False + +USE_XFORMERS_OPS = True + +def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int: + assert image_size % patch_size == 0 + return image_size // patch_size + + +def get_blip_num_patches(*, image_size: int, patch_size: int) -> int: + grid_length = get_blip_patch_grid_length(image_size=image_size, + patch_size=patch_size) + return grid_length * grid_length + + +def get_blip_image_feature_size( + hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int: + return get_blip_num_patches(image_size=hf_config.image_size, + patch_size=hf_config.patch_size) + + +def get_max_blip_image_tokens( + hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int: + return get_blip_image_feature_size(hf_config) + + +def dummy_seq_data_for_blip( + hf_config: Union[BlipVisionConfig, Blip2VisionConfig], + seq_len: int, + num_images: int, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + if image_feature_size_override is None: + image_feature_size = get_blip_image_feature_size(hf_config) + else: + image_feature_size = image_feature_size_override + + return SequenceData.from_token_counts( + (image_token_id, image_feature_size * num_images), + (0, seq_len - image_feature_size * num_images), + ) + + +def dummy_image_for_blip( + hf_config: Union[BlipVisionConfig, Blip2VisionConfig], + num_images: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + width = height = hf_config.image_size + if image_width_override is not None: + width = image_width_override + if image_height_override is not None: + height = image_height_override + + image = Image.new("RGB", (width, height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def input_processor_for_blip( + model_config: ModelConfig, + hf_config: Union[BlipVisionConfig, Blip2VisionConfig], + llm_inputs: LLMInputs, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + if image_feature_size_override is None: + image_feature_size = get_blip_image_feature_size(hf_config) + else: + image_feature_size = image_feature_size_override + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=image_token_id, + repeat_count=image_feature_size, + ) + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa +class BlipVisionEmbeddings(nn.Module): + + def __init__(self, config: BlipVisionConfig): + super().__init__() + + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) + + self.patch_embedding = nn.Conv2d( + in_channels=3, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + ) + + self.num_patches = get_blip_num_patches(image_size=self.image_size, + patch_size=self.patch_size) + self.num_positions = self.num_patches + 1 + + self.position_embedding = nn.Parameter( + torch.randn(1, self.num_positions, self.embed_dim)) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to( + dtype=target_dtype)) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + + position_embeds = self.position_embedding.to(target_dtype) + embeddings = embeddings + position_embeds[:, :embeddings.size(1), :] + + return embeddings + + +class BlipParallelAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: BlipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + "embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads}).") + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + self.qkv = QKVParallelLinear( + self.embed_dim, + self.head_dim, + self.num_heads, + bias=config.qkv_bias, + quant_config=quant_config, + ) + self.projection = RowParallelLinear( + self.embed_dim, + self.embed_dim, + quant_config=quant_config, + ) + + self.tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = divide(self.num_heads, self.tp_size) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, + self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + ): + """Input shape: Batch x Time x Channel""" + bsz, tgt_len, _ = hidden_states.size() + + qkv_states, _ = self.qkv(hidden_states) + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1) + query_states = query_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + key_states = key_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + value_states = value_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + query_states = query_states.transpose(1,2) + key_states = key_states.transpose(1,2) + value_states = value_states.transpose(1,2) + attn = (query_states @ (key_states * self.scale).permute(0,1,3,2)).float() + attn = torch.softmax(attn, dim=-1).to(value_states.dtype) + out = attn @ value_states + out = out.transpose(1,2).reshape(bsz, tgt_len, -1) + # TODO: use use F.attention when supported + # out = xops.memory_efficient_attention_forward(query_states, + # key_states, + # value_states, + # p=self.dropout, + # scale=self.scale) + # out = out.view(bsz, tgt_len, -1) + attn_output, _ = self.projection(out) + + return attn_output, None + + +class BlipMLP(nn.Module): + + def __init__(self, + config: BlipVisionConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + + self.config = config + + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear(config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config) + self.fc2 = RowParallelLinear(config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + + return hidden_states + + +class BlipEncoderLayer(nn.Module): + + def __init__(self, + config: BlipVisionConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + + # fallback to sdpa attention if tp unavailable + num_heads = config.num_attention_heads + tp_size = get_tensor_model_parallel_world_size() + if USE_XFORMERS_OPS and num_heads % tp_size == 0: + self.self_attn = BlipParallelAttention(config, + quant_config=quant_config) + else: + # Blip doesn't have SDPA attention implemented in transformers + # use eager attention instead for cpu backend + self.self_attn = BlipAttention(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.mlp = BlipMLP(config, quant_config=quant_config) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn(hidden_states=hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class BlipEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self + attention layers. Each layer is a [`BlipEncoderLayer`]. + + Args: + config: BlipConfig + """ + + def __init__(self, + config: BlipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None): + super().__init__() + + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + + self.layers = nn.ModuleList([ + BlipEncoderLayer(config=config, quant_config=quant_config) + for _ in range(num_hidden_layers) + ]) + + def forward(self, inputs_embeds: torch.Tensor): + hidden_states = inputs_embeds + for encoder_layer in self.layers: + hidden_states = encoder_layer(hidden_states) + + return hidden_states + + +class BlipVisionModel(nn.Module): + config_class = BlipVisionConfig + main_input_name = "pixel_values" + + def __init__(self, + config: BlipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None): + super().__init__() + + tp_size = get_tensor_model_parallel_world_size() + num_heads = config.num_attention_heads + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0 + + self.config = config + + self.embeddings = BlipVisionEmbeddings(config) + self.encoder = BlipEncoder( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + ) + + if len(self.encoder.layers) > config.num_hidden_layers: + raise ValueError( + f"The original encoder only has {config.num_hidden_layers} " + f"layers, but you requested {len(self.encoder.layers)} layers." + ) + elif len(self.encoder.layers) == config.num_hidden_layers: + self.post_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + else: + # post_layernorm is unused when we extract intermediate features + # In this case, we can skip it to conserve memory + self.post_layernorm = None + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + hidden_states = self.embeddings(pixel_values) + hidden_states = self.encoder(inputs_embeds=hidden_states) + + if self.post_layernorm is None: + return hidden_states + + return self.post_layernorm(hidden_states) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] if self.shard_weight else [] + params_dict = dict(self.named_parameters()) + layer_count = len(self.encoder.layers) + + for name, loaded_weight in weights: + # post_layernorm is not needed in BlipVisionModel + if (name.startswith("post_layernorm") + and self.post_layernorm is None): + continue + + # omit layers when num_hidden_layers_override is set + if name.startswith("encoder.layers"): + layer_idx = int(name.split(".")[2]) + if layer_idx >= layer_count: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + + param = params_dict[name.replace(weight_name, param_name)] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py new file mode 100644 index 00000000..3ab23575 --- /dev/null +++ b/vllm/model_executor/models/blip2.py @@ -0,0 +1,690 @@ +from functools import cached_property +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +import torch.nn as nn +from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig, + apply_chunking_to_forward) + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import IntermediateTensors, SequenceData + +from .blip import (BlipVisionModel, dummy_image_for_blip, + get_max_blip_image_tokens) +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (AutoWeightsLoader, init_vllm_registered_model, + merge_multimodal_embeddings) + +# We use this internally as placeholders since there is no image token +# defined on the HuggingFace repo +BLIP2_IMAGE_TOKEN = "" +BLIP2_IMAGE_TOKEN_ID = 50265 + + +class Blip2ImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: `(batch_size * num_images, num_channels, height, width)`""" + + +class Blip2ImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +Blip2ImageInputs = Union[Blip2ImagePixelInputs, Blip2ImageEmbeddingInputs] + + +class Blip2QFormerMultiHeadAttention(nn.Module): + + def __init__( + self, + config: Blip2QFormerConfig, + *, + quant_config: Optional[QuantizationConfig], + cache_config: Optional[CacheConfig], + is_cross_attention: bool = False, + ) -> None: + super().__init__() + + self.config = config + + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({config.hidden_size}) is not a multiple of " + f"the number of attention heads ({config.num_attention_heads})" + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = (config.hidden_size // + config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + self.scaling = self.attention_head_size**-0.5 + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + kv_hidden_size = config.encoder_hidden_size + else: + kv_hidden_size = config.hidden_size + self.key = nn.Linear(kv_hidden_size, self.all_head_size) + self.value = nn.Linear(kv_hidden_size, self.all_head_size) + + self.position_embedding_type = getattr(config, + "position_embedding_type", + "absolute") + if self.position_embedding_type != "absolute": + raise NotImplementedError("Unsupported position_embedding_type: " + f"{self.position_embedding_type}") + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + x = x.view(*x.size()[:-1], self.num_attention_heads, + self.attention_head_size) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + ): + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + mixed_query_layer = self.query(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_probs = torch.softmax(attention_scores * self.scaling, + dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + context_layer = context_layer.view(*context_layer.size()[:-2], + self.all_head_size) + + return context_layer + + +class Blip2QFormerSelfOutput(nn.Module): + + def __init__(self, config: Blip2QFormerConfig) -> None: + super().__init__() + + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward( + self, + hidden_states: torch.Tensor, + input_tensor: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class Blip2QFormerAttention(nn.Module): + + def __init__( + self, + config: Blip2QFormerConfig, + *, + quant_config: Optional[QuantizationConfig], + cache_config: Optional[CacheConfig], + is_cross_attention: bool = False, + ) -> None: + super().__init__() + + self.attention = Blip2QFormerMultiHeadAttention( + config, + quant_config=quant_config, + cache_config=cache_config, + is_cross_attention=is_cross_attention, + ) + + self.output = Blip2QFormerSelfOutput(config) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.Tensor]: + self_output = self.attention( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + ) + attention_output = self.output(self_output, hidden_states) + + return attention_output + + +class Blip2QFormerIntermediate(nn.Module): + + def __init__(self, config: Blip2QFormerConfig) -> None: + super().__init__() + + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + self.intermediate_act_fn = get_act_fn(config.hidden_act) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class Blip2QFormerOutput(nn.Module): + + def __init__(self, config: Blip2QFormerConfig) -> None: + super().__init__() + + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward( + self, + hidden_states: torch.Tensor, + input_tensor: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class Blip2QFormerLayer(nn.Module): + + def __init__( + self, + config: Blip2QFormerConfig, + *, + quant_config: Optional[QuantizationConfig], + cache_config: Optional[CacheConfig], + layer_idx: int, + ) -> None: + super().__init__() + + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = Blip2QFormerAttention(config, + quant_config=quant_config, + cache_config=cache_config) + + self.layer_idx = layer_idx + + if layer_idx % config.cross_attention_frequency == 0: + self.crossattention = Blip2QFormerAttention( + config, + quant_config=quant_config, + cache_config=cache_config, + is_cross_attention=True) + self.has_cross_attention = True + else: + self.has_cross_attention = False + + self.intermediate_query = Blip2QFormerIntermediate(config) + self.output_query = Blip2QFormerOutput(config) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: torch.FloatTensor, + query_length: int, + ): + attention_output = self.attention(hidden_states) + + if query_length > 0: + query_attention_output = attention_output[:, :query_length, :] + + if self.has_cross_attention: + query_attention_output = self.crossattention( + query_attention_output, + encoder_hidden_states=encoder_hidden_states, + ) + + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk_query, + self.chunk_size_feed_forward, + self.seq_len_dim, + query_attention_output, + ) + + if attention_output.shape[1] > query_length: + layer_output_text = apply_chunking_to_forward( + self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output[:, query_length:, :], + ) + layer_output = torch.cat([layer_output, layer_output_text], + dim=1) + else: + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output, + ) + + return layer_output + + def feed_forward_chunk(self, + attention_output: torch.Tensor) -> torch.Tensor: + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + def feed_forward_chunk_query( + self, attention_output: torch.Tensor) -> torch.Tensor: + intermediate_output = self.intermediate_query(attention_output) + layer_output = self.output_query(intermediate_output, attention_output) + return layer_output + + +class Blip2QFormerEncoder(nn.Module): + + def __init__( + self, + config: Blip2QFormerConfig, + *, + quant_config: Optional[QuantizationConfig], + cache_config: Optional[CacheConfig], + ) -> None: + super().__init__() + + self.config = config + + self.layer = nn.ModuleList([ + Blip2QFormerLayer(config, + quant_config=quant_config, + cache_config=cache_config, + layer_idx=layer_idx) + for layer_idx in range(config.num_hidden_layers) + ]) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: torch.FloatTensor, + query_length: int, + ) -> torch.Tensor: + for i in range(self.config.num_hidden_layers): + layer_module = self.layer[i] + + hidden_states = layer_module( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + query_length=query_length, + ) + + return hidden_states + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1025 +class Blip2QFormerModel(nn.Module): + + def __init__( + self, + config: Blip2QFormerConfig, + *, + quant_config: Optional[QuantizationConfig], + cache_config: Optional[CacheConfig], + ) -> None: + super().__init__() + + self.config = config + + self.layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + self.encoder = Blip2QFormerEncoder(config, + quant_config=quant_config, + cache_config=cache_config) + + def forward( + self, + query_embeds: torch.FloatTensor, + encoder_hidden_states: torch.FloatTensor, + ) -> torch.Tensor: + query_length = query_embeds.shape[1] + + embedding_output = self.layernorm(query_embeds) + embedding_output = self.dropout(embedding_output) + + sequence_output = self.encoder( + embedding_output, + encoder_hidden_states=encoder_hidden_states, + query_length=query_length, + ) + + return sequence_output + + +def get_blip2_image_feature_size(hf_config: Blip2Config) -> int: + return hf_config.num_query_tokens + + +def get_max_blip2_image_tokens(ctx: InputContext): + hf_config = ctx.get_hf_config(Blip2Config) + vision_config = hf_config.vision_config + + if isinstance(vision_config, Blip2VisionConfig): + return get_max_blip_image_tokens(vision_config) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def dummy_seq_data_for_blip2( + hf_config: Blip2Config, + seq_len: int, + num_images: int, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + if image_feature_size_override is None: + image_feature_size = get_blip2_image_feature_size(hf_config) + else: + image_feature_size = image_feature_size_override + + return SequenceData.from_token_counts( + (image_token_id, image_feature_size * num_images), + (0, seq_len - image_feature_size * num_images), + ) + + +def dummy_data_for_blip2(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(Blip2Config) + vision_config = hf_config.vision_config + num_images = mm_counts["image"] + + seq_data = dummy_seq_data_for_blip2( + hf_config, + seq_len, + num_images, + image_token_id=BLIP2_IMAGE_TOKEN_ID, + ) + + if isinstance(vision_config, Blip2VisionConfig): + mm_data = dummy_image_for_blip(vision_config, num_images) + + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + hf_config = ctx.get_hf_config(Blip2Config) + image_feature_size = get_blip2_image_feature_size(hf_config) + + # The original model places image tokens at the front + # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514 + new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size + new_token_ids += llm_inputs["prompt_token_ids"] + + new_prompt = llm_inputs.get("prompt") + if new_prompt is not None: + new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt + + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2) +@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2) +class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: Blip2Config, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + # TODO: Optionally initializes this for supporting embeddings. + self.vision_model = BlipVisionModel(config.vision_config) + + self.query_tokens = nn.Parameter( + torch.zeros(1, config.num_query_tokens, + config.qformer_config.hidden_size)) + + self.qformer = Blip2QFormerModel(config.qformer_config, + cache_config=cache_config, + quant_config=quant_config) + + self.language_projection = nn.Linear( + config.qformer_config.hidden_size, + config.text_config.hidden_size, + bias=True, + ) + + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + actual_dims = tuple(data.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("batch_size", *map(str, expected_dims)) + raise ValueError( + f"The expected shape of pixel values is {expected_expr}. " + f"You supplied {tuple(data.shape)}.") + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[Blip2ImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, torch.Tensor): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + # Remove the N dimension until multiple images are supported. + pixel_values = pixel_values.squeeze(1) + + return Blip2ImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values(pixel_values), + ) + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + # Remove the N dimension until multiple images are supported. + image_embeds = image_embeds.squeeze(1) + + return Blip2ImageEmbeddingInputs( + type="image_embeds", + data=image_embeds, + ) + + raise AssertionError("This line should be unreachable.") + + def _image_pixels_to_features(self, vision_model: BlipVisionModel, + pixel_values: torch.Tensor) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + image_features = vision_model(pixel_values) + + return image_features + + def _process_image_pixels(self, + inputs: Blip2ImagePixelInputs) -> torch.Tensor: + assert self.vision_model is not None + + pixel_values = inputs["data"] + + return self._image_pixels_to_features(self.vision_model, pixel_values) + + def _process_image_input(self, + image_input: Blip2ImageInputs) -> torch.Tensor: + + if image_input["type"] == "image_embeds": + return image_input["data"] + + assert self.vision_model is not None + image_features = self._process_image_pixels(image_input) + + query_tokens = self.query_tokens.expand(image_features.shape[0], -1, + -1) + query_output = self.qformer( + query_embeds=query_tokens, + encoder_hidden_states=image_features, + ) + + return self.language_projection(query_output) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[SamplerOutput, IntermediateTensors]: + """Run forward pass for BLIP-2. + + One key thing to understand is the `input_ids` already accounts for the + positions of the to-be-inserted image embeddings. + + Concretely, consider a text prompt: + `"Question: What's the content of the image? Answer:"`. + + Tokenizer outputs: + `[2, 45641, 35, 653, 18, 5, 1383, 9, 5, 2274, 116, 31652, 35]`. + + To reserve space in KV cache, we have to insert placeholder tokens + before they are inputted to the model, so the input processor prepends + dummy tokens (denoted as `50265`), resulting in: + `[50265, ..., 50265, 2, 45641, 35, ..., 31652, 35]`. + + We insert 32 tokens since it corresponds to the number of query + embeddings outputted by the Q-Former and inputted to the language model. + + This way, the `positions` and `attn_metadata` are consistent + with the `input_ids`. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + pixel_values: The pixels in each input image. + + See also: + :class:`Blip2ImageInputs` + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + BLIP2_IMAGE_TOKEN_ID) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model( + input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/bloom.py b/vllm/model_executor/models/bloom.py new file mode 100644 index 00000000..b2c9e221 --- /dev/null +++ b/vllm/model_executor/models/bloom.py @@ -0,0 +1,362 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py +# Copyright 2023 The vLLM team. +# Copyright 2022 HuggingFace Inc. team and BigScience workshop. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only BLOOM model compatible with HuggingFace weights.""" +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import BloomConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor: + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads)) + base = torch.tensor( + 2**(-(2**-(math.log2(closest_power_of_2) - 3))), + dtype=torch.float32, + ) + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32) + slopes = torch.pow(base, powers) + + if closest_power_of_2 != total_num_heads: + extra_base = torch.tensor( + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))), + dtype=torch.float32, + ) + num_remaining_heads = min(closest_power_of_2, + total_num_heads - closest_power_of_2) + extra_powers = torch.arange(start=1, + end=1 + 2 * num_remaining_heads, + step=2, + dtype=torch.int32) + slopes = torch.cat( + [slopes, torch.pow(extra_base, extra_powers)], dim=0) + return slopes + + +class BloomAttention(nn.Module): + + def __init__( + self, + config: BloomConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.total_num_heads = config.n_head + self.head_dim = self.hidden_size // self.total_num_heads + assert self.head_dim * self.total_num_heads == self.hidden_size + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + self.query_key_value = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + bias=True, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config, + ) + + # Create the alibi slopes and slice them. + tp_rank = get_tensor_model_parallel_rank() + head_start = tp_rank * self.num_heads + head_end = (tp_rank + 1) * self.num_heads + alibi_slopes = _get_alibi_slopes(self.total_num_heads) + alibi_slopes = alibi_slopes[head_start:head_end].tolist() + + scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + scaling, + alibi_slopes=alibi_slopes, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + del position_ids # Unused. + qkv, _ = self.query_key_value(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.dense(attn_output) + return output + + +class BloomMLP(nn.Module): + + def __init__( + self, + config: BloomConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + self.dense_h_to_4h = ColumnParallelLinear( + hidden_size, + 4 * hidden_size, + quant_config=quant_config, + ) + self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size) + self.dense_4h_to_h = RowParallelLinear( + 4 * hidden_size, + hidden_size, + quant_config=quant_config, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.dense_h_to_4h(x) + x = self.gelu_impl(x) + x, _ = self.dense_4h_to_h(x) + return x + + +class BloomBlock(nn.Module): + + def __init__( + self, + config: BloomConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + + self.input_layernorm = nn.LayerNorm(hidden_size, + eps=config.layer_norm_epsilon) + self.self_attention = BloomAttention(config, cache_config, + quant_config) + self.post_attention_layernorm = nn.LayerNorm( + hidden_size, eps=config.layer_norm_epsilon) + self.mlp = BloomMLP(config, quant_config) + self.apply_residual_connection_post_layernorm = ( + config.apply_residual_connection_post_layernorm) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + + # Layer norm post the self attention. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + # Self attention. + attention_output = self.self_attention( + position_ids=position_ids, + hidden_states=layernorm_output, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + attention_output = attention_output + residual + layernorm_output = self.post_attention_layernorm(attention_output) + + # Get residual + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = attention_output + + # MLP. + output = self.mlp(layernorm_output) + residual + return output + + +class BloomModel(nn.Module): + + def __init__( + self, + config: BloomConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.embed_dim = config.hidden_size + + # Embedding + LN Embedding + self.word_embeddings = VocabParallelEmbedding( + config.vocab_size, + self.embed_dim, + ) + self.word_embeddings_layernorm = nn.LayerNorm( + self.embed_dim, eps=config.layer_norm_epsilon) + + # Transformer blocks + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: BloomBlock(config, cache_config, quant_config), + prefix=f"{prefix}.h") + + # Final Layer Norm + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.word_embeddings(input_ids) + hidden_states = self.word_embeddings_layernorm(hidden_states) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer( + position_ids, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class BloomForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: BloomConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.transformer = BloomModel(config, cache_config, quant_config) + if self.config.tie_word_embeddings: + self.lm_head = self.transformer.word_embeddings + else: + self.lm_head = ParallelLMHead(self.config.vocab_size, + self.config.hidden_size) + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if name == "lm_head.weight": + continue + if not name.startswith("transformer."): + name = "transformer." + name + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + + if "query_key_value" in name: + # NOTE: BLOOM's fused QKV's output_dim has the shape of + # (num_heads * 3 * head_size), while the + # required shape is (3 * num_heads * head_size). + # Thus, we need weight conversion. + output_dim = getattr(param, "output_dim", None) + num_heads = self.config.num_attention_heads + if output_dim is not None: + loaded_weight_shape = loaded_weight.shape + loaded_weight = loaded_weight.view( + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) + + loaded_weight_shape[output_dim + 1:]) + loaded_weight = loaded_weight.transpose( + output_dim, output_dim + 1) + loaded_weight = loaded_weight.reshape(loaded_weight_shape) + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py new file mode 100644 index 00000000..03c7419f --- /dev/null +++ b/vllm/model_executor/models/chameleon.py @@ -0,0 +1,1104 @@ +from functools import cached_property +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, + Tuple, TypedDict, Union) + +import torch +import torch.nn.functional as F +from PIL import Image +from torch import nn +from transformers import ChameleonConfig, ChameleonVQVAEConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, row_parallel_weight_loader) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import IntermediateTensors, SequenceData +from vllm.utils import print_warning_once + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +# These configs are not part of the model config but the preprocessor +# and processor files, so we hardcode them in the model file for now. +CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512 +CHAMELEON_IMAGE_SEQ_LENGTH = 1024 +CHAMELEON_IMAGE_TOKEN_ID = 8711 +CHAMELEON_IMAGE_START_TOKEN_ID = 8197 +CHAMELEON_IMAGE_END_TOKEN_ID = 8196 +CHAMELEON_SEP_TOKEN_ID = 8710 + + +class ChameleonImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: `(batch_size * num_images, num_channels, height, width)`""" + + +def get_max_chameleon_image_tokens(ctx: InputContext): + return CHAMELEON_IMAGE_SEQ_LENGTH + + +def dummy_seq_data_for_chameleon( + seq_len: int, + num_images: int, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + if image_feature_size_override is None: + image_feature_size = CHAMELEON_IMAGE_SEQ_LENGTH + else: + image_feature_size = image_feature_size_override + + return SequenceData.from_token_counts( + (image_token_id, image_feature_size * num_images), + (0, seq_len - image_feature_size * num_images), + ) + + +def dummy_image_for_chameleon( + num_images: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + width = CHAMELEON_CROP_SIZE_WIDTH + height = CHAMELEON_CROP_SIZE_HEIGHT + if image_width_override is not None: + width = image_width_override + if image_height_override is not None: + height = image_height_override + + image = Image.new("RGB", (width, height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def dummy_data_for_chameleon(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + num_images = mm_counts["image"] + + seq_data = dummy_seq_data_for_chameleon( + seq_len, + num_images, + image_token_id=CHAMELEON_IMAGE_TOKEN_ID, + ) + + mm_data = dummy_image_for_chameleon(num_images) + return seq_data, mm_data + + +def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs): + + """ + Processing input prompt to insert required tokens for image placeholder. + + See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58 + """ # noqa + + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + tokenizer = cached_get_tokenizer(model_config.tokenizer) + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID, + repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH, + pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID, + pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID, + ) + + # Appending sep token for chat mode to follow default processor + # behavior + if new_prompt is not None: + new_prompt += tokenizer.sep_token + new_token_ids += [CHAMELEON_SEP_TOKEN_ID] + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +class ChameleonLayerNorm(nn.LayerNorm): + + def __init__(self, hidden_size, *args, **kwargs): + super().__init__(hidden_size, *args, **kwargs) + self.normalized_shape = (hidden_size[-1], ) + + set_weight_attrs(self.weight, + {"weight_loader": row_parallel_weight_loader}) + set_weight_attrs(self.bias, + {"weight_loader": row_parallel_weight_loader}) + + def forward(self, hidden_states): + hidden_states = F.layer_norm(hidden_states, + self.normalized_shape, + None, + None, + eps=1e-5) + hidden_states = hidden_states * self.weight + self.bias + return hidden_states + + +# Copied from vllm.model_executor.models.llama.LlamaMLP -> ChameleonMLP +class ChameleonMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config) + self.down_proj = RowParallelLinear(input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +# Modified from vllm.model_executor.models.llama.LlamaAttention -> ChameleonAttention #noqa +class ChameleonAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 4096, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + ) + self.q_norm = ChameleonLayerNorm((self.num_heads, self.head_dim)) + self.k_norm = ChameleonLayerNorm((self.num_kv_heads, self.head_dim)) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def _apply_qk_norm(self, q: torch.Tensor, + k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + # reshape for layernorm + q = q.reshape(-1, self.num_heads, self.head_dim) + k = k.reshape(-1, self.num_kv_heads, self.head_dim) + q = self.q_norm(q) + k = self.k_norm(k) + q = q.view(*q.shape[:-2], -1) + k = k.view(*k.shape[:-2], -1) + return q, k + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self._apply_qk_norm(q, k) + + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class ChameleonDecoderLayer(nn.Module): + + def __init__( + self, + config: ChameleonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 4096) + + self.self_attn = ChameleonAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=False, + cache_config=cache_config, + ) + self.mlp = ChameleonMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + +class ChameleonSwinDecoderLayer(nn.Module): + + def __init__( + self, + config: ChameleonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 4096) + + self.self_attn = ChameleonAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=False, + cache_config=cache_config, + ) + self.mlp = ChameleonMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + + residual = hidden_states + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + hidden_states = self.input_layernorm(hidden_states) + hidden_states = hidden_states + residual + + # Fully Connected + residual = hidden_states + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states, residual + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEVectorQuantizer #noqa +class ChameleonVQVAEVectorQuantizer(nn.Module): + + def __init__(self, config: ChameleonVQVAEConfig): + super().__init__() + self.num_embeddings = config.num_embeddings + self.embedding_dim = config.embed_dim + self.beta = getattr(config, "beta", 0.25) + + self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim) + self.re_embed = self.num_embeddings + + def forward(self, hidden_state: torch.Tensor): + hidden_state = hidden_state.permute(0, 2, 3, 1).contiguous() + hidden_state_flattened = hidden_state.view(-1, self.embedding_dim) + + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + distances = ( + torch.sum(hidden_state_flattened**2, dim=1, keepdim=True) + + torch.sum(self.embedding.weight**2, dim=1) - + 2 * torch.einsum("bd,dn->bn", hidden_state_flattened, + self.embedding.weight.transpose(0, 1))) + + min_encoding_indices = torch.argmin(distances, dim=1) + hidden_state_quant = self.embedding(min_encoding_indices).view( + hidden_state.shape) + + # compute loss for embedding + loss = torch.mean((hidden_state_quant.detach() - hidden_state)** + 2) + self.beta * torch.mean( + (hidden_state_quant - hidden_state.detach())**2) + + # preserve gradients + hidden_state_quant = hidden_state + (hidden_state_quant - + hidden_state).detach() + + # reshape back to match original input shape + hidden_state_quant = hidden_state_quant.permute(0, 3, 1, + 2).contiguous() + + return hidden_state_quant, loss, min_encoding_indices + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderConvDownsample #noqa +class ChameleonVQVAEEncoderConvDownsample(nn.Module): + + def __init__(self, in_channels: int): + super().__init__() + self.conv = nn.Conv2d(in_channels, + in_channels, + kernel_size=3, + stride=2, + padding=0) + + def forward(self, hidden_states: torch.Tensor): + # no asymmetric padding in torch conv, must do it ourselves + hidden_states = F.pad(hidden_states, + pad=(0, 1, 0, 1), + mode="constant", + value=0) + hidden_states = self.conv(hidden_states) + return hidden_states + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderResnetBlock #noqa +class ChameleonVQVAEEncoderResnetBlock(nn.Module): + + def __init__( + self, + config: ChameleonVQVAEConfig, + in_channels: int, + out_channels=None, + conv_shortcut=False, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None \ + else out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = torch.nn.GroupNorm(num_groups=32, + num_channels=in_channels, + eps=1e-6, + affine=True) + self.conv1 = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + self.norm2 = torch.nn.GroupNorm(num_groups=32, + num_channels=out_channels, + eps=1e-6, + affine=True) + self.dropout = torch.nn.Dropout(config.dropout) + self.conv2 = torch.nn.Conv2d(out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, hidden_states: torch.Tensor): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states *= torch.sigmoid(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states *= torch.sigmoid(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + residual = self.conv_shortcut(residual) + else: + residual = self.nin_shortcut(residual) + + return residual + hidden_states + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderAttnBlock #noqa +class ChameleonVQVAEEncoderAttnBlock(nn.Module): + + def __init__(self, in_channels: int): + super().__init__() + self.in_channels = in_channels + + self.norm = torch.nn.GroupNorm(num_groups=32, + num_channels=in_channels, + eps=1e-6, + affine=True) + self.q = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.k = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.v = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.proj_out = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, hidden_states: torch.Tensor): + residual = hidden_states + hidden_states = self.norm(hidden_states) + query_states = self.q(hidden_states) + key_states = self.k(hidden_states) + value_states = self.v(hidden_states) + + # compute attention + batch_size, channels, height, width = query_states.shape + query_states = query_states.reshape(batch_size, channels, + height * width).permute(0, 2, 1) + key_states = key_states.reshape(batch_size, channels, height * width) + attn_weights = torch.bmm(query_states, key_states) + attn_weights = attn_weights * (int(channels)**(-0.5)) + attn_weights = F.softmax(attn_weights, dim=2) + + # attend to values + value_states = value_states.reshape(batch_size, channels, + height * width) + attn_weights = attn_weights.permute(0, 2, 1) + attn_output = torch.bmm(value_states, + attn_weights).reshape(batch_size, channels, + height, width) + + attn_output = self.proj_out(attn_output) + return residual + attn_output + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoder #noqa +class ChameleonVQVAEEncoder(nn.Module): + + def __init__(self, config: ChameleonVQVAEConfig): + super().__init__() + + self.num_resolutions = len(config.channel_multiplier) + self.num_res_blocks = config.num_res_blocks + base_channels = config.base_channels + resolution = config.resolution + in_channels = config.in_channels + double_latent = config.double_latent + latent_channels = config.latent_channels + channel_multiplier = config.channel_multiplier + + self.conv_in = torch.nn.Conv2d(in_channels, + base_channels, + kernel_size=3, + stride=1, + padding=1) + + curr_res = resolution + in_channel_multiplier = (1, ) + tuple(channel_multiplier) + self.in_channel_multiplier = in_channel_multiplier + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = base_channels * in_channel_multiplier[i_level] + block_out = base_channels * channel_multiplier[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ChameleonVQVAEEncoderResnetBlock( + config=config, + in_channels=block_in, + out_channels=block_out, + )) + block_in = block_out + if (config.attn_resolutions is not None + and curr_res in config.attn_resolutions + and config.attn_type == "vanilla"): + attn.append(ChameleonVQVAEEncoderAttnBlock(block_in)) + + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = ChameleonVQVAEEncoderConvDownsample(block_in) + curr_res = curr_res // 2 + self.down.append(down) + + self.mid = nn.Module() + self.mid.block_1 = ChameleonVQVAEEncoderResnetBlock( + config=config, + in_channels=block_in, + out_channels=block_in, + ) + self.mid.attn_1 = ChameleonVQVAEEncoderAttnBlock( + block_in) if config.attn_type == "vanilla" else nn.Identity() + self.mid.block_2 = ChameleonVQVAEEncoderResnetBlock( + config=config, + in_channels=block_in, + out_channels=block_in, + ) + + self.norm_out = torch.nn.GroupNorm(num_groups=32, + num_channels=block_in, + eps=1e-6, + affine=True) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * latent_channels if double_latent else latent_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, pixel_values: torch.Tensor): + pixel_values = pixel_values.to(self.conv_in.weight.dtype) + + # downsampling + hidden_states = [self.conv_in(pixel_values)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + hidden_state = self.down[i_level].block[i_block]( + hidden_states[-1], ) + if len(self.down[i_level].attn) > 0: + hidden_state = self.down[i_level].attn[i_block]( + hidden_state) + hidden_states.append(hidden_state) + if i_level != self.num_resolutions - 1: + hidden_states.append(self.down[i_level].downsample( + hidden_states[-1])) + + # middle + last_hidden_state = hidden_states[-1] + last_hidden_state = self.mid.block_1(last_hidden_state) + last_hidden_state = self.mid.attn_1(last_hidden_state) + last_hidden_state = self.mid.block_2(last_hidden_state) + + # end + last_hidden_state = self.norm_out(last_hidden_state) + last_hidden_state *= torch.sigmoid(last_hidden_state) + last_hidden_state = self.conv_out(last_hidden_state) + return last_hidden_state + + +# Adapted from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAE #noqa +class ChameleonVQVAE(nn.Module): + + def __init__(self, config: ChameleonVQVAEConfig): + super().__init__() + self.encoder = ChameleonVQVAEEncoder(config) + self.quantize = ChameleonVQVAEVectorQuantizer(config) + self.quant_conv = torch.nn.Conv2d(config.latent_channels, + config.embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(config.embed_dim, + config.latent_channels, 1) + self.eval() # Chameleon's VQ model is frozen + + def encode( + self, pixel_values: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quant, emb_loss, indices = self.quantize(hidden_states) + return quant, emb_loss, indices + + +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonImageVocabularyMapping #noqa +class ChameleonImageVocabularyMapping: + """ + A class for mapping discrete image tokens from VQGAN to BPE tokens. + """ + + def __init__(self, vocab_map: Dict[str, int]): + self.vocab_map = vocab_map + self.image_token_id = vocab_map.get("") + + @cached_property + def val2name(self): + return {v: k for k, v in self.vocab_map.items()} + + @cached_property + def image_tokens(self): + return sorted([ + val for name, val in self.vocab_map.items() + if name.startswith("IMGIMG") + ]) + + @cached_property + def bpe2img(self): + img_tkn_chr_mapping = {chr(ord("A") + i): str(i) for i in range(10)} + + def remap(old_name: str) -> str: + return "".join( + img_tkn_chr_mapping.get(c, c) + for c in old_name[len("IMGIMG"):-1]) + + return { + tok: int(remap(self.val2name[tok])) + for tok in self.image_tokens + } + + @cached_property + def img2bpe(self): + return {v: k for k, v in self.bpe2img.items()} + + @cached_property + def bpe2img_search_tensors(self): + return torch.tensor(sorted(self.bpe2img.keys())), torch.tensor( + sorted(self.bpe2img.values())) + + @cached_property + def img2bpe_mapping_tensor(self): + mapping = torch.zeros(max(self.img2bpe.keys()) + 1, dtype=torch.int) + for k, v in self.img2bpe.items(): + mapping[k] = v + return mapping + + def convert_img2bpe(self, img_batch: torch.Tensor) -> torch.Tensor: + device = img_batch.device + img_tokens = self.img2bpe_mapping_tensor[img_batch.to("cpu")] + return img_tokens.to(device) + + +class ChameleonModel(nn.Module): + + def __init__( + self, + config: ChameleonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + ) + self.vocabulary_mapping = ChameleonImageVocabularyMapping( + config.vocabulary_map) + decoder_layer = ChameleonDecoderLayer if not self.config.swin_norm \ + else ChameleonSwinDecoderLayer + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: decoder_layer(config=config, + cache_config=cache_config, + quant_config=quant_config), + prefix=f"{prefix}.layers", + ) + + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.vqmodel = ChameleonVQVAE(config.vq_config) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def get_image_tokens(self, pixel_values: torch.Tensor) -> torch.Tensor: + """ + Tokenizes images into discrete tokens with VQGAN module. Converts + obtained image tokens into BPE tokens and wraps with "boi" and "eoi" + special tokens. + """ + batch_size = pixel_values.shape[0] + _, _, image_toks = self.vqmodel.encode(pixel_values) + bpe_toks = self.vocabulary_mapping.convert_img2bpe(image_toks) + bpe_toks = bpe_toks.view(batch_size, -1) + return bpe_toks + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon) +@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon) +class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__( + self, + config: ChameleonConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.multimodal_config = multimodal_config + self.model = ChameleonModel(config, cache_config, quant_config) + self.unpadded_vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, logit_scale) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + + expected_dims = (3, CHAMELEON_CROP_SIZE_HEIGHT, + CHAMELEON_CROP_SIZE_WIDTH) + actual_dims = tuple(data.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("batch_size", *map(str, expected_dims)) + raise ValueError( + f"The expected shape of pixel values is {expected_expr}. " + f"You supplied {tuple(data.shape)}.") + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[ChameleonImagePixelInputs]: + pixel_values = kwargs.pop("pixel_values", None) + + if pixel_values is None: + return None + + if not isinstance(pixel_values, torch.Tensor): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + # Remove the N dimension until multiple images are supported. + pixel_values = pixel_values.squeeze(1) + + return ChameleonImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values(pixel_values), + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs, + ) -> Union[torch.Tensor, IntermediateTensors]: + + if intermediate_tensors is not None: + input_ids = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + assert self.model.vqmodel is not None + image_tokens = self.model.get_image_tokens( + image_input["data"].to(self.config.torch_dtype)) + image_token_id = self.model.vocabulary_mapping.image_token_id + special_image_mask = input_ids == image_token_id + image_tokens = image_tokens.to(input_ids.device, + input_ids.dtype) + input_ids = input_ids.masked_scatter(special_image_mask, + image_tokens) + + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + + # Disallow image tokens which does not include special + # begin-image and end-image tokens + if logits is not None: + image_tokens = self.model.vocabulary_mapping.image_tokens + logits[:, image_tokens] = torch.finfo(logits.dtype).min + + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + + # With tie_word_embeddings, we can skip lm_head.weight + # The weight might appear unnecessarily in the files if the model is + # processed with quantization, LoRA, fine-tuning, etc. + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + + use_default_weight_loading = False + if "vqmodel" in name: + if self.model.vqmodel is not None: + # We only do sharding for language model and + # not vqvae for now. + use_default_weight_loading = True + else: + for (param_name, weight_name, + shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + if name.endswith("kv_scale"): + remapped_kv_scale_name = name.replace( + ".kv_scale", ".attn.kv_scale") + if remapped_kv_scale_name not in params_dict: + print_warning_once( + "Found kv scale in the checkpoint (e.g. " + f"{name}), but not found the expected name in " + f"the model (e.g. {remapped_kv_scale_name}). " + "kv-scale is not loaded.") + continue + else: + name = remapped_kv_scale_name + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + if use_default_weight_loading and name in params_dict: + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/chatglm.py b/vllm/model_executor/models/chatglm.py new file mode 100644 index 00000000..f26c9f95 --- /dev/null +++ b/vllm/model_executor/models/chatglm.py @@ -0,0 +1,671 @@ +# coding=utf-8 +# Adapted from +# https://github.com/THUDM/GLM-4 +"""Inference-only ChatGLM model compatible with THUDM weights.""" +from argparse import Namespace +from array import array +from typing import Dict, Iterable, List, Mapping, Optional, Tuple, TypedDict + +import torch +from PIL import Image +from torch import nn +from torch.nn import LayerNorm + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict, + MultiModalInputs) +from vllm.multimodal.base import MultiModalData +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors, + SequenceData) +from vllm.transformers_utils.configs import ChatGLMConfig + +from .interfaces import SupportsLoRA, SupportsMultiModal + +logger = init_logger(__name__) + + +def calculate_image_placeholder(vision_config): + return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2 + + +def mm_input_mapper_for_glmv( + ctx: InputContext, + data: MultiModalData[object], +) -> Dict: + model_config = ctx.model_config + tokenizer = cached_get_tokenizer(model_config.tokenizer, + trust_remote_code=True) + if tokenizer is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + try: + raw_batch_data = tokenizer.apply_chat_template( + conversation=[{ + "role": "user", + "image": data + }], + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True).data + except Exception: + logger.error("Failed to process image (%s)", data) + raise + pixel_values = raw_batch_data['images'] + + return MultiModalInputs({'pixel_values': pixel_values}) + + +def merge_glm_vision_embeddings( + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, + vision_embeddings: torch.Tensor, + boi_token_id: int, + eoi_token_id: int, +) -> torch.Tensor: + + boi_positions = (input_ids == boi_token_id).nonzero(as_tuple=True)[0] + eoi_positions = (input_ids == eoi_token_id).nonzero(as_tuple=True)[0] + + mask = torch.zeros_like(input_ids, dtype=torch.bool) + + for boi_pos, eoi_pos in zip(boi_positions, eoi_positions): + assert boi_pos < eoi_pos + mask[boi_pos:eoi_pos + 1] = True + inputs_embeds[mask] = vision_embeddings.view(-1, + vision_embeddings.shape[-1]) + return inputs_embeds + + +class GLMImagePixelInputs(TypedDict): + pixel_values: torch.Tensor + """Shape: `(batch_size, num_channels, height, width)`""" + + +def get_max_glmv_image_tokens(ctx: InputContext): + hf_config = ctx.get_hf_config(ChatGLMConfig) + + vision_config = getattr(hf_config, 'vision_config', None) + if vision_config is None: + return 1 + elif isinstance(vision_config, dict): + return calculate_image_placeholder(vision_config) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def dummy_data_for_glmv( + ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int] +) -> Tuple[SequenceData, Optional[MultiModalDataDict]]: + hf_config = ctx.get_hf_config(ChatGLMConfig) + vision_config = getattr(hf_config, 'vision_config', None) + + if vision_config is None: + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len) + seq_data = SequenceData(token_ids) + return seq_data, None + elif isinstance(vision_config, dict): + image_size = vision_config["image_size"] + image_placeholder_length = calculate_image_placeholder(vision_config) + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [hf_config.boi_token_id] + + [0] * image_placeholder_length + + [hf_config.eoi_token_id]) + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, + [0] * (seq_len - image_placeholder_length - 2)) + seq_data = SequenceData(token_ids) + + mm_data = { + "image": Image.new("RGB", (image_size, image_size), color=0) + } + + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def find_all_positions(input_ids: List[int], target: int) -> List[int]: + return [index for index, value in enumerate(input_ids) if value == target] + + +def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs): + hf_config = ctx.get_hf_config(ChatGLMConfig) + vision_config = getattr(hf_config, 'vision_config', None) + + if vision_config is None: + return llm_inputs + elif isinstance(vision_config, dict): + image_placeholder_length = calculate_image_placeholder(vision_config) + else: + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + input_ids = llm_inputs.get("prompt_token_ids") + position_ids = llm_inputs.get("position_ids") + tokenizer = cached_get_tokenizer( + ctx.model_config.model, + trust_remote_code=ctx.model_config.trust_remote_code) + + try: + raw_batch_data = tokenizer.apply_chat_template( + conversation=[{ + "role": "user", + "image": llm_inputs['multi_modal_data']["image"], + "content": llm_inputs['prompt'] + }], + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True).data + except Exception: + logger.error("Failed to process content (%s)", llm_inputs['prompt']) + raise + input_ids = raw_batch_data['input_ids'][0].tolist() + + if position_ids is None: + position_ids = list(range(len(input_ids))) + boi_token_id = hf_config.boi_token_id + eoi_token_id = hf_config.eoi_token_id + boi_positions = find_all_positions(input_ids, boi_token_id) + eoi_positions = find_all_positions(input_ids, eoi_token_id) + + assert len(boi_positions) == len(eoi_positions) + + new_input_ids = [] + new_position_ids = [] + final_processed_position = 0 + final_processed_position = 0 + + for boi_position, eoi_position in zip(boi_positions, eoi_positions): + assert boi_position < eoi_position + new_input_ids.extend(input_ids[final_processed_position:boi_position + + 1]) + new_position_ids.extend( + list(range(final_processed_position, boi_position + 1))) + new_input_ids.extend([input_ids[boi_position + 1]] * + image_placeholder_length) + new_position_ids.extend([boi_position + 1] * image_placeholder_length) + final_processed_position = eoi_position + + new_input_ids.extend(input_ids[final_processed_position:]) + new_position_ids.extend( + list(range(final_processed_position, len(input_ids)))) + + assert len(new_input_ids) == len(new_position_ids) + + llm_inputs["prompt_token_ids"] = new_input_ids + llm_inputs["position_ids"] = new_position_ids + return llm_inputs + + +class GLMAttention(nn.Module): + + def __init__( + self, + config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.multi_query_attention = config.multi_query_attention + self.total_num_kv_heads = (config.multi_query_group_num + if config.multi_query_attention else + config.num_attention_heads) + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = config.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + + self.query_key_value = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.add_bias_linear or config.add_qkv_bias, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + self.total_num_heads * self.head_dim, + config.hidden_size, + bias=config.add_bias_linear, + quant_config=quant_config, + ) + + # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141 + rope_ratio = getattr(config, "rope_ratio", 1.0) + max_positions = getattr(config, "seq_length", 8192) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim // 2, + max_position=max_positions, + base=10000 * rope_ratio, + is_neox_style=False, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.query_key_value(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(position_ids, q, k) + context_layer = self.attn( + q, + k, + v, + kv_cache, + attn_metadata, + ) + attn_output, _ = self.dense(context_layer) + return attn_output + + +class GLMMLP(nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.add_bias = config.add_bias_linear + + # Project to 4h. + self.dense_h_to_4h = MergedColumnParallelLinear( + config.hidden_size, + [config.ffn_hidden_size] * 2, + bias=config.add_bias_linear, + quant_config=quant_config, + ) + + self.activation_func = SiluAndMul() + + # Project back to h. + self.dense_4h_to_h = RowParallelLinear( + config.ffn_hidden_size, + config.hidden_size, + bias=config.add_bias_linear, + quant_config=quant_config, + ) + + def forward(self, hidden_states): + # [s, b, 4hp] + intermediate_parallel, _ = self.dense_h_to_4h(hidden_states) + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output, _ = self.dense_4h_to_h(intermediate_parallel) + return output + + +class GLMBlock(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__( + self, + config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.apply_residual_connection_post_layernorm = ( + config.apply_residual_connection_post_layernorm) + + self.fp32_residual_connection = config.fp32_residual_connection + + layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm + # Layernorm on the input data. + self.input_layernorm = layer_norm_func(config.hidden_size, + eps=config.layernorm_epsilon) + + # Self attention. + self.self_attention = GLMAttention(config, cache_config, quant_config) + self.hidden_dropout = config.hidden_dropout + + # Layernorm on the attention output + self.post_attention_layernorm = layer_norm_func( + config.hidden_size, eps=config.layernorm_epsilon) + + # MLP + self.mlp = GLMMLP(config, quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # hidden_states: [num_tokens, h] + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output = self.self_attention( + hidden_states=layernorm_output, + position_ids=position_ids, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + layernorm_input = residual + attention_output + + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + + # Second residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + output = self.mlp(layernorm_output) + residual + + return output + + +class GLMTransformer(nn.Module): + """Transformer class.""" + + def __init__( + self, + config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.post_layer_norm = config.post_layer_norm + + # Number of layers. + self.num_layers = config.num_layers + + # Transformer layers. + self.layers = nn.ModuleList([ + GLMBlock(config, cache_config, quant_config) + for i in range(self.num_layers) + ]) + + if self.post_layer_norm: + layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm + # Final layer norm before output. + self.final_layernorm = layer_norm_func( + config.hidden_size, eps=config.layernorm_epsilon) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + for i in range(self.num_layers): + layer = self.layers[i] + hidden_states = layer( + hidden_states=hidden_states, + position_ids=position_ids, + kv_cache=kv_caches[i], + attn_metadata=attn_metadata, + ) + # Final layer norm. + if self.post_layer_norm: + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class ChatGLMModel(nn.Module): + + def __init__( + self, + config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.config = config + + self.embedding = VocabParallelEmbedding(config.padded_vocab_size, + config.hidden_size, + quant_config=quant_config) + + self.num_layers = config.num_layers + self.multi_query_group_num = config.multi_query_group_num + self.kv_channels = config.kv_channels + self.encoder = GLMTransformer(config, cache_config, quant_config) + + self.output_layer = ParallelLMHead(config.padded_vocab_size, + config.hidden_size, + quant_config=quant_config) + + vision_config_flag = getattr(config, 'vision_config', None) + if vision_config_flag is not None: + self.vision_config = Namespace(**config.vision_config) + self.vision = EVA2CLIPModel(self.config, quant_config) + else: + self.vision = None + + def _parse_and_validate_image_input( + self, **kwargs: object) -> GLMImagePixelInputs: + + pixel_values = kwargs.pop("pixel_values", None) + if pixel_values is not None and self.vision is not None: + if isinstance(pixel_values, torch.Tensor): + if pixel_values.ndim > 2: + pixel_values = torch.concat(list(pixel_values)) + elif isinstance(pixel_values, list): + return torch.concat(pixel_values) + else: + raise TypeError("""pixel_values must be a torch.Tensor + or a list of torch.Tensor + """) + return GLMImagePixelInputs(pixel_values=pixel_values) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> torch.Tensor: + + inputs_embeds = self.embedding(input_ids) + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input["pixel_values"] is not None: + pixel_values = image_input["pixel_values"].to( + dtype=inputs_embeds.dtype) + image_embeds = self.vision(pixel_values) + + boi_token_id = self.config.boi_token_id + eoi_token_id = self.config.eoi_token_id + + inputs_embeds = merge_glm_vision_embeddings( + input_ids=input_ids, + inputs_embeds=inputs_embeds, + vision_embeddings=image_embeds, + boi_token_id=boi_token_id, + eoi_token_id=eoi_token_id) + + # Run encoder. + hidden_states = self.encoder( + hidden_states=inputs_embeds, + position_ids=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + ) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv) +@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv) +class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal): + packed_modules_mapping = { + "query_key_value": ["query_key_value"], + "dense_h_to_4h": ["dense_h_to_4h"] + } + # LoRA specific attributes + supported_lora_modules = [ + "query_key_value", + "dense", + "dense_h_to_4h", + "dense_4h_to_h", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: ChatGLMConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__() + + self.config = config + self.lora_config = lora_config + self.multimodal_config = multimodal_config + + self.quant_config = quant_config + self.max_position_embeddings = getattr(config, "max_sequence_length", + 8192) + self.transformer = ChatGLMModel(config, cache_config, quant_config) + if self.config.tie_word_embeddings: + self.transformer.output_layer.weight = ( + self.transformer.embedding.weight) + self.lm_head = self.transformer.output_layer + self.logits_processor = LogitsProcessor(config.padded_vocab_size) + self.sampler = Sampler() + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs) -> torch.Tensor: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, **kwargs) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + # Merge two ColumnParallelLinear into one MergedColumnParallelLinear + merged_weights_dict: Dict[str, Dict[str, Optional[torch.Tensor]]] = { + "transformer.vision.linear_proj.merged_proj.weight": { + "transformer.vision.linear_proj.gate_proj.weight": None, + "transformer.vision.linear_proj.dense_h_to_4h.weight": None, + } + } + + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + is_weight_to_be_merge = False + for _, merged_weight_dict in merged_weights_dict.items(): + if name in merged_weight_dict: + assert merged_weight_dict[name] is None + merged_weight_dict[name] = loaded_weight + is_weight_to_be_merge = True + if is_weight_to_be_merge: + continue + if "rotary_pos_emb.inv_freq" in name: + continue + if "word_embeddings" in name: + name = name.replace(".word_embeddings", "") + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + for combined_name, merged_weight_dict in merged_weights_dict.items(): + if combined_name in params_dict: + param = params_dict[combined_name] + combined_weight = torch.cat(list(merged_weight_dict.values()), + dim=0) + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, combined_weight) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py new file mode 100644 index 00000000..0fb0bf2c --- /dev/null +++ b/vllm/model_executor/models/clip.py @@ -0,0 +1,474 @@ +"""Minimal implementation of CLIPVisionModel intended to be only used +within a vision language model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from transformers import CLIPVisionConfig +from transformers.models.clip.modeling_clip import CLIPSdpaAttention + +from vllm.config import ModelConfig +from vllm.distributed import divide, get_tensor_model_parallel_world_size +from vllm.inputs import LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import SequenceData + +try: + from xformers import ops as xops + USE_XFORMERS_OPS = True +except ImportError: + USE_XFORMERS_OPS = False + +USE_XFORMERS_OPS = True + + +def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int: + assert image_size % patch_size == 0 + return image_size // patch_size + + +def get_clip_num_patches(*, image_size: int, patch_size: int) -> int: + grid_length = get_clip_patch_grid_length(image_size=image_size, + patch_size=patch_size) + return grid_length * grid_length + + +def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int: + return get_clip_num_patches(image_size=hf_config.image_size, + patch_size=hf_config.patch_size) + 1 + + +def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int: + return get_clip_image_feature_size(hf_config) + + +def dummy_seq_data_for_clip( + hf_config: CLIPVisionConfig, + seq_len: int, + num_images: int, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + if image_feature_size_override is None: + image_feature_size = get_clip_image_feature_size(hf_config) + else: + image_feature_size = image_feature_size_override + + return SequenceData.from_token_counts( + (image_token_id, image_feature_size * num_images), + (0, seq_len - image_feature_size * num_images), + ) + + +def dummy_image_for_clip( + hf_config: CLIPVisionConfig, + num_images: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + width = height = hf_config.image_size + if image_width_override is not None: + width = image_width_override + if image_height_override is not None: + height = image_height_override + + image = Image.new("RGB", (width, height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def dummy_video_for_clip( + hf_config: CLIPVisionConfig, + num_frames: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + pil_frame = dummy_image_for_clip( + hf_config, + num_images=1, + image_width_override=image_width_override, + image_height_override=image_height_override) + np_frame = np.array(pil_frame["image"]) + mm_data_per_video = np.repeat([np_frame], num_frames, axis=0) + mm_data = {"video": mm_data_per_video} + return mm_data + + +def input_processor_for_clip( + model_config: ModelConfig, + hf_config: CLIPVisionConfig, + llm_inputs: LLMInputs, + *, + image_token_id: int, + image_feature_size_override: Optional[Union[int, List[int]]] = None, +): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + if image_feature_size_override is None: + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + image_feature_size = get_clip_image_feature_size(hf_config) + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + else: + image_feature_size = image_feature_size_override + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=image_token_id, + repeat_count=image_feature_size, + ) + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa +class CLIPVisionEmbeddings(nn.Module): + + def __init__(self, config: CLIPVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(self.embed_dim)) + + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + bias=False, + ) + + self.num_patches = get_clip_num_patches(image_size=self.image_size, + patch_size=self.patch_size) + self.num_positions = self.num_patches + 1 + self.position_embedding = nn.Embedding(self.num_positions, + self.embed_dim) + self.register_buffer("position_ids", + torch.arange(self.num_positions).expand((1, -1)), + persistent=False) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to( + dtype=target_dtype)) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + embeddings = embeddings + self.position_embedding(self.position_ids) + + return embeddings + + +class CLIPParallelAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + "embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads}).") + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + self.qkv_proj = QKVParallelLinear( + hidden_size=self.embed_dim, + head_size=self.head_dim, + total_num_heads=self.num_heads, + quant_config=quant_config, + ) + + self.out_proj = RowParallelLinear( + input_size=self.embed_dim, + output_size=self.embed_dim, + quant_config=quant_config, + ) + + self.tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = divide(self.num_heads, self.tp_size) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, + self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + ): + """Input shape: Batch x Time x Channel""" + bsz, tgt_len, _ = hidden_states.size() + + qkv_states, _ = self.qkv_proj(hidden_states) + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1) + + query_states = query_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + key_states = key_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + value_states = value_states.view(bsz, tgt_len, + self.num_heads_per_partition, + self.head_dim) + query_states = query_states.transpose(1,2) + key_states = key_states.transpose(1,2) + value_states = value_states.transpose(1,2) + attn = (query_states @ (key_states * self.scale).permute(0,1,3,2)).float() + attn = torch.softmax(attn, dim=-1).to(value_states.dtype) + out = attn @ value_states + out = out.transpose(1,2).reshape(bsz, tgt_len, -1) + # TODO: use F.attention when supported + # out = xops.memory_efficient_attention_forward(query_states, + # key_states, + # value_states, + # p=self.dropout, + # scale=self.scale) + attn_output, _ = self.out_proj(out) + + return attn_output, None + + +class CLIPMLP(nn.Module): + + def __init__(self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear(config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config) + self.fc2 = RowParallelLinear(config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + + return hidden_states + + +class CLIPEncoderLayer(nn.Module): + + def __init__(self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + + num_heads = config.num_attention_heads + tp_size = get_tensor_model_parallel_world_size() + if USE_XFORMERS_OPS and num_heads % tp_size == 0: + self.self_attn = CLIPParallelAttention(config, + quant_config=quant_config) + else: + self.self_attn = CLIPSdpaAttention(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.mlp = CLIPMLP(config, quant_config=quant_config) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn(hidden_states=hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class CLIPEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self + attention layers. Each layer is a [`CLIPEncoderLayer`]. + + Args: + config: CLIPConfig + """ + + def __init__(self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None): + super().__init__() + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + self.layers = nn.ModuleList([ + CLIPEncoderLayer(config=config, quant_config=quant_config) + for _ in range(num_hidden_layers) + ]) + + def forward(self, inputs_embeds: torch.Tensor): + + hidden_states = inputs_embeds + for encoder_layer in self.layers: + hidden_states = encoder_layer(hidden_states) + + return hidden_states + + +class CLIPVisionTransformer(nn.Module): + + def __init__(self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.embeddings = CLIPVisionEmbeddings(config) + + # NOTE: This typo of "layrnorm" is not fixed on purpose to match + # the original transformers code and name of the model weights. + self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + self.encoder = CLIPEncoder( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override) + + if len(self.encoder.layers) > config.num_hidden_layers: + raise ValueError( + f"The original encoder only has {config.num_hidden_layers} " + f"layers, but you requested {len(self.encoder.layers)} layers." + ) + elif len(self.encoder.layers) == config.num_hidden_layers: + self.post_layernorm = nn.LayerNorm(embed_dim, + eps=config.layer_norm_eps) + else: + # post_layernorm is unused when we extract intermediate features + # In this case, we can skip it to conserve memory + self.post_layernorm = None + + def forward( + self, + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + hidden_states = self.embeddings(pixel_values) + hidden_states = self.pre_layrnorm(hidden_states) + hidden_states = self.encoder(inputs_embeds=hidden_states) + + if self.post_layernorm is None: + return hidden_states + + return self.post_layernorm(hidden_states) + + +class CLIPVisionModel(nn.Module): + + config_class = CLIPVisionConfig + main_input_name = "pixel_values" + + def __init__(self, + config: CLIPVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None): + super().__init__() + + tp_size = get_tensor_model_parallel_world_size() + num_heads = config.num_attention_heads + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0 + + self.vision_model = CLIPVisionTransformer( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + return self.vision_model(pixel_values) + + @property + def device(self): + return next(self.parameters()).device + + # (TODO) Add prefix argument for filtering out weights to be loaded + # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] if self.shard_weight else [] + params_dict = dict(self.named_parameters()) + layer_count = len(self.vision_model.encoder.layers) + + for name, loaded_weight in weights: + # post_layernorm is not needed in CLIPVisionModel + if (name.startswith("vision_model.post_layernorm") + and self.vision_model.post_layernorm is None): + continue + + # omit layers when num_hidden_layers_override is set + if name.startswith("vision_model.encoder.layers"): + layer_idx = int(name.split(".")[3]) + if layer_idx >= layer_count: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + + param = params_dict[name.replace(weight_name, param_name)] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py new file mode 100644 index 00000000..578cd2f0 --- /dev/null +++ b/vllm/model_executor/models/commandr.py @@ -0,0 +1,441 @@ +# coding=utf-8 +# Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is based on the LLama model definition file in transformers +"""PyTorch Cohere model.""" +from typing import Iterable, List, Optional, Set, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from transformers import CohereConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name, + row_parallel_weight_loader) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +@torch.compile +def layer_norm_func(hidden_states, weight, variance_epsilon): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + mean = hidden_states.mean(-1, keepdim=True) + variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True) + hidden_states = (hidden_states - mean) * torch.rsqrt(variance + + variance_epsilon) + hidden_states = weight.to(torch.float32) * hidden_states + return hidden_states.to(input_dtype) + + +class LayerNorm(nn.Module): + + def __init__(self, param_shape=None, eps=1e-5): + super().__init__() + self.weight = nn.Parameter(torch.ones(param_shape)) + self.variance_epsilon = eps + set_weight_attrs(self.weight, + {"weight_loader": row_parallel_weight_loader}) + + def forward(self, hidden_states, residuals=None): + hidden_states = layer_norm_func(hidden_states, self.weight, + self.variance_epsilon) + return hidden_states, residuals + + +# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere +class CohereMLP(nn.Module): + + def __init__( + self, + config: CohereConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_up_proj = MergedColumnParallelLinear( + self.hidden_size, + [self.intermediate_size] * 2, + bias=False, + quant_config=quant_config, + ) + self.down_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=False, + quant_config=quant_config, + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class CohereAttention(nn.Module): + + def __init__( + self, + config: CohereConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + tp_size = get_tensor_model_parallel_world_size() + self.config = config + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.total_num_heads = config.num_attention_heads + self.num_heads = self.total_num_heads // tp_size + self.head_dim = self.hidden_size // self.total_num_heads + self.total_num_kv_heads = config.num_key_value_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.max_position_embeddings = getattr( + config, "model_max_length", None) or getattr( + config, "max_position_embeddings", 8192) + self.rope_theta = config.rope_theta + self.rope_scaling = getattr(config, "rope_scaling", None) + self.use_qk_norm = getattr(config, "use_qk_norm", False) + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + rope_scaling=self.rope_scaling, + is_neox_style=False, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + if self.use_qk_norm: + self.q_norm = LayerNorm(param_shape=(self.num_heads, + self.head_dim), + eps=config.layer_norm_eps) + self.k_norm = LayerNorm(param_shape=(self.num_kv_heads, + self.head_dim), + eps=config.layer_norm_eps) + + def _apply_qk_norm(self, q, k): + q = q.view(*q.shape[:-1], -1, self.head_dim) + k = k.view(*k.shape[:-1], -1, self.head_dim) + q, _ = self.q_norm(q) + k, _ = self.k_norm(k) + q = q.view(*q.shape[:-2], -1) + k = k.view(*k.shape[:-2], -1) + return q, k + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.use_qk_norm: + q, k = self._apply_qk_norm(q, k) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class CohereDecoderLayer(nn.Module): + + def __init__(self, + config: CohereConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = CohereAttention(config, + cache_config, + quant_config=quant_config) + + self.mlp = CohereMLP(config, quant_config=quant_config) + self.input_layernorm = LayerNorm(param_shape=(config.hidden_size), + eps=config.layer_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + residual = hidden_states + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states_attention = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states_mlp = self.mlp(hidden_states) + # Add everything together + hidden_states = residual + hidden_states_attention + hidden_states_mlp + + return hidden_states, residual + + +class CohereModel(nn.Module): + + def __init__( + self, + config: CohereConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: CohereDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + self.norm = LayerNorm(param_shape=(config.hidden_size), + eps=config.layer_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens" + ] + embedding_modules = {"embed_tokens": "input_embeddings"} + embedding_padding_modules = [] + + def __init__( + self, + config: CohereConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + self.config = config + # currently all existing command R models have `tie_word_embeddings` + # enabled + assert config.tie_word_embeddings + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.quant_config = quant_config + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + scale=config.logit_scale) + self.model = CohereModel(config, + cache_config, + quant_config, + lora_config=lora_config) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + is_not_lora = hasattr(self.model.embed_tokens, 'weight') + if is_not_lora: + logits = self.logits_processor(self.model.embed_tokens, + hidden_states, sampling_metadata) + else: + logits = self.logits_processor(self.model.embed_tokens.base_layer, + hidden_states, sampling_metadata) + + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: Set[str] = set() + for name, loaded_weight in weights: + for param_name, shard_name, shard_id in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # lm_head is not used in vllm as it is tied with embed_token. + # To prevent errors, skip loading lm_head.weight. + if "lm_head.weight" in name: + continue + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) diff --git a/vllm/model_executor/models/dbrx.py b/vllm/model_executor/models/dbrx.py new file mode 100644 index 00000000..aae7ab73 --- /dev/null +++ b/vllm/model_executor/models/dbrx.py @@ -0,0 +1,439 @@ +# coding=utf-8 +from typing import Iterable, List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.dbrx import DbrxConfig + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class DbrxRouter(nn.Module): + """A Router implementation for DBRX that returns logits for each expert + per token. + """ + + def __init__( + self, + config: DbrxConfig, + params_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.num_total_experts = config.ffn_config.moe_num_experts + self.d_model = config.d_model + self.layer = ReplicatedLinear( + self.d_model, + self.num_total_experts, + bias=False, + params_dtype=params_dtype, + quant_config=None, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + router_logits, _ = self.layer(hidden_states) + return router_logits + + +class DbrxExperts(FusedMoE): + + def __init__( + self, + config: DbrxConfig, + quant_config: Optional[QuantizationConfig] = None, + params_dtype: Optional[torch.dtype] = None, + ): + super().__init__( + num_experts=config.ffn_config.moe_num_experts, + top_k=config.ffn_config.moe_top_k, + hidden_size=config.d_model, + intermediate_size=config.ffn_config.ffn_hidden_size, + params_dtype=params_dtype, + reduce_results=True, + renormalize=True, + quant_config=quant_config, + tp_size=get_tensor_model_parallel_world_size(), + ) + self.config = config + self.tp_size = get_tensor_model_parallel_world_size() + self.d_model = config.d_model + self.intermediate_size = (self.config.ffn_config.ffn_hidden_size // + self.tp_size) + + # Define custom weight loader for dbrx model + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, + weight_name: str): + tp_rank = get_tensor_model_parallel_rank() + param_data = param.data + shard_size = self.intermediate_size + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + # DBRX uses GLU for each experts. + # GLU has 3 linear layers: w1, v1 and w2. + if weight_name.endswith("w1"): + loaded_weight = torch.reshape( + loaded_weight, + [-1, self.intermediate_size * self.tp_size, self.d_model], + ) + param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :] + if weight_name.endswith("v1"): + loaded_weight = torch.reshape( + loaded_weight, + [-1, self.intermediate_size * self.tp_size, self.d_model], + ) + param_data[:, + shard_size:2 * shard_size, :] = loaded_weight[:, + shard, :] + if weight_name.endswith("w2"): + loaded_weight = torch.reshape( + loaded_weight, + [-1, self.intermediate_size * self.tp_size, self.d_model], + ).transpose(1, 2) + param_data[:] = loaded_weight[:, :, shard] + + +class DbrxMoE(nn.Module): + """A tensor-parallel MoE implementation for DBRX. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__( + self, + config: DbrxConfig, + quant_config: Optional[QuantizationConfig] = None, + params_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.d_model = config.d_model + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + + self.router = DbrxRouter(config, self.params_dtype) + + self.experts = DbrxExperts(config=config, + quant_config=quant_config, + params_dtype=self.params_dtype) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.d_model) + # router_logits: (num_tokens, n_experts) + router_logits = self.router(hidden_states) + final_hidden_states = self.experts(hidden_states, router_logits) + return final_hidden_states.view(orig_shape) + + +class DbrxAttention(nn.Module): + + def __init__( + self, + config: DbrxConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.total_num_heads = config.n_heads + self.head_dim = self.d_model // self.total_num_heads + self.total_num_kv_heads = config.attn_config.kv_n_heads + self.clip_qkv = config.attn_config.clip_qkv + self.rope_theta = config.attn_config.rope_theta + self.max_position = config.max_seq_len + + # pylint: disable=invalid-name + self.Wqkv = QKVParallelLinear( + self.d_model, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.out_proj = RowParallelLinear( + self.d_model, + self.d_model, + bias=False, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position, + base=int(self.rope_theta), + is_neox_style=True, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + self.tp_size = tp_world_size + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + if self.total_num_kv_heads >= tp_world_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_world_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_world_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.Wqkv(hidden_states) + if self.clip_qkv is not None: + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(position_ids, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + hidden_states, _ = self.out_proj(attn_output) + return hidden_states + + +class DbrxFusedNormAttention(nn.Module): + + def __init__( + self, + config: DbrxConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.attn = DbrxAttention(config, cache_config, quant_config) + self.norm_1 = nn.LayerNorm(self.d_model) + self.norm_2 = nn.LayerNorm(self.d_model) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.norm_1(hidden_states) + x = self.attn( + position_ids=position_ids, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + x + residual = hidden_states + hidden_states = self.norm_2(hidden_states) + return hidden_states, residual + + +class DbrxBlock(nn.Module): + + def __init__( + self, + config: DbrxConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config, + quant_config) + self.ffn = DbrxMoE(config, quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + hidden_states, residual = self.norm_attn_norm( + position_ids=position_ids, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = self.ffn(hidden_states) + hidden_states = hidden_states + residual + return hidden_states + + +class DbrxModel(nn.Module): + + def __init__( + self, + config: DbrxConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.wte = VocabParallelEmbedding( + config.vocab_size, + config.d_model, + ) + self.start_layer, self.end_layer, self.blocks = make_layers( + config.n_layers, + lambda prefix: DbrxBlock(config, cache_config, quant_config), + prefix=f"{prefix}.blocks", + ) + self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5) + for module in self.modules(): + if hasattr(module, "bias") and isinstance(module.bias, + nn.Parameter): + # Remove the bias term in Linear and LayerNorm. + module.register_parameter("bias", None) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.d_model)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.wte(input_ids) + else: + assert intermediate_tensors + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + block = self.blocks[i] + hidden_states = block( + position_ids, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.norm_f(hidden_states) + return hidden_states + + +class DbrxForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: DbrxConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + if config.tie_word_embeddings: + raise ValueError( + "tie_word_embeddings is not supported for Dbrx models.") + self.quant_config = quant_config + self.unpadded_vocab_size = config.vocab_size + self.transformer = DbrxModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.d_model, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + quant_config=quant_config, + ) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + expert_params_mapping = [( + "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight", + f"mlp.{weight_name}", + ) for weight_name in ["w1", "v1", "w2"]] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + for param_name, weight_name in expert_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, weight_name) + break + else: + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/decilm.py b/vllm/model_executor/models/decilm.py new file mode 100644 index 00000000..7ed2b96e --- /dev/null +++ b/vllm/model_executor/models/decilm.py @@ -0,0 +1,129 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 DeciAI Research Team. All rights reserved. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on MistralAI GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only DeciLM model compatible with HuggingFace weights.""" + +from typing import Iterable, Optional, Tuple + +import torch +from transformers import LlamaConfig + +from vllm.config import CacheConfig, LoRAConfig +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.llama import LlamaForCausalLM + +from .utils import is_pp_missing_parameter + + +class DeciLMForCausalLM(LlamaForCausalLM): + """ + Implementation for https://huggingface.co/Deci/DeciLM-7b-instruct. + Based on the llama executor. + + The main difference is that DeciLM uses Variable Grouped Query Attention. + The constant number of GQA heads in the decoder is overridden with a value + per layer. + + Usually, in the HuggingFace implementation, instead of + "config.num_key_value_heads", we use + "config.num_key_value_heads_per_layer[i]" which varies. + + Currently, PagedAttention does not work well with variable GQA, so we + normalize the weights upon loading, and use uniform GQA with the max value + instead. + """ + + def __init__( + self, + config: LlamaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + config.num_key_value_heads = max(config.num_key_value_heads_per_layer) + delattr(config, "num_key_value_heads_per_layer") + super().__init__(config=config, + cache_config=cache_config, + quant_config=quant_config, + lora_config=lora_config) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + if "k_proj" in name or "v_proj" in name: + loaded_weight = self._degroup_weight(loaded_weight) + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + def _degroup_weight(self, loaded_weight: torch.Tensor) -> torch.Tensor: + hidden_size = self.config.hidden_size + head_size = self.config.hidden_size // self.config.num_attention_heads + target_num_kv_heads = self.config.num_key_value_heads + num_kv_heads = loaded_weight.shape[0] // head_size + n_repeats = target_num_kv_heads / num_kv_heads + assert n_repeats == int(n_repeats) + + n_repeats = int(n_repeats) + loaded_weight = loaded_weight.view(num_kv_heads, head_size, + hidden_size) + loaded_weight = torch.repeat_interleave(loaded_weight, + repeats=n_repeats, + dim=0) + loaded_weight = loaded_weight.reshape(target_num_kv_heads * head_size, + hidden_size) + + return loaded_weight diff --git a/vllm/model_executor/models/deepseek.py b/vllm/model_executor/models/deepseek.py new file mode 100644 index 00000000..5b4db8f2 --- /dev/null +++ b/vllm/model_executor/models/deepseek.py @@ -0,0 +1,480 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Deepseek model.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import fused_moe +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class DeepseekMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class DeepseekMoE(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + self.n_routed_experts = config.n_routed_experts + self.top_k = config.num_experts_per_tok + if self.tp_size > self.n_routed_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {self.n_routed_experts}.") + + self.experts = nn.ModuleList([ + DeepseekMLP(hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False) + for idx in range(self.n_routed_experts) + ]) + self.pack_params() + + self.gate = ReplicatedLinear(config.hidden_size, + self.n_routed_experts, + bias=False, + quant_config=None) + + if config.n_shared_experts is not None: + intermediate_size = (config.moe_intermediate_size * + config.n_shared_experts) + self.shared_experts = DeepseekMLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + ) + + def pack_params(self): + w1 = [] + w2 = [] + for expert in self.experts: + w1.append(expert.gate_up_proj.weight) + w2.append(expert.down_proj.weight) + self.w1 = torch._utils._flatten_dense_tensors(w1) + w1s = torch._utils._unflatten_dense_tensors(self.w1, w1) + for data, param in zip(w1s, w1): + param.data = data + self.w1 = self.w1.view(len(w1), *w1s[0].shape) + + self.w2 = torch._utils._flatten_dense_tensors(w2) + w2s = torch._utils._unflatten_dense_tensors(self.w2, w2) + for data, param in zip(w2s, w2): + param.data = data + + self.w2 = self.w2.view(len(w2), *w2s[0].shape) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + if self.config.n_shared_experts is not None: + shared_output = self.shared_experts(hidden_states) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = fused_moe(hidden_states, + self.w1, + self.w2, + router_logits, + self.top_k, + renormalize=self.config.norm_topk_prob, + inplace=True) + + if self.config.n_shared_experts is not None: + final_hidden_states = final_hidden_states + shared_output + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states.view(num_tokens, hidden_dim) + + +class DeepseekAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class DeepseekDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = DeepseekAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + if (config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % config.moe_layer_freq == 0): + self.mlp = DeepseekMoE(config=config, quant_config=quant_config) + else: + self.mlp = DeepseekMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class DeepseekModel(nn.Module): + + fall_back_to_pt_during_load = False + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: DeepseekDecoderLayer(config, + int(prefix.split(".")[-1]), + cache_config, + quant_config=quant_config), + prefix=f"{prefix}.layers") + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class DeepseekForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = DeepseekModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip experts that are not assigned to this worker. + if (("mlp.experts." in name or "mlp.shared_experts." in name) + and name not in params_dict): + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip experts that are not assigned to this worker. + if (("mlp.experts." in name or "mlp.shared_experts." in name) + and name not in params_dict): + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py new file mode 100644 index 00000000..702be7b7 --- /dev/null +++ b/vllm/model_executor/models/deepseek_v2.py @@ -0,0 +1,617 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only DeepseekV2 model.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class DeepseekV2MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj") + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class DeepseekV2MoE(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.routed_scaling_factor = config.routed_scaling_factor + self.n_shared_experts = config.n_shared_experts + self.routed_scaling_factor = config.routed_scaling_factor + if self.tp_size > config.n_routed_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.n_routed_experts}.") + + if config.hidden_act != "silu": + raise ValueError(f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now.") + + self.experts = FusedMoE(num_experts=config.n_routed_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + reduce_results=False, + renormalize=config.norm_topk_prob, + quant_config=quant_config, + use_grouped_topk=True, + num_expert_group=config.n_group, + topk_group=config.topk_group, + prefix=f"{prefix}.experts") + + self.gate = ReplicatedLinear(config.hidden_size, + config.n_routed_experts, + bias=False, + quant_config=None, + prefix=f"{prefix}.gate") + if config.n_shared_experts is not None: + intermediate_size = (config.moe_intermediate_size * + config.n_shared_experts) + self.shared_experts = DeepseekV2MLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + if self.n_shared_experts is not None: + shared_output = self.shared_experts(hidden_states) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts( + hidden_states=hidden_states, + router_logits=router_logits) * self.routed_scaling_factor + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states.view(num_tokens, hidden_dim) + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + import math + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +class DeepseekV2Attention(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: int, + kv_lora_rank: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.num_heads = num_heads + tp_size = get_tensor_model_parallel_world_size() + assert num_heads % tp_size == 0 + self.num_local_heads = num_heads // tp_size + self.scaling = self.qk_head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + if self.q_lora_rank is not None: + self.q_a_proj = ReplicatedLinear(self.hidden_size, + self.q_lora_rank, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_a_proj") + self.q_a_layernorm = RMSNorm(self.q_lora_rank, + eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear(q_lora_rank, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_b_proj") + else: + self.q_proj = ColumnParallelLinear(self.hidden_size, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj") + + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_a_proj_with_mqa") + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, + eps=config.rms_norm_eps) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_b_proj") + # O projection. + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj") + rope_scaling['type'] = 'deepseek_yarn' + self.rotary_emb = get_rope(qk_rope_head_dim, + rotary_dim=qk_rope_head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=False) + + if rope_scaling: + mscale_all_dim = rope_scaling.get("mscale_all_dim", False) + scaling_factor = rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale + + # self.attn = Attention(self.num_heads, + # self.qk_head_dim, + # self.scaling, + # num_kv_heads=self.num_heads) + + # TODO, support head_size 192 + self.attn = Attention(self.num_local_heads, + 256, + self.scaling, + num_kv_heads=self.num_local_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + if self.q_lora_rank is not None: + q = self.q_a_proj(hidden_states)[0] + q = self.q_a_layernorm(q) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, + self.qk_head_dim) + else: + q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads, + self.qk_head_dim) + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], + dim=-1) + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] + kv_a, _ = latent_cache.split( + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + latent_cache = latent_cache.unsqueeze(1) + kv_a = self.kv_a_layernorm(kv_a.contiguous()) + kv = self.kv_b_proj(kv_a)[0] + kv = kv.view(-1, self.num_local_heads, + self.qk_nope_head_dim + self.v_head_dim) + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + k_pe = latent_cache[:, :, self.kv_lora_rank:] + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) + q[..., self.qk_nope_head_dim:] = q_pe + k = torch.empty_like(q) + k[..., :self.qk_nope_head_dim] = k_nope + k[..., self.qk_nope_head_dim:] = k_pe + q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim], + value=0).view(-1, + self.num_local_heads * 256) + k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim], + value=0).view(-1, + self.num_local_heads * 256) + v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim], + value=0).view(-1, + self.num_local_heads * 256) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output = attn_output.view( + -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape( + -1, self.num_local_heads * self.v_head_dim) + output, _ = self.o_proj(attn_output) + return output + + +class DeepseekV2DecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + prefix: str, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # DecoderLayers are created with `make_layers` which passes the prefix + # with the layer's index. + layer_idx = int(prefix.split(sep='.')[-1]) + self.self_attn = DeepseekV2Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + q_lora_rank=config.q_lora_rank + if hasattr(config, "q_lora_rank") else None, + kv_lora_rank=config.kv_lora_rank, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + if (config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % config.moe_layer_freq == 0): + self.mlp = DeepseekV2MoE( + config=config, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + else: + self.mlp = DeepseekV2MLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class DeepseekV2Model(nn.Module): + + fall_back_to_pt_during_load = False + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + if get_pp_group().is_first_rank: + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + else: + self.embed_tokens = PPMissingLayer() + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: DeepseekV2DecoderLayer( + config, + prefix, + cache_config=cache_config, + quant_config=quant_config, + ), + prefix=f"{prefix}.layers") + + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class DeepseekV2ForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = DeepseekV2Model(config, + cache_config, + quant_config, + prefix="model") + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def make_empty_intermediate_tensors( + self, batch_size: int, dtype: torch.dtype, + device: torch.device) -> IntermediateTensors: + return IntermediateTensors({ + "hidden_states": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + "residual": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + }) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.n_routed_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if (("mlp.experts." in name) and name not in params_dict): + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/eagle.py b/vllm/model_executor/models/eagle.py new file mode 100644 index 00000000..13811d33 --- /dev/null +++ b/vllm/model_executor/models/eagle.py @@ -0,0 +1,170 @@ +from typing import Iterable, List, Optional, Tuple + +import torch +import torch.nn as nn + +from vllm.attention.backends.abstract import AttentionMetadata +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models import ModelRegistry +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.eagle import EAGLEConfig + + +class EAGLE(nn.Module): + """This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077 + Reference implementation: https://github.com/SafeAILab/EAGLE + + Differences from reference implementation: + 1. In reference, LlamaDecoderLayer implementation doesn't have + input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427) + but we do as HF implementation also does. + 2. We allow any decoder layer to be used in EAGLE whereas in reference + decoder layer is fixed to be LlamaDecoderLayer. + 3. We have an optional token_map which reduces draft vocab to most + frequently used tokens to give some additional speed-up by reducing + sampling overhead. This is disabled unless the checkpoint file has + explicit token_map tensor and config has an optional attribute + truncated_vocab_size < vocab_size. To use this technique, one has to find + the top-k most frequent tokens in target dataset and add that as a tensor + in the draft checkpoint (using key token_map). Also, the draft config + needs to have truncated_vocab_size (=k) as an attribute.""" + + def __init__(self, config: EAGLEConfig, *args, **kwargs) -> None: + super().__init__() + self.config = config + + architectures = getattr(self.config.model, "architectures", []) + model_cls, _ = ModelRegistry.resolve_model_cls(architectures) + + self.model = model_cls(self.config.model, *args, **kwargs) + self.fc = nn.Linear(config.model.hidden_size * 2, + config.model.hidden_size, + bias=getattr(self.config, "bias", False)) + + self.orig_vocab_size = config.vocab_size + self.truncated_vocab_size = config.truncated_vocab_size + self.unpadded_vocab_size = self.truncated_vocab_size + + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=self.truncated_vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + ) + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + self.truncated_vocab_size, + logit_scale) + + # Token map is a idx to token mapping to reduce the vocab size for + # the draft model. Using smaller vocab size for draft, containing + # only most frequent tokens reduces the speculation overhead. This + # doesn't affect the acceptance rate much and thus gives more speed + # -up. By default, this is disabled and is only used if the EAGLE + # checkpoint file has token_map tensor. + self.token_map = None + + @property + def sampler(self): + return self.model.sampler + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + previous_hidden_states: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> torch.Tensor: + + tok_embeds = self.model.model.embed_tokens(input_ids) + inputs_embeds = self.fc( + torch.cat([tok_embeds, previous_hidden_states], dim=-1)) + + inputs_embeds[positions == 0] = 0 # masking inputs at position=0 + + hidden_states = self.model.model( + input_ids=None, + inputs_embeds=inputs_embeds, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + + if self.token_map is not None: + _logits = logits + logits = -torch.inf * torch.ones( + size=(*_logits.shape[:-1], self.orig_vocab_size), + device=_logits.device, + dtype=_logits.dtype) + + logits[..., self.token_map] = _logits + + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + # This implementation is incompitable with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B + # due to missing lm_head weights and its config being that of a + # Llama model. Here's a compatible version with the same weights: + # https://huggingface.co/abhigoyal/EAGLE-LLaMA3-Instruct-8B-vllm + # Also, here's an example script for converting trained EAGLE + # checkpoint to vLLM compatible version: https://gist.github.com/abhigoyal1997/1e7a4109ccb7704fbc67f625e86b2d6d + model_weights = {} + for name, loaded_weight in weights: + if name == "token_map": + if self.config.truncated_vocab_size < self.config.vocab_size: + self.token_map = nn.Parameter(loaded_weight, + requires_grad=False) + elif name.startswith("fc.weight"): + weight_loader = getattr(self.fc.weight, "weight_loader", + default_weight_loader) + weight_loader(self.fc.weight, loaded_weight) + elif name.startswith("fc.bias"): + if self.fc.bias is not None: + weight_loader = getattr(self.fc.bias, "weight_loader", + default_weight_loader) + weight_loader(self.fc.bias, loaded_weight) + else: + raise ValueError("Found bias in the loaded weights " + "but the model config doesn't have bias") + elif name.startswith("model.lm_head.") or name.startswith( + "model.model."): + model_weights[name.split("model.", 1)[-1]] = loaded_weight + elif name.startswith("lm_head.") or name.startswith("model."): + model_weights[name] = loaded_weight + else: + model_weights[f"model.{name}"] = loaded_weight + + lm_head_weight = model_weights.pop("lm_head.weight") + + if self.token_map is not None and\ + lm_head_weight.shape[0] > self.token_map.shape[0]: + + lm_head_weight = lm_head_weight[self.token_map] + + weight_loader = getattr(self.lm_head.weight, "weight_loader", + default_weight_loader) + weight_loader(self.lm_head.weight, lm_head_weight) + + self.model.load_weights(model_weights.items()) diff --git a/vllm/model_executor/models/exaone.py b/vllm/model_executor/models/exaone.py new file mode 100644 index 00000000..dfb8fe55 --- /dev/null +++ b/vllm/model_executor/models/exaone.py @@ -0,0 +1,606 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/modeling_exaone.py +# Copyright 2024 The LG U+ CTO AI Tech Lab. +# Copyright 2021 The LG AI Research EXAONE Lab +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Exaone model compatible with HuggingFace weights.""" + +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + get_compressed_tensors_cache_scale) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.exaone import ExaoneConfig +from vllm.utils import is_hip + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class ExaoneGatedMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.c_proj = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.c_proj", + ) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.c_proj(x) + return x + + +class ExaoneAttention(nn.Module): + + def __init__( + self, + config: ExaoneConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", + self.hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.out_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.out_proj", + ) + + is_neox_style = True + if quant_config is not None and quant_config.get_name() == "gguf": + is_neox_style = False + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=is_neox_style, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.out_proj(attn_output) + return output + + +class ExaoneBlockAttention(nn.Module): + + def __init__( + self, + config: ExaoneConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.attention = ExaoneAttention( + config=config, + hidden_size=hidden_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=bias, + cache_config=cache_config, + prefix=prefix, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + return self.attention( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + +class ExaoneDecoderLayer(nn.Module): + + def __init__( + self, + config: ExaoneConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + self.attn = ExaoneBlockAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + cache_config=cache_config, + prefix=f"{prefix}.attn", + ) + self.mlp = ExaoneGatedMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.activation_function, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + else: + hidden_states, residual = self.ln_1(hidden_states, residual) + hidden_states = self.attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.ln_2(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class ExaoneModel(nn.Module): + + def __init__( + self, + config: ExaoneConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = ((lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0) + self.vocab_size = config.vocab_size + lora_vocab + self.wte = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.wte = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=quant_config, + ) + else: + self.wte = PPMissingLayer() + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: ExaoneDecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), + prefix=f"{prefix}.h", + ) + if get_pp_group().is_last_rank: + self.ln_f = RMSNorm(config.hidden_size, + eps=config.layer_norm_epsilon) + else: + self.ln_f = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.wte(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.ln_f(hidden_states, residual) + return hidden_states + + +class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "c_fc_0", + "c_fc_1", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "out_proj", + "gate_up_proj", + "c_proj", + "wte", + "lm_head", + ] + embedding_modules = { + "wte": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "c_fc_0": ("gate_up_proj", 0), + "c_fc_1": ("gate_up_proj", 1), + } + + def __init__( + self, + config: ExaoneConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.transformer = ExaoneModel( + config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model", + ) + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.transformer.wte.weight + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + self.sampler = Sampler() + else: + self.lm_head = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return model_output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".c_fc_0", 0), + (".gate_up_proj", ".c_fc_1", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + # With tie_word_embeddings, we can skip lm_head.weight + # The weight might appear unnecessarily in the files if the model is + # processed with quantization, LoRA, fine-tuning, etc. + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + if scale_name := get_compressed_tensors_cache_scale(name): + # Loading kv cache scales for compressed-tensors quantization + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + # If this function is called, it should always initialize KV cache scale + # factors (or else raise an exception). Thus, handled exceptions should + # make sure to leave KV cache scale factors in a known good (dummy) state + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + for layer_idx, scaling_factor in kv_cache_scales_loader( + quantization_param_path, + tp_rank, + tp_size, + self.config.num_hidden_layers, + self.config.__class__.model_type, + ): + if not isinstance(self.transformer.h[layer_idx], nn.Identity): + layer_self_attn = self.transformer.h[layer_idx].attn + + if is_hip(): + # The scaling factor convention we are assuming is + # quantized_value * scaling_factor ~= true_value + # which is consistent with the practice of setting + # scaling_factor = tensor_amax / FPtype_max + scaling_factor *= 2 + if hasattr(layer_self_attn, "kv_scale"): + layer_self_attn.attn._kv_scale = scaling_factor + else: + raise RuntimeError("Self attention has no KV cache scaling " + "factor attribute!") diff --git a/vllm/model_executor/models/falcon.py b/vllm/model_executor/models/falcon.py new file mode 100644 index 00000000..467a3350 --- /dev/null +++ b/vllm/model_executor/models/falcon.py @@ -0,0 +1,508 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/a5cc30d72ae2dc19af534e4b35c986cc28db1275/src/transformers/models/falcon/modeling_falcon.py +# Copyright 2023 The vLLM team. +# Copyright 2023 the Falcon authors and HuggingFace Inc. team. All rights +# reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Falcon model.""" + +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from torch.nn import LayerNorm +from transformers import FalconConfig as HF_FalconConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs import RWConfig + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +FalconConfig = Union[HF_FalconConfig, RWConfig] + + +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor: + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads)) + base = torch.tensor(2**(-(2**-(math.log2(closest_power_of_2) - 3))), + dtype=torch.float32) + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32) + slopes = torch.pow(base, powers) + + if closest_power_of_2 != total_num_heads: + extra_base = torch.tensor( + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))), + dtype=torch.float32) + num_remaining_heads = min(closest_power_of_2, + total_num_heads - closest_power_of_2) + extra_powers = torch.arange(1, + 1 + 2 * num_remaining_heads, + 2, + dtype=torch.int32) + slopes = torch.cat( + [slopes, torch.pow(extra_base, extra_powers)], dim=0) + + return slopes + + +class FalconAttention(nn.Module): + + def __init__( + self, + config: FalconConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.hidden_size = config.hidden_size + tp_size = get_tensor_model_parallel_world_size() + + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.head_dim = self.hidden_size // self.total_num_heads + assert self.head_dim * self.total_num_heads == self.hidden_size + + self.new_decoder_architecture = config.new_decoder_architecture + self.multi_query = config.multi_query + + if self.new_decoder_architecture: + self.total_num_kv_heads = config.num_kv_heads + elif self.multi_query: + self.total_num_kv_heads = 1 + else: + self.total_num_kv_heads = self.total_num_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + + self.query_key_value = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.bias, + skip_bias_add=True, + quant_config=quant_config, + ) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + self.reduce_row_parallel_results = not (config.new_decoder_architecture + or config.parallel_attn) + self.dense = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=config.bias, + skip_bias_add=True, + quant_config=quant_config, + reduce_results=self.reduce_row_parallel_results) + + self.use_rotary = config.rotary + self.use_alibi = config.alibi + assert not (self.use_rotary and self.use_alibi), ( + "Rotary and alibi are mutually exclusive.") + + if self.use_rotary: + rope_theta = getattr(config, "rope_theta", 10000) + max_position_embeddings = getattr(config, + "max_position_embeddings", 8192) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.inv_norm_factor, + num_kv_heads=self.num_kv_heads, + quant_config=quant_config) + elif self.use_alibi: + tp_rank = get_tensor_model_parallel_rank() + head_start = tp_rank * self.num_heads + head_end = (tp_rank + 1) * self.num_heads + alibi_slopes = (_get_alibi_slopes(self.total_num_heads) * + self.inv_norm_factor) + alibi_slopes = alibi_slopes[head_start:head_end].tolist() + self.attn = Attention(self.num_heads, + self.head_dim, + self.inv_norm_factor, + num_kv_heads=self.num_kv_heads, + alibi_slopes=alibi_slopes, + quant_config=quant_config) + else: + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.inv_norm_factor, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, bias = self.query_key_value(hidden_states) + if bias is not None: + qkv += bias + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.use_rotary: + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output, bias = self.dense(attn_output) + return attn_output, bias + + +class FalconMLP(nn.Module): + + def __init__( + self, + config: FalconConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + + self.dense_h_to_4h = ColumnParallelLinear(hidden_size, + 4 * hidden_size, + bias=config.bias, + skip_bias_add=True, + quant_config=quant_config) + self.act = get_act_fn("gelu", quant_config, 4 * hidden_size) + self.reduce_row_parallel_results = not (config.new_decoder_architecture + or config.parallel_attn) + self.dense_4h_to_h = RowParallelLinear( + 4 * hidden_size, + hidden_size, + bias=config.bias, + skip_bias_add=True, + reduce_results=self.reduce_row_parallel_results, + quant_config=quant_config) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # NOTE(zhuohan): Following huggingface, we do not fuse bias add here. + x, bias = self.dense_h_to_4h(x) + if bias is not None: + x += bias + x = self.act(x) + x, bias = self.dense_4h_to_h(x) + return x, bias + + +class FalconDecoderLayer(nn.Module): + + def __init__( + self, + config: FalconConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.self_attention = FalconAttention(config, cache_config, + quant_config) + self.mlp = FalconMLP(config, quant_config) + self.config = config + + if (config.num_ln_in_parallel_attn is None + and config.new_decoder_architecture): + config.num_ln_in_parallel_attn = 2 + + if not config.parallel_attn: + self.post_attention_layernorm = LayerNorm( + hidden_size, eps=config.layer_norm_epsilon) + self.input_layernorm = LayerNorm(hidden_size, + eps=config.layer_norm_epsilon) + else: + if config.num_ln_in_parallel_attn == 2: + # The layer norm before self-attention + self.ln_attn = LayerNorm(hidden_size, + eps=config.layer_norm_epsilon) + # The layer norm before the MLP + self.ln_mlp = LayerNorm(hidden_size, + eps=config.layer_norm_epsilon) + else: + self.input_layernorm = LayerNorm(hidden_size, + eps=config.layer_norm_epsilon) + + self.reduce_row_parallel_results = not (config.new_decoder_architecture + or config.parallel_attn) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + + if self.config.num_ln_in_parallel_attn == 2: + attention_layernorm_out = self.ln_attn(hidden_states) + mlp_layernorm_out = self.ln_mlp(hidden_states) + else: + attention_layernorm_out = self.input_layernorm(hidden_states) + + # Self attention. + attention_output, attention_bias = self.self_attention( + positions=positions, + hidden_states=attention_layernorm_out, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + if self.reduce_row_parallel_results and attention_bias is not None: + attention_output += attention_bias + + if not self.config.new_decoder_architecture: + if self.config.parallel_attn: + mlp_layernorm_out = attention_layernorm_out + else: + residual += attention_output + mlp_layernorm_out = self.post_attention_layernorm(residual) + + if (self.config.new_decoder_architecture and self.config.parallel_attn + and self.config.num_ln_in_parallel_attn == 1): + mlp_layernorm_out = attention_layernorm_out + + # MLP. + mlp_output, mlp_bias = self.mlp(mlp_layernorm_out) + if self.reduce_row_parallel_results and mlp_bias is not None: + mlp_output += mlp_bias + + if not self.reduce_row_parallel_results: + # When MLP and Attention layers are parallel, we can use + # only one all-reduce operator to reduce the results from + # both MLP and Attention layers. + mlp_output += attention_output + mlp_output = tensor_model_parallel_all_reduce(mlp_output) + if attention_bias is not None: + mlp_output += attention_bias + if mlp_bias is not None: + mlp_output += mlp_bias + + output = mlp_output + residual + return output + + +class FalconModel(nn.Module): + + def __init__( + self, + config: FalconConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.use_alibi = config.alibi + + # Embedding + LN Embedding + self.word_embeddings = VocabParallelEmbedding( + config.vocab_size, + self.embed_dim, + ) + + # Transformer blocks + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: FalconDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.h") + + # Final Layer Norm + self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.word_embeddings(input_ids) + else: + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class FalconForCausalLM(nn.Module, SupportsPP): + + # BitandBytes specific attributes + bitsandbytes_stacked_params_mapping = {} + default_bitsandbytes_target_modules = [ + ".query_key_value.", + ".dense.", + ".dense_h_to_4h.", + ".dense_4h_to_h.", + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".dense_4h_to_h.", ".dense."] + + def __init__( + self, + config: FalconConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.transformer = FalconModel(config, cache_config, quant_config) + # only Falcon-11B doesn't share lm_head weight with word embeddings + # and previous Falcon model doesn't have tie_word_embeddings config + # so we set tie_word_embeddings to True by default + self.tie_word_embeddings = (config.tie_word_embeddings + if config.tie_word_embeddings is not None + else True) + if self.tie_word_embeddings: + self.lm_head = self.transformer.word_embeddings + else: + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.LongTensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> torch.Tensor: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + total_num_heads = self.config.num_attention_heads + if self.config.new_decoder_architecture: + total_num_kv_heads = self.config.num_kv_heads + elif self.config.multi_query: + total_num_kv_heads = 1 + else: + total_num_kv_heads = total_num_heads + num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if name == "lm_head.weight" and self.tie_word_embeddings: + # Falcon uses tied embeddings except Falcon-11b. + continue + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + if "query_key_value" in name: + output_dim = getattr(param, "output_dim", None) + loaded_weight_shape = loaded_weight.shape + if output_dim is not None: + loaded_weight = loaded_weight.view( + loaded_weight_shape[:output_dim] + + (total_num_kv_heads, num_query_heads_per_kv_head + 2, + -1) + loaded_weight_shape[output_dim + 1:]) + wq = loaded_weight.narrow( + output_dim + 1, 0, + num_query_heads_per_kv_head).reshape( + *loaded_weight_shape[:output_dim], -1, + *loaded_weight_shape[output_dim + 1:]) + wk = loaded_weight.narrow( + output_dim + 1, num_query_heads_per_kv_head, + 1).reshape(*loaded_weight_shape[:output_dim], -1, + *loaded_weight_shape[output_dim + 1:]) + wv = loaded_weight.narrow( + output_dim + 1, num_query_heads_per_kv_head + 1, + 1).reshape(*loaded_weight_shape[:output_dim], -1, + *loaded_weight_shape[output_dim + 1:]) + loaded_weight = torch.cat([wq, wk, wv], dim=output_dim) + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/fuyu.py b/vllm/model_executor/models/fuyu.py new file mode 100644 index 00000000..62a1b1f8 --- /dev/null +++ b/vllm/model_executor/models/fuyu.py @@ -0,0 +1,351 @@ +# coding=utf-8 +# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py +# Copyright 2023 The vLLM team. +# Copyright 2023 HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Fuyu model.""" +import math +from array import array +from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from PIL import Image +from transformers import FuyuConfig, FuyuImageProcessor + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.linear import ColumnParallelLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.models.persimmon import PersimmonForCausalLM +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs +from vllm.multimodal.image import cached_get_image_processor +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors, + SequenceData) + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings + +# Cannot find the following 2 numbers from hf config. +_IMAGE_TOKEN_ID = 71011 +_NEWLINE_TOKEN_ID = 71019 + +MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080 +MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920 + + +class FuyuImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """ + Shape: + (batch_size, num_patches, patch_size_x * patch_size_y * num_channels) + """ + + +def _calculate_num_image_tokens( + height: int, + width: int, +) -> Tuple[int, int]: + """ + calculate number of image tokens needed for a given image size + The expected Fuyu image prompts is in format: + (image_token * ncols + newline_token) * nrows + args: + image_size: Tuple[int, int] - (width, height) of the image + returns: + ncols: int - number of image tokens in x direction + nrows: int - number of image tokens in y direction + """ + ncol = math.ceil(width / 30) + nrow = math.ceil(height / 30) + return ncol, nrow + + +def get_max_fuyu_image_feature_size(): + + return _calculate_num_image_tokens( + height=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + width=MAX_IMAGE_FEATURE_SIZE_WIDTH, + ) + + +def get_max_fuyu_image_tokens(ctx: InputContext): + ncol, nrow = get_max_fuyu_image_feature_size() + return (ncol + 1) * nrow + + +def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int): + ncol, nrow = get_max_fuyu_image_feature_size() + image_feature_size = get_max_fuyu_image_tokens(ctx) + + image_token_ids = ( + array(VLLM_TOKEN_ID_ARRAY_TYPE, [_IMAGE_TOKEN_ID]) * ncol + + array(VLLM_TOKEN_ID_ARRAY_TYPE, [_NEWLINE_TOKEN_ID])) * nrow + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, + [0]) * (seq_len - image_feature_size * num_images) + return SequenceData(token_ids) + + +def dummy_image_for_fuyu( + num_images: int, + *, + image_width: int, + image_height: int, +): + image = Image.new("RGB", (image_width, image_height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def dummy_data_for_fuyu(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + num_images = mm_counts["image"] + seq_data = dummy_seq_data_for_fuyu(ctx, seq_len, num_images) + mm_data = dummy_image_for_fuyu(num_images, + image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH, + image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT) + return seq_data, mm_data + + +def _fuyu_image_preprocess(image_processor: FuyuImageProcessor, + data: Image.Image): + image_encoding = image_processor.preprocess(data, return_tensors="pt") + batch_images = torch.stack([img[0] for img in image_encoding["images"] + ]).unsqueeze(1) + image_unpadded_heights = torch.tensor( + image_encoding["image_unpadded_heights"]) + image_unpadded_widths = torch.tensor( + image_encoding["image_unpadded_widths"]) + + batch_size = len(image_encoding["images"]) + image_present = torch.ones(batch_size, 1, 1) + model_image_input = image_processor.preprocess_with_tokenizer_info( + image_input=batch_images, + image_present=image_present, + image_unpadded_h=image_unpadded_heights, + image_unpadded_w=image_unpadded_widths, + image_placeholder_id=_IMAGE_TOKEN_ID, + image_newline_id=_NEWLINE_TOKEN_ID, + variable_sized=True, + ) + return model_image_input + + +def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + image_data = multi_modal_data["image"] + new_multi_modal_data = {} + # process image data + if isinstance(image_data, Image.Image): + # Fuyu's image_processor can also finish token padding + image_processor: FuyuImageProcessor = cached_get_image_processor( + model_config.model) + + model_image_input = _fuyu_image_preprocess(image_processor, image_data) + image_patches = torch.cat([ + image_patch[0] + for image_patch in model_image_input["image_patches"] + ]) + new_multi_modal_data["image"] = image_patches + + elif isinstance(image_data, torch.Tensor): + raise NotImplementedError("Embeddings input is not supported yet") + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + # process prompts + prompt = llm_inputs.get("prompt") + prompt_token_ids = llm_inputs["prompt_token_ids"] + tokenizer = cached_get_tokenizer(model_config.model) + # dim0 is batch_size, dim1 is subseq_size which will always be 1 + image_input_ids: List[List[ + torch.Tensor]] = model_image_input["image_input_ids"] + image_input_ids = image_input_ids[0][0].tolist() + bos_token = tokenizer.encode("", add_special_tokens=False)[1:] + boa_token = tokenizer.encode("\x04", add_special_tokens=False)[1:] + + new_prompt = prompt + "\x04" + new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[ + 1:] + boa_token + + return LLMInputs(prompt=new_prompt, + prompt_token_ids=new_prompt_token_ids, + multi_modal_data=new_multi_modal_data) + + +def input_mapper_for_fuyu(ctx: InputContext, data: object): + model_config = ctx.model_config + if isinstance(data, Image.Image): + # Fuyu's image_processor can also finish token padding + image_processor: FuyuImageProcessor = cached_get_image_processor( + model_config.model) + + model_image_input = _fuyu_image_preprocess(image_processor, data) + data = torch.stack([ + image_patch[0] + for image_patch in model_image_input["image_patches"] + ]) + + # image has been processed with prompt in input processor + return MultiModalInputs({"pixel_values": data}) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu) +@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu) +class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: FuyuConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.config = config + self.multimodal_config = multimodal_config + + self.padding_idx = config.pad_token_id + self.vocab_size = config.text_config.vocab_size + self.image_token_id = _IMAGE_TOKEN_ID + self.image_feature_size = config.patch_size**2 * config.num_channels + + self.vision_embed_tokens = ColumnParallelLinear( + self.image_feature_size, + config.hidden_size, + quant_config=quant_config, + gather_output=True, + ) + self.language_model = PersimmonForCausalLM(config.text_config, + cache_config=cache_config, + quant_config=quant_config) + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @property + def sampler(self): + return self.language_model.sampler + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + + h = w = self.config.patch_size + num_channels = self.config.num_channels + expected_dims = num_channels * h * w + + def _validate_shape(d: torch.Tensor): + actual_dims = d.size(-1) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + "The expected shape of pixel values per image per batch " + f" per patch is {expected_expr}. " + f"You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data.to(self.vision_embed_tokens.weight.dtype) + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[FuyuImagePixelInputs]: + pixel_values = kwargs.pop("pixel_values", None) + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of image patches. " + f"Got type: {type(pixel_values)}") + + return FuyuImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values( + flatten_bn(pixel_values, concat=True)), + ) + + return None + + def _process_image_input( + self, image_input: FuyuImagePixelInputs) -> torch.Tensor: + + assert self.vision_embed_tokens is not None + vision_embeddings, _ = self.vision_embed_tokens(image_input["data"]) + return vision_embeddings + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ): + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.embed_tokens( + input_ids) + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.image_token_id) + + else: + inputs_embeds = None + + hidden_states = self.language_model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.language_model.logits_processor( + self.language_model.lm_head, hidden_states, sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.language_model.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/gemma.py b/vllm/model_executor/models/gemma.py new file mode 100644 index 00000000..91e556db --- /dev/null +++ b/vllm/model_executor/models/gemma.py @@ -0,0 +1,455 @@ +# coding=utf-8 +# Copyright 2023 The vLLM team. +# Copyright (c) Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Gemma model compatible with HuggingFace weights.""" +from functools import lru_cache +from typing import Iterable, List, Optional, Set, Tuple, Union + +import torch +from torch import nn +from transformers import GemmaConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import GeluAndMul +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +logger = init_logger(__name__) + + +@lru_cache(maxsize=None) +def _get_gemma_act_fn( + hidden_act: Optional[str], + hidden_activation: Optional[str], +) -> nn.Module: + if hidden_activation is None: + if hidden_act is not None: + logger.warning( + "Gemma's activation function was incorrectly set to exact GeLU " + "in the config JSON file when it was initially released. " + "Changing the activation function to approximate GeLU " + "(`gelu_pytorch_tanh`). If you want to use the legacy " + "`%s`, edit the config JSON to set " + "`hidden_activation=%s` instead of `hidden_act`. " + "See https://github.com/huggingface/transformers/pull/29402 " + "for more details.", hidden_act, hidden_act) + return GeluAndMul(approximate="tanh") + elif hidden_activation == "gelu_pytorch_tanh": + return GeluAndMul(approximate="tanh") + elif hidden_activation == "gelu": + return GeluAndMul(approximate="none") + else: + raise ValueError(f"Activation function {hidden_act} is not " + "supported for Gemma models.") + + +class GemmaMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: Optional[str] = None, + hidden_activation: Optional[str] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation) + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class GemmaAttention(nn.Module): + + def __init__(self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + max_position_embeddings: int = 8192, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=self.rope_theta, + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class GemmaDecoderLayer(nn.Module): + + def __init__( + self, + config: GemmaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = GemmaAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + max_position_embeddings=config.max_position_embeddings, + rope_theta=config.rope_theta, + cache_config=cache_config, + quant_config=quant_config, + ) + self.mlp = GemmaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + hidden_activation=getattr(config, "hidden_activation", None), + quant_config=quant_config, + ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class GemmaModel(nn.Module): + + def __init__( + self, + config: GemmaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: GemmaDecoderLayer(config, cache_config, quant_config + ), + prefix=f"{prefix}.layers") + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + # Normalize the embedding by sqrt(hidden_size) + # The normalizer's data type should be downcasted to the model's + # data type such as bfloat16, not float32. + # See https://github.com/huggingface/transformers/pull/29402 + normalizer = self.config.hidden_size**0.5 + self.register_buffer("normalizer", torch.tensor(normalizer)) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + hidden_states *= self.normalizer + residual = None + else: + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + ] + + # BitandBytes specific attributes + default_bitsandbytes_target_modules = [ + ".gate_proj.", + ".down_proj.", + ".up_proj.", + ".q_proj.", + ".k_proj.", + ".v_proj.", + ".o_proj.", + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".down_proj.", ".o_proj."] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + # Gemma does not apply LoRA to the embedding layer. + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: GemmaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + # currently all existing Gemma models have `tie_word_embeddings` enabled + assert config.tie_word_embeddings + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = GemmaModel(config, cache_config, quant_config) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.model.embed_tokens, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: Set[str] = set() + for name, loaded_weight in weights: + for (param_name, shard_name, shard_id) in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # lm_head is not used in vllm as it is tied with embed_token. + # To prevent errors, skip loading lm_head.weight. + if "lm_head.weight" in name: + continue + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + unloaded_params = params_dict.keys() - loaded_params + if unloaded_params: + logger.warning( + "Some weights are not initialized from checkpoints: %s", + unloaded_params) diff --git a/vllm/model_executor/models/gemma2.py b/vllm/model_executor/models/gemma2.py new file mode 100644 index 00000000..bcb03ef5 --- /dev/null +++ b/vllm/model_executor/models/gemma2.py @@ -0,0 +1,463 @@ +# coding=utf-8 +# Copyright 2024 The vLLM team. +# Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Iterable, List, Optional, Set, Tuple, Union + +import torch +from torch import nn +from transformers import Gemma2Config + +from vllm.attention import Attention, AttentionMetadata +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import GeluAndMul +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (AutoWeightsLoader, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +logger = init_logger(__name__) + + +class Gemma2MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + hidden_activation: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if not (hidden_act == hidden_activation == "gelu_pytorch_tanh"): + raise ValueError( + "Gemma2 uses `gelu_pytorch_tanh` as the hidden activation " + "function. Please set `hidden_act` and `hidden_activation` to " + "`gelu_pytorch_tanh`.") + self.act_fn = GeluAndMul(approximate="tanh") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Gemma2Attention(nn.Module): + + def __init__(self, + layer_idx: int, + config: Gemma2Config, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + max_position_embeddings: int, + rope_theta: float, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + attn_logits_soft_cap: Optional[float] = None) -> None: + super().__init__() + self.layer_idx = layer_idx + self.config = config + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = config.query_pre_attn_scalar**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.attention_bias, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=config.attention_bias, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=self.rope_theta, + is_neox_style=True, + ) + + # FIXME(woosuk): While Gemma 2 uses sliding window attention for every + # odd layer, vLLM currently ignores it and uses global attention for + # all layers. + use_sliding_window = (layer_idx % 2 == 1 + and config.sliding_window is not None) + del use_sliding_window # Unused. + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + logits_soft_cap=attn_logits_soft_cap) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Gemma2DecoderLayer(nn.Module): + + def __init__( + self, + layer_idx: int, + config: Gemma2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Gemma2Attention( + layer_idx=layer_idx, + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + max_position_embeddings=config.max_position_embeddings, + rope_theta=config.rope_theta, + cache_config=cache_config, + quant_config=quant_config, + attn_logits_soft_cap=config.attn_logit_softcapping, + ) + self.hidden_size = config.hidden_size + self.mlp = Gemma2MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + hidden_activation=config.hidden_activation, + quant_config=quant_config, + ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = self.post_attention_layernorm(hidden_states) + + hidden_states, residual = self.pre_feedforward_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_feedforward_layernorm(hidden_states) + return hidden_states, residual + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": 0, + "inputs_embeds": 0, + "intermediate_tensors": 0, + }) +class Gemma2Model(nn.Module): + + def __init__( + self, + config: Gemma2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[ + -1]), config, cache_config, quant_config), + prefix=f"{prefix}.layers") + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + # Normalize the embedding by sqrt(hidden_size) + # The normalizer's data type should be downcasted to the model's + # data type such as bfloat16, not float32. + # See https://github.com/huggingface/transformers/pull/29402 + normalizer = self.config.hidden_size**0.5 + self.register_buffer("normalizer", torch.tensor(normalizer)) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_tokens(input_ids) + hidden_states *= self.normalizer + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: Set[str] = set() + for name, loaded_weight in weights: + for (param_name, shard_name, shard_id) in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + unloaded_params = params_dict.keys() - loaded_params + if unloaded_params: + logger.warning( + "Some weights are not initialized from checkpoints: %s", + unloaded_params) + + +class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + ] + # Gemma does not apply LoRA to the embedding layer. + embedding_modules = {} + embedding_padding_modules = [] + + # BitandBytes specific attributes + default_bitsandbytes_target_modules = [ + ".gate_proj.", + ".down_proj.", + ".up_proj.", + ".q_proj.", + ".k_proj.", + ".v_proj.", + ".o_proj.", + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".down_proj.", ".o_proj."] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__( + self, + config: Gemma2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + del lora_config # Unused. + super().__init__() + self.config = config + # currently all existing Gemma models have `tie_word_embeddings` enabled + assert config.tie_word_embeddings + self.quant_config = quant_config + self.model = Gemma2Model(config, cache_config, quant_config) + self.logits_processor = LogitsProcessor( + config.vocab_size, soft_cap=config.final_logit_softcapping) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.model.embed_tokens, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/gemma2_embedding.py b/vllm/model_executor/models/gemma2_embedding.py new file mode 100644 index 00000000..e8e10598 --- /dev/null +++ b/vllm/model_executor/models/gemma2_embedding.py @@ -0,0 +1,57 @@ +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import AttentionMetadata +from vllm.model_executor.layers.pooler import Pooler, PoolingType +from vllm.model_executor.pooling_metadata import PoolingMetadata +from vllm.sequence import IntermediateTensors, PoolerOutput + +from .gemma2 import Gemma2Model +from .interfaces import SupportsPP + + +class Gemma2EmbeddingModel(nn.Module, SupportsPP): + """A model that uses Gemma2 with additional embedding functionalities. + + This class encapsulates the Gemma2Model and provides an interface for + embedding operations and customized pooling functions. + + Attributes: + model: An instance of Gemma2Model used for forward operations. + _pooler: An instance of Pooler used for pooling operations. + """ + + def __init__( + self, + **kwargs, + ) -> None: + super().__init__() + self.model = Gemma2Model(**kwargs) + self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + return self.model(input_ids, positions, kv_caches, attn_metadata, + intermediate_tensors, inputs_embeds) + + def pooler( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> Optional[PoolerOutput]: + return self._pooler(hidden_states, pooling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + self.model.load_weights(weights) diff --git a/vllm/model_executor/models/glm4.py b/vllm/model_executor/models/glm4.py new file mode 100644 index 00000000..e74fd855 --- /dev/null +++ b/vllm/model_executor/models/glm4.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2025 The Zhipu AI team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GLM-4-0414 model compatible with HuggingFace weights.""" +from typing import Iterable, Optional, Set, Tuple, Union, List + +import torch +from torch import nn +from transformers import Glm4Config + +from vllm.attention import Attention, AttentionType, AttentionMetadata +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig , LoRAConfig #, VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import SamplerOutput, Sampler #get_sampler +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .llama import LlamaMLP as Glm4MLP +from .llama import LlamaModel +from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix + + +class Glm4Attention(nn.Module): + + def __init__(self, + config: Glm4Config, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + head_dim: Optional[int] = None, + qkv_bias: bool = False, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + rope_scaling: Optional[Tuple] = None, + prefix: str = "", + attn_type: str = AttentionType.DECODER) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + partial_rotary_factor = getattr(config, "partial_rotary_factor", 0.5) + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim or hidden_size // self.total_num_heads + self.rotary_dim = self.head_dim #int(partial_rotary_factor * self.head_dim) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.rotary_dim, + max_position=max_position, + base=self.rope_theta, + rope_scaling=rope_scaling, + partial_rotary_factor=partial_rotary_factor, + is_neox_style=False, + ) + # self.attn = Attention(self.num_heads, + # self.head_dim, + # self.scaling, + # num_kv_heads=self.num_kv_heads, + # cache_config=cache_config, + # quant_config=quant_config, + # prefix=f"{prefix}.attn", + # attn_type=attn_type) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn") + + + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Glm4DecoderLayer(nn.Module): + + def __init__( + self, + config: Glm4Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 1000000) + rope_scaling = getattr(config, "rope_scaling", None) + + self.self_attn = Glm4Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + qkv_bias=getattr(config, 'attention_bias', False), + head_dim=getattr(config, 'head_dim', None), + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=rope_scaling, + prefix=f"{prefix}.self_attn", + attn_type=AttentionType.DECODER, + ) + self.mlp = Glm4MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_self_attn_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_mlp_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata + ) + + hidden_states = self.post_self_attn_layernorm(hidden_states) + # hidden_states = residual + hidden_states + + # Fully Connected + # hidden_states = self.post_attention_layernorm(hidden_states, residual) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_mlp_layernorm(hidden_states) + # hidden_states = residual + hidden_states + + # Fully Connected + # residual = hidden_states + # hidden_states = self.post_attention_layernorm(hidden_states) + # hidden_states = self.mlp(hidden_states) + # hidden_states = self.post_mlp_layernorm(hidden_states) + # hidden_states = residual + hidden_states + + return hidden_states, residual + + +ALL_DECODER_LAYER_TYPES = { + "attention": Glm4DecoderLayer, +} + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + }) +class Glm4Model(LlamaModel): + + # def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + # super().__init__(vllm_config=vllm_config, + # prefix=prefix, + # layer_type=Glm4DecoderLayer) + + def __init__(self, *, + config: Glm4Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = ""): + super().__init__(config=config, + cache_config=cache_config, + quant_config=quant_config, + lora_config=lora_config, + prefix=prefix, + layer_type=Glm4DecoderLayer) + + +class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + def __init__(self, *, + config: Glm4Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = ""): + super().__init__() + # config = vllm_config.model_config.hf_config + # quant_config = vllm_config.quant_config + # lora_config = vllm_config.lora_config + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = Glm4Model(config=config, + cache_config=cache_config, + quant_config=quant_config, + lora_config=lora_config, + prefix=maybe_prefix(prefix, "model")) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix( + prefix, "lm_head")) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, attn_metadata, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, + torch.Tensor]]) -> Set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) \ No newline at end of file diff --git a/vllm/model_executor/models/glm4_vision_encoder.py b/vllm/model_executor/models/glm4_vision_encoder.py new file mode 100644 index 00000000..2ce89392 --- /dev/null +++ b/vllm/model_executor/models/glm4_vision_encoder.py @@ -0,0 +1,298 @@ +# coding=utf-8 +# Adapted from +# https://github.com/THUDM/GLM-4 +"""Inference-only GLM-4v model visual encoder compatible with THUDM weights.""" +from argparse import Namespace +from typing import Optional + +import torch +from torch import nn +from torch.nn import LayerNorm + +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) + + +class PatchEmbedding(nn.Module): + + def __init__(self, config): + super().__init__() + self.proj = nn.Conv2d(config.in_channels, + config.hidden_size, + kernel_size=config.patch_size, + stride=config.patch_size) + self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size)) + self.position_embedding = nn.Embedding(config.num_positions, + config.hidden_size) + + def forward(self, images: torch.Tensor) -> torch.Tensor: + """ + Parameters: + images : torch.Tensor + Input image tensor with shape (B, C, H, W) + + Returns: + torch.Tensor + Transformed tensor with shape (B, L, D) + """ + images = images.to(self.proj.weight.device) + x = self.proj(images) + x = x.flatten(2).transpose(1, 2) + cls_token = self.cls_embedding.expand(x.shape[0], -1, -1) + x = torch.cat((cls_token, x), dim=1) + x += self.position_embedding.weight.unsqueeze(0) + return x + + +class Attention(nn.Module): + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_rank = config.num_heads // self.tp_size + self.head_dim = config.hidden_size // config.num_heads + self.scale = self.head_dim**-0.5 + + self.query_key_value = QKVParallelLinear( + config.hidden_size, + self.head_dim, + config.num_heads, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + config.hidden_size, + config.hidden_size, + quant_config=quant_config, + ) + + self.output_dropout = torch.nn.Dropout(config.dropout_prob) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, L, _ = x.shape + qkv, _ = self.query_key_value(x) # B, L, 3 * H * D + q, k, v = qkv.chunk(3, dim=-1) + q = q.reshape(B, L, self.num_heads_per_rank, + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D + k = k.reshape(B, L, self.num_heads_per_rank, + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D + v = v.reshape(B, L, self.num_heads_per_rank, + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D + + out = torch.nn.functional.scaled_dot_product_attention(q, + k, + v, + attn_mask=None, + dropout_p=0., + is_causal=False) + + output, _ = self.dense(out.transpose(1, 2).contiguous().view(B, L, -1)) + output = self.output_dropout(output) + return output + + +class MLP(nn.Module): + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + quant_config=quant_config, + ) + self.fc2 = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + quant_config=quant_config, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc1(x) + x = self.activation_fn(x) + x, _ = self.fc2(x) + return x + + +class TransformerLayer(nn.Module): + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.input_layernorm = LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.attention = Attention(config, quant_config=quant_config) + self.mlp = MLP(config, quant_config=quant_config) + self.post_attention_layernorm = LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + + def forward(self, hidden_states): + attention_input = hidden_states + attention_output = self.input_layernorm( + self.attention(attention_input)) + hidden_states = attention_input + attention_output + mlp_input = hidden_states + mlp_output = self.post_attention_layernorm(self.mlp(mlp_input)) + output = mlp_input + mlp_output + return output + + +class Transformer(nn.Module): + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.layers = nn.ModuleList([ + TransformerLayer(config, quant_config=quant_config) + for _ in range(config.num_hidden_layers) + ]) + + def forward(self, hidden_states): + for layer_module in self.layers: + hidden_states = layer_module(hidden_states) + return hidden_states + + +class GLU(nn.Module): + + def __init__( + self, + config, + in_features, + quant_config: Optional[QuantizationConfig] = None, + ): + """ + The original implementation is the same as: + ```python + self.dense_h_to_4h = ColumnParallelLinear( + config.hidden_size, + config.ffn_hidden_size, + bias=False, + quant_config=quant_config + ) + + self.gate_proj = ColumnParallelLinear( + config.hidden_size, + config.ffn_hidden_size, + bias=False, + quant_config=quant_config + ) + ``` + ``` + gate_proj_output, _ = self.gate_proj(x) + dense_h_to_4h_output, _ = self.dense_h_to_4h(x) + x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1) + ``` + + We merge two ColumnParallelLinear into one MergedColumnParallelLinear: + ``` + self.merged_proj = MergedColumnParallelLinear( + config.hidden_size, + [config.ffn_hidden_size] * 2, + bias=False, + quant_config=quant_config + ) + ``` + ``` + x, _ = self.merged_proj(x) + ``` + """ + super().__init__() + self.linear_proj = ReplicatedLinear(in_features, + config.hidden_size, + bias=False, + quant_config=quant_config) + self.norm1 = nn.LayerNorm(config.hidden_size) + self.act1 = nn.GELU() + self.act2 = SiluAndMul() + + self.merged_proj = MergedColumnParallelLinear( + config.hidden_size, [config.ffn_hidden_size] * 2, + bias=False, + quant_config=quant_config) + + self.dense_4h_to_h = RowParallelLinear(config.ffn_hidden_size, + config.hidden_size, + bias=False, + quant_config=quant_config) + + def forward(self, x): + x, _ = self.linear_proj(x) + x = self.act1(self.norm1(x)) + x, _ = self.merged_proj(x) + x = self.act2(x) + x, _ = self.dense_4h_to_h(x) + return x + + +class EVA2CLIPModel(nn.Module): + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + vision_config = Namespace(**config.vision_config) + self.patch_embedding = PatchEmbedding(vision_config) + self.transformer = Transformer(vision_config, + quant_config=quant_config) + self.linear_proj = GLU(config, + in_features=config.hidden_size, + quant_config=quant_config) + self.conv = nn.Conv2d(in_channels=vision_config.hidden_size, + out_channels=config.hidden_size, + kernel_size=2, + stride=2) + self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) + self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) + self.scaling_factor = vision_config.scaling_factor + + def forward(self, images: torch.Tensor) -> torch.Tensor: + """ + Parameters: + images : torch.Tensor + Input image tensor with shape (B, C, H, W) + + Returns: + torch.Tensor + Transformed tensor with shape (B, L, D) + """ + x = self.patch_embedding(images) + x = self.transformer(x) + x = x[:, 1:] + + b, s, h = x.shape + grid_size = int(s**0.5) + x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2) + x = self.conv(x) + + x = x.flatten(2).transpose(1, 2) + x = self.linear_proj(x) + boi = self.boi.expand(x.shape[0], -1, -1) + eoi = self.eoi.expand(x.shape[0], -1, -1) + x = torch.cat((boi, x, eoi), dim=1) + x = x / self.scaling_factor + return x diff --git a/vllm/model_executor/models/gpt2.py b/vllm/model_executor/models/gpt2.py new file mode 100644 index 00000000..97550234 --- /dev/null +++ b/vllm/model_executor/models/gpt2.py @@ -0,0 +1,324 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py +# Copyright 2023 The vLLM team. +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GPT-2 model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import GPT2Config + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed.parallel_state import ( + get_pp_group, get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class GPT2Attention(nn.Module): + + def __init__( + self, + config: GPT2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = config.hidden_size + total_num_heads = config.num_attention_heads + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + assert total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = total_num_heads // tensor_model_parallel_world_size + self.head_dim = self.hidden_size // total_num_heads + self.scale = self.head_dim**-0.5 + + self.c_attn = QKVParallelLinear( + self.hidden_size, + self.head_dim, + total_num_heads, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.c_attn", + ) + self.c_proj = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.c_proj", + ) + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scale, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.c_attn(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output, _ = self.c_proj(attn_output) + return attn_output + + +class GPT2MLP(nn.Module): + + def __init__( + self, + intermediate_size: int, + config: GPT2Config, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + hidden_size = config.hidden_size + self.c_fc = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.c_fc", + ) + self.c_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.c_proj", + ) + self.act = get_act_fn(config.activation_function, quant_config, + intermediate_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.c_fc(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.c_proj(hidden_states) + return hidden_states + + +class GPT2Block(nn.Module): + + def __init__( + self, + config: GPT2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + hidden_size = config.hidden_size + inner_dim = (config.n_inner if config.n_inner is not None else 4 * + hidden_size) + + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.attn = GPT2Attention(config, + cache_config, + quant_config, + prefix=f"{prefix}.attn") + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.mlp = GPT2MLP(inner_dim, + config, + quant_config, + prefix=f"{prefix}.mlp") + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + attn_output = self.attn( + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + # residual connection + hidden_states = attn_output + residual + + residual = hidden_states + hidden_states = self.ln_2(hidden_states) + feed_forward_hidden_states = self.mlp(hidden_states) + # residual connection + hidden_states = residual + feed_forward_hidden_states + return hidden_states + + +class GPT2Model(nn.Module): + + def __init__( + self, + config: GPT2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + assert not config.add_cross_attention + assert not config.scale_attn_by_inverse_layer_idx + assert not config.reorder_and_upcast_attn + self.embed_dim = config.hidden_size + self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim) + self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim) + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: GPT2Block( + config, cache_config, quant_config, prefix=prefix), + prefix=f"{prefix}.h") + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.n_embd)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + inputs_embeds = self.wte(input_ids) + position_embeds = self.wpe(position_ids) + hidden_states = inputs_embeds + position_embeds + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer(hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class GPT2LMHeadModel(nn.Module, SupportsPP): + + def __init__( + self, + config: GPT2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.transformer = GPT2Model(config, + cache_config, + quant_config, + prefix="transformer") + if self.config.tie_word_embeddings: + self.lm_head = self.transformer.wte + else: + self.lm_head = ParallelLMHead(self.config.vocab_size, + self.config.hidden_size) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "lm_head.weight" in name: + # GPT-2 ties the weights of the embedding layer and the final + # linear layer. + continue + if ".attn.bias" in name or ".attn.masked_bias" in name: + # Skip attention mask. + # NOTE: "c_attn.bias" should not be skipped. + continue + if not name.startswith("transformer."): + name = "transformer." + name + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + # The HF's GPT-2 implementation uses Conv1D instead of Linear. + # Because of this, we need to transpose the weights. + # Note(zhuohan): the logic below might break quantized models. + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: + if conv1d_weight_name not in name: + continue + if not name.endswith(".weight"): + continue + loaded_weight = loaded_weight.t() + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/gpt_bigcode.py b/vllm/model_executor/models/gpt_bigcode.py new file mode 100644 index 00000000..6c4a0466 --- /dev/null +++ b/vllm/model_executor/models/gpt_bigcode.py @@ -0,0 +1,341 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py +# Copyright 2023 The vLLM team. +# Copyright 2023 CTranslate2, and Michael Feil +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GPTBigCode model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import GPTBigCodeConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class GPTBigCodeAttention(nn.Module): + + def __init__( + self, + config: GPTBigCodeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + total_num_heads = config.num_attention_heads + self.tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + assert total_num_heads % self.tensor_model_parallel_world_size == 0 + self.num_heads = (total_num_heads // + self.tensor_model_parallel_world_size) + self.head_dim = self.hidden_size // total_num_heads + self.scale = self.head_dim**-0.5 + + self.multi_query = config.multi_query + if self.multi_query: + total_num_kv_heads = 1 + self.num_kv_heads = 1 + else: + total_num_kv_heads = total_num_heads + self.num_kv_heads = self.num_heads + self.kv_dim = self.head_dim * self.num_kv_heads + self.c_attn = QKVParallelLinear( + self.hidden_size, + self.head_dim, + total_num_heads, + total_num_kv_heads, + bias=True, + quant_config=quant_config, + ) + + self.c_proj = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scale, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.c_attn(hidden_states) + q, k, v = qkv.split( + [ + self.hidden_size // self.tensor_model_parallel_world_size, + self.kv_dim, self.kv_dim + ], + dim=-1, + ) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output, _ = self.c_proj(attn_output) + return attn_output + + +class GPTBigMLP(nn.Module): + + def __init__( + self, + intermediate_size: int, + config: GPTBigCodeConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + self.c_fc = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + ) + self.c_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + ) + self.act = get_act_fn(config.activation_function, quant_config, + intermediate_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.c_fc(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.c_proj(hidden_states) + return hidden_states + + +class GPTBigCodeBlock(nn.Module): + + def __init__( + self, + config: GPTBigCodeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + inner_dim = (config.n_inner if config.n_inner is not None else 4 * + hidden_size) + + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.attn = GPTBigCodeAttention(config, cache_config, quant_config) + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.mlp = GPTBigMLP(inner_dim, config, quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + attn_output = self.attn( + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + # residual connection + hidden_states = attn_output + residual + + residual = hidden_states + hidden_states = self.ln_2(hidden_states) + feed_forward_hidden_states = self.mlp(hidden_states) + # residual connection + hidden_states = residual + feed_forward_hidden_states + return hidden_states + + +class GPTBigCodeModel(nn.Module): + + def __init__( + self, + config: GPTBigCodeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + assert not config.add_cross_attention + + self.embed_dim = config.hidden_size + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.wte = VocabParallelEmbedding(self.vocab_size, + self.embed_dim, + org_num_embeddings=config.vocab_size) + self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim) + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: GPTBigCodeBlock(config, cache_config, quant_config), + prefix=f"{prefix}.h", + ) + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.n_embd)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + inputs_embeds = self.wte(input_ids) + position_embeds = self.wpe(position_ids) + hidden_states = inputs_embeds + position_embeds + else: + hidden_states = intermediate_tensors["hidden_states"] + + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer(hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = {"c_attn": ["c_attn"]} + + supported_lora_modules = ["c_fc", "c_proj", "wte", "c_attn"] + + embedding_modules = { + "wte": "input_embeddings", + "lm_head": "output_embeddings", + } + + embedding_padding_modules = [] + + def __init__( + self, + config: GPTBigCodeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.transformer = GPTBigCodeModel(config, cache_config, quant_config, + lora_config) + if self.config.tie_word_embeddings: + self.lm_head = self.transformer.wte + else: + self.lm_head = ParallelLMHead( + self.transformer.vocab_size, + self.transformer.embed_dim, + org_num_embeddings=self.config.vocab_size) + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "lm_head.weight" in name: + continue + if ".attn.bias" in name: + # Skip attention mask. + # NOTE: "c_attn.bias" should not be skipped. + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + # TODO (@robertgshaw2-neuralmagic): move to fp8 linear method + if "c_attn.input_scale" in name or "c_attn.weight_scale" in name: + weight_loader(param, loaded_weight, 'q') + weight_loader(param, loaded_weight, 'k') + weight_loader(param, loaded_weight, 'v') + else: + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/gpt_j.py b/vllm/model_executor/models/gpt_j.py new file mode 100644 index 00000000..d40bf8c8 --- /dev/null +++ b/vllm/model_executor/models/gpt_j.py @@ -0,0 +1,317 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gptj/modeling_gptj.py +# Copyright 2023 The vLLM team. +# Copyright 2021 The EleutherAI and HuggingFace Teams. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GPT-J model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import GPTJConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class GPTJAttention(nn.Module): + + def __init__( + self, + config: GPTJConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.total_num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.total_num_heads + + self.qkv_proj = QKVParallelLinear( + config.hidden_size, + self.head_size, + self.total_num_heads, + bias=False, + quant_config=quant_config, + ) + self.out_proj = RowParallelLinear( + config.hidden_size, + config.hidden_size, + bias=False, + quant_config=quant_config, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + scaling = self.head_size**-0.5 + assert getattr(config, "rotary", True) + assert config.rotary_dim % 2 == 0 + rope_theta = getattr(config, "rope_theta", 10000) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.rotary_emb = get_rope( + self.head_size, + rotary_dim=config.rotary_dim, + max_position=max_position_embeddings, + base=rope_theta, + is_neox_style=False, + ) + self.attn = Attention(self.num_heads, + self.head_size, + scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + q, k = self.rotary_emb(position_ids, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output, _ = self.out_proj(attn_output) + return attn_output + + +class GPTJMLP(nn.Module): + + def __init__( + self, + intermediate_size: int, + config: GPTJConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.n_embd + self.fc_in = ColumnParallelLinear( + hidden_size, + intermediate_size, + quant_config=quant_config, + ) + self.fc_out = RowParallelLinear( + intermediate_size, + hidden_size, + quant_config=quant_config, + ) + self.act = get_act_fn(config.activation_function, quant_config, + intermediate_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc_in(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.fc_out(hidden_states) + return hidden_states + + +class GPTJBlock(nn.Module): + + def __init__( + self, + config: GPTJConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + inner_dim = (4 * config.n_embd + if config.n_inner is None else config.n_inner) + self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + self.attn = GPTJAttention(config, cache_config, quant_config) + self.mlp = GPTJMLP(inner_dim, config, quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + attn_output = self.attn( + position_ids=position_ids, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + mlp_output = self.mlp(hidden_states) + hidden_states = attn_output + mlp_output + residual + return hidden_states + + +class GPTJModel(nn.Module): + + def __init__( + self, + config: GPTJConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.embed_dim = config.n_embd + self.wte = VocabParallelEmbedding( + config.vocab_size, + self.embed_dim, + ) + self.start_layer, self.end_layer, self.h = make_layers( + config.n_layer, + lambda prefix: GPTJBlock(config, cache_config, quant_config), + prefix=f"{prefix}.h", + ) + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.n_embd)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.wte(input_ids) + else: + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer( + position_ids, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class GPTJForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: GPTJConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + assert not config.tie_word_embeddings + self.transformer = GPTJModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.n_embd, + bias=True, + quant_config=quant_config, + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata, self.lm_head.bias) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "attn.bias" in name or "attn.masked_bias" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/gpt_neox.py b/vllm/model_executor/models/gpt_neox.py new file mode 100644 index 00000000..23a1ca06 --- /dev/null +++ b/vllm/model_executor/models/gpt_neox.py @@ -0,0 +1,329 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GPT-NeoX model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import GPTNeoXConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class GPTNeoXAttention(nn.Module): + + def __init__( + self, + config: GPTNeoXConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.total_num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.total_num_heads + self.bias = getattr(config, "attention_bias", True) + + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + assert self.total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = (self.total_num_heads // + tensor_model_parallel_world_size) + + self.query_key_value = QKVParallelLinear( + config.hidden_size, + self.head_size, + self.total_num_heads, + bias=self.bias, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + config.hidden_size, + config.hidden_size, + bias=self.bias, + quant_config=quant_config, + ) + scaling = self.head_size**-0.5 + rotary_dim = int(self.head_size * config.rotary_pct) + assert rotary_dim % 2 == 0 + rope_theta = getattr(config, "rope_theta", 10000) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.rotary_emb = get_rope( + self.head_size, + rotary_dim=rotary_dim, + max_position=max_position_embeddings, + base=rope_theta, + ) + self.attn = Attention(self.num_heads, + self.head_size, + scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.query_key_value(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + q, k = self.rotary_emb(position_ids, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.dense(attn_output) + return output + + +class GPTNeoXMLP(nn.Module): + + def __init__( + self, + config: GPTNeoXConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.dense_h_to_4h = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + quant_config=quant_config, + ) + self.dense_4h_to_h = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + quant_config=quant_config, + ) + self.act = get_act_fn(config.hidden_act, quant_config, + config.intermediate_size) + + def forward(self, hidden_states): + hidden_states, _ = self.dense_h_to_4h(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.dense_4h_to_h(hidden_states) + return hidden_states + + +class GPTNeoXLayer(nn.Module): + + def __init__( + self, + config: GPTNeoXConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.use_parallel_residual = config.use_parallel_residual + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.attention = GPTNeoXAttention(config, cache_config, quant_config) + self.mlp = GPTNeoXMLP(config, quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + attn_input = self.input_layernorm(hidden_states) + attn_output = self.attention( + position_ids=position_ids, + hidden_states=attn_input, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + if self.use_parallel_residual: + # pseudocode: + # x = x + attn(ln1(x)) + mlp(ln2(x)) + mlp_input = self.post_attention_layernorm(hidden_states) + mlp_output = self.mlp(mlp_input) + hidden_states = mlp_output + attn_output + hidden_states + else: + # pseudocode: + # x = x + attn(ln1(x)) + # x = x + mlp(ln2(x)) + attn_output = attn_output + hidden_states + mlp_input = self.post_attention_layernorm(attn_output) + mlp_output = self.mlp(mlp_input) + hidden_states = mlp_output + attn_output + return hidden_states + + +class GPTNeoXModel(nn.Module): + + def __init__( + self, + config: GPTNeoXConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + + self.embed_in = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: GPTNeoXLayer(config, cache_config, quant_config), + prefix=f"{prefix}.layers", + ) + self.final_layer_norm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_in(input_ids) + else: + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer( + position_ids, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.final_layer_norm(hidden_states) + return hidden_states + + +class GPTNeoXForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: GPTNeoXConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config) + self.embed_out = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + ) + if self.config.tie_word_embeddings: + self.embed_out.weight = self.gpt_neox.embed_in.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.gpt_neox.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.gpt_neox(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.embed_out, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if ("attention.bias" in name or "attention.masked_bias" in name + or "rotary_emb.inv_freq" in name): + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using OpenRLHF may include + # these tensors in the checkpoint. Skip them. + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + + if "query_key_value" in name: + # NOTE: GPT-NeoX's fused QKV's output_dim has the shape of + # (num_heads * 3 * head_size), while the + # required shape is (3 * num_heads * head_size). + # Thus, we need weight conversion. + output_dim = getattr(param, "output_dim", None) + num_heads = self.config.num_attention_heads + if output_dim is not None: + loaded_weight_shape = loaded_weight.shape + loaded_weight = loaded_weight.view( + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) + + loaded_weight_shape[output_dim + 1:]) + loaded_weight = loaded_weight.transpose( + output_dim, output_dim + 1) + loaded_weight = loaded_weight.reshape(loaded_weight_shape) + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/granite.py b/vllm/model_executor/models/granite.py new file mode 100644 index 00000000..dcf4f5b2 --- /dev/null +++ b/vllm/model_executor/models/granite.py @@ -0,0 +1,545 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only IBM Granite model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import GraniteConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + get_compressed_tensors_cache_scale) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import is_hip + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers + + +class GraniteMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj") + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class GraniteAttention(nn.Module): + + def __init__( + self, + config: GraniteConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", + self.hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = config.attention_multiplier + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class GraniteDecoderLayer(nn.Module): + + def __init__( + self, + config: GraniteConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.residual_multiplier = config.residual_multiplier + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + self.self_attn = GraniteAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + cache_config=cache_config, + prefix=f"{prefix}.self_attn", + ) + + self.mlp = GraniteMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states * self.residual_multiplier + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states * self.residual_multiplier + return hidden_states + + +class GraniteModel(nn.Module): + + def __init__( + self, + config: GraniteConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=quant_config, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: GraniteDecoderLayer(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix), + prefix=f"{prefix}.layers") + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + + hidden_states *= self.config.embedding_multiplier + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states = self.norm(hidden_states) + return hidden_states + + +class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens", + "lm_head" + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__( + self, + config: GraniteConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = GraniteModel(config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model") + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + logit_scale = getattr(config, "logit_scale", 1.0) + + if hasattr(config, "logits_scaling"): + logit_scale /= config.logits_scaling + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + scale=logit_scale) + self.sampler = Sampler() + else: + self.lm_head = PPMissingLayer() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return model_output + + def compute_logits( + self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def make_empty_intermediate_tensors( + self, batch_size: int, dtype: torch.dtype, + device: torch.device) -> IntermediateTensors: + return IntermediateTensors({ + "hidden_states": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + "residual": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + }) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + # With tie_word_embeddings, we can skip lm_head.weight + # The weight might appear unnecessarily in the files if the model is + # processed with quantization, LoRA, fine-tuning, etc. + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + if scale_name := get_compressed_tensors_cache_scale(name): + # Loading kv cache scales for compressed-tensors quantization + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + # If this function is called, it should always initialize KV cache scale + # factors (or else raise an exception). Thus, handled exceptions should + # make sure to leave KV cache scale factors in a known good (dummy) state + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + for layer_idx, scaling_factor in kv_cache_scales_loader( + quantization_param_path, tp_rank, tp_size, + self.config.num_hidden_layers, + self.config.__class__.model_type): + if not isinstance(self.model.layers[layer_idx], nn.Identity): + layer_self_attn = self.model.layers[layer_idx].self_attn + + if is_hip(): + # The scaling factor convention we are assuming is + # quantized_value * scaling_factor ~= true_value + # which is consistent with the practice of setting + # scaling_factor = tensor_amax / FPtype_max + scaling_factor *= 2 + if hasattr(layer_self_attn, "kv_scale"): + layer_self_attn.attn._kv_scale = scaling_factor + else: + raise RuntimeError("Self attention has no KV cache scaling " + "factor attribute!") diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py new file mode 100644 index 00000000..52669517 --- /dev/null +++ b/vllm/model_executor/models/granitemoe.py @@ -0,0 +1,448 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GraniteMoe model.""" +from typing import Iterable, List, Optional, Tuple + +import torch +from torch import nn +from transformers.models.granitemoe import GraniteMoeConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from . import mixtral +from .interfaces import SupportsLoRA, SupportsPP +from .utils import make_layers + + +class GraniteMoeMoE(nn.Module): + """A tensor-parallel MoE implementation for GraniteMoe that shards each + expert across all ranks. + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__(self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + prefix: str = ""): + super().__init__() + self.hidden_size = hidden_size + + # Gate always runs at half / full precision for now. + self.gate = ReplicatedLinear(hidden_size, + num_experts, + bias=False, + params_dtype=params_dtype, + quant_config=None, + prefix=f"{prefix}.gate") + + self.experts = FusedMoE(num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + params_dtype=params_dtype, + reduce_results=True, + renormalize=True, + quant_config=quant_config, + tp_size=tp_size, + prefix=f"{prefix}.experts") + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states, router_logits) + return final_hidden_states.view(orig_shape) + + +class GraniteMoeAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + attention_multiplier: Optional[float] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = (attention_multiplier if attention_multiplier + is not None else self.head_dim**-1) + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=int(self.rope_theta), + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class GraniteMoeDecoderLayer(nn.Module): + + def __init__( + self, + config: GraniteMoeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 10000) + self.self_attn = GraniteMoeAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + attention_multiplier=config.attention_multiplier) + self.block_sparse_moe = GraniteMoeMoE( + num_experts=config.num_local_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.block_sparse_moe") + + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + self.residual_multiplier = config.residual_multiplier + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states * self.residual_multiplier + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.block_sparse_moe(hidden_states) + hidden_states = residual + hidden_states * self.residual_multiplier + + return hidden_states + + +class GraniteMoeModel(nn.Module): + + def __init__( + self, + config: GraniteMoeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + self.embedding_multiplier = config.embedding_multiplier + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: GraniteMoeDecoderLayer( + config, cache_config, quant_config=quant_config, prefix=prefix + ), + prefix=f"{prefix}.layers") + + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> torch.Tensor: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + hidden_states *= self.embedding_multiplier + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + fall_back_to_pt_during_load = False + + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: GraniteMoeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = GraniteMoeModel(config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model") + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + scale=1 / + self.config.logits_scaling) + + self.sampler = Sampler() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def make_empty_intermediate_tensors( + self, batch_size: int, dtype: torch.dtype, + device: torch.device) -> IntermediateTensors: + return IntermediateTensors({ + "hidden_states": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + "residual": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + }) + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + new_weights = {} + for n, p in weights: + if n.endswith('.block_sparse_moe.input_linear.weight'): + for e in range(p.size(0)): + w1_name = n.replace( + '.block_sparse_moe.input_linear.weight', + ".block_sparse_moe.experts.%d.w1.weight" % e) + w3_name = n.replace( + '.block_sparse_moe.input_linear.weight', + ".block_sparse_moe.experts.%d.w3.weight" % e) + w1_param, w3_param = p[e].chunk(2, dim=0) + assert w1_name not in new_weights + assert w3_name not in new_weights + new_weights[w1_name] = w1_param + new_weights[w3_name] = w3_param + elif n.endswith('.block_sparse_moe.output_linear.weight'): + for e in range(p.size(0)): + w2_name = n.replace( + '.block_sparse_moe.output_linear.weight', + ".block_sparse_moe.experts.%d.w2.weight" % e) + w2_param = p[e] + assert w2_name not in new_weights + new_weights[w2_name] = w2_param + elif n.endswith('.block_sparse_moe.router.layer.weight'): + gate_name = n.replace('.block_sparse_moe.router.layer.weight', + ".block_sparse_moe.gate.weight") + assert gate_name not in new_weights + new_weights[gate_name] = p + elif n == 'lm_head.weight' and self.config.tie_word_embeddings: + pass + else: + new_weights[n] = p + mixtral.MixtralForCausalLM.load_weights(self, new_weights.items()) diff --git a/vllm/model_executor/models/idefics2_vision_model.py b/vllm/model_executor/models/idefics2_vision_model.py new file mode 100644 index 00000000..3b0b6feb --- /dev/null +++ b/vllm/model_executor/models/idefics2_vision_model.py @@ -0,0 +1,302 @@ +# coding=utf-8 + +# adapted from https://github.com/huggingface/transformers/blob/v4.43.2/src/transformers/models/idefics2/modeling_idefics2.py +# Copyright 2024 The vLLM team. +# Copyright 2024 the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Idefics2 model.""" + +from typing import Optional + +import torch +from torch import nn +from transformers.models.idefics2.configuration_idefics2 import ( + Idefics2Config, Idefics2VisionConfig) +from xformers import ops as xops + +from vllm.distributed import divide, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig + + +class Idefics2VisionEmbeddings(nn.Module): + """ + This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings + ` to enable images of variable + resolution. + + The modifications are adapted from [Patch n' Pack: NaViT, a Vision + Transformer for any Aspect Ratio and Resolution](https://arxiv.org/abs/2307.06304) + which allows treating images in their native aspect ratio and without the + need to resize them to the same fixed size. In particular, we start from the + original pre-trained SigLIP model(which uses images of fixed-size square + images) and adapt it by training on images of variable resolutions. + """ + + def __init__(self, config: Idefics2VisionConfig): + super().__init__() + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + padding="valid", + ) + self.num_patches_per_side = self.image_size // self.patch_size + self.num_patches = self.num_patches_per_side**2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, + self.embed_dim) + + def forward(self, + pixel_values: torch.FloatTensor, + patch_attention_mask: torch.BoolTensor, + tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor: + batch_size, _, max_im_h, max_im_w = pixel_values.shape + patch_embeds = self.patch_embedding(pixel_values) + embeddings = patch_embeds.flatten(2).transpose(1, 2) + max_nb_patches_h, max_nb_patches_w = ( + max_im_h // self.patch_size, + max_im_w // self.patch_size, + ) + boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, + 1 / self.num_patches_per_side) + position_ids = torch.full(size=(batch_size, + max_nb_patches_h * max_nb_patches_w), + fill_value=0) + + for batch_idx, p_attn_mask in enumerate(patch_attention_mask): + + if tgt_sizes is not None: + nb_patches_h = tgt_sizes[batch_idx][0] + nb_patches_w = tgt_sizes[batch_idx][1] + else: + nb_patches_h = p_attn_mask[:, 0].sum() + nb_patches_w = p_attn_mask[0].sum() + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h) + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w) + bucket_coords_h = torch.bucketize(fractional_coords_h, + boundaries, + right=True) + bucket_coords_w = torch.bucketize(fractional_coords_w, + boundaries, + right=True) + pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + + bucket_coords_w).flatten() + position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids + position_ids = position_ids.to(self.position_embedding.weight.device) + embeddings = embeddings + self.position_embedding(position_ids) + return embeddings + + +class Idefics2VisionAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: Idefics2Config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" # noqa: E501 + f" {self.num_heads}).") + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + self.qkv_proj = QKVParallelLinear( + self.embed_dim, + self.head_dim, + self.num_heads, + quant_config=quant_config, + ) + self.out_proj = RowParallelLinear( + self.embed_dim, + self.embed_dim, + bias=True, + quant_config=quant_config, + ) + self.tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = divide(self.num_heads, self.tp_size) + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + batch_size, q_len, _ = hidden_states.size() + qkv, _ = self.qkv_proj( + hidden_states + ) # batch_size, q_len, 3 * num_heads_per_partition * head_dim + query_states, key_states, value_states = qkv.chunk(3, dim=-1) + query_states = query_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + key_states = key_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + value_states = value_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + # see: https://facebookresearch.github.io/xformers/components/ops.html + out = xops.memory_efficient_attention_forward( + query_states, + key_states, + value_states, + p=self.dropout, + scale=self.scale, + ) + out = out.view(batch_size, q_len, -1) + attn_output, _ = self.out_proj(out) + return attn_output + + +class Idefics2VisionMLP(nn.Module): + + def __init__( + self, + config: Idefics2Config, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config, + ) + self.fc2 = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + return hidden_states + + +class Idefics2EncoderLayer(nn.Module): + + def __init__(self, config: Idefics2Config): + super().__init__() + self.embed_dim = config.hidden_size + self.self_attn = Idefics2VisionAttention(config) + self.layer_norm1 = nn.LayerNorm(self.embed_dim, + eps=config.layer_norm_eps) + self.mlp = Idefics2VisionMLP(config) + self.layer_norm2 = nn.LayerNorm(self.embed_dim, + eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): + Input to the layer of shape `(batch, seq_len, embed_dim)`. + + """ + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn(hidden_states) + hidden_states = residual + hidden_states + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +class Idefics2Encoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention + layers. Each layer is a + [`Idefics2EncoderLayer`]. + + Args: + config: Idefics2Config + """ + + def __init__(self, config: Idefics2Config): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + Idefics2EncoderLayer(config) + for _ in range(config.num_hidden_layers) + ]) + + def forward( + self, + inputs_embeds: torch.Tensor, + ) -> torch.Tensor: + r""" + Args: + inputs_embeds (torch.Tensor): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. + This is useful if you want more control over how to convert + `input_ids` indices into associated vectorsthan the model's + internal embedding lookup matrix. + """ + hidden_states = inputs_embeds + for encoder_layer in self.layers: + layer_outputs = encoder_layer(hidden_states) + hidden_states = layer_outputs + return hidden_states + + +class Idefics2VisionTransformer(nn.Module): + + def __init__(self, config: Idefics2VisionConfig): + super().__init__() + embed_dim = config.hidden_size + self.config = config + self.embeddings = Idefics2VisionEmbeddings(config) + self.encoder = Idefics2Encoder(config) + self.post_layernorm = nn.LayerNorm(embed_dim, + eps=config.layer_norm_eps) + + def get_input_embeddings(self): + return self.embeddings + + def forward( + self, + pixel_values, + patch_attention_mask: Optional[torch.BoolTensor] = None, + tgt_sizes: Optional[torch.IntTensor] = None, + ) -> torch.Tensor: + hidden_states = self.embeddings( + pixel_values=pixel_values, + patch_attention_mask=patch_attention_mask, + tgt_sizes=tgt_sizes) + encoder_outputs = self.encoder(hidden_states) + last_hidden_state = self.post_layernorm(encoder_outputs) + return last_hidden_state diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py new file mode 100644 index 00000000..dcead651 --- /dev/null +++ b/vllm/model_executor/models/interfaces.py @@ -0,0 +1,352 @@ +from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional, + Protocol, Type, Union, overload, runtime_checkable) + +import torch +from typing_extensions import TypeIs + +from vllm.logger import init_logger +from vllm.utils import supports_kw + +if TYPE_CHECKING: + from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig + from vllm.sequence import IntermediateTensors + +logger = init_logger(__name__) + + +@runtime_checkable +class SupportsMultiModal(Protocol): + """The interface required for all multi-modal models.""" + + supports_multimodal: ClassVar[Literal[True]] = True + """ + A flag that indicates this model supports multi-modal inputs. + + Note: + There is no need to redefine this flag if this class is in the + MRO of your model class. + """ + + def __init__(self, *, multimodal_config: "MultiModalConfig") -> None: + ... + + +# We can't use runtime_checkable with ClassVar for issubclass checks +# so we need to treat the class as an instance and use isinstance instead +@runtime_checkable +class _SupportsMultiModalType(Protocol): + supports_multimodal: Literal[True] + + def __call__(self, *, multimodal_config: "MultiModalConfig") -> None: + ... + + +@overload +def supports_multimodal( + model: Type[object]) -> TypeIs[Type[SupportsMultiModal]]: + ... + + +@overload +def supports_multimodal(model: object) -> TypeIs[SupportsMultiModal]: + ... + + +def supports_multimodal( + model: Union[Type[object], object], +) -> Union[TypeIs[Type[SupportsMultiModal]], TypeIs[SupportsMultiModal]]: + if isinstance(model, type): + return isinstance(model, _SupportsMultiModalType) + + return isinstance(model, SupportsMultiModal) + + +@runtime_checkable +class SupportsLoRA(Protocol): + """The interface required for all models that support LoRA.""" + + supports_lora: ClassVar[Literal[True]] = True + """ + A flag that indicates this model supports LoRA. + + Note: + There is no need to redefine this flag if this class is in the + MRO of your model class. + """ + + packed_modules_mapping: ClassVar[Dict[str, List[str]]] + supported_lora_modules: ClassVar[List[str]] + embedding_modules: ClassVar[Dict[str, str]] + embedding_padding_modules: ClassVar[List[str]] + + # lora_config is None when LoRA is not enabled + def __init__(self, *, lora_config: Optional["LoRAConfig"] = None) -> None: + ... + + +# We can't use runtime_checkable with ClassVar for issubclass checks +# so we need to treat the class as an instance and use isinstance instead +@runtime_checkable +class _SupportsLoRAType(Protocol): + supports_lora: Literal[True] + + packed_modules_mapping: Dict[str, List[str]] + supported_lora_modules: List[str] + embedding_modules: Dict[str, str] + embedding_padding_modules: List[str] + + def __call__(self, *, lora_config: Optional["LoRAConfig"] = None) -> None: + ... + + +@overload +def supports_lora(model: Type[object]) -> TypeIs[Type[SupportsLoRA]]: + ... + + +@overload +def supports_lora(model: object) -> TypeIs[SupportsLoRA]: + ... + + +def supports_lora( + model: Union[Type[object], object], +) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]: + result = _supports_lora(model) + + if not result: + lora_attrs = ( + "packed_modules_mapping", + "supported_lora_modules", + "embedding_modules", + "embedding_padding_modules", + ) + missing_attrs = tuple(attr for attr in lora_attrs + if not hasattr(model, attr)) + + if getattr(model, "supports_lora", False): + if missing_attrs: + logger.warning( + "The model (%s) sets `supports_lora=True`, " + "but is missing LoRA-specific attributes: %s", + model, + missing_attrs, + ) + else: + if not missing_attrs: + logger.warning( + "The model (%s) contains all LoRA-specific attributes, " + "but does not set `supports_lora=True`.", model) + + return result + + +def _supports_lora(model: Union[Type[object], object]) -> bool: + if isinstance(model, type): + return isinstance(model, _SupportsLoRAType) + + return isinstance(model, SupportsLoRA) + + +@runtime_checkable +class SupportsPP(Protocol): + """The interface required for all models that support pipeline parallel.""" + + supports_pp: ClassVar[Literal[True]] = True + """ + A flag that indicates this model supports pipeline parallel. + + Note: + There is no need to redefine this flag if this class is in the + MRO of your model class. + """ + + def make_empty_intermediate_tensors( + self, + batch_size: int, + dtype: torch.dtype, + device: torch.device, + ) -> "IntermediateTensors": + """Called when PP rank > 0 for profiling purposes.""" + ... + + def forward( + self, + *, + intermediate_tensors: Optional["IntermediateTensors"], + ) -> Union[torch.Tensor, "IntermediateTensors"]: + """ + Accept :class:`IntermediateTensors` when PP rank > 0. + + Return :class:`IntermediateTensors` only for the last PP rank. + """ + ... + + +# We can't use runtime_checkable with ClassVar for issubclass checks +# so we need to treat the class as an instance and use isinstance instead +@runtime_checkable +class _SupportsPPType(Protocol): + supports_pp: Literal[True] + + def make_empty_intermediate_tensors( + self, + batch_size: int, + dtype: torch.dtype, + device: torch.device, + ) -> "IntermediateTensors": + ... + + def forward( + self, + *, + intermediate_tensors: Optional["IntermediateTensors"], + ) -> Union[torch.Tensor, "IntermediateTensors"]: + ... + + +@overload +def supports_pp(model: Type[object]) -> TypeIs[Type[SupportsPP]]: + ... + + +@overload +def supports_pp(model: object) -> TypeIs[SupportsPP]: + ... + + +def supports_pp( + model: Union[Type[object], object], +) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]: + supports_attributes = _supports_pp_attributes(model) + supports_inspect = _supports_pp_inspect(model) + + if supports_attributes and not supports_inspect: + logger.warning( + "The model (%s) sets `supports_pp=True`, but does not accept " + "`intermediate_tensors` in its `forward` method", model) + + if not supports_attributes: + pp_attrs = ("make_empty_intermediate_tensors", ) + missing_attrs = tuple(attr for attr in pp_attrs + if not hasattr(model, attr)) + + if getattr(model, "supports_pp", False): + if missing_attrs: + logger.warning( + "The model (%s) sets `supports_pp=True`, " + "but is missing PP-specific attributes: %s", + model, + missing_attrs, + ) + else: + if not missing_attrs: + logger.warning( + "The model (%s) contains all PP-specific attributes, " + "but does not set `supports_pp=True`.", model) + + return supports_attributes and supports_inspect + + +def _supports_pp_attributes(model: Union[Type[object], object]) -> bool: + if isinstance(model, type): + return isinstance(model, _SupportsPPType) + + return isinstance(model, SupportsPP) + + +def _supports_pp_inspect(model: Union[Type[object], object]) -> bool: + model_forward = getattr(model, "forward", None) + if not callable(model_forward): + return False + + return supports_kw(model_forward, "intermediate_tensors") + + +@runtime_checkable +class HasInnerState(Protocol): + """The interface required for all models that has inner state.""" + + has_inner_state: ClassVar[Literal[True]] = True + """ + A flag that indicates this model has inner state. + Models that has inner state usually need access to the scheduler_config + for max_num_seqs, etc. True for e.g. both Mamba and Jamba. + """ + + def __init__(self, + *, + scheduler_config: Optional["SchedulerConfig"] = None) -> None: + ... + + +@runtime_checkable +class _HasInnerStateType(Protocol): + has_inner_state: ClassVar[Literal[True]] + + def __init__(self, + *, + scheduler_config: Optional["SchedulerConfig"] = None) -> None: + ... + + +@overload +def has_inner_state(model: object) -> TypeIs[HasInnerState]: + ... + + +@overload +def has_inner_state(model: Type[object]) -> TypeIs[Type[HasInnerState]]: + ... + + +def has_inner_state( + model: Union[Type[object], object] +) -> Union[TypeIs[Type[HasInnerState]], TypeIs[HasInnerState]]: + if isinstance(model, type): + return isinstance(model, _HasInnerStateType) + + return isinstance(model, HasInnerState) + + +@runtime_checkable +class IsAttentionFree(Protocol): + """The interface required for all models like Mamba that lack attention, + but do have state whose size is constant wrt the number of tokens.""" + + is_attention_free: ClassVar[Literal[True]] = True + """ + A flag that indicates this model has no attention. + Used for block manager and attention backend selection. + True for Mamba but not Jamba. + """ + + def __init__(self) -> None: + ... + + +@runtime_checkable +class _IsAttentionFreeType(Protocol): + is_attention_free: ClassVar[Literal[True]] + + def __init__(self) -> None: + ... + + +@overload +def is_attention_free(model: object) -> TypeIs[IsAttentionFree]: + ... + + +@overload +def is_attention_free(model: Type[object]) -> TypeIs[Type[IsAttentionFree]]: + ... + + +def is_attention_free( + model: Union[Type[object], object] +) -> Union[TypeIs[Type[IsAttentionFree]], TypeIs[IsAttentionFree]]: + if isinstance(model, type): + return isinstance(model, _IsAttentionFreeType) + + return isinstance(model, IsAttentionFree) diff --git a/vllm/model_executor/models/interfaces_base.py b/vllm/model_executor/models/interfaces_base.py new file mode 100644 index 00000000..8d2d422f --- /dev/null +++ b/vllm/model_executor/models/interfaces_base.py @@ -0,0 +1,191 @@ +from typing import (TYPE_CHECKING, List, Optional, Protocol, Type, Union, + overload, runtime_checkable) + +import torch +import torch.nn as nn +from transformers import PretrainedConfig +from typing_extensions import TypeIs, TypeVar + +from vllm.logger import init_logger +from vllm.utils import supports_kw + +if TYPE_CHECKING: + from vllm.attention import AttentionMetadata + from vllm.config import CacheConfig + from vllm.model_executor.layers.pooler import PoolerOutput + from vllm.model_executor.layers.quantization import QuantizationConfig + from vllm.model_executor.layers.sampler import SamplerOutput + from vllm.model_executor.pooling_metadata import PoolingMetadata + from vllm.model_executor.sampling_metadata import SamplingMetadata + +logger = init_logger(__name__) + +# The type of HF config +C_co = TypeVar("C_co", bound=PretrainedConfig, covariant=True) + +# The type of hidden states +# Currently, T = torch.Tensor for all models except for Medusa +# which has T = List[torch.Tensor] +T = TypeVar("T", default=torch.Tensor) +T_co = TypeVar("T_co", default=torch.Tensor, covariant=True) + +# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags +# for the base interfaces to avoid breaking OOT registration for existing models +# that don't inherit from the base interface classes + + +@runtime_checkable +class VllmModel(Protocol[C_co, T_co]): + + def __init__( + self, + config: C_co, + *, + cache_config: Optional["CacheConfig"], + quant_config: Optional["QuantizationConfig"], + ) -> None: + ... + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: "AttentionMetadata", + ) -> T_co: + ... + + +def _check_vllm_model_init(model: Union[Type[object], object]) -> bool: + model_init = model.__init__ + vllm_kws = ("cache_config", "quant_config") + missing_kws = tuple(kw for kw in vllm_kws + if not supports_kw(model_init, kw)) + + if missing_kws and (isinstance(model, type) + and issubclass(model, nn.Module)): + logger.warning( + "The model (%s) is missing " + "vLLM-specific keywords from its initializer: %s", + model, + missing_kws, + ) + + return len(missing_kws) == 0 + + +def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool: + model_forward = getattr(model, "forward", None) + if not callable(model_forward): + return False + + vllm_kws = ("input_ids", "positions", "kv_caches", "attn_metadata") + missing_kws = tuple(kw for kw in vllm_kws + if not supports_kw(model_forward, kw)) + + if missing_kws and (isinstance(model, type) + and issubclass(model, nn.Module)): + logger.warning( + "The model (%s) is missing " + "vLLM-specific keywords from its initializer: %s", + model, + missing_kws, + ) + + return len(missing_kws) == 0 + + +@overload +def is_vllm_model(model: Type[object]) -> TypeIs[Type[VllmModel]]: + ... + + +@overload +def is_vllm_model(model: object) -> TypeIs[VllmModel]: + ... + + +def is_vllm_model( + model: Union[Type[object], object], +) -> Union[TypeIs[Type[VllmModel]], TypeIs[VllmModel]]: + return _check_vllm_model_init(model) and _check_vllm_model_forward(model) + + +@runtime_checkable +class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]): + + def compute_logits( + self, + hidden_states: T, + sampling_metadata: "SamplingMetadata", + ) -> Optional[T]: + """Return `None` if TP rank > 0.""" + ... + + def sample( + self, + logits: T, + sampling_metadata: "SamplingMetadata", + ) -> "SamplerOutput": + """Only called on TP rank 0.""" + ... + + +@overload +def is_text_generation_model( + model: Type[object]) -> TypeIs[Type[VllmModelForTextGeneration]]: + ... + + +@overload +def is_text_generation_model( + model: object) -> TypeIs[VllmModelForTextGeneration]: + ... + + +def is_text_generation_model( + model: Union[Type[object], object], +) -> Union[TypeIs[Type[VllmModelForTextGeneration]], + TypeIs[VllmModelForTextGeneration]]: + if not is_vllm_model(model): + return False + + if isinstance(model, type): + return isinstance(model, VllmModelForTextGeneration) + + return isinstance(model, VllmModelForTextGeneration) + + +@runtime_checkable +class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]): + + def pooler( + self, + hidden_states: T, + pooling_metadata: "PoolingMetadata", + ) -> "PoolerOutput": + """Only called on TP rank 0.""" + ... + + +@overload +def is_embedding_model( + model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]: + ... + + +@overload +def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]: + ... + + +def is_embedding_model( + model: Union[Type[object], object], +) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]: + if not is_vllm_model(model): + return False + + if isinstance(model, type): + return isinstance(model, VllmModelForEmbedding) + + return isinstance(model, VllmModelForEmbedding) diff --git a/vllm/model_executor/models/intern_vit.py b/vllm/model_executor/models/intern_vit.py new file mode 100644 index 00000000..35be1cec --- /dev/null +++ b/vllm/model_executor/models/intern_vit.py @@ -0,0 +1,428 @@ +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from functools import partial +from typing import Iterable, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig + +from vllm.distributed import (divide, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +try: + from xformers import ops as xops + USE_XFORMERS_OPS = True +except ImportError: + USE_XFORMERS_OPS = False + +NORM2FN = { + 'rms_norm': RMSNorm, + 'layer_norm': nn.LayerNorm, +} + + +class InternVisionEmbeddings(nn.Module): + + def __init__(self, config: PretrainedConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) + + self.patch_embedding = nn.Conv2d(in_channels=3, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size) + + self.num_patches = (self.image_size // self.patch_size)**2 + self.num_positions = self.num_patches + 1 + + self.position_embedding = nn.Parameter( + torch.randn(1, self.num_positions, self.embed_dim)) + + def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int): + target_dtype = pos_embed.dtype + pos_embed = pos_embed.float().reshape( + 1, self.image_size // self.patch_size, + self.image_size // self.patch_size, -1).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, + size=(H, W), + mode='bicubic', + align_corners=False) + return pos_embed.reshape(1, -1, H * W).permute(0, 2, + 1).to(target_dtype) + + def _get_position_embedding(self, H: int, W: int) -> torch.Tensor: + position_embedding = self.position_embedding + if self.num_patches == H * W: + return position_embedding + + return torch.cat( + [ + position_embedding[:, :1, :], + self._get_pos_embed(position_embedding[:, 1:, :], H, W), + ], + dim=1, + ) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to( + target_dtype)) # shape = [*, channel, width, height] + batch_size, _, height, width = patch_embeds.shape + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + class_embeds = self.class_embedding.expand(batch_size, 1, + -1).to(target_dtype) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + position_embedding = self._get_position_embedding(height, width) + embeddings = embeddings + position_embedding.to(target_dtype) + return embeddings + + +class InternParallelAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_dummy_heads: int = 0, + ) -> None: + super().__init__() + + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f'embed_dim must be divisible by num_heads ' + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:' + f' {self.num_heads}).') + + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + + # Additional dummy heads are used to enable TP for common GPU counts. + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim + self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads, + self.tp_size) + + self.scale = self.head_dim**-0.5 + self.qkv = QKVParallelLinear( + self.embed_dim, + self.head_dim, + num_dummy_heads + self.num_heads, + bias=config.qkv_bias, + quant_config=quant_config, + ) + + self.qk_normalization = config.qk_normalization + + if self.qk_normalization: + self.q_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + self.k_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + + self.proj = RowParallelLinear( + self.dummy_dim, + self.embed_dim, + quant_config=quant_config, + ) + + def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor): + if self.tp_size > 1: + q = tensor_model_parallel_all_gather(q.contiguous()) + k = tensor_model_parallel_all_gather(k.contiguous()) + q = self.q_norm.forward_native(q) + k = self.k_norm.forward_native(k) + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + return q, k + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, N, _ = x.shape + qkv, _ = self.qkv(x) + q, k, v = qkv.chunk(3, dim=-1) + + if self.qk_normalization: + q, k = self._apply_qk_norm(q, k) + + q = q.view(B, N, self.num_heads_per_partition, self.head_dim) + k = k.view(B, N, self.num_heads_per_partition, self.head_dim) + v = v.view(B, N, self.num_heads_per_partition, self.head_dim) + + x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale) + x = x.view(B, N, -1) + + x, _ = self.proj(x) + return x + + +class InternSdpaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: PretrainedConfig, + *, + num_dummy_heads: int = 0, + ) -> None: + super().__init__() + + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f'embed_dim must be divisible by num_heads ' + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:' + f' {self.num_heads}).') + + # Additional dummy heads are used to enable TP for common GPU counts. + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim + + self.scale = self.head_dim**-0.5 + self.qkv = nn.Linear(self.embed_dim, + 3 * self.dummy_dim, + bias=config.qkv_bias) + + self.qk_normalization = config.qk_normalization + + if self.qk_normalization: + self.q_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + self.k_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + + self.proj = nn.Linear(self.dummy_dim, self.embed_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, N, C = x.shape + qkv = self.qkv(x) + q, k, v = qkv.chunk(3, dim=-1) + + q = q.view(B, N, self.num_heads, self.head_dim) + k = k.view(B, N, self.num_heads, self.head_dim) + v = v.view(B, N, self.num_heads, self.head_dim) + + if self.qk_normalization: + B_, N_, H_, D_ = q.shape + q = self.q_norm.forward_native(q.flatten(-2, + -1)).view(B_, N_, H_, D_) + k = self.k_norm.forward_native(k.flatten(-2, + -1)).view(B_, N_, H_, D_) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + x = x.transpose(1, 2).view(B, N, -1) + + x = self.proj(x) + return x + + +class InternMLP(nn.Module): + + def __init__(self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear(config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config) + self.fc2 = RowParallelLinear(config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + + return hidden_states + + +class InternVisionEncoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_dummy_heads: int = 0, + ) -> None: + super().__init__() + + self.embed_dim = config.hidden_size + self.intermediate_size = config.intermediate_size + self.norm_type = config.norm_type + + self.attn = self._init_attn(config, + quant_config, + num_dummy_heads=num_dummy_heads) + + self.mlp = InternMLP(config, quant_config=quant_config) + self.norm1 = NORM2FN[self.norm_type](self.embed_dim, + eps=config.layer_norm_eps) + self.norm2 = NORM2FN[self.norm_type](self.embed_dim, + eps=config.layer_norm_eps) + + self.ls1 = nn.Parameter(config.initializer_factor * + torch.ones(self.embed_dim)) + self.ls2 = nn.Parameter(config.initializer_factor * + torch.ones(self.embed_dim)) + + def _init_attn( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig], + *, + num_dummy_heads: int, + ): + # fallback to sdpa attention if tp unavailable + tp_size = get_tensor_model_parallel_world_size() + num_heads = config.num_attention_heads + + if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0: + return InternParallelAttention(config, + quant_config=quant_config, + num_dummy_heads=num_dummy_heads) + + return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads) + + def forward( + self, + hidden_states: torch.Tensor, + ): + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states)) * self.ls1 + + hidden_states = hidden_states + self.mlp( + self.norm2(hidden_states)) * self.ls2 + + return hidden_states + + +class InternVisionEncoder(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_hidden_layers_override: Optional[int] = None, + num_dummy_heads: int = 0, + ): + super().__init__() + + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + + self.layers = nn.ModuleList([ + InternVisionEncoderLayer(config, + quant_config, + num_dummy_heads=num_dummy_heads) + for _ in range(num_hidden_layers) + ]) + + def forward(self, inputs_embeds: torch.Tensor): + + hidden_states = inputs_embeds + for encoder_layer in self.layers: + hidden_states = encoder_layer(hidden_states) + + return hidden_states + + +class InternVisionModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_hidden_layers_override: Optional[int] = None, + num_dummy_heads: int = 0, + ): + super().__init__() + + self.config = config + + self.embeddings = InternVisionEmbeddings(config) + self.encoder = InternVisionEncoder( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + num_dummy_heads=num_dummy_heads, + ) + + def get_input_embeddings(self): + return self.embeddings + + def forward( + self, + pixel_values: Optional[torch.Tensor] = None, + pixel_embeds: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + if pixel_values is None and pixel_embeds is None: + raise ValueError( + 'You have to specify pixel_values or pixel_embeds') + + if pixel_embeds is not None: + hidden_states = pixel_embeds + elif pixel_values is not None: + if pixel_values.ndim == 4: + hidden_states = self.embeddings(pixel_values) + else: + raise ValueError( + f'wrong pixel_values size: {pixel_values.shape}') + + encoder_outputs = self.encoder(inputs_embeds=hidden_states) + + return encoder_outputs + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/internlm2.py b/vllm/model_executor/models/internlm2.py new file mode 100644 index 00000000..f6cde44e --- /dev/null +++ b/vllm/model_executor/models/internlm2.py @@ -0,0 +1,383 @@ +# -*- coding: utf-8 -*- +from functools import partial +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class InternLM2MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.w2 = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.w2(x) + return x + + +class InternLM2Attention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + self.total_num_heads = num_heads + assert self.total_num_heads % self.tp_size == 0 + self.num_heads = self.total_num_heads // self.tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= self.tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % self.tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert self.tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.key_value_groups = int(self.num_heads / self.num_kv_heads) + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.wqkv = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.wo = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def split_qkv(self, qkv: torch.Tensor): + seq_len = qkv.shape[0] + if self.tp_size > 1: + qkv_map = [self.q_size, self.kv_size, self.kv_size] * self.tp_size + qkv = tensor_model_parallel_all_gather(qkv) + qkv = torch.split(qkv, qkv_map, dim=-1) + qkv = qkv[::3] + qkv[1::3] + qkv[2::3] + qkv = torch.cat(qkv, dim=-1) + + qkv = qkv.view(seq_len, self.total_num_kv_heads, + self.key_value_groups + 2, self.head_dim) + q, k, v = torch.split(qkv, [self.key_value_groups, 1, 1], dim=-2) + q = q.reshape(seq_len, self.q_size * self.tp_size) + k = k.reshape(seq_len, self.kv_size * self.tp_size) + v = v.reshape(seq_len, self.kv_size * self.tp_size) + + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + v = splitter(v)[self.tp_rank] + return q, k, v + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.wqkv(hidden_states) + q, k, v = self.split_qkv(qkv) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.wo(attn_output) + return output + + +class InternLMDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.attention = InternLM2Attention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + self.feed_forward = InternLM2MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.attention_norm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.attention_norm(hidden_states) + else: + hidden_states, residual = self.attention_norm( + hidden_states, residual) + hidden_states = self.attention( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.ffn_norm(hidden_states, residual) + hidden_states = self.feed_forward(hidden_states) + return hidden_states, residual + + +class InternLM2Model(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.tok_embeddings = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: InternLMDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.tok_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.tok_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class InternLM2ForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = InternLM2Model(config, cache_config, quant_config) + self.output = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.output.weight = self.model.tok_embeddings.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.output, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "w1", 0), + ("gate_up_proj", "w3", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py new file mode 100644 index 00000000..203badb3 --- /dev/null +++ b/vllm/model_executor/models/internvl.py @@ -0,0 +1,612 @@ +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +import regex as re +from functools import cached_property, partial +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +import torch.nn as nn +import torchvision.transforms as T +from PIL import Image +from transformers import PretrainedConfig + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.models.intern_vit import InternVisionModel +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip, + get_clip_num_patches) +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, + merge_multimodal_embeddings) + +IMG_START = '' +IMG_END = '' +IMG_CONTEXT = '' + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +class InternVLImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """ + Shape: + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)` + """ + + +class InternVLImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +InternVLImageInputs = Union[InternVLImagePixelInputs, + InternVLImageEmbeddingInputs] + + +# copied from https://huggingface.co/OpenGVLab/InternVL2-1B +def build_transform(input_size): + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD + transform = T.Compose([ + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), + T.Resize((input_size, input_size), + interpolation=T.InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=MEAN, std=STD) + ]) + return transform + + +# copied from https://huggingface.co/OpenGVLab/InternVL2-1B +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, + image_size): + best_ratio_diff = float('inf') + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + + +def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int, + max_num: int, image_size: int, + use_thumbnail: bool) -> Tuple[int, int, int]: + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set((i, j) for n in range(min_num, max_num + 1) + for i in range(1, n + 1) for j in range(1, n + 1) + if i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, + target_ratios, orig_width, + orig_height, image_size) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + # add thumbnail image if num_blocks > 1 + if use_thumbnail and blocks > 1: + blocks += 1 + return blocks, target_width, target_height + + +def calculate_num_blocks_wrapper(hf_config: PretrainedConfig, + max_dynamic_patch: Optional[int] = None): + if max_dynamic_patch is None: + max_dynamic_patch = hf_config.max_dynamic_patch + min_num = hf_config.min_dynamic_patch + image_size = hf_config.vision_config.image_size + use_thumbnail = hf_config.use_thumbnail + return partial(calculate_num_blocks, + min_num=min_num, + max_num=max_dynamic_patch, + image_size=image_size, + use_thumbnail=use_thumbnail) + + +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B +def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int, + image_size: int, + use_thumbnail: bool) -> List[Image.Image]: + orig_width, orig_height = image.size + + # calculate the number of blocks without thumbnail + blocks, target_width, target_height = calculate_num_blocks( + orig_width, + orig_height, + min_num, + max_num, + image_size, + use_thumbnail=False) + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ((i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B +def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int, + max_num: int, use_thumbnail: bool) -> torch.Tensor: + transform = build_transform(input_size=input_size) + images = dynamic_preprocess(image, + min_num=min_num, + max_num=max_num, + image_size=input_size, + use_thumbnail=use_thumbnail) + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values + + +def image_to_pixel_values_wrapper(hf_config: PretrainedConfig, + max_dynamic_patch: Optional[int] = None): + image_size = hf_config.vision_config.image_size + min_num = hf_config.min_dynamic_patch + if max_dynamic_patch is None: + max_dynamic_patch = hf_config.max_dynamic_patch + use_thumbnail = hf_config.use_thumbnail + return partial(image_to_pixel_values, + input_size=image_size, + min_num=min_num, + max_num=max_dynamic_patch, + use_thumbnail=use_thumbnail) + + +def get_internvl_num_patches(hf_config: PretrainedConfig): + vision_config = hf_config.vision_config + downsample_ratio = hf_config.downsample_ratio + image_size = vision_config.image_size + patch_size = vision_config.patch_size + return int( + get_clip_num_patches(image_size=image_size, patch_size=patch_size) * + (downsample_ratio**2)) + + +def get_max_internvl_image_tokens(ctx: InputContext, + *, + max_dynamic_patch: Optional[int] = None): + hf_config = ctx.get_hf_config() + + if max_dynamic_patch is None: + max_dynamic_patch = hf_config.max_dynamic_patch + use_thumbnail = hf_config.use_thumbnail + if use_thumbnail and max_dynamic_patch > 1: + max_dynamic_patch += 1 + + num_patches = get_internvl_num_patches(hf_config) + return num_patches * max_dynamic_patch + + +def get_max_internvl_image_size(ctx: InputContext, + *, + max_dynamic_patch: Optional[int] = None): + hf_config = ctx.get_hf_config() + image_size = hf_config.vision_config.image_size + + if max_dynamic_patch is None: + max_dynamic_patch = hf_config.max_dynamic_patch + use_thumbnail = hf_config.use_thumbnail + if use_thumbnail and max_dynamic_patch > 1: + max_dynamic_patch += 1 + width = image_size * max_dynamic_patch + height = image_size + return width, height + + +class InternVLInputPipeline: + + def __init__( + self, + img_start_token: str, + img_end_token: str, + img_context_token: str, + ) -> None: + super().__init__() + + self.img_start_token = img_start_token + self.img_end_token = img_end_token + self.img_context_token = img_context_token + + def _create_image_prompt(self, feature_size: int, num_patches: int) -> str: + return (self.img_start_token + self.img_context_token * feature_size + + self.img_end_token) + + def _expand_image_prompt( + self, + prompt: str, + feature_sizes: List[int], + num_patches: int, + ) -> str: + image_idx = sorted( + map(int, re.findall(r"Image-(\d+): \n", prompt))) + + new_prompt = prompt + for idx, feature_size in enumerate(feature_sizes, start=1): + image_prompt = self._create_image_prompt(feature_size, num_patches) + if not image_idx: + image_prompt = f"Image-{idx}: {image_prompt}" + + new_prompt = new_prompt.replace('', image_prompt, 1) + + return new_prompt + + def input_processor( + self, + ctx: InputContext, + llm_inputs: LLMInputs, + *, + max_dynamic_patch: Optional[int] = None, + ) -> LLMInputs: + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_config() + + image_data = multi_modal_data["image"] + num_patches = get_internvl_num_patches(hf_config) + num_blocks_calculator = calculate_num_blocks_wrapper( + hf_config, max_dynamic_patch) + if isinstance(image_data, Image.Image): + width, height = image_data.size + num_blocks, _, _ = num_blocks_calculator(width, height) + image_feature_sizes = [num_blocks * num_patches] + elif is_list_of(image_data, Image.Image): + image_feature_sizes = [] + for image in image_data: + width, height = image.size + num_blocks, _, _ = num_blocks_calculator(width, height) + image_feature_sizes.append(num_blocks * num_patches) + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + image_feature_sizes = [image_feature_size] + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + + prompt = llm_inputs.get("prompt") + prompt_token_ids = llm_inputs["prompt_token_ids"] + if prompt is None: + prompt = tokenizer.decode(prompt_token_ids) + + new_prompt = self._expand_image_prompt(prompt, image_feature_sizes, + num_patches) + new_prompt_token_ids = tokenizer.encode(new_prompt) + + return LLMInputs(prompt=prompt, + prompt_token_ids=new_prompt_token_ids, + multi_modal_data=multi_modal_data) + + def input_mapper( + self, + ctx: InputContext, + data: object, + *, + max_dynamic_patch: Optional[int] = None, + ): + hf_config = ctx.get_hf_config() + + image_pixel_values_mapper = image_to_pixel_values_wrapper( + hf_config, max_dynamic_patch) + if isinstance(data, Image.Image): + data = image_pixel_values_mapper(data) + # Add an N dimension for number of images per prompt (currently 1). + data = data.unsqueeze(0) + elif is_list_of(data, Image.Image): + # we can't stack here because images may have different num_patches + data = [image_pixel_values_mapper(img) for img in data] + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + image_token_id = tokenizer.encode(self.img_context_token, + add_special_tokens=False, + return_tensors="pt")[0] + + return MultiModalInputs({ + "pixel_values": data, + "image_token_id": image_token_id + }) + + def dummy_data( + self, + ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], + *, + max_dynamic_patch: Optional[int] = None, + ): + num_images = mm_counts["image"] + + hf_config = ctx.get_hf_config() + + image_feature_size = get_max_internvl_image_tokens( + ctx, max_dynamic_patch=max_dynamic_patch) + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + + seq_data = dummy_seq_data_for_clip( + hf_config.vision_config, + seq_len, + num_images, + image_token_id=tokenizer.encode(self.img_context_token, + add_special_tokens=False)[0], + image_feature_size_override=image_feature_size, + ) + + max_image_width, max_image_height = get_max_internvl_image_size( + ctx, max_dynamic_patch=max_dynamic_patch) + + mm_data = dummy_image_for_clip( + hf_config.vision_config, + num_images, + image_width_override=max_image_width, + image_height_override=max_image_height, + ) + + return seq_data, mm_data + + +input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens) +@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data) +@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor) +class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + image_size = config.force_image_size or config.vision_config.image_size + patch_size = config.vision_config.patch_size + self.patch_size = patch_size + self.select_layer = config.select_layer + self.num_image_token = int( + (image_size // patch_size)**2 * (config.downsample_ratio**2)) + self.downsample_ratio = config.downsample_ratio + self.ps_version = config.ps_version + + vision_feature_layer = self.select_layer + if vision_feature_layer < 0: + num_hidden_layers = config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + self.vision_model = self._init_vision_model(config, num_hidden_layers) + + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.mlp1 = self._init_mlp1(config) + + self.img_context_token_id = None + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _init_vision_model(self, config: PretrainedConfig, + num_hidden_layers: int): + return InternVisionModel(config.vision_config, + num_hidden_layers_override=num_hidden_layers) + + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential: + vit_hidden_size = config.vision_config.hidden_size + llm_hidden_size = config.text_config.hidden_size + + return nn.Sequential( + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2), + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2, + llm_hidden_size), + nn.GELU(), + nn.Linear(llm_hidden_size, llm_hidden_size), + ) + + def pixel_shuffle(self, x, scale_factor=0.5): + n, w, h, c = x.size() + # N, W, H, C --> N, W, H * scale, C // scale + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor)) + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale + x = x.permute(0, 2, 1, 3).contiguous() + x = x.view(n, int(h * scale_factor), int(w * scale_factor), + int(c / (scale_factor * scale_factor))) + if self.ps_version == 'v1': + pass + else: + x = x.permute(0, 2, 1, 3).contiguous() + return x + + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor: + vit_embeds = self.vision_model(pixel_values=pixel_values) + vit_embeds = vit_embeds[:, 1:, :] + + h = w = int(vit_embeds.shape[1]**0.5) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1) + vit_embeds = self.pixel_shuffle(vit_embeds, + scale_factor=self.downsample_ratio) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, + vit_embeds.shape[-1]) + vit_embeds = self.mlp1(vit_embeds) + return vit_embeds + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + "The expected shape of pixel values per image per batch " + f" per patch is {expected_expr}. " + f"You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[InternVLImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_token_id = kwargs.pop("image_token_id", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + return InternVLImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds), + ) + + self.img_context_token_id = image_token_id[0] + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + # We need to flatten (B, N, P) to (B*N*P), + # so we call flatten_bn twice. + return InternVLImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values( + flatten_bn(flatten_bn(pixel_values), concat=True)), + ) + + raise AssertionError("This line should be unreachable.") + + def _process_image_input( + self, + image_input: InternVLImageInputs, + ) -> torch.Tensor: + if image_input["type"] == "image_embeds": + return image_input["data"] + + assert self.vision_model is not None + image_embeds = self.extract_feature(image_input["data"]) + + return image_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[SamplerOutput, IntermediateTensors]: + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + if image_input is not None: + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.img_context_token_id) + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/jais.py b/vllm/model_executor/models/jais.py new file mode 100644 index 00000000..c5e53934 --- /dev/null +++ b/vllm/model_executor/models/jais.py @@ -0,0 +1,374 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/core42/jais-30b-chat-v3/blob/main/modeling_jais.py +# Copyright 2023 The vLLM team. +# Copyright 2023 the Jais authors and HuggingFace Inc. team. All rights +# reserved. +# Copyright 2023 Cerebras Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Jais model compatible with HuggingFace weights.""" + +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs import JAISConfig + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class SwiGLUActivation(nn.Module): + + def forward(self, x1: torch.Tensor, x2: torch.Tensor) -> torch.Tensor: + return x1 * nn.functional.silu(x2) + + +def _get_alibi_slopes(n): + + def get_slopes_power_of_2(n): + start = 2**(-(2**-(math.log2(n) - 3))) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2(n) + else: + closest_power_of_2 = 2**math.floor(math.log2(n)) + return (get_slopes_power_of_2(closest_power_of_2) + _get_alibi_slopes( + 2 * closest_power_of_2)[0::2][:n - closest_power_of_2]) + + +class JAISAttention(nn.Module): + + def __init__( + self, + config: JAISConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + total_num_heads = config.num_attention_heads + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + assert total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = total_num_heads // tensor_model_parallel_world_size + self.head_dim = self.hidden_size // total_num_heads + if hasattr(config, "scale_qk_dot_by_d"): + config.mup_scale_qk_dot_by_d = config.scale_qk_dot_by_d + self.attn_scale_power = 1.0 if config.mup_scale_qk_dot_by_d else 0.5 + self.scale = self.head_dim**-self.attn_scale_power + + self.c_attn = QKVParallelLinear( + self.hidden_size, + self.head_dim, + total_num_heads, + bias=True, + quant_config=quant_config, + ) + self.c_proj = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config, + ) + + tp_rank = get_tensor_model_parallel_rank() + head_start = tp_rank * self.num_heads + head_end = (tp_rank + 1) * self.num_heads + alibi_slopes = _get_alibi_slopes(total_num_heads) + alibi_slopes = alibi_slopes[head_start:head_end] + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scale, + alibi_slopes=alibi_slopes, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.c_attn(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output, _ = self.c_proj(attn_output) + return attn_output + + +class JAISMLP(nn.Module): + + def __init__( + self, + intermediate_size: int, + config: JAISConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + self.swiglu = config.activation_function == "swiglu" + self.c_fc = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + ) + self.c_fc2 = (ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + ) if self.swiglu else None) + self.c_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + ) + + self.act = SwiGLUActivation() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.swiglu: + hidden_states2, _ = self.c_fc2(hidden_states) + hidden_states, _ = self.c_fc(hidden_states) + hidden_states = (self.act(hidden_states, hidden_states2) + if self.swiglu else self.act(hidden_states)) + hidden_states, _ = self.c_proj(hidden_states) + return hidden_states + + +class JAISBlock(nn.Module): + + def __init__( + self, + config: JAISConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.hidden_size + inner_dim = (config.n_inner if config.n_inner is not None else 4 * + hidden_size) + + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.attn = JAISAttention(config, cache_config, quant_config) + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.mlp = JAISMLP(inner_dim, config, quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + attn_output = self.attn( + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + # residual connection + hidden_states = attn_output + residual + + residual = hidden_states + hidden_states = self.ln_2(hidden_states) + feed_forward_hidden_states = self.mlp(hidden_states) + # residual connection + hidden_states = residual + feed_forward_hidden_states + return hidden_states + + +class JAISModel(nn.Module): + + def __init__( + self, + config: JAISConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + assert not config.add_cross_attention + assert not config.scale_attn_by_inverse_layer_idx + assert not config.reorder_and_upcast_attn + self.embed_dim = config.hidden_size + self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim) + self.wpe = (nn.Embedding(config.max_position_embeddings, + self.embed_dim) + if config.position_embedding_type != "alibi" else None) + if hasattr(config, "embeddings_scale"): + self.embeddings_scale = config.embeddings_scale + else: + self.embeddings_scale = config.mup_embeddings_scale + + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: JAISBlock(config=config, + cache_config=cache_config, + quant_config=quant_config), + prefix=f"{prefix}.h", + ) + + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.n_embd)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[IntermediateTensors, torch.Tensor]: + if get_pp_group().is_first_rank: + inputs_embeds = self.wte(input_ids) + if self.wpe is not None: + position_embeds = self.wpe(position_ids) + hidden_states = inputs_embeds + position_embeds + else: + hidden_states = inputs_embeds + hidden_states *= torch.tensor(float(self.embeddings_scale), + dtype=hidden_states.dtype) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states = layer(hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class JAISLMHeadModel(nn.Module, SupportsPP): + + def __init__( + self, + config: JAISConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.transformer = JAISModel(config, cache_config, quant_config) + if self.config.tie_word_embeddings: + self.lm_head = self.transformer.wte + else: + self.lm_head = ParallelLMHead(self.config.vocab_size, + self.config.hidden_size) + if hasattr(config, "width_scale"): + self.output_logits_scale = config.width_scale + else: + self.output_logits_scale = (config.mup_output_alpha * + config.mup_width_scale) + self.logits_processor = LogitsProcessor(vocab_size=config.vocab_size, + scale=self.output_logits_scale) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[IntermediateTensors, torch.Tensor]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "lm_head.weight" in name: + # GPT-2 ties the weights of the embedding layer and the final + # linear layer. + continue + if ".attn.bias" in name or ".attn.masked_bias" in name: + # Skip attention mask. + # NOTE: "c_attn.bias" should not be skipped. + continue + if "relative_pe" in name: + continue + if not name.startswith("transformer."): + name = "transformer." + name + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + # The HF's GPT-2 implementation uses Conv1D instead of Linear. + # Because of this, we need to transpose the weights. + # Note(zhuohan): the logic below might break quantized models. + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: + if conv1d_weight_name not in name: + continue + if not name.endswith(".weight"): + continue + loaded_weight = loaded_weight.t() + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/jamba.py b/vllm/model_executor/models/jamba.py new file mode 100644 index 00000000..ac251b88 --- /dev/null +++ b/vllm/model_executor/models/jamba.py @@ -0,0 +1,717 @@ +# coding=utf-8 +"""Inference-only Jamba model.""" +from dataclasses import dataclass +from typing import Iterable, List, Optional, Tuple + +import torch +from torch import nn +from transformers import JambaConfig + +from vllm.attention.backends.abstract import AttentionMetadata +from vllm.attention.layer import Attention +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( + causal_conv1d_fn, causal_conv1d_update) +from vllm.model_executor.layers.mamba.ops.mamba_ssm import ( + selective_scan_fn, selective_state_update) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + composed_weight_loader, default_weight_loader, sharded_weight_loader) +from vllm.model_executor.models.mamba_cache import MambaCacheManager +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors +from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE, + _get_graph_batch_size) + +from .interfaces import HasInnerState, SupportsLoRA + +KVCache = Tuple[torch.Tensor, torch.Tensor] + + +@dataclass +class MambaCacheParams: + is_prompt: bool = False + conv_state: torch.Tensor = torch.Tensor() + ssm_state: torch.Tensor = torch.Tensor() + + +# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer +class JambaMambaMixer(nn.Module): + """ + Compute ∆, A, B, C, and D the state space parameters and compute + the `contextualized_states`. A, D are input independent + (see Mamba paper [1] Section 3.5.2 "Interpretation of A" + for why A isn't selective) ∆, B, C are input-dependent + (this is a key difference between Mamba and the linear time + invariant S4, and is why Mamba is called + **selective** state spaces) + """ + + def __init__(self, config: JambaConfig, layer_idx): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.ssm_state_size = config.mamba_d_state + self.conv_kernel_size = config.mamba_d_conv + self.intermediate_size = config.mamba_expand * config.hidden_size + self.time_step_rank = config.mamba_dt_rank + self.use_conv_bias = config.mamba_conv_bias + self.use_bias = config.mamba_proj_bias + self.conv1d = ColumnParallelLinear( + input_size=self.conv_kernel_size, + output_size=self.intermediate_size, + bias=self.use_conv_bias, + ) + # unsqueeze to fit conv1d weights shape into the linear weights shape. + # Can't do this in `weight_loader` since it already exists in + # `ColumnParallelLinear` and `set_weight_attrs` + # doesn't allow to override it + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1) + + self.in_proj = MergedColumnParallelLinear(self.hidden_size, + [self.intermediate_size] * 2, + bias=self.use_bias) + # selective projection used to make dt, B and C input dependent + self.x_proj = RowParallelLinear( + self.intermediate_size, + self.time_step_rank + self.ssm_state_size * 2, + bias=False, + ) + # time step projection (discretization) - + # In the forward we need to apply dt_proj without the bias, + # as the bias is added in the selective scan kernel. + self.dt_proj = ColumnParallelLinear(self.time_step_rank, + self.intermediate_size, + bias=True, + skip_bias_add=True) + + tp_size = get_tensor_model_parallel_world_size() + self.A = nn.Parameter( + torch.empty( + self.intermediate_size // tp_size, + self.ssm_state_size, + dtype=torch.float32, + )) + self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size)) + + set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)}) + a_weight_loader = composed_weight_loader( + sharded_weight_loader(0), lambda x: -torch.exp(x.float())) + set_weight_attrs(self.A, {"weight_loader": a_weight_loader}) + + self.out_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=self.use_bias, + input_is_parallel=True, + ) + self.activation = config.hidden_act + + self.dt_layernorm = RMSNorm(self.time_step_rank, + eps=config.rms_norm_eps) + self.b_layernorm = RMSNorm(self.ssm_state_size, + eps=config.rms_norm_eps) + self.c_layernorm = RMSNorm(self.ssm_state_size, + eps=config.rms_norm_eps) + + def forward(self, hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, conv_state: torch.Tensor, + ssm_state: torch.Tensor): + + # 1. Gated MLP's linear projection + projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1) + hidden_states, gate = projected_states.chunk(2, dim=-2) + + # 2. Convolution sequence transformation + conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0), + self.conv1d.weight.size(2)) + + if attn_metadata.query_start_loc is not None \ + and attn_metadata.context_lens_tensor is not None: + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ---------------------| + # |-- query_len ---| + hidden_states = causal_conv1d_fn( + hidden_states, + conv_weights, + self.conv1d.bias, + activation=self.activation, + conv_states=conv_state, + has_initial_state=attn_metadata.context_lens_tensor > 0, + query_start_loc=attn_metadata.query_start_loc) + else: + hidden_states = causal_conv1d_update( + hidden_states.transpose(0, 1), + conv_state, + conv_weights, + self.conv1d.bias, + self.activation, + ) + hidden_states = hidden_states.transpose(0, 1) + + # 3. State Space Model sequence transformation + # 3.a. input varying initialization of time_step, B and C + ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0] + + time_step, B, C = torch.split( + ssm_parameters, + [self.time_step_rank, self.ssm_state_size, self.ssm_state_size], + dim=-1, + ) + time_step = self.dt_layernorm(time_step.contiguous()) + B = self.b_layernorm(B.contiguous()) + C = self.c_layernorm(C.contiguous()) + + discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1) + # 3.c perform the recurrence y ← SSM(A, B, C)(x) + time_proj_bias = (self.dt_proj.bias.float() if hasattr( + self.dt_proj, "bias") else None) + + if attn_metadata.query_start_loc is not None \ + and attn_metadata.context_lens_tensor is not None: + scan_outputs = selective_scan_fn( + hidden_states, + ssm_state, + discrete_time_step, + self.A, + B.transpose(-2, -1), + C.transpose(-2, -1), + self.D.float(), + gate, + time_proj_bias, + delta_softplus=True, + has_initial_state=attn_metadata.context_lens_tensor > 0, + query_start_loc=attn_metadata.query_start_loc) + else: + scan_outputs = selective_state_update( + ssm_state, + hidden_states.transpose(0, 1), + discrete_time_step.transpose(0, 1), + self.A, + B, + C, + self.D, + gate.transpose(0, 1), + time_proj_bias, + dt_softplus=True, + ) + scan_outputs = scan_outputs.transpose(0, 1) + + # 4. Final linear projection + contextualized_states = self.out_proj(scan_outputs.transpose(-2, + -1))[0] + return contextualized_states + + +class JambaMoE(nn.Module): + + def __init__(self, + config: JambaConfig, + num_experts: Optional[int] = None, + top_k: Optional[int] = None, + params_dtype: Optional[torch.dtype] = None, + tp_size: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.num_total_experts = num_experts or config.num_experts + self.top_k = top_k or config.num_experts_per_tok + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + + if self.num_total_experts > 1: + self.router = ReplicatedLinear(self.hidden_size, + self.num_total_experts, + bias=False, + quant_config=None, + params_dtype=params_dtype) + + self.experts = FusedMoE(self.num_total_experts, + self.top_k, + self.hidden_size, + self.intermediate_size, + tp_size=tp_size, + params_dtype=params_dtype, + reduce_results=True, + renormalize=False, + use_grouped_topk=False, + quant_config=quant_config) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (batch * sequence_length, n_experts) + if self.num_total_experts > 1: + router_logits, _ = self.router(hidden_states) + else: + router_logits = torch.ones((hidden_states.shape[0], 1), + device=hidden_states.device, + dtype=hidden_states.dtype) + hidden_states = self.experts(hidden_states, router_logits) + return hidden_states.view(orig_shape) + + +class JambaMLP(JambaMoE): + + def __init__(self, + config: JambaConfig, + params_dtype: Optional[torch.dtype] = None, + tp_size: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__(config, + num_experts=1, + top_k=1, + params_dtype=params_dtype, + tp_size=tp_size, + quant_config=quant_config) + + +class JambaMambaDecoderLayer(nn.Module): + + def __init__(self, + config: JambaConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.layer_idx = layer_idx + self.config = config + self.mamba = JambaMambaMixer(config, layer_idx) + + num_experts = config.layers_num_experts[layer_idx] + ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP + self.feed_forward = ffn_layer_class(config, quant_config=quant_config) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.pre_ff_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + **kwargs, + ): + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + + hidden_states = self.mamba(hidden_states, attn_metadata, conv_state, + ssm_state) + # Fully Connected + hidden_states, residual = self.pre_ff_layernorm( + hidden_states, residual) + hidden_states = self.feed_forward(hidden_states) + return hidden_states, residual + + +class JambaAttentionDecoderLayer(nn.Module): + + def __init__( + self, + config: JambaConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = config.num_key_value_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = config.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + config.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim, + config.hidden_size, + bias=False, + quant_config=quant_config) + + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + ) + + num_experts = config.layers_num_experts[layer_idx] + ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP + self.feed_forward = ffn_layer_class(config, quant_config=quant_config) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.pre_ff_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def self_attention( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + **kwargs, + ): + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + + hidden_states = self.self_attention( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + # Fully Connected + hidden_states, residual = self.pre_ff_layernorm( + hidden_states, residual) + hidden_states = self.feed_forward(hidden_states) + return hidden_states, residual + + +ALL_DECODER_LAYER_TYPES = { + "attention": JambaAttentionDecoderLayer, + "mamba": JambaMambaDecoderLayer +} + + +class JambaModel(nn.Module): + + def __init__( + self, + config: JambaConfig, + quant_config: Optional[QuantizationConfig] = None, + cache_config: Optional[CacheConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = ((lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0) + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + + decoder_layers = [] + for i in range(config.num_hidden_layers): + layer_class = ALL_DECODER_LAYER_TYPES[config.layers_block_type[i]] + decoder_layers.append( + layer_class(config, + layer_idx=i, + cache_config=cache_config, + quant_config=quant_config)) + self.layers = nn.ModuleList(decoder_layers) + self.final_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + residual = None + + for i in range(len(self.layers)): + layer = self.layers[i] + kv_cache = None + current_ssm_state = None + current_conv_state = None + if isinstance(layer, JambaAttentionDecoderLayer): + kv_cache = kv_caches[(i - self.config.attn_layer_offset) // + self.config.attn_layer_period] + if isinstance(layer, JambaMambaDecoderLayer): + current_state_layer = i - (1 + + (i - self.config.attn_layer_offset) + // self.config.attn_layer_period) + current_ssm_state = ssm_state[current_state_layer] + current_conv_state = conv_state[current_state_layer] + + hidden_states, residual = layer( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + residual=residual, + conv_state=current_conv_state, + ssm_state=current_ssm_state, + ) + hidden_states, _ = self.final_layernorm(hidden_states, residual) + return hidden_states + + +class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: JambaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + scheduler_config: Optional[SchedulerConfig] = None, + ) -> None: + assert not cache_config.enable_prefix_caching, \ + "Jamba currently does not support prefix caching" + + super().__init__() + self.config = config + self.scheduler_config = scheduler_config + self.model = JambaModel(config, + cache_config=cache_config, + quant_config=quant_config, + lora_config=lora_config) + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + ) + # Used to track and store by the Mamba cache between steps. + self.mamba_cache: Optional[MambaCacheManager] = None + + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[KVCache], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs): + if self.mamba_cache is None: + max_batch_size = (_get_graph_batch_size( + self.scheduler_config.max_num_seqs) if self.scheduler_config + else max(_BATCH_SIZES_TO_CAPTURE) + 2) + + layers_type = self.config.layers_block_type + num_mamba_layers = sum( + [layer_type == "mamba" for layer_type in layers_type]) + + self.mamba_cache = MambaCacheManager( + self.lm_head.weight.dtype, num_mamba_layers, max_batch_size, + *self._get_mamba_cache_shape()) + + mamba_cache_tensors = self.mamba_cache.current_run_tensors( + input_ids, attn_metadata, **kwargs) + + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, mamba_cache_tensors[0], + mamba_cache_tensors[1]) + return hidden_states + + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + return self.mamba_cache.copy_inputs_before_cuda_graphs( + input_buffers, **kwargs) + + def get_seqlen_agnostic_capture_inputs(self, batch_size: int): + return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size) + + def _get_mamba_cache_shape( + self) -> Tuple[Tuple[int, int], Tuple[int, int]]: + world_size = get_tensor_model_parallel_world_size() + hidden_size = self.config.hidden_size + conv_state_shape = ( + self.config.mamba_expand * hidden_size // world_size, + self.config.mamba_d_conv - 1, + ) + temporal_state_shape = ( + self.config.mamba_expand * hidden_size // world_size, + self.config.mamba_d_state, + ) + return conv_state_shape, temporal_state_shape + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + if "A_log" in name: + name = name.replace("A_log", "A") + + if ".self_attn." in name: + name = name.replace(".self_attn", "") + + if "feed_forward" in name and not _is_moe_layer(name): + ## map MLP layers to expert with ID=0 + name = name.replace("feed_forward", "feed_forward.experts.0") + + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + if 'experts' in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for ( + param_name, + weight_name, + expert_id, + shard_id, + ) in expert_params_mapping: + if weight_name not in name: + continue + + name = name.replace(weight_name, param_name) + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + +def _is_moe_layer(name: str): + return any( + [experts_name in name for experts_name in [ + "experts", + "router", + ]]) diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py new file mode 100644 index 00000000..a6464fdc --- /dev/null +++ b/vllm/model_executor/models/llama.py @@ -0,0 +1,637 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only LLaMA model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, Type + +import torch +from torch import nn +from transformers import LlamaConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + get_compressed_tensors_cache_scale) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import is_hip + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class LlamaMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class LlamaAttention(nn.Module): + + def __init__( + self, + config: LlamaConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", + self.hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + is_neox_style = True + if quant_config is not None and quant_config.get_name() == "gguf": + is_neox_style = False + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=is_neox_style, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class LlamaDecoderLayer(nn.Module): + + def __init__( + self, + config: LlamaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + self.self_attn = LlamaAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + cache_config=cache_config, + prefix=f"{prefix}.self_attn", + ) + self.mlp = LlamaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn(positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": 0, + "inputs_embeds": 0, + "intermediate_tensors": 0, + }) +class LlamaModel(nn.Module): + + def __init__( + self, + config: LlamaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + layer_type: Type[LlamaDecoderLayer] = LlamaDecoderLayer + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=quant_config, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + # lambda prefix: LlamaDecoderLayer(config=config, + # cache_config=cache_config, + # quant_config=quant_config, + # prefix=prefix), + lambda prefix: layer_type(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix), + prefix=f"{prefix}.layers", + ) + + # from ixformer.inference.overlap import create_vllm_llama_decoder_layer + from vllm.distributed import get_tp_group + + # for layer_idx in range(self.start_layer, self.end_layer): + # self.layers[layer_idx] = create_vllm_llama_decoder_layer( + # self.layers[layer_idx], + # model_id = id(self), + # layer_idx = layer_idx, + # enable_overlap = True, + # group = get_tp_group(), + # quant_config=quant_config, + # activation_dtype=torch.get_default_dtype() + # ) + + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + if scale_name := get_compressed_tensors_cache_scale(name): + # Loading kv cache scales for compressed-tensors quantization + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + # If this function is called, it should always initialize KV cache scale + # factors (or else raise an exception). Thus, handled exceptions should + # make sure to leave KV cache scale factors in a known good (dummy) state + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + for layer_idx, scaling_factor in kv_cache_scales_loader( + quantization_param_path, tp_rank, tp_size, + self.config.num_hidden_layers, + self.config.__class__.model_type): + if not isinstance(self.layers[layer_idx], nn.Identity): + layer_self_attn = self.layers[layer_idx].self_attn + + if is_hip(): + # The scaling factor convention we are assuming is + # quantized_value * scaling_factor ~= true_value + # which is consistent with the practice of setting + # scaling_factor = tensor_amax / FPtype_max + scaling_factor *= 2 + if hasattr(layer_self_attn, "kv_scale"): + layer_self_attn.attn._kv_scale = scaling_factor + else: + raise RuntimeError("Self attention has no KV cache scaling " + "factor attribute!") + + +class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"] + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens", + "lm_head" + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings" + } + embedding_padding_modules = ["lm_head"] + + # BitandBytes specific attributes + default_bitsandbytes_target_modules = [ + ".gate_proj.", + ".down_proj.", + ".up_proj.", + ".q_proj.", + ".k_proj.", + ".v_proj.", + ".o_proj.", + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".down_proj.", ".o_proj."] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + # Mistral/Llama models can also be loaded with --load-format mistral + # from consolidated.safetensors checkpoints + mistral_mapping = { + "layers": "model.layers", + "attention": "self_attn", + "wq": "q_proj", + "wk": "k_proj", + "wv": "v_proj", + "wo": "o_proj", + "attention_norm": "input_layernorm", + "feed_forward": "mlp", + "w1": "gate_proj", + "w2": "down_proj", + "w3": "up_proj", + "ffn_norm": "post_attention_layernorm", + "tok_embeddings": "model.embed_tokens", + "output": "lm_head", + "norm": "model.norm" + } + + def __init__( + self, + config: LlamaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = LlamaModel(config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model") + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=( + DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else + lora_config.lora_vocab_padding_size), + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights( + self.model.embed_tokens) + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + self.sampler = Sampler() + else: + self.lm_head = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return model_output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample(self, logits: torch.Tensor, + sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + loader.load_weights( + self.maybe_remap_mistral(name, loaded_weight) + for name, loaded_weight in weights) + + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + self.model.load_kv_cache_scales(quantization_param_path) + + # This function is used to remap the mistral format as + # used by Mistral and Llama <=2 + def maybe_remap_mistral( + self, + name: str, + loaded_weight: torch.Tensor, + ) -> Tuple[str, torch.Tensor]: + + def permute(w: torch.Tensor, n_heads: int): + attn_in = self.config.head_dim * n_heads + attn_out = self.config.hidden_size + + return w.view(n_heads, attn_in // n_heads // 2, 2, + attn_out).transpose(1, 2).reshape(attn_in, attn_out) + + mapping = self.mistral_mapping + modules = name.split(".") + + # rotary embeds should be sliced + if "wk" in modules: + loaded_weight = permute(loaded_weight, + self.config.num_key_value_heads) + elif "wq" in modules: + loaded_weight = permute(loaded_weight, + self.config.num_attention_heads) + + for item in modules: + if item in mapping and mapping[item] not in name: + name = name.replace(item, mapping[item]) + + return name, loaded_weight \ No newline at end of file diff --git a/vllm/model_executor/models/llama_embedding.py b/vllm/model_executor/models/llama_embedding.py new file mode 100644 index 00000000..13574e84 --- /dev/null +++ b/vllm/model_executor/models/llama_embedding.py @@ -0,0 +1,59 @@ +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import AttentionMetadata +from vllm.model_executor.layers.pooler import Pooler, PoolingType +from vllm.model_executor.pooling_metadata import PoolingMetadata +from vllm.sequence import IntermediateTensors, PoolerOutput + +from .interfaces import SupportsPP +from .llama import LlamaModel + + +class LlamaEmbeddingModel(nn.Module, SupportsPP): + """A model that uses Llama with additional embedding functionalities. + + This class encapsulates the LlamaModel and provides an interface for + embedding operations and customized pooling functions. + + Attributes: + model: An instance of LlamaModel used for forward operations. + _pooler: An instance of Pooler used for pooling operations. + """ + + def __init__( + self, + **kwargs, + ) -> None: + super().__init__() + self.model = LlamaModel(**kwargs) + self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + return self.model(input_ids, positions, kv_caches, attn_metadata, + intermediate_tensors, inputs_embeds) + + def pooler( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> Optional[PoolerOutput]: + return self._pooler(hidden_states, pooling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + self.model.load_weights(weights) + + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + self.model.load_kv_cache_scales(quantization_param_path) diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py new file mode 100644 index 00000000..864b9ff6 --- /dev/null +++ b/vllm/model_executor/models/llava.py @@ -0,0 +1,411 @@ +from functools import cached_property +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +import torch.nn as nn +from PIL import Image +from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import (CLIPVisionModel, dummy_image_for_clip, + dummy_seq_data_for_clip, get_max_clip_image_tokens, + input_processor_for_clip) +from .interfaces import SupportsMultiModal, SupportsPP +from .siglip import (SiglipVisionModel, dummy_image_for_siglip, + dummy_seq_data_for_siglip, get_max_siglip_image_tokens, + input_processor_for_siglip) +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, + merge_multimodal_embeddings) + + +class LlavaImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: `(batch_size * num_images, num_channels, height, width)`""" + + +class LlavaImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs] + + +# TODO(xwjiang): Run benchmark and decide if TP. +class LlavaMultiModalProjector(nn.Module): + + def __init__(self, vision_hidden_size: int, text_hidden_size: int, + projector_hidden_act: str): + super().__init__() + + self.linear_1 = nn.Linear(vision_hidden_size, + text_hidden_size, + bias=True) + self.act = get_act_fn(projector_hidden_act) + self.linear_2 = nn.Linear(text_hidden_size, + text_hidden_size, + bias=True) + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + hidden_states = self.linear_1(image_features) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +def get_max_llava_image_tokens(ctx: InputContext): + hf_config = ctx.get_hf_config(LlavaConfig) + vision_config = hf_config.vision_config + + if isinstance(vision_config, CLIPVisionConfig): + num_image_tokens = get_max_clip_image_tokens(vision_config) + elif isinstance(vision_config, SiglipVisionConfig): + num_image_tokens = get_max_siglip_image_tokens(vision_config) + else: + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + strategy = hf_config.vision_feature_select_strategy + if strategy == "default": + return num_image_tokens - 1 + elif strategy == "full": + return num_image_tokens + else: + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + +def dummy_data_for_llava(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(LlavaConfig) + vision_config = hf_config.vision_config + num_images = mm_counts["image"] + + image_feature_size = get_max_llava_image_tokens(ctx) + + if isinstance(vision_config, CLIPVisionConfig): + seq_data = dummy_seq_data_for_clip( + vision_config, + seq_len, + num_images, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + mm_data = dummy_image_for_clip(vision_config, num_images) + return seq_data, mm_data + elif isinstance(vision_config, SiglipVisionConfig): + seq_data = dummy_seq_data_for_siglip( + vision_config, + seq_len, + num_images, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + mm_data = dummy_image_for_siglip(vision_config, num_images) + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(LlavaConfig) + vision_config = hf_config.vision_config + + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + image_feature_size = get_max_llava_image_tokens(ctx) + elif is_list_of(image_data, Image.Image): + image_feature_size = [get_max_llava_image_tokens(ctx) + ] * len(image_data) + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + elif is_list_of(image_data, torch.Tensor): + image_feature_size = [item.shape[1] for item in image_data] + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + if isinstance(vision_config, CLIPVisionConfig): + return input_processor_for_clip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return input_processor_for_siglip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def _init_vision_tower(hf_config: LlavaConfig): + vision_config = hf_config.vision_config + + # Initialize the vision tower only up to the required feature layer + vision_feature_layer = hf_config.vision_feature_layer + if vision_feature_layer < 0: + num_hidden_layers = hf_config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + + if isinstance(vision_config, CLIPVisionConfig): + return CLIPVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return SiglipVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava) +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava) +class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: LlavaConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + # TODO: Optionally initializes this for supporting embeddings. + self.vision_tower = _init_vision_tower(config) + self.multi_modal_projector = LlavaMultiModalProjector( + vision_hidden_size=config.vision_config.hidden_size, + text_hidden_size=config.text_config.hidden_size, + projector_hidden_act=config.projector_hidden_act) + + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + actual_dims = tuple(data.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("batch_size", *map(str, expected_dims)) + raise ValueError( + f"The expected shape of pixel values is {expected_expr}. " + f"You supplied {tuple(data.shape)}.") + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[LlavaImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + return LlavaImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values( + flatten_bn(pixel_values, concat=True)), + ) + + if image_embeds is not None: + if not isinstance(image_embeds, (torch.Tensor, list)): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + return LlavaImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds, concat=True), + ) + + raise AssertionError("This line should be unreachable.") + + def _select_image_features(self, image_features: torch.Tensor, *, + strategy: str) -> torch.Tensor: + # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa + if strategy == "default": + return image_features[:, 1:] + elif strategy == "full": + return image_features + + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + def _image_pixels_to_features( + self, + vision_tower: Union[CLIPVisionModel, SiglipVisionModel], + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + image_features = vision_tower(pixel_values) + + return self._select_image_features( + image_features, + strategy=self.config.vision_feature_select_strategy, + ) + + def _process_image_pixels(self, + inputs: LlavaImagePixelInputs) -> torch.Tensor: + assert self.vision_tower is not None + + pixel_values = inputs["data"] + + return self._image_pixels_to_features(self.vision_tower, pixel_values) + + def _process_image_input(self, + image_input: LlavaImageInputs) -> torch.Tensor: + + if image_input["type"] == "image_embeds": + return image_input["data"] + + assert self.vision_tower is not None + image_features = self._process_image_pixels(image_input) + return self.multi_modal_projector(image_features) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for LLaVA-1.5. + + One key thing to understand is the `input_ids` already accounts for the + positions of the to-be-inserted image embeddings. + + Concretely, consider a text prompt: + `"USER: \\nWhat's the content of the image?\\nASSISTANT:"`. + + Tokenizer outputs: + `[1, 3148, 1001, 29901, 29871, 32000, 29871, 13, 5618, 29915, 29879, + 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901]`. + + To reserve space in KV cache, we have to insert placeholder tokens + before they are inputted to the model, so the input processor prepends + additional image tokens (denoted as `32000`), resulting in: + `[1, 3148, 1001, 29901, 29871, 32000, ..., 32000, 29871, 13, 5618, + 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, + 29901]`. + + We insert 575 tokens so that including the original image token in the + input, there are a total of 576 (24 * 24) image tokens, which + corresponds to the number of image tokens inputted to the language + model, i.e. the number of image tokens outputted by the visual encoder. + + This way, the `positions` and `attn_metadata` are consistent + with the `input_ids`. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + pixel_values: The pixels in each input image. + + See also: + :class:`LlavaImageInputs` + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + # always pass the input via `inputs_embeds` + # to make sure the computation graph is consistent + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.config.image_token_index) + else: + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + input_ids = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py new file mode 100644 index 00000000..766f6a4c --- /dev/null +++ b/vllm/model_executor/models/llava_next.py @@ -0,0 +1,645 @@ +from functools import cached_property +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +import torch.nn as nn +from PIL import Image +from transformers import CLIPVisionConfig, LlavaNextConfig, SiglipVisionConfig +from transformers.models.llava_next.modeling_llava_next import ( + get_anyres_image_grid_shape, unpad_image) +from typing_extensions import NotRequired + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import (CLIPVisionModel, dummy_image_for_clip, + dummy_seq_data_for_clip, get_clip_image_feature_size, + get_clip_patch_grid_length, input_processor_for_clip) +from .interfaces import SupportsMultiModal, SupportsPP +from .llava import LlavaMultiModalProjector +from .siglip import (SiglipVisionModel, dummy_image_for_siglip, + dummy_seq_data_for_siglip, get_siglip_image_feature_size, + get_siglip_patch_grid_length, input_processor_for_siglip) +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, + merge_multimodal_embeddings) + +# Result in the max possible feature size (2x2 grid of 336x336px tiles) +MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448 + + +class LlavaNextImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: Union[torch.Tensor, List[torch.Tensor]] + """ + Shape: + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)` + + Note that `num_patches` may be different per batch and image, + in which case the data is passed as a list instead of a batched tensor. + """ + + image_sizes: NotRequired[torch.Tensor] + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(height, width)` format. + """ + + +class LlavaNextImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +LlavaNextImageInputs = Union[LlavaNextImagePixelInputs, + LlavaNextImageEmbeddingInputs] + + +# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79 +def _get_llava_next_num_unpadded_features( + original_height: int, + original_width: int, + npatches: int, + num_patch_height: int, + num_patch_width: int, +) -> Tuple[int, int]: + current_height = npatches * num_patch_height + current_width = npatches * num_patch_width + + original_aspect_ratio = original_width / original_height + current_aspect_ratio = current_width / current_height + + if original_aspect_ratio > current_aspect_ratio: + scale_factor = current_width / original_width + new_height = int(original_height * scale_factor) + padding = (current_height - new_height) // 2 + current_height -= 2 * padding + else: + scale_factor = current_height / original_height + new_width = int(original_width * scale_factor) + padding = (current_width - new_width) // 2 + current_width -= 2 * padding + + unpadded_features = current_height * current_width + newline_features = current_height + return (unpadded_features, newline_features) + + +# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106 +def get_llava_next_image_feature_size( + hf_config: LlavaNextConfig, + *, + input_height: int, + input_width: int, +) -> int: + vision_config = hf_config.vision_config + + if isinstance(vision_config, CLIPVisionConfig): + num_patches = get_clip_patch_grid_length( + image_size=vision_config.image_size, + patch_size=vision_config.patch_size, + ) + base_feature_size = get_clip_image_feature_size(vision_config) + elif isinstance(vision_config, SiglipVisionConfig): + num_patches = get_siglip_patch_grid_length( + image_size=vision_config.image_size, + patch_size=vision_config.patch_size, + ) + base_feature_size = get_siglip_image_feature_size(vision_config) + else: + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + strategy = hf_config.vision_feature_select_strategy + if strategy == "default": + base_feature_size -= 1 + elif strategy == "full": + pass + else: + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + num_patch_height, num_patch_width = get_anyres_image_grid_shape( + image_size=(input_height, input_width), + grid_pinpoints=hf_config.image_grid_pinpoints, + patch_size=vision_config.image_size, + ) + + ( + unpadded_feature_size, + newline_feature_size, + ) = _get_llava_next_num_unpadded_features(input_height, input_width, + num_patches, num_patch_height, + num_patch_width) + + return unpadded_feature_size + newline_feature_size + base_feature_size + + +def get_max_llava_next_image_tokens(ctx: InputContext): + return get_llava_next_image_feature_size( + ctx.get_hf_config(LlavaNextConfig), + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH, + ) + + +def dummy_data_for_llava_next(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(LlavaNextConfig) + vision_config = hf_config.vision_config + num_images = mm_counts["image"] + + image_feature_size = get_max_llava_next_image_tokens(ctx) + + if isinstance(vision_config, CLIPVisionConfig): + seq_data = dummy_seq_data_for_clip( + vision_config, + seq_len, + num_images, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + mm_data = dummy_image_for_clip( + vision_config, + num_images, + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH, + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + ) + + return seq_data, mm_data + elif isinstance(vision_config, SiglipVisionConfig): + seq_data = dummy_seq_data_for_siglip( + vision_config, + seq_len, + num_images, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + mm_data = dummy_image_for_siglip( + vision_config, + num_images, + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH, + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + ) + + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(LlavaNextConfig) + vision_config = hf_config.vision_config + + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + width, height = image_data.size + + image_feature_size = get_llava_next_image_feature_size( + hf_config, + input_height=height, + input_width=width, + ) + elif is_list_of(image_data, Image.Image): + image_feature_size = [ + get_llava_next_image_feature_size(hf_config, + input_height=img.height, + input_width=img.width) + for img in image_data + ] + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + elif is_list_of(image_data, torch.Tensor): + image_feature_size = [item.shape[1] for item in image_data] + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + vision_config = hf_config.vision_config + + if isinstance(vision_config, CLIPVisionConfig): + return input_processor_for_clip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return input_processor_for_siglip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def _init_vision_tower(hf_config: LlavaNextConfig): + vision_config = hf_config.vision_config + + # Initialize the vision tower only up to the required feature layer + vision_feature_layer = hf_config.vision_feature_layer + if vision_feature_layer < 0: + num_hidden_layers = hf_config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + + if isinstance(vision_config, CLIPVisionConfig): + return CLIPVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return SiglipVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next) +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next) +class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: LlavaNextConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + # TODO: Optionally initializes this for supporting embeddings. + self.vision_tower = _init_vision_tower(config) + self.image_newline = nn.Parameter( + torch.empty(config.text_config.hidden_size)) + self.multi_modal_projector = LlavaMultiModalProjector( + vision_hidden_size=config.vision_config.hidden_size, + text_hidden_size=config.text_config.hidden_size, + projector_hidden_act=config.projector_hidden_act) + + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor: + expected_dims = (2, ) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + f"The expected shape of image sizes per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _validate_pixel_values( + self, data: Union[torch.Tensor, List[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor]]: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("num_patches", *map(str, expected_dims)) + raise ValueError( + "The expected shape of pixel values per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[LlavaNextImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_sizes = kwargs.pop("image_sizes", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + if not isinstance(image_sizes, (torch.Tensor, list)): + raise ValueError("Incorrect type of image sizes. " + f"Got type: {type(image_sizes)}") + + return LlavaNextImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values(flatten_bn(pixel_values)), + image_sizes=self._validate_image_sizes( + flatten_bn(image_sizes, concat=True)), + ) + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeds. " + f"Got type: {type(image_embeds)}") + + return LlavaNextImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds), + ) + + raise AssertionError("This line should be unreachable.") + + def _select_image_features(self, image_features: torch.Tensor, *, + strategy: str) -> torch.Tensor: + # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa + if strategy == "default": + return image_features[:, 1:] + elif strategy == "full": + return image_features + + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + def _image_pixels_to_features( + self, + vision_tower: Union[CLIPVisionModel, SiglipVisionModel], + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + image_features = vision_tower(pixel_values) + + return self._select_image_features( + image_features, + strategy=self.config.vision_feature_select_strategy, + ) + + # Based on: https://github.com/haotian-liu/LLaVA/blob/main/llava/model/llava_arch.py + def _merge_image_patch_embeddings(self, image_size: torch.Tensor, + patch_embeddings: torch.Tensor, *, + strategy: str) -> torch.Tensor: + if strategy == "flat": + return patch_embeddings.flatten(0, 1) + + if strategy.startswith("spatial"): + height = width = self.config.vision_config.image_size \ + // self.config.vision_config.patch_size + + base_patch_embeds = patch_embeddings[0] + if height * width != base_patch_embeds.shape[0]: + raise ValueError( + "The number of patches is not consistent with the " + "image size.") + + if patch_embeddings.shape[0] > 1: + other_patch_embeds = patch_embeddings[1:] + + # Move to CPU to avoid floating-point errors + orig_height, orig_width = image_size.tolist() + + # image_aspect_ratio == "anyres" + num_patch_height, num_patch_width = get_anyres_image_grid_shape( + (orig_height, orig_width), + self.config.image_grid_pinpoints, + self.config.vision_config.image_size, + ) + num_patches = num_patch_height * num_patch_width + + # Image patches might be padded for batch processing + other_patch_embeds = other_patch_embeds[:num_patches] \ + .view(num_patch_height, num_patch_width, height, width, -1) + + if "unpad" in strategy: + other_patch_embeds = other_patch_embeds \ + .permute(4, 0, 2, 1, 3).contiguous() \ + .flatten(1, 2).flatten(2, 3) + other_patch_embeds = unpad_image(other_patch_embeds, + (orig_height, orig_width)) + other_patch_embeds = torch.cat(( + other_patch_embeds, + self.image_newline[:, None, None] \ + .expand(*other_patch_embeds.shape[:-1], 1) \ + .to(other_patch_embeds.device), + ), dim=-1) + other_patch_embeds = other_patch_embeds \ + .flatten(1, 2).transpose(0, 1) + else: + other_patch_embeds = other_patch_embeds \ + .permute(0, 2, 1, 3, 4).contiguous() \ + .flatten(0, 3) + + merged_patch_embeddings = torch.cat( + (base_patch_embeds, other_patch_embeds), dim=0) + else: + if "unpad" in strategy: + merged_patch_embeddings = torch.cat( + (base_patch_embeds, + self.image_newline[None] \ + .to(base_patch_embeds.device) + ), dim=0) + else: + merged_patch_embeddings = base_patch_embeds + + return merged_patch_embeddings + + raise ValueError(f"Unexpected patch merge strategy: {strategy}") + + def _process_image_pixels( + self, + inputs: LlavaNextImagePixelInputs, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + assert self.vision_tower is not None + + pixel_values = inputs["data"] + + if isinstance(pixel_values, torch.Tensor): + b, num_patches, c, h, w = pixel_values.shape + stacked_pixel_values = pixel_values.view(b * num_patches, c, h, w) + stacked_image_features = self._image_pixels_to_features( + self.vision_tower, stacked_pixel_values) + stacked_patch_embeddings = self.multi_modal_projector( + stacked_image_features) + + return stacked_patch_embeddings.view( + b, num_patches, *stacked_patch_embeddings.shape[1:]) + + num_patches_per_batch = [v.shape[0] for v in pixel_values] + stacked_pixel_values = torch.cat(pixel_values) + stacked_image_features = self._image_pixels_to_features( + self.vision_tower, stacked_pixel_values) + + return [ + self.multi_modal_projector(image_features) for image_features in + torch.split(stacked_image_features, num_patches_per_batch) + ] + + def _process_image_input( + self, + image_input: LlavaNextImageInputs, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + if image_input["type"] == "image_embeds": + return [image_input["data"]] + + patch_embeddings = self._process_image_pixels(image_input) + + image_sizes = image_input.get("image_sizes") + if image_sizes is None: + batch_size = len(image_input["data"]) + vision_config = self.config.vision_config + default_height = default_width = vision_config.image_size + image_sizes = torch.as_tensor([[default_height, default_width] + for _ in range(batch_size)]) + + return [ + self._merge_image_patch_embeddings(image_sizes[i], + patch_features_batch, + strategy="spatial_unpad") + for i, patch_features_batch in enumerate(patch_embeddings) + ] + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for LlaVA-NeXT. + + One key thing to understand is the `input_ids` already accounts for the + positions of the to-be-inserted image embeddings. + + Concretely, consider a text prompt: + `"A chat between a curious human and an artificial intelligence + assistant. The assistant gives helpful, detailed, and polite answers to + the human's questions. + USER: \\nWhat is shown in this image? ASSISTANT:"`. + + Tokenizer outputs: + `[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255, + 29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568, + 6089, 304, 278, 5199, 29915, 29879, 5155, 29889, 3148, 1001, 29901, + 29871, 32000, 13, 5618, 338, 4318, 297, 445, 1967, 29973, 319, 1799, + 9047, 13566, 29901]`. + + To reserve space in KV cache, we have to insert placeholder tokens + before they are inputted to the model, so the input processor prepends + additional image tokens (denoted as `32000`), resulting in: + `[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255, + 29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568, + 6089, 304, 278, 5199, 29915, 29879, 5155, 29889, 3148, 1001, 29901, + 29871, 32000, ..., 32000, 13, 5618, 338, 4318, 297, 445, 1967, 29973, + 319, 1799, 9047, 13566, 29901]`. + + Unlike in LLaVA-1.5, the number of image tokens inputted to the language + model depends on the original size of the input image. Including the + original image token in the input, the required number of image tokens + is given by :func:`get_llava_next_image_feature_size`. + + This way, the `positions` and `attn_metadata` are consistent + with the `input_ids`. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + pixel_values: The pixels in each grid patch for each input image. + image_sizes: The original `(height, width)` for each input image. + + See also: + :class:`LlavaNextImageInputs` + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.config.image_token_index) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py new file mode 100644 index 00000000..e10c1f9e --- /dev/null +++ b/vllm/model_executor/models/llava_next_video.py @@ -0,0 +1,465 @@ +import math +from functools import cached_property +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import numpy as np +import torch +import torch.nn as nn +from transformers import (CLIPVisionConfig, LlavaNextVideoConfig, + SiglipVisionConfig) + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.models.clip import CLIPVisionModel +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip +from .interfaces import SupportsMultiModal, SupportsPP +from .siglip import (SiglipVisionModel, dummy_image_for_siglip, + dummy_seq_data_for_siglip) +from .utils import (AutoWeightsLoader, init_vllm_registered_model, + merge_multimodal_embeddings) + +# For profile run +_MAX_FRAMES_PER_VIDEO = 32 +_MAX_NUM_VIDEOS = 1 + + +class LlavaNextVideoPixelInputs(TypedDict): + type: Literal["pixel_values_videos"] + data: Union[torch.Tensor, List[torch.Tensor]] + """ + Shape: `(batch_size, num_frames, num_channels, height, width)` + + Note that `num_frames` may be different for each batch, in which case + the data is passed as a list instead of a batched tensor. + + Note that it only supports one video input for one batch. + """ + + +def get_llava_next_video_frame_feature_size( + hf_config: LlavaNextVideoConfig) -> int: + # Support both CLIPVisionConfig and SiglipVisionConfig + image_size = hf_config.vision_config.image_size + patch_size = hf_config.vision_config.patch_size + spatial_pool_stride = hf_config.spatial_pool_stride + + return int((image_size / patch_size / spatial_pool_stride)**2) + + +def _get_max_llm_tokens(ctx: InputContext) -> int: + """ + Calculated from the maximum video frames under the context length + constraints of the language model. + """ + hf_text_config = ctx.model_config.hf_text_config + model_config = ctx.model_config + max_tokens = model_config.max_model_len + rope_scaling = model_config.rope_scaling + + if rope_scaling: + rope_scaling_factor = hf_text_config.rope_scaling["factor"] + else: + rope_scaling_factor = 1 + + max_tokens *= rope_scaling_factor + + return max_tokens + + +def get_max_llava_next_video_tokens(ctx: InputContext) -> int: + # Currently set to 32 frames + # TODO: max_tokens = _get_max_llm_tokens(ctx) + hf_config = ctx.get_hf_config(LlavaNextVideoConfig) + tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config) + return _MAX_FRAMES_PER_VIDEO * tokens_per_frame + + +def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(LlavaNextVideoConfig) + vision_config = hf_config.vision_config + + # TODO: support multiple videos + num_videos = mm_counts["video"] + if num_videos != _MAX_NUM_VIDEOS: + raise NotImplementedError( + f"Only {_MAX_NUM_VIDEOS} videos are supported") + + # TODO: support configuring the number of frames + frames_per_video = _MAX_FRAMES_PER_VIDEO + # num_images = num_videos * frames_per_video + + # fills the sequence with as longer video data as possible + tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config) + video_feature_size = frames_per_video * tokens_per_frame + + if isinstance(vision_config, CLIPVisionConfig): + seq_data = dummy_seq_data_for_clip( + vision_config, + seq_len, + num_videos, + image_token_id=hf_config.video_token_index, + image_feature_size_override=video_feature_size, + ) + + pil_frame = dummy_image_for_clip(vision_config, num_images=1) + np_frame = np.array(pil_frame["image"]) + mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0) + mm_data = {"video": mm_data_per_video} + return seq_data, mm_data + elif isinstance(vision_config, SiglipVisionConfig): + seq_data = dummy_seq_data_for_siglip( + vision_config, + seq_len, + num_videos, + image_token_id=hf_config.video_token_index, + image_feature_size_override=video_feature_size, + ) + + pil_frame = dummy_image_for_siglip(vision_config, num_images=1) + np_frame = np.array(pil_frame["image"]) + mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0) + mm_data = {"video": mm_data_per_video} + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_for_llava_next_video(ctx: InputContext, + llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "video" not in multi_modal_data: + return llm_inputs + video_data = multi_modal_data["video"] + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(LlavaNextVideoConfig) + vision_config = hf_config.vision_config + + if isinstance(video_data, np.ndarray): + # Supports both CLIP and Siglip + num_frames = video_data.shape[0] + frame_feature_size = \ + get_llava_next_video_frame_feature_size(hf_config) + video_feature_size = num_frames * frame_feature_size + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=hf_config.video_token_index, + repeat_count=video_feature_size, + ) + + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + elif is_list_of(video_data, np.ndarray): + raise NotImplementedError( + "Processing multiple videos is not supported") + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def _init_vision_tower(hf_config: LlavaNextVideoConfig): + vision_config = hf_config.vision_config + + # Initialize the vision tower only up to the required feature layer + vision_feature_layer = hf_config.vision_feature_layer + if vision_feature_layer < 0: + num_hidden_layers = hf_config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + + if isinstance(vision_config, CLIPVisionConfig): + return CLIPVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return SiglipVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +# adopted from transformers modeling_llava_next_video.py +class LlavaNextVideoPooler(nn.Module): + + def __init__(self, config): + super().__init__() + + mode = config.spatial_pool_mode + stride = config.spatial_pool_stride + image_size = config.vision_config.image_size + patch_size = config.vision_config.patch_size + self.image_size = image_size // patch_size**2 + + if mode == "average": + self.pool = nn.AvgPool2d(kernel_size=stride, stride=stride) + elif mode == "max": + self.pool = nn.MaxPool2d(kernel_size=stride, stride=stride) + else: + # TODO: Support Conv2d pooling layer, need to load weights + raise ValueError( + f"Unknown pooling mode: {mode}. Expected [`average`, `max`]") + + def forward(self, image_features): + ori_width = int( + math.sqrt(image_features.shape[1] * self.image_size // + self.image_size)) + ori_height = int(ori_width * self.image_size // self.image_size) + + batch_size, _, dim = image_features.shape + image_features_spatial = image_features \ + .view(batch_size, ori_height, ori_height, dim) \ + .permute(0, 3, 1, 2) + image_features_spatial = self.pool(image_features_spatial) + + return image_features_spatial.flatten(2).transpose(1, 2).contiguous() + + +class LlavaNextMultiModalProjector(nn.Module): + + def __init__(self, vision_hidden_size: int, text_hidden_size: int, + projector_hidden_act: str): + super().__init__() + + self.linear_1 = nn.Linear(vision_hidden_size, + text_hidden_size, + bias=True) + self.act = get_act_fn(projector_hidden_act) + self.linear_2 = nn.Linear(text_hidden_size, + text_hidden_size, + bias=True) + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + hidden_states = self.linear_1(image_features) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_input_mapper("video") +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "video", get_max_llava_next_video_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next_video) +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next_video) +class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: LlavaNextVideoConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + # Initialize the vision tower only up to the required feature layer + self.vision_tower = _init_vision_tower(config) + self.vision_resampler = LlavaNextVideoPooler(config) + self.multi_modal_projector = LlavaNextMultiModalProjector( + vision_hidden_size=config.vision_config.hidden_size, + text_hidden_size=config.text_config.hidden_size, + projector_hidden_act=config.projector_hidden_act) + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.make_empty_intermediate_tensors = ( + self.language_model.model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_video_pixel_values( + self, data: Union[torch.Tensor, List[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor]]: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape[2:]) + + if actual_dims != expected_dims: + expected_expr = ("num_frames", *map(str, expected_dims)) + raise ValueError( + "The expected shape of pixel values in each video frame " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_video_input( + self, **kwargs: object) -> Optional[LlavaNextVideoPixelInputs]: + """ + A legal video input should have the following dimensions: + { + "pixel_values_videos" : + List[b, Tensor(nb_frames, nb_channels, height, width)] + } + """ + pixel_values = kwargs.pop("pixel_values_videos", None) + + if pixel_values is None: + return None + + if not (is_list_of(pixel_values, + (torch.Tensor)) # different shape videos + or isinstance(pixel_values, + torch.Tensor)): # same shape videos + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + return LlavaNextVideoPixelInputs( + type="pixel_values_videos", + data=pixel_values, + ) + + def _select_image_features(self, image_features: torch.Tensor, *, + strategy: str) -> torch.Tensor: + if strategy == "default": + return image_features[:, 1:] + elif strategy == "full": + return image_features + + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + def _video_pixels_to_features( + self, + vision_tower: Union[CLIPVisionModel, SiglipVisionModel], + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + image_features = vision_tower(pixel_values) + image_features = self._select_image_features( + image_features, + strategy=self.config.vision_feature_select_strategy, + ) + image_features = self.vision_resampler(image_features) + image_features = self.multi_modal_projector(image_features) + return image_features + + def _process_video_pixels(self, inputs: LlavaNextVideoPixelInputs): + assert self.vision_tower is not None + + video_pixels = inputs["data"] + + if isinstance(video_pixels, torch.Tensor): + # TODO: support multiple videos per input + b, num_videos, num_frames, c, h, w = video_pixels.shape + assert (num_videos == 1) + stacked_pixels = video_pixels.view(b * num_videos * num_frames, c, + h, w) + stacked_embeddings = self._video_pixels_to_features( + self.vision_tower, stacked_pixels) + return stacked_embeddings.view(b, num_frames, + *stacked_embeddings.shape[1:]) + + elif is_list_of(video_pixels, torch.Tensor): + frames_per_videos = [v.shape[0] for v in video_pixels] + stacked_pixels = torch.cat(video_pixels, dim=0) + stacked_embeddings = self._video_pixels_to_features( + self.vision_tower, stacked_pixels) + return torch.split(stacked_embeddings, frames_per_videos, dim=0) + + else: + raise ValueError( + f"Unsupported type of video input {type(video_pixels)}") + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for LlaVA-NeXT-Video. + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + pixel_values_videos: Pixels in each frames for each input videos. + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + video_input = self._parse_and_validate_video_input(**kwargs) + if video_input is not None: + video_embeddings = self._process_video_pixels(video_input) + inputs_embeds = self.language_model \ + .model.get_input_embeddings(input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, video_embeddings, + self.config.video_token_index) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader( + self, + # This model doesn't support images for now + ignore_unexpected_prefixes=["image_newline"], + ) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py new file mode 100644 index 00000000..46e97e78 --- /dev/null +++ b/vllm/model_executor/models/llava_onevision.py @@ -0,0 +1,875 @@ +import math +from functools import cached_property +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from transformers import (CLIPVisionConfig, LlavaOnevisionConfig, + SiglipVisionConfig) +from transformers.models.llava_onevision.modeling_llava_onevision import ( + get_anyres_image_grid_shape, unpad_image) +from typing_extensions import NotRequired + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import (CLIPVisionModel, dummy_seq_data_for_clip, + dummy_video_for_clip, get_clip_image_feature_size, + get_clip_patch_grid_length, input_processor_for_clip) +from .interfaces import SupportsMultiModal, SupportsPP +from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip, + dummy_video_for_siglip, get_siglip_image_feature_size, + get_siglip_patch_grid_length, input_processor_for_siglip) +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model, + merge_multimodal_embeddings) + +logger = init_logger(__name__) + +# Result in the max possible feature size (2x2 grid of 336x336px tiles) +MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448 + +# For profile run +_MAX_FRAMES_PER_VIDEO = 16 +_MAX_NUM_VIDEOS = 1 + + +class LlavaOnevisionVideoPixelInputs(TypedDict): + type: Literal["pixel_values_videos"] + data: Union[torch.Tensor, List[torch.Tensor]] + """ + Shape: `(batch_size, num_frames, num_channels, height, width)` + + Note that `num_frames` may be different for each batch, in which case + the data is passed as a list instead of a batched tensor. + + Note that it only supports one video input for one batch. + """ + + +class LlavaOnevisionImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: Union[torch.Tensor, List[torch.Tensor]] + """ + Shape: + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)` + + Note that `num_patches` may be different per batch and image, + in which case the data is passed as a list instead of a batched tensor. + """ + + image_sizes: NotRequired[torch.Tensor] + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(height, width)` format. + """ + + +class LlavaOnevisionImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +LlavaOnevisionImageInputs = Union[LlavaOnevisionImagePixelInputs, + LlavaOnevisionImageEmbeddingInputs] + +LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs, + LlavaOnevisionVideoPixelInputs] + + +def _get_llava_onevision_image_unppaded_feature_size(height, width, patches, + scale_height, + scale_width): + current_height = patches * scale_height + current_width = patches * scale_width + + original_aspect_ratio = width / height + current_aspect_ratio = current_width / current_height + if original_aspect_ratio > current_aspect_ratio: + new_height = int(height * (current_width / width)) + padding = (current_height - new_height) // 2 + current_height -= padding * 2 + else: + new_width = int(width * (current_height / height)) + padding = (current_width - new_width) // 2 + current_width -= padding * 2 + + unpadded_features = current_height * current_width + newline_features = current_height + + ratio = math.sqrt(current_height * current_width / (9 * patches**2)) + if ratio > 1.1: + unpadded_features = int(current_height // ratio) * int( + current_width // ratio) + newline_features = int(current_height // ratio) + + return (unpadded_features, newline_features) + + +def get_llava_onevision_image_feature_size( + hf_config: LlavaOnevisionConfig, + *, + input_height: int, + input_width: int, +) -> int: + vision_config = hf_config.vision_config + + if isinstance(vision_config, CLIPVisionConfig): + num_patches = get_clip_patch_grid_length( + image_size=vision_config.image_size, + patch_size=vision_config.patch_size, + ) + base_feature_size = get_clip_image_feature_size(vision_config) + elif isinstance(vision_config, SiglipVisionConfig): + num_patches = get_siglip_patch_grid_length( + image_size=vision_config.image_size, + patch_size=vision_config.patch_size, + ) + base_feature_size = get_siglip_image_feature_size(vision_config) + else: + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + strategy = hf_config.vision_feature_select_strategy + if strategy == "default": + base_feature_size -= 1 + elif strategy == "full": + pass + else: + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + num_patch_height, num_patch_width = get_anyres_image_grid_shape( + image_size=(input_height, input_width), + grid_pinpoints=hf_config.image_grid_pinpoints, + patch_size=vision_config.image_size, + ) + + ( + unpadded_feature_size, + newline_feature_size, + ) = _get_llava_onevision_image_unppaded_feature_size( + input_height, input_width, num_patches, num_patch_height, + num_patch_width) + + return unpadded_feature_size + newline_feature_size + base_feature_size + + +def get_max_llava_onevision_image_tokens(ctx: InputContext): + return get_llava_onevision_image_feature_size( + ctx.get_hf_config(LlavaOnevisionConfig), + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH, + ) + + +def get_llava_onevision_video_frame_feature_size( + hf_config: LlavaOnevisionConfig) -> int: + # Support both CLIPVisionConfig and SiglipVisionConfig + image_size = hf_config.vision_config.image_size + patch_size = hf_config.vision_config.patch_size + spatial_pool_stride = hf_config.spatial_pool_stride if hasattr( + hf_config, "spatial_pool_stride") else 2 + + height = width = image_size // patch_size + return math.ceil(height / spatial_pool_stride) * math.ceil( + width / spatial_pool_stride) + + +def get_llava_onevision_video_tokens(ctx: InputContext, + num_frames: int) -> int: + hf_config = ctx.get_hf_config(LlavaOnevisionConfig) + + # TODO: support configuring (not supported by HF right now) + num_token_image_newline = 1 + tokens_per_frame = get_llava_onevision_video_frame_feature_size(hf_config) + video_feature_size = num_frames * tokens_per_frame + num_token_image_newline + + return video_feature_size + + +def get_max_llava_onevision_video_tokens(ctx: InputContext) -> int: + return get_llava_onevision_video_tokens(ctx, _MAX_FRAMES_PER_VIDEO) + + +def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(LlavaOnevisionConfig) + vision_config = hf_config.vision_config + + # TODO: support multiple videos + num_videos = mm_counts["video"] + if num_videos > _MAX_NUM_VIDEOS: + raise NotImplementedError( + f"Only {_MAX_NUM_VIDEOS} videos are supported") + + # TODO: support configuring the number of frames + num_frames = _MAX_FRAMES_PER_VIDEO + video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames) + + if isinstance(vision_config, CLIPVisionConfig): + seq_data = dummy_seq_data_for_clip( + vision_config, + seq_len, + num_videos, + image_token_id=hf_config.video_token_index, + image_feature_size_override=video_feature_size, + ) + + mm_data = dummy_video_for_clip(vision_config, num_frames=num_frames) + return seq_data, mm_data + elif isinstance(vision_config, SiglipVisionConfig): + seq_data = dummy_seq_data_for_siglip( + vision_config, + seq_len, + num_videos, + image_token_id=hf_config.video_token_index, + image_feature_size_override=video_feature_size, + ) + + mm_data = dummy_video_for_siglip(vision_config, num_frames=num_frames) + return seq_data, mm_data + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_when_multimodal_input_image(ctx: InputContext, + llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(LlavaOnevisionConfig) + vision_config = hf_config.vision_config + + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + width, height = image_data.size + + image_feature_size = get_llava_onevision_image_feature_size( + hf_config, + input_height=height, + input_width=width, + ) + elif is_list_of(image_data, Image.Image): + image_feature_size = [ + get_llava_onevision_image_feature_size(hf_config, + input_height=img.height, + input_width=img.width) + for img in image_data + ] + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + elif is_list_of(image_data, torch.Tensor): + image_feature_size = [item.shape[1] for item in image_data] + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + vision_config = hf_config.vision_config + + if isinstance(vision_config, CLIPVisionConfig): + return input_processor_for_clip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return input_processor_for_siglip( + model_config, + vision_config, + llm_inputs, + image_token_id=hf_config.image_token_index, + image_feature_size_override=image_feature_size, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_when_multimodal_input_video(ctx: InputContext, + llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "video" not in multi_modal_data: + return llm_inputs + video_data = multi_modal_data["video"] + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(LlavaOnevisionConfig) + vision_config = hf_config.vision_config + + if isinstance(video_data, np.ndarray): + # Supports both CLIP and Siglip + num_frames = video_data.shape[0] + video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames) + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=hf_config.video_token_index, + repeat_count=video_feature_size, + ) + + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + elif is_list_of(video_data, np.ndarray): + raise NotImplementedError( + "Processing multiple videos is not supported") + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +def input_processor_for_llava_onevision(ctx: InputContext, + llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or ("video" not in multi_modal_data + and "image" not in multi_modal_data): + return llm_inputs + if "image" in multi_modal_data: + return input_processor_when_multimodal_input_image(ctx, llm_inputs) + if "video" in multi_modal_data: + return input_processor_when_multimodal_input_video(ctx, llm_inputs) + + msg = "Unsupported multi data type" + raise NotImplementedError(msg) + + +def _init_vision_tower(hf_config: LlavaOnevisionConfig): + vision_config = hf_config.vision_config + + # Initialize the vision tower only up to the required feature layer + vision_feature_layer = hf_config.vision_feature_layer + if vision_feature_layer < 0: + num_hidden_layers = hf_config.vision_config.num_hidden_layers \ + + vision_feature_layer + 1 + else: + num_hidden_layers = vision_feature_layer + 1 + + if isinstance(vision_config, CLIPVisionConfig): + return CLIPVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + elif isinstance(vision_config, SiglipVisionConfig): + return SiglipVisionModel( + vision_config, + num_hidden_layers_override=num_hidden_layers, + ) + + msg = f"Unsupported vision config: {type(vision_config)}" + raise NotImplementedError(msg) + + +class LlavaOnevisionMultiModalProjector(nn.Module): + + def __init__(self, config: LlavaOnevisionConfig): + super().__init__() + + self.linear_1 = nn.Linear(config.vision_config.hidden_size, + config.text_config.hidden_size, + bias=True) + self.act = get_act_fn(config.projector_hidden_act) + self.linear_2 = nn.Linear(config.text_config.hidden_size, + config.text_config.hidden_size, + bias=True) + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + hidden_states = self.linear_1(image_features) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_input_mapper("video") +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "image", get_max_llava_onevision_image_tokens) +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "video", get_max_llava_onevision_video_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_onevision) +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_onevision) +class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: LlavaOnevisionConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + # Initialize the vision tower only up to the required feature layer + self.vision_tower = _init_vision_tower(config) + self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config) + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + self.image_newline = nn.Parameter( + torch.empty(config.text_config.hidden_size)) + + self.make_empty_intermediate_tensors = ( + self.language_model.model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor: + expected_dims = (2, ) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + f"The expected shape of image sizes per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _validate_image_pixel_values( + self, data: Union[torch.Tensor, List[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor]]: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("num_patches", *map(str, expected_dims)) + raise ValueError( + "The expected shape of pixel values per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[LlavaOnevisionImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_sizes = kwargs.pop("image_sizes", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + if not isinstance(image_sizes, (torch.Tensor, list)): + raise ValueError("Incorrect type of image sizes. " + f"Got type: {type(image_sizes)}") + + return LlavaOnevisionImagePixelInputs( + type="pixel_values", + data=self._validate_image_pixel_values( + flatten_bn(pixel_values)), + image_sizes=self._validate_image_sizes( + flatten_bn(image_sizes, concat=True)), + ) + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeds. " + f"Got type: {type(image_embeds)}") + + return LlavaOnevisionImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds), + ) + + raise AssertionError("This line should be unreachable.") + + def _validate_video_pixel_values( + self, data: Union[torch.Tensor, List[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor]]: + + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape[2:]) + + if actual_dims != expected_dims: + expected_expr = ("num_frames", *map(str, expected_dims)) + raise ValueError( + "The expected shape of pixel values in each video frame " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_video_input( + self, + **kwargs: object) -> Optional[LlavaOnevisionVideoPixelInputs]: + """ + A legal video input should have the following dimensions: + { + "pixel_values_videos" : + List[b, Tensor(nb_frames, nb_channels, height, width)] + } + """ + pixel_values = kwargs.pop("pixel_values_videos", None) + + if pixel_values is None: + return None + + if not (is_list_of(pixel_values, + (torch.Tensor)) # different shape videos + or isinstance(pixel_values, + torch.Tensor)): # same shape videos + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + return LlavaOnevisionVideoPixelInputs( + type="pixel_values_videos", + data=pixel_values, + ) + + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: + modalities = {} + + if "pixel_values" in kwargs: + modalities["images"] = self._parse_and_validate_image_input( + **kwargs) + + if "pixel_values_videos" in kwargs: + modalities["videos"] = self._parse_and_validate_video_input( + **kwargs) + + return modalities + + def _select_image_features(self, image_features: torch.Tensor, *, + strategy: str) -> torch.Tensor: + if strategy == "default": + return image_features[:, 1:] + elif strategy == "full": + return image_features + + raise ValueError(f"Unexpected select feature strategy: {strategy}") + + def _image_pixels_to_features( + self, + vision_tower: Union[CLIPVisionModel, SiglipVisionModel], + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + image_features = vision_tower(pixel_values) + return self._select_image_features( + image_features, + strategy=self.config.vision_feature_select_strategy, + ) + + # Based on: https://github.com/haotian-liu/LLaVA/blob/main/llava/model/llava_arch.py + def _merge_image_patch_embeddings(self, + image_size: torch.Tensor, + patch_embeddings: torch.Tensor, + *, + image_newline=None, + vision_aspect_ratio="anyres_max_9", + strategy: str) -> torch.Tensor: + if strategy == "flat": + return patch_embeddings.flatten(0, 1) + + if strategy.startswith("spatial"): + height = width = self.config.vision_config.image_size \ + // self.config.vision_config.patch_size + + base_patch_embeds = patch_embeddings[0] + if height * width != base_patch_embeds.shape[0]: + raise ValueError( + "The number of patches is not consistent with the " + "image size.") + + if patch_embeddings.shape[0] > 1: + other_patch_embeds = patch_embeddings[1:] + + # Move to CPU to avoid floating-point errors + orig_height, orig_width = image_size.tolist() + + # image_aspect_ratio == "anyres" + num_patch_height, num_patch_width = get_anyres_image_grid_shape( + (orig_height, orig_width), + self.config.image_grid_pinpoints, + self.config.vision_config.image_size, + ) + num_patches = num_patch_height * num_patch_width + + # Image patches might be padded for batch processing + other_patch_embeds = other_patch_embeds[:num_patches] \ + .view(num_patch_height, num_patch_width, height, width, -1) + + if "unpad" in strategy: + other_patch_embeds = other_patch_embeds \ + .permute(4, 0, 2, 1, 3).contiguous() \ + .flatten(1, 2).flatten(2, 3) + other_patch_embeds = unpad_image(other_patch_embeds, + (orig_height, orig_width)) + max_num_patches = int( + vision_aspect_ratio.removeprefix("anyres_max_")) + channels, curr_height, curr_width = other_patch_embeds.shape + ratio = math.sqrt(curr_height * curr_width / + (max_num_patches * height**2)) + if ratio > 1.1: + other_patch_embeds = other_patch_embeds[None] + other_patch_embeds = nn.functional.interpolate( + other_patch_embeds, [ + int(curr_height // ratio), + int(curr_width // ratio) + ], + mode="bilinear")[0] + if image_newline is not None: + other_patch_embeds = torch.cat( + ( + other_patch_embeds, + image_newline[:, None, None] \ + .expand(*other_patch_embeds.shape[:-1], 1) \ + .to(other_patch_embeds.device), + ), + dim=-1) + other_patch_embeds = other_patch_embeds \ + .flatten(1, 2).transpose(0, 1) + else: + other_patch_embeds = other_patch_embeds \ + .permute(0, 2, 1, 3, 4).contiguous() \ + .flatten(0, 3) + + merged_patch_embeddings = torch.cat( + (base_patch_embeds, other_patch_embeds), dim=0) + else: + if "unpad" in strategy: + merged_patch_embeddings = torch.cat( + (base_patch_embeds, + self.image_newline[None] \ + .to(base_patch_embeds.device) + ), dim=0) + else: + merged_patch_embeddings = base_patch_embeds + + return merged_patch_embeddings + + raise ValueError(f"Unexpected patch merge strategy: {strategy}") + + def _process_image_pixels( + self, + inputs: LlavaOnevisionImagePixelInputs, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + assert self.vision_tower is not None + + pixel_values = inputs["data"] + + if isinstance(pixel_values, torch.Tensor): + b, num_patches, c, h, w = pixel_values.shape + stacked_pixel_values = pixel_values.view(b * num_patches, c, h, w) + stacked_image_features = self._image_pixels_to_features( + self.vision_tower, stacked_pixel_values) + stacked_patch_embeddings = self.multi_modal_projector( + stacked_image_features) + + return stacked_patch_embeddings.view( + b, num_patches, *stacked_patch_embeddings.shape[1:]) + + num_patches_per_batch = [v.shape[0] for v in pixel_values] + stacked_pixel_values = torch.cat(pixel_values) + stacked_image_features = self._image_pixels_to_features( + self.vision_tower, stacked_pixel_values) + + return [ + self.multi_modal_projector(image_features) for image_features in + torch.split(stacked_image_features, num_patches_per_batch) + ] + + def _process_image_input( + self, + image_input: LlavaOnevisionImageInputs, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + if image_input["type"] == "image_embeds": + return [image_input["data"]] + + patch_embeddings = self._process_image_pixels(image_input) + + image_sizes = image_input.get("image_sizes") + if image_sizes is None: + batch_size = len(image_input["data"]) + vision_config = self.config.vision_config + default_height = default_width = vision_config.image_size + image_sizes = torch.as_tensor([[default_height, default_width] + for _ in range(batch_size)]) + + return [ + self._merge_image_patch_embeddings( + image_sizes[i], + patch_features_batch, + image_newline=self.image_newline, + strategy="spatial_unpad") + for i, patch_features_batch in enumerate(patch_embeddings) + ] + + def _video_pixels_to_features( + self, + vision_tower: Union[CLIPVisionModel, SiglipVisionModel], + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the vision tower + b, num_videos, frames, c, h, w = pixel_values.shape + assert (num_videos == _MAX_NUM_VIDEOS) + pixel_values = pixel_values.reshape(b * num_videos * frames, c, h, w) + video_features = vision_tower(pixel_values) + video_features = self._select_image_features( + video_features, + strategy=self.config.vision_feature_select_strategy, + ) + video_features = self.multi_modal_projector(video_features) + video_features = self.apply_pooling(video_features) + video_features = video_features.reshape( + b, frames * video_features.shape[1], -1) + image_newline = self.image_newline[None, None, :].repeat(b, 1, 1).to( + video_features.device) + video_features = torch.cat((video_features, image_newline), dim=1) + video_features = video_features.flatten(0, 1) + + return video_features + + def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs): + assert self.vision_tower is not None + + video_pixels = inputs["data"] + + # TODO: support multiple videos per input + if isinstance(video_pixels, torch.Tensor): + stacked_embeddings = self._video_pixels_to_features( + self.vision_tower, video_pixels) + return stacked_embeddings + else: + raise ValueError( + f"Unsupported type of video input {type(video_pixels)}") + + def apply_pooling(self, image_features, stride=2): + vision_config = self.config.vision_config + height = width = vision_config.image_size // vision_config.patch_size + batch_frames, _, dim = image_features.shape + image_features = image_features.view(batch_frames, height, width, -1) + image_features = image_features.permute(0, 3, 1, 2) + + # TODO support other pooling types config + height, width = image_features.shape[2:] + scaled_shape = [math.ceil(height / stride), math.ceil(width / stride)] + image_feature = nn.functional.interpolate(image_features, + size=scaled_shape, + mode='bilinear') + image_feature = image_feature.permute(0, 2, 3, 1) + image_feature = image_feature.view(batch_frames, -1, dim) + return image_feature + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for LlaVA-Onevision. + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + pixel_values_videos: Pixels in each frames for each input videos. + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + modalities = self._parse_and_validate_multimodal_inputs(**kwargs) + if modalities: + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + if "images" in modalities: + image_input = modalities["images"] + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.config.image_token_index) + if "videos" in modalities: + video_input = modalities["videos"] + video_embeddings = self._process_video_pixels(video_input) + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, video_embeddings, + self.config.video_token_index) + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/mamba.py b/vllm/model_executor/models/mamba.py new file mode 100644 index 00000000..1112a218 --- /dev/null +++ b/vllm/model_executor/models/mamba.py @@ -0,0 +1,499 @@ +# coding=utf-8 +"""PyTorch MAMBA model.""" +from dataclasses import dataclass +from typing import Iterable, List, Optional, Tuple + +import torch +from torch import nn +from transformers import MambaConfig + +from vllm.attention.backends.abstract import AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( + causal_conv1d_fn, causal_conv1d_update) +from vllm.model_executor.layers.mamba.ops.mamba_ssm import ( + selective_scan_fn, selective_state_update) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + composed_weight_loader, default_weight_loader, sharded_weight_loader) +from vllm.model_executor.models.interfaces import (HasInnerState, + IsAttentionFree) +from vllm.model_executor.models.mamba_cache import MambaCacheManager +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors +from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE, + _get_graph_batch_size) + +KVCache = Tuple[torch.Tensor, torch.Tensor] + + +@dataclass +class MambaCacheParams: + is_prompt: bool = False + conv_state: torch.Tensor = torch.Tensor() + ssm_state: torch.Tensor = torch.Tensor() + + +# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer +class MambaMixer(nn.Module): + """ + Compute ∆, A, B, C, and D the state space parameters and compute + the `contextualized_states`. A, D are input independent + (see Mamba paper [1] Section 3.5.2 "Interpretation of A" + for why A isn't selective) ∆, B, C are input-dependent + (this is a key difference between Mamba and the linear time + invariant S4, and is why Mamba is called + **selective** state spaces) + """ + + def __init__(self, config: MambaConfig, layer_idx): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.ssm_state_size = config.state_size + self.conv_kernel_size = config.conv_kernel + self.intermediate_size = config.intermediate_size + self.time_step_rank = int(config.time_step_rank) + + self.conv1d = ColumnParallelLinear( + input_size=self.conv_kernel_size, + output_size=self.intermediate_size, + bias=config.use_conv_bias, + ) + # unsqueeze to fit conv1d weights shape into the linear weights shape. + # Can't do this in `weight_loader` since it already exists in + # `ColumnParallelLinear` and `set_weight_attrs` + # doesn't allow to override it + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1) + + self.in_proj = MergedColumnParallelLinear(self.hidden_size, + [self.intermediate_size] * 2, + bias=config.use_bias) + # selective projection used to make dt, B and C input dependent + self.x_proj = RowParallelLinear( + self.intermediate_size, + self.time_step_rank + self.ssm_state_size * 2, + bias=False, + ) + # time step projection (discretization) - + # In the forward we need to apply dt_proj without the bias, + # as the bias is added in the selective scan kernel. + self.dt_proj = ColumnParallelLinear(self.time_step_rank, + self.intermediate_size, + bias=True, + skip_bias_add=True) + + tp_size = get_tensor_model_parallel_world_size() + self.A = nn.Parameter( + torch.empty( + self.intermediate_size // tp_size, + self.ssm_state_size, + dtype=torch.float32, + )) + self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size)) + + set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)}) + a_weight_loader = composed_weight_loader( + sharded_weight_loader(0), lambda x: -torch.exp(x.float())) + set_weight_attrs(self.A, {"weight_loader": a_weight_loader}) + + self.out_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=config.use_bias, + input_is_parallel=True, + ) + self.activation = config.hidden_act + + def forward(self, hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, conv_state: torch.Tensor, + ssm_state: torch.Tensor): + + # 1. Gated MLP's linear projection + projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1) + hidden_states, gate = projected_states.chunk(2, dim=-2) + + # 2. Convolution sequence transformation + conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0), + self.conv1d.weight.size(2)) + + if attn_metadata.query_start_loc is not None \ + and attn_metadata.context_lens_tensor is not None: + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ---------------------| + # |-- query_len ---| + hidden_states = causal_conv1d_fn( + hidden_states, + conv_weights, + self.conv1d.bias, + activation=self.activation, + conv_states=conv_state, + has_initial_state=attn_metadata.context_lens_tensor > 0, + query_start_loc=attn_metadata.query_start_loc) + else: + hidden_states = causal_conv1d_update( + hidden_states.transpose(0, 1), + conv_state, + conv_weights, + self.conv1d.bias, + self.activation, + ) + hidden_states = hidden_states.transpose(0, 1) + + # 3. State Space Model sequence transformation + # 3.a. input varying initialization of time_step, B and C + ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0] + + time_step, B, C = torch.split( + ssm_parameters, + [self.time_step_rank, self.ssm_state_size, self.ssm_state_size], + dim=-1, + ) + + # Note that Jamba normalizes B, C, and time_step here but Mamba doesn't. + + discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1) + # 3.c perform the recurrence y ← SSM(A, B, C)(x) + time_proj_bias = (self.dt_proj.bias.float() if hasattr( + self.dt_proj, "bias") else None) + + if attn_metadata.query_start_loc is not None \ + and attn_metadata.context_lens_tensor is not None: + scan_outputs = selective_scan_fn( + hidden_states, + ssm_state, + discrete_time_step, + self.A, + B.transpose(-2, -1), + C.transpose(-2, -1), + self.D.float(), + gate, + time_proj_bias, + delta_softplus=True, + has_initial_state=attn_metadata.context_lens_tensor > 0, + query_start_loc=attn_metadata.query_start_loc) + else: + scan_outputs = selective_state_update( + ssm_state, + hidden_states.transpose(0, 1), + discrete_time_step.transpose(0, 1), + self.A, + B, + C, + self.D, + gate.transpose(0, 1), + time_proj_bias, + dt_softplus=True, + ) + scan_outputs = scan_outputs.transpose(0, 1) + + # 4. Final linear projection + contextualized_states = self.out_proj(scan_outputs.transpose(-2, + -1))[0] + return contextualized_states + + +class MambaMLP(nn.Module): + + def __init__( + self, + config: MambaConfig, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + hidden_size = config.hidden_size + intermediate_size = config.intermediate_size + hidden_act = config.hidden_act + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class MambaDecoderLayer(nn.Module): + + def __init__(self, + config: MambaConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.layer_idx = layer_idx + self.config = config + self.mixer = MambaMixer(config, layer_idx) + + self.feed_forward = MambaMLP(config, quant_config=quant_config) + self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.pre_ff_layernorm = RMSNorm(config.hidden_size, + eps=config.layer_norm_epsilon) + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + **kwargs, + ): + if residual is None: + residual = hidden_states + hidden_states = self.norm(hidden_states) + else: + hidden_states, residual = self.norm(hidden_states, residual) + + hidden_states = self.mixer(hidden_states, attn_metadata, conv_state, + ssm_state) + # Fully Connected + hidden_states, residual = self.pre_ff_layernorm( + hidden_states, residual) + hidden_states = self.feed_forward(hidden_states) + return hidden_states, residual + + +class MambaModel(nn.Module): + + def __init__( + self, + config: MambaConfig, + quant_config: Optional[QuantizationConfig] = None, + cache_config: Optional[CacheConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = ((lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0) + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embeddings = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + + decoder_layers = [] + for i in range(config.num_hidden_layers): + decoder_layers.append( + MambaDecoderLayer(config, + layer_idx=i, + cache_config=cache_config, + quant_config=quant_config)) + self.layers = nn.ModuleList(decoder_layers) + self.norm_f = RMSNorm(config.hidden_size, + eps=config.layer_norm_epsilon) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.embeddings(input_ids) + residual = None + + for i in range(len(self.layers)): + layer = self.layers[i] + current_ssm_state = ssm_state[i] + current_conv_state = conv_state[i] + + hidden_states, residual = layer( + positions=positions, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + residual=residual, + conv_state=current_conv_state, + ssm_state=current_ssm_state, + ) + hidden_states, _ = self.norm_f(hidden_states, residual) + + return hidden_states + + +class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embeddings": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: MambaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + scheduler_config: Optional[SchedulerConfig] = None, + ) -> None: + assert not cache_config.enable_prefix_caching, \ + "Mamba does not support prefix caching" + + super().__init__() + self.config = config + self.scheduler_config = scheduler_config + self.backbone = MambaModel(config, + cache_config=cache_config, + quant_config=quant_config, + lora_config=lora_config) + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + + self.lm_head = self.backbone.embeddings + + # Used to track and store by the Mamba cache between steps. + self.mamba_cache: Optional[MambaCacheManager] = None + + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[KVCache], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs): + if self.mamba_cache is None: + max_batch_size = (_get_graph_batch_size( + self.scheduler_config.max_num_seqs) if self.scheduler_config + else max(_BATCH_SIZES_TO_CAPTURE) + 2) + self.mamba_cache = MambaCacheManager( + self.lm_head.weight.dtype, self.config.num_hidden_layers, + max_batch_size, *self._get_mamba_cache_shape()) + + mamba_cache_tensors = self.mamba_cache.current_run_tensors( + input_ids, attn_metadata, **kwargs) + + hidden_states = self.backbone(input_ids, positions, kv_caches, + attn_metadata, mamba_cache_tensors[0], + mamba_cache_tensors[1]) + + return hidden_states + + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + return self.mamba_cache.copy_inputs_before_cuda_graphs( + input_buffers, **kwargs) + + def get_seqlen_agnostic_capture_inputs(self, batch_size: int): + return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size) + + def _get_mamba_cache_shape( + self) -> Tuple[Tuple[int, int], Tuple[int, int]]: + world_size = get_tensor_model_parallel_world_size() + conv_state_shape = ( + self.config.intermediate_size // world_size, + self.config.conv_kernel - 1, + ) + temporal_state_shape = ( + self.config.intermediate_size // world_size, + self.config.state_size, + ) + return conv_state_shape, temporal_state_shape + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + if "A_log" in name: + name = name.replace("A_log", "A") + + if ".self_attn." in name: + name = name.replace(".self_attn", "") + + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/mamba_cache.py b/vllm/model_executor/models/mamba_cache.py new file mode 100644 index 00000000..8d1ba373 --- /dev/null +++ b/vllm/model_executor/models/mamba_cache.py @@ -0,0 +1,222 @@ +from typing import Dict, List, Optional + +import torch + +from vllm.attention.backends.abstract import AttentionMetadata + + +class MambaCacheManager: + + def __init__(self, dtype, num_mamba_layers, max_batch_size, + conv_state_shape, temporal_state_shape): + + conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) + + conv_state_shape, + dtype=dtype, + device="cuda") + temporal_state = torch.empty(size=(num_mamba_layers, max_batch_size) + + temporal_state_shape, + dtype=dtype, + device="cuda") + + self.mamba_cache = (conv_state, temporal_state) + + # Maps between the request id and a dict that maps between the seq_id + # and its index inside the self.mamba_cache + self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {} + + def current_run_tensors(self, input_ids: torch.Tensor, + attn_metadata: AttentionMetadata, **kwargs): + """ + Return the tensors for the current run's conv and ssm state. + """ + if "seqlen_agnostic_capture_inputs" not in kwargs: + # We get here only on Prefill/Eager mode runs + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"] + finished_requests_ids = kwargs["finished_requests_ids"] + + self._release_finished_requests(finished_requests_ids) + mamba_cache_tensors = self._prepare_current_run_mamba_cache( + request_ids_to_seq_ids, finished_requests_ids) + + else: + # CUDA graph capturing runs + mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"] + + return mamba_cache_tensors + + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + """ + Copy the relevant Mamba cache into the CUDA graph input buffer + that was provided during the capture runs + (JambaForCausalLM.mamba_gc_cache_buffer). + """ + assert all( + key in kwargs + for key in ["request_ids_to_seq_ids", "finished_requests_ids"]) + finished_requests_ids = kwargs["finished_requests_ids"] + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"] + + self._release_finished_requests(finished_requests_ids) + self._prepare_current_run_mamba_cache(request_ids_to_seq_ids, + finished_requests_ids) + + def get_seqlen_agnostic_capture_inputs(self, batch_size: int): + """ + Provide the CUDA graph capture runs with a buffer in adjusted size. + The buffer is used to maintain the Mamba Cache during the CUDA graph + replay runs. + """ + return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache) + + def _swap_mamba_cache(self, from_index: int, to_index: int): + assert len(self.mamba_cache) > 0 + for cache_t in self.mamba_cache: + cache_t[:, [to_index,from_index]] = \ + cache_t[:, [from_index,to_index]] + + def _copy_mamba_cache(self, from_index: int, to_index: int): + assert len(self.mamba_cache) > 0 + for cache_t in self.mamba_cache: + cache_t[:, to_index].copy_(cache_t[:, from_index], + non_blocking=True) + + def _move_out_if_already_occupied(self, index: int, + all_occupied_indices: List[int]): + if index in all_occupied_indices: + first_free_index = self._first_free_index_in_mamba_cache() + # In case occupied, move the occupied to a new empty block + self._move_cache_index_and_mappings(from_index=index, + to_index=first_free_index) + + def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str, + seq_id: int, + destination_index: int): + """ + Assign (req_id,seq_id) pair to a `destination_index` index, if + already occupied, move the occupying index to a free index. + """ + all_occupied_indices = self._get_all_occupied_indices() + if cur_rid not in self.mamba_cache_indices_mapping: + self._move_out_if_already_occupied( + index=destination_index, + all_occupied_indices=all_occupied_indices) + self.mamba_cache_indices_mapping[cur_rid] = { + seq_id: destination_index + } + elif seq_id not in (seq_ids2indices := + self.mamba_cache_indices_mapping[cur_rid]): + # parallel sampling , where n > 1, assume prefill have + # already happened now we only need to copy the already + # existing cache into the siblings seq_ids caches + self._move_out_if_already_occupied( + index=destination_index, + all_occupied_indices=all_occupied_indices) + index_exists = list(seq_ids2indices.values())[0] + # case of decoding n>1, copy prefill cache to decoding indices + self._copy_mamba_cache(from_index=index_exists, + to_index=destination_index) + self.mamba_cache_indices_mapping[cur_rid][ + seq_id] = destination_index + else: + # already exists + cache_index_already_exists = self.mamba_cache_indices_mapping[ + cur_rid][seq_id] + if cache_index_already_exists != destination_index: + # In case the seq id already exists but not in + # the right destination, swap it with what's occupying it + self._swap_pair_indices_and_mappings( + from_index=cache_index_already_exists, + to_index=destination_index) + + def _prepare_current_run_mamba_cache( + self, request_ids_to_seq_ids: Dict[str, list[int]], + finished_requests_ids: List[str]): + running_indices = [] + request_ids_to_seq_ids_flatten = [ + (req_id, seq_id) + for req_id, seq_ids in request_ids_to_seq_ids.items() + for seq_id in seq_ids + ] + batch_size = len(request_ids_to_seq_ids_flatten) + for dest_index, (request_id, + seq_id) in enumerate(request_ids_to_seq_ids_flatten): + if request_id in finished_requests_ids: + # Do not allocate cache index for requests that run + # and finish right after + continue + self._assign_seq_id_to_mamba_cache_in_specific_dest( + request_id, seq_id, dest_index) + running_indices.append(dest_index) + + self._clean_up_first_bs_blocks(batch_size, running_indices) + conv_state = self.mamba_cache[0][:, :batch_size] + temporal_state = self.mamba_cache[1][:, :batch_size] + + return (conv_state, temporal_state) + + def _get_all_occupied_indices(self): + return [ + cache_idx + for seq_ids2indices in self.mamba_cache_indices_mapping.values() + for cache_idx in seq_ids2indices.values() + ] + + def _clean_up_first_bs_blocks(self, batch_size: int, + indices_for_current_run: List[int]): + # move out all of the occupied but currently not running blocks + # outside of the first n blocks + destination_indices = range(batch_size) + max_possible_batch_size = self.mamba_cache[0].shape[1] + for destination_index in destination_indices: + if destination_index in self._get_all_occupied_indices() and \ + destination_index not in indices_for_current_run: + # move not running indices outside of the batch + all_other_indices = list( + range(batch_size, max_possible_batch_size)) + first_avail_index = self._first_free_index_in_mamba_cache( + all_other_indices) + self._swap_indices(from_index=destination_index, + to_index=first_avail_index) + + def _move_cache_index_and_mappings(self, from_index: int, to_index: int): + self._copy_mamba_cache(from_index=from_index, to_index=to_index) + self._update_mapping_index(from_index=from_index, to_index=to_index) + + def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int): + self._swap_mamba_cache(from_index=from_index, to_index=to_index) + self._swap_mapping_index(from_index=from_index, to_index=to_index) + + def _swap_mapping_index(self, from_index: int, to_index: int): + for seq_ids2index in self.mamba_cache_indices_mapping.values(): + for seq_id, index in seq_ids2index.items(): + if from_index == index: + seq_ids2index.update({seq_id: to_index}) + elif to_index == index: + seq_ids2index.update({seq_id: from_index}) + + def _update_mapping_index(self, from_index: int, to_index: int): + for seq_ids2index in self.mamba_cache_indices_mapping.values(): + for seq_id, index in seq_ids2index.items(): + if from_index == index: + seq_ids2index.update({seq_id: to_index}) + return + + def _release_finished_requests(self, + finished_seq_groups_req_ids: List[str]): + for req_id in finished_seq_groups_req_ids: + if req_id in self.mamba_cache_indices_mapping: + self.mamba_cache_indices_mapping.pop(req_id) + + def _first_free_index_in_mamba_cache( + self, indices_range: Optional[List[int]] = None) -> int: + assert self.mamba_cache is not None + if indices_range is None: + max_possible_batch_size = self.mamba_cache[0].shape[1] + indices_range = list(range(max_possible_batch_size)) + all_occupied_indices = self._get_all_occupied_indices() + for i in indices_range: + if i not in all_occupied_indices: + return i + raise Exception("Couldn't find a free spot in the mamba cache! This" + "should never happen") diff --git a/vllm/model_executor/models/medusa.py b/vllm/model_executor/models/medusa.py new file mode 100644 index 00000000..619a5cd0 --- /dev/null +++ b/vllm/model_executor/models/medusa.py @@ -0,0 +1,184 @@ +from typing import Iterable, List, Optional, Tuple + +import torch +import torch.nn as nn + +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.transformers_utils.configs.medusa import MedusaConfig + + +class ResidualBlock(nn.Module): + + def __init__(self, hidden_size: int, num_layers: int) -> None: + super().__init__() + + self.layers = nn.ModuleList([ + nn.Linear(hidden_size, hidden_size, bias=False) + for _ in range(num_layers) + ]) + self.act = nn.SiLU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + for layer in self.layers: + x = x + self.act(layer(x)) + return x + + +class Medusa(nn.Module): + """This class implements the Medusa draft model from the paper: https://arxiv.org/abs/2401.10774 + Reference implementation: https://github.com/FasterDecoding/Medusa + + Differences from reference implementation: + 1. Currently this only supports generating proposals from top-1 tokens. + 2. We have an optional token_map which reduces draft vocab to most + frequently used tokens to give some additional speed-up by reducing + sampling overhead. This is disabled unless the checkpoint file has + explicit token_map tensor and config has an optional attribute + truncated_vocab_size < vocab_size. To use this technique, one has to find + the top-k most frequent tokens in target dataset and add that as a tensor + in the draft checkpoint (using key token_map). Also, the draft config + needs to have truncated_vocab_size (=k) as an attribute.""" + + def __init__(self, config: MedusaConfig, **_) -> None: + super().__init__() + self.config = config + self.blocks = nn.ModuleList([ + ResidualBlock(hidden_size=self.config.hidden_size, + num_layers=self.config.num_hidden_layers) + for _ in range(self.config.num_heads) + ]) + self.orig_vocab_size = config.vocab_size + self.truncated_vocab_size = config.truncated_vocab_size + self.unpadded_vocab_size = self.truncated_vocab_size + + self.lm_heads = nn.ModuleList([ + ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=self.truncated_vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + ) for _ in range(self.config.num_heads) + ]) + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + self.truncated_vocab_size, + logit_scale) + + # Token map is a idx to token mapping to reduce the vocab size for + # the draft model. Using smaller vocab size for draft, containing + # only most frequent tokens reduces the speculation overhead. This + # doesn't affect the acceptance rate much and thus gives more speed + # -up. By default, this is disabled and is only used if the EAGLE + # checkpoint file has token_map tensor. + self.token_map = None + + def forward(self, hidden_states: torch.Tensor) -> List[torch.Tensor]: + return [block(hidden_states) for block in self.blocks] + + def compute_logits( + self, hidden_states: List[torch.Tensor], + sampling_metadata: SamplingMetadata) -> List[torch.Tensor]: + logits_lst: List[torch.Tensor] = [] + + for hs, lm_head in zip(hidden_states, self.lm_heads): + _logits = self.logits_processor(lm_head, hs, sampling_metadata) + + if _logits is None: + # _logits should only be None on rank > 0, in which case + # it should remain true for every lm_head + assert len(logits_lst) == 0 + continue + + if self.token_map is None: + logits_lst.append(_logits) + else: + logits_lst.append(-torch.inf * torch.ones( + size=(*_logits.shape[:-1], self.orig_vocab_size), + device=_logits.device, + dtype=_logits.dtype)) + + logits_lst[-1][..., self.token_map] = _logits + + return logits_lst + + def sample( + self, + logits: List[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> List[SamplerOutput]: + logits = torch.stack(logits, dim=0).float() + logprobs = torch.log_softmax(logits, dim=-1) + token_ids = logits.argmax(-1) # support only top-1 for now + probs = torch.softmax(logits, dim=-1) + + token_id_list = [] + token_prob_list = [] + token_logprob_list = [] + + for idx, seq_group in enumerate(sampling_metadata.seq_groups): + token_id_list.append(token_ids[:, seq_group.sample_indices]) + token_prob_list.append(probs[:, seq_group.sample_indices]) + token_logprob_list.append(logprobs[:, seq_group.sample_indices]) + + outputs: List[Optional[SamplerOutput]] = [] + for idx in range(len(sampling_metadata.seq_groups)): + outputs.append( + SamplerOutput( + outputs=None, + sampled_token_probs=token_prob_list[idx].squeeze(1), + logprobs=token_logprob_list[idx].squeeze(1), + sampled_token_ids=token_id_list[idx].squeeze(1), + )) + + return outputs + + def generate_proposals( + self, + previous_hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> List[SamplerOutput]: + return self.sample( + logits=self.compute_logits( + hidden_states=self.forward(previous_hidden_states), + sampling_metadata=sampling_metadata, + ), + sampling_metadata=sampling_metadata, + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters()) + + weights_map = {} + + for name, loaded_weight in weights: + name = name.replace("medusa_heads.", "") + + if name == "token_map": + if self.truncated_vocab_size < self.orig_vocab_size: + self.token_map = nn.Parameter(loaded_weight, + requires_grad=False) + elif name in params_dict: + weights_map[name] = loaded_weight + + for name, loaded_weight in weights_map.items(): + if "lm_head" in name and self.token_map is not None and\ + loaded_weight.shape[0] > self.token_map.shape[0]: + + loaded_weight = loaded_weight[self.token_map] + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + if self.token_map is not None: + self.token_map.to(device=self.lm_heads[0].weight.device) + + assert (self.truncated_vocab_size + == self.orig_vocab_size) or (self.token_map is not None) diff --git a/vllm/model_executor/models/minicpm.py b/vllm/model_executor/models/minicpm.py new file mode 100644 index 00000000..41c28771 --- /dev/null +++ b/vllm/model_executor/models/minicpm.py @@ -0,0 +1,599 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only MiniCPM model compatible with HuggingFace weights.""" +import math +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import fused_moe +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.model_executor.utils import set_weight_attrs +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class MiniCPMMoE(nn.Module): + """A tensor-parallel MoE implementation that shards each expert + across all ranks. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + tp_size: Optional[int] = None, + ): + super().__init__() + self.tp_size = tp_size or get_tensor_model_parallel_world_size() + self.num_total_experts = num_experts + self.top_k = top_k + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size // self.tp_size + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + + self.gate = ReplicatedLinear(self.hidden_size, + self.num_total_experts, + bias=False, + params_dtype=self.params_dtype, + quant_config=None) + + self.ws = nn.Parameter( + torch.empty(self.num_total_experts, + 2 * self.intermediate_size, + self.hidden_size, + device="cuda", + dtype=self.params_dtype)) + self.w2s = nn.Parameter( + torch.empty(self.num_total_experts, + self.hidden_size, + self.intermediate_size, + device="cuda", + dtype=self.params_dtype)) + + set_weight_attrs(self.ws, { + "weight_loader": self.weight_loader, + }) + set_weight_attrs(self.w2s, { + "weight_loader": self.weight_loader, + }) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, + weight_name: str, expert_id: int): + tp_rank = get_tensor_model_parallel_rank() + param_data = param.data + shard_size = self.intermediate_size + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + if weight_name.endswith("w1.weight"): + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :] + if weight_name.endswith("w3.weight"): + param_data[expert_id, + shard_size:2 * shard_size, :] = loaded_weight[shard, :] + if weight_name.endswith("w2.weight"): + param_data[expert_id, :, :] = loaded_weight[:, shard] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_size = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = fused_moe(hidden_states, + self.ws, + self.w2s, + router_logits, + self.top_k, + renormalize=True, + inplace=True) + + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states.view(num_tokens, hidden_size) + + +class MiniCPMMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class MiniCPMAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + # set rope as fp32 instead of bf16 + self.rotary_emb.cos_sin_cache = self.rotary_emb._compute_cos_sin_cache( + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + orig_dtype = q.dtype + q, k = q.float(), k.float() + q, k = self.rotary_emb(positions, q, k) + q, k = q.to(orig_dtype), k.to(orig_dtype) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class MiniCPMDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.cache_config = cache_config + self.quant_config = quant_config + self.hidden_size = config.hidden_size + self.rope_theta = getattr(config, "rope_theta", 10000) + self.rope_scaling = getattr(config, "rope_scaling", None) + self.max_position_embeddings = getattr(config, + "max_position_embeddings", 8192) + self._init_attn_block() + self._init_ffn_block() + + def _init_attn_block(self): + self.input_layernorm = RMSNorm(self.config.hidden_size, + eps=self.config.rms_norm_eps) + self.self_attn = MiniCPMAttention( + hidden_size=self.hidden_size, + num_heads=self.config.num_attention_heads, + num_kv_heads=self.config.num_key_value_heads, + rope_theta=self.rope_theta, + rope_scaling=self.rope_scaling, + max_position_embeddings=self.max_position_embeddings, + cache_config=self.cache_config, + quant_config=self.quant_config, + ) + + def _init_ffn_block(self): + self.post_attention_layernorm = RMSNorm(self.config.hidden_size, + eps=self.config.rms_norm_eps) + self.num_experts = getattr(self.config, "num_experts", 0) + if self.num_experts == 0: + self.mlp = MiniCPMMLP( + hidden_size=self.hidden_size, + intermediate_size=self.config.intermediate_size, + hidden_act=self.config.hidden_act, + quant_config=self.quant_config, + ) + else: + self.mlp = MiniCPMMoE( + num_experts=self.config.num_experts, + top_k=self.config.num_experts_per_tok, + hidden_size=self.config.hidden_size, + intermediate_size=self.config.intermediate_size) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states * \ + (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers)) + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states * \ + (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers)) + + return hidden_states, None + + +class MiniCPMModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.cache_config = cache_config + self.quant_config = quant_config + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + self._init_layers(prefix, config, cache_config, quant_config) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], self.config.hidden_size)) + + def _init_layers( + self, + prefix: str, + config: PretrainedConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig], + ): + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: MiniCPMDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + embedding = self.embed_tokens(input_ids) + return embedding * self.config.scale_emb + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + self.cache_config = cache_config + self.quant_config = quant_config + + self.num_experts = getattr(self.config, "num_experts", 0) + self._init_model() + unpadded_vocab_size = config.vocab_size + if lora_config: + unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) + self.scale_width = self.config.hidden_size / self.config.dim_model_base + + self.logits_processor = LogitsProcessor(unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def _init_model(self): + self.model = MiniCPMModel(config=self.config, + cache_config=self.cache_config, + quant_config=self.quant_config, + lora_config=self.lora_config) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + hidden_states = hidden_states / self.scale_width + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + expert_params_mapping = [ + # (param_name, weight_name, expert_id) + ("ws" if weight_name in ["w1", "w3"] else "w2s", + f"experts.{expert_id}.{weight_name}.weight", expert_id) + for expert_id in range(self.num_experts) + for weight_name in ["w1", "w2", "w3"] + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + # With tie_word_embeddings, we can skip lm_head.weight + # The weight might appear unnecessarily in the files if the model is + # processed with quantization, LoRA, fine-tuning, etc. + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for param_name, weight_name, expert_id in expert_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + weight_name, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/minicpm3.py b/vllm/model_executor/models/minicpm3.py new file mode 100644 index 00000000..e4aaf951 --- /dev/null +++ b/vllm/model_executor/models/minicpm3.py @@ -0,0 +1,340 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2024 The ModelBest team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only MiniCPM3 model compatible with HuggingFace weights.""" +from typing import Any, Dict, Optional, Union, List, Tuple +import math + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer, + MiniCPMForCausalLM, + MiniCPMModel) +from vllm.sequence import IntermediateTensors +from vllm.distributed import get_pp_group + +from .utils import make_layers + + +class MiniCPM3Attention(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: int, + kv_lora_rank: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.num_heads = num_heads + + tp_size = get_tensor_model_parallel_world_size() + assert self.num_heads % tp_size == 0 + self.num_local_heads = num_heads // tp_size + + self.scaling = self.qk_head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.q_a_proj = ReplicatedLinear(self.hidden_size, + self.q_lora_rank, + bias=False, + quant_config=quant_config) + self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear(q_lora_rank, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config) + + self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size, + self.kv_lora_rank + + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config) + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, + eps=config.rms_norm_eps) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config) + # O projection. + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config) + + self.rotary_emb = get_rope( + self.qk_rope_head_dim, + rotary_dim=self.qk_rope_head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_local_heads, + self.qk_head_dim, + self.scaling, + num_kv_heads=self.num_local_heads, + cache_config=cache_config, + quant_config=quant_config) + self.merge_q_kv_a = False + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + long_prompt_offset: torch.Tensor, + long_short_cos_sin_cache: torch.Tensor, + ) -> torch.Tensor: + import ixformer.inference.functions as ixf + if hidden_states.dtype == torch.float16 or hidden_states.dtype == torch.bfloat16: + if not self.merge_q_kv_a: + self.qkv_weight = torch.cat([self.q_a_proj.weight, self.kv_a_proj_with_mqa.weight], dim=0) + del self.q_a_proj + del self.kv_a_proj_with_mqa + self.merge_q_kv_a = True + q_latent_cache = ixf.linear(hidden_states, self.qkv_weight) + q, latent_cache = q_latent_cache.split([self.q_lora_rank, + self.kv_lora_rank + self.qk_rope_head_dim], + dim=-1) + else: + q, _ = self.q_a_proj(hidden_states) + latent_cache, _ = self.kv_a_proj_with_mqa(hidden_states) + + q = self.q_a_layernorm(q) + q, _ = self.q_b_proj(q) + q = q.view(-1, self.num_local_heads, self.qk_head_dim) + _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], + dim=-1) + + kv_a, _ = latent_cache.split( + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + latent_cache = latent_cache.unsqueeze(1) + kv_a = self.kv_a_layernorm(kv_a) + kv, _ = self.kv_b_proj(kv_a) + kv = kv.view(-1, self.num_local_heads, + self.qk_nope_head_dim + self.v_head_dim) + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + + q_pe, k_pe = ixf.minicpm3_fused_rope( + positions, + long_prompt_offset, + long_short_cos_sin_cache, + q_pe, latent_cache[:, :, self.kv_lora_rank:], + out_query = q[..., self.qk_nope_head_dim:] + ) + + q = q.view(-1, self.num_local_heads * self.qk_head_dim) + + k, v = ixf.minicpm3_fused_copy_kv(k_nope, k_pe, v) + + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + attn_output = attn_output.view(-1, self.num_local_heads, self.qk_head_dim) + new_attn_output = attn_output.new_empty([attn_output.shape[0], attn_output.shape[1], self.v_head_dim]) + new_attn_output[:, :, :] = attn_output[:, :, :self.v_head_dim] + attn_output = new_attn_output.view(-1, self.num_local_heads * self.v_head_dim) + + output, _ = self.o_proj(attn_output) + return output + + +class MiniCPM3DecoderLayer(MiniCPMDecoderLayer): + def __init__(self, config: PretrainedConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None) -> None: + super().__init__(config, cache_config, quant_config) + self.hidden_scale = config.scale_depth / math.sqrt(config.num_hidden_layers) + + def _init_attn_block(self): + self.input_layernorm = RMSNorm(self.config.hidden_size, + eps=self.config.rms_norm_eps) + self.self_attn = MiniCPM3Attention( + config=self.config, + hidden_size=self.hidden_size, + num_heads=self.config.num_attention_heads, + qk_nope_head_dim=self.config.qk_nope_head_dim, + qk_rope_head_dim=self.config.qk_rope_head_dim, + v_head_dim=self.config.v_head_dim, + q_lora_rank=self.config.q_lora_rank, + kv_lora_rank=self.config.kv_lora_rank, + rope_theta=self.rope_theta, + rope_scaling=self.rope_scaling, + max_position_embeddings=self.max_position_embeddings, + cache_config=self.cache_config, + quant_config=self.quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + long_prompt_offset: Optional[torch.Tensor], + long_short_cos_sin_cache: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(residual, hidden_states, self.hidden_scale) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + long_prompt_offset=long_prompt_offset, + long_short_cos_sin_cache=long_short_cos_sin_cache, + ) + + hidden_states, residual = self.post_attention_layernorm(residual, hidden_states, self.hidden_scale) + + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + +class MiniCPM3Model(MiniCPMModel): + + def _init_layers( + self, + prefix: str, + config: PretrainedConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig], + ): + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: MiniCPM3DecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + k = self.layers[self.start_layer].self_attn.rotary_emb.original_max_position_embeddings + long_prompt_offset = (torch.any(positions > k).float() * + torch.full_like(positions, k)).long() + long_short_cos_sin_cache = ( + self.layers[self.start_layer].self_attn.rotary_emb.long_short_cos_sin_cache.to(input_ids.device)) + + + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + long_prompt_offset=long_prompt_offset, + long_short_cos_sin_cache=long_short_cos_sin_cache, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, residual = self.norm(residual, hidden_states, self.layers[self.start_layer].hidden_scale) + + return hidden_states + + +class MiniCPM3ForCausalLM(MiniCPMForCausalLM): + packed_modules_mapping = { + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "kv_a_proj_with_mqa", + "q_a_proj", + "q_b_proj", + "kv_b_proj", + "o_proj", + "gate_up_proj", + "down_proj", + "embed_tokens", + "lm_head", + ] + + # `embedding_modules` and `embedding_padding_modules` + # are inherited from MiniCPMForCausalLM + + def _init_model(self): + self.model = MiniCPM3Model(config=self.config, + cache_config=self.cache_config, + quant_config=self.quant_config, + lora_config=self.lora_config) diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py new file mode 100644 index 00000000..0236fc44 --- /dev/null +++ b/vllm/model_executor/models/minicpmv.py @@ -0,0 +1,1044 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only MiniCPM-V model compatible with HuggingFace weights.""" +import math +import regex as re +from functools import partial +from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional, + Tuple, TypedDict, Union) + +import torch +import torch.types +from PIL import Image +from torch import nn +from transformers import PretrainedConfig +from typing_extensions import NotRequired + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2, + get_2d_sincos_pos_embed) +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.model_loader.utils import set_default_torch_dtype +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.llama import LlamaModel +from vllm.model_executor.models.minicpm import MiniCPMModel +from vllm.model_executor.models.module_mapping import MultiModelKeys +from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.model_executor.models.utils import LLMWrapper +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs +from vllm.multimodal.image import cached_get_image_processor +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import IntermediateTensors, SequenceData + +from .idefics2_vision_model import Idefics2VisionTransformer +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP +from .utils import is_pp_missing_parameter + +_KEYS_TO_MODIFY_MAPPING = { + "llm.lm_head": "lm_head", +} + +RawImageType = Union[Image.Image, torch.Tensor] + + +class MiniCPMVRawImageInput(TypedDict): + """Input mapper input with auxiliary data for computing image bounds.""" + image: RawImageType + + # Image bounds token ids in 0-dim scaler tensor. + im_start_id: torch.Tensor + im_end_id: torch.Tensor + slice_start_id: NotRequired[torch.Tensor] + slice_end_id: NotRequired[torch.Tensor] + + +class MiniCPMVImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: List[torch.Tensor] + """ + Shape: `(batch_size * num_images, num_channels, height, width)` + + Note that the image size may vary, so we pass it as a list + instead of a batched tensor. + """ + + image_bounds: torch.Tensor + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(start, stop)` format. + """ + + tgt_sizes: torch.Tensor + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(height, width)` format. + """ + + +class MiniCPMVImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """ + Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + instead of a batched tensor. + """ + + image_bounds: torch.Tensor + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(start, stop)` format. + """ + + +MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs, + MiniCPMVImageEmbeddingInputs] + +DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6) + + +class Resampler2_5(BaseResampler): + + def __init__( + self, + num_queries: int, + embed_dim: int, + num_heads: int, + kv_dim: Optional[int] = None, + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN, + max_size: Tuple[int, int] = (70, 70), + ) -> None: + super().__init__(num_queries, embed_dim, num_heads, kv_dim, norm_layer) + + self.max_size = max_size + self._set_2d_pos_cache(self.max_size) + + self.apply(self._init_weights) + + def _set_2d_pos_cache(self, + max_size: Tuple[int, int], + device: torch.types.Device = "cpu") -> None: + pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim, + max_size, + version=(2, 5)) + pos_embed = torch.from_numpy(pos_embed_arr).float().to(device) + self.register_buffer("pos_embed", pos_embed, persistent=False) + + def _adjust_pos_cache(self, tgt_sizes: torch.Tensor, + device: torch.types.Device) -> None: + max_h = tgt_sizes[:, 0].max().item() + max_w = tgt_sizes[:, 1].max().item() + assert isinstance(max_h, int) and isinstance(max_w, int) + + if max_h > self.max_size[0] or max_w > self.max_size[1]: + self.max_size = ( + max(max_h, self.max_size[0]), + max(max_w, self.max_size[1]), + ) + self._set_2d_pos_cache(self.max_size, device) + + def forward(self, x: torch.Tensor, + tgt_sizes: torch.Tensor) -> torch.Tensor: + assert x.shape[0] == tgt_sizes.shape[0] + bs = x.shape[0] + + device = x.device + dtype = x.dtype + + patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1] + + self._adjust_pos_cache(tgt_sizes, device=device) + + max_patch_len = patch_len.max().item() + assert isinstance(max_patch_len, int) + + key_padding_mask = torch.zeros((bs, max_patch_len), + dtype=torch.bool, + device=device) + + pos_embed = [] + for i in range(bs): + tgt_h, tgt_w = tgt_sizes[i].tolist() + pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape( + (tgt_h * tgt_w, -1)).to(dtype)) # patches * D + key_padding_mask[i, patch_len[i]:] = True + pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed, + batch_first=True, + padding_value=0.0).permute( + 1, 0, + 2) # BLD => L * B * D + x, _ = self.kv_proj(x) # B * L * D + x = self.ln_kv(x).permute(1, 0, 2) # L * B * D + + q = self.ln_q(self.query) # Q * D + + out = self.attn( + self._repeat(q, bs), # Q * B * D + x + pos_embed, # L * B * D + L * B * D + x, + key_padding_mask=key_padding_mask, + )[0] + # out: Q * B * D + x = out.permute(1, 0, 2) # B * Q * D + + x = self.ln_post(x) + x = x @ self.proj + return x + + +def _build_image_input(ctx: InputContext, + image: RawImageType) -> MiniCPMVRawImageInput: + tokenizer = cached_get_tokenizer( + ctx.model_config.tokenizer, + trust_remote_code=ctx.model_config.trust_remote_code) + if hasattr(tokenizer, "slice_start_id"): + return MiniCPMVRawImageInput( + image=image, + im_start_id=torch.tensor(tokenizer.im_start_id), + im_end_id=torch.tensor(tokenizer.im_end_id), + slice_start_id=torch.tensor(tokenizer.slice_start_id), + slice_end_id=torch.tensor(tokenizer.slice_end_id)) + else: + return MiniCPMVRawImageInput( + image=image, + im_start_id=torch.tensor(tokenizer.im_start_id), + im_end_id=torch.tensor(tokenizer.im_end_id)) + + +def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]: + version_float = getattr(config, "version", None) + + # The old configs do not include version number + # TODO: Remove this after the HF repos are updated + if version_float is None: + if config.hidden_size == 2304 and config.query_num == 64: + return (2, 0) + return (2, 5) + + version_str = str(version_float) + return tuple(int(x) for x in version_str.split(".")) + + +def get_max_minicpmv_image_tokens(ctx: InputContext): + hf_config = ctx.get_hf_config() + return getattr(hf_config, "query_num", 64) + + +def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int): + return SequenceData.from_token_counts((0, seq_len)) + + +def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig, + num_images: int): + width = height = hf_config.image_size + image = _build_image_input(ctx, + image=Image.new("RGB", (width, height), + color=0)) + return {"image": [image] if num_images == 1 else [image] * num_images} + + +def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config() + num_images = mm_counts["image"] + + seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images) + mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images) + + return seq_data, mm_data + + +def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + model_config = ctx.model_config + version = get_version_by_config(model_config.hf_config) + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + image_processor = cached_get_image_processor(model_config.tokenizer) + + def get_placeholder(image_size: Tuple[int, int], num_image: int): + if version == (2, 0) or version == (2, 5): + return image_processor. \ + get_slice_image_placeholder(image_size) + return image_processor. \ + get_slice_image_placeholder(image_size, num_image) + + prompt = llm_inputs.get("prompt") + token_ids = llm_inputs.get("prompt_token_ids") + if prompt is None: + prompt = tokenizer.decode(token_ids) + + pattern = "(./)" + images = multi_modal_data["image"] + image_tags = re.findall(pattern, prompt) + if len(image_tags) == 0: + new_token_ids = token_ids + new_prompt = prompt + else: + if isinstance(images, dict): + image_size_list = images.get("image_size_list") + images = [images.get("image_embeds")] + else: + if isinstance(images, Image.Image): + images = [images] + image_size_list = [image.size for image in images] + + text_chunks = prompt.split(pattern) + new_prompt_chunks: List[str] = [] + for i in range(len(image_size_list)): + new_prompt_chunks += [ + text_chunks[i], + get_placeholder(image_size_list[i], i) + ] + new_prompt_chunks.append(text_chunks[-1]) + new_prompt = "".join(new_prompt_chunks) + new_token_ids = tokenizer.encode(new_prompt) + + multi_modal_data["image"] = [ + _build_image_input(ctx, image) for image in images + ] + + llm_inputs = LLMInputs( + prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data, + ) + return llm_inputs + + +def input_mapper_for_minicpmv(ctx: InputContext, data: object): + model_config = ctx.model_config + + image_processor = cached_get_image_processor( + model_config.model, trust_remote_code=model_config.trust_remote_code) + if image_processor is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + + if not isinstance(data, list): + raise ValueError( + "Image input must be list of MiniCPMVImageInput, got (%s)", data) + + if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor): + batch_data = { + "image_embeds": data[0]['image'], + } + else: + batch_data = image_processor \ + .preprocess([img["image"] for img in data], return_tensors="pt") \ + .data + + if len(data) > 0: + batch_data["im_start_id"] = data[0]["im_start_id"] + batch_data["im_end_id"] = data[0]["im_end_id"] + if "slice_start_id" in data[0]: + batch_data["slice_start_id"] = data[0]["slice_start_id"] + batch_data["slice_end_id"] = data[0]["slice_end_id"] + + return MultiModalInputs(batch_data) + + +class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): + """ + The abstract class of MiniCPMV can only be inherited, but cannot be + instantiated. + """ + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + # All MiniCPM-V models disable `tie_word_embeddings` but + # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot + # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model + # and config class + self.config = config + self.multimodal_config = multimodal_config + + self.version = get_version_by_config(self.config) + self.llm = self.init_llm(config, cache_config, quant_config) + self.vpm = self.init_vision_module() + param_dtype = torch.get_default_dtype() + self.vpm.to(dtype=param_dtype) + self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else + self.vpm.embeddings.embed_dim) + self.embed_dim = self.config.hidden_size + self.resampler = self.init_resampler(self.embed_dim, self.vision_dim) + self.resampler.to(device="cuda", dtype=param_dtype) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + + self.make_empty_intermediate_tensors = ( + self.llm.make_empty_intermediate_tensors) + + def get_embedding( + self, + input_ids: torch.Tensor, + image_inputs: Optional[MiniCPMVImageInputs], + ) -> Tuple[torch.Tensor, torch.Tensor]: + vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids) + if hasattr(self.config, "scale_emb"): + vlm_embedding *= self.config.scale_emb + + if image_inputs is None: # No image + vision_hidden_states = torch.tensor([], device=input_ids.device) + else: + if image_inputs["type"] == "image_embeds": + vision_hidden_states = (image_inputs["data"].type( + vlm_embedding.dtype).to(vlm_embedding.device)) + else: + vision_hidden_states = self.get_vision_hidden_states( + image_inputs) + + # See NOTE in _parse_and_validate_inputs + image_bounds = image_inputs["image_bounds"] + if len(image_bounds) > 0: + image_indices = torch.stack([ + torch.arange(start, end, dtype=torch.long) + for start, end in image_bounds.tolist() + ]).to(vlm_embedding.device) + vlm_embedding.scatter_( + 0, + image_indices.view(-1, 1).repeat(1, + vlm_embedding.shape[-1]), + vision_hidden_states.view(-1, + vision_hidden_states.shape[-1]), + ) + + return vlm_embedding, vision_hidden_states + + def _get_image_bounds( + self, + input_ids: torch.Tensor, + im_start_id: torch.Tensor, + im_end_id: torch.Tensor, + slice_start_id: Optional[torch.Tensor] = None, + slice_end_id: Optional[torch.Tensor] = None) -> torch.Tensor: + # All the images in the batch should share the same special image + # bound token ids. + start_cond = input_ids == im_start_id[0] + end_cond = input_ids == im_end_id[0] + if slice_start_id is not None: + start_cond |= (input_ids == slice_start_id[0]) + end_cond |= (input_ids == slice_end_id[0]) + + image_start_tokens, = torch.where(start_cond) + image_start_tokens += 1 + image_end_tokens, = torch.where(end_cond) + valid_image_nums = max(len(image_start_tokens), len(image_end_tokens)) + + if valid_image_nums == 0: + return torch.zeros((0, 2), device=input_ids.device) + + return torch.hstack([ + image_start_tokens[:valid_image_nums].unsqueeze(-1), + image_end_tokens[:valid_image_nums].unsqueeze(-1), + ]) + + def _parse_and_validate_inputs( + self, + input_ids: torch.Tensor, + **kwargs: object, + ) -> Optional[MiniCPMVImageInputs]: + pixel_values = kwargs.pop("pixel_values", []) + tgt_sizes = kwargs.pop("tgt_sizes", []) + im_start_id = kwargs.pop("im_start_id", None) + im_end_id = kwargs.pop("im_end_id", None) + slice_start_id = kwargs.pop("slice_start_id", None) + slice_end_id = kwargs.pop("slice_end_id", None) + image_embeds = kwargs.pop("image_embeds", None) + + if image_embeds is not None: + return MiniCPMVImageEmbeddingInputs( + image_bounds=self._get_image_bounds(input_ids, im_start_id, + im_end_id, slice_start_id, + slice_end_id), + data=image_embeds, + type="image_embeds", + ) + + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + if not isinstance(tgt_sizes, (torch.Tensor, list)): + raise ValueError("Incorrect type of target sizes. " + f"Got type: {type(tgt_sizes)}") + + if len(pixel_values) != len(tgt_sizes): + raise ValueError("Inconsistent batch lengths, found: " + f"{len(pixel_values)} vs. {len(tgt_sizes)}") + + pixel_values_flat: List[torch.Tensor] = [] + tgt_sizes_flat: List[torch.Tensor] = [] + for pixel_b, tgt_b in zip(pixel_values, tgt_sizes): + if len(pixel_b) != len(tgt_b): + raise ValueError("Inconsistent N lengths, found: " + f"{len(pixel_b)} vs {len(tgt_b)}") + + for pixel_n, tgt_n in zip(pixel_b, tgt_b): + pixel_values_flat += pixel_n + tgt_sizes_flat += tgt_n + + # NOTE: Input IDs does not contain image tokens during memory profiling, + # so we allow it to be empty + if len(pixel_values_flat) != len(tgt_sizes_flat): + raise ValueError("Inconsistent flattened lengths, found: " + f"{len(pixel_values_flat)} vs. " + f"{len(tgt_sizes_flat)}") + + if len(pixel_values_flat) == 0: + return None + + if im_start_id is None: + return None + + return MiniCPMVImagePixelInputs( + image_bounds=self._get_image_bounds(input_ids, im_start_id, + im_end_id, slice_start_id, + slice_end_id), + data=pixel_values_flat, + tgt_sizes=torch.stack(tgt_sizes_flat), + type="pixel_values", + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: Any, + ) -> torch.Tensor: + if intermediate_tensors is not None: + vlm_embeddings = None + else: + image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs) + + vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs) + + output = self.llm( + input_ids=None, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=vlm_embeddings, + ) + return output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items(): + if key_to_modify in name: + name = name.replace(key_to_modify, new_key) + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + use_default_weight_loading = False + if self.is_default_weight_loading(name): + use_default_weight_loading = True + else: + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + if is_pp_missing_parameter( + name.replace(weight_name, param_name), self): + continue + param = params_dict[name.replace(weight_name, param_name)] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + use_default_weight_loading = True + if use_default_weight_loading: + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + def get_mm_mapping(self) -> MultiModelKeys: + """ + Get the module prefix in multimodal models + """ + return MultiModelKeys.from_string_field(language_model="llm", + connector="resampler", + tower_model="vpm") + + def init_llm( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> nn.Module: + raise NotImplementedError + + def init_vision_module(self) -> nn.Module: + raise NotImplementedError + + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module: + raise NotImplementedError + + def get_vision_embedding( + self, + pixel_values: List[torch.Tensor], + patch_attn_mask: Optional[torch.Tensor] = None, + tgt_sizes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def get_vision_hidden_states(self, + data: MiniCPMVImageInputs) -> torch.Tensor: + raise NotImplementedError + + def is_default_weight_loading(self, name: str) -> bool: + raise NotImplementedError + + +class MiniCPMV2_0(MiniCPMVBaseModel): + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__(config, multimodal_config, cache_config, quant_config) + assert self.version == (2, 0) + + def init_llm( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> nn.Module: + + return LLMWrapper(MiniCPMModel(config, + cache_config=cache_config, + quant_config=quant_config), + name="model") + + def init_vision_module(self) -> nn.Module: + # TODO :refactor this vision model + try: + import timm + except ImportError: + raise ImportError("Please install timm==0.9.10") from ImportError + with set_default_torch_dtype(torch.float16): + model = timm.create_model( + "vit_so400m_patch14_siglip_384.webli", + pretrained=False, + num_classes=0, + dynamic_img_size=True, + dynamic_img_pad=True, + ) + + if (isinstance(model, timm.models.VisionTransformer) + and model.attn_pool is not None): + model.attn_pool = torch.nn.Identity() + + if self.config.drop_vision_last_layer: + model.blocks = model.blocks[:-1] + + return model + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_tokens(input_ids) + + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module: + with set_default_torch_dtype(torch.float16): + resampler = Resampler2( + embed_dim=embed_dim, + num_heads=embed_dim // 128, + grid_size=int(math.sqrt(self.config.query_num)), + kv_dim=vision_dim, + adaptive=False, + do_post_projection=True, + ) + + return resampler + + def get_vision_embedding( + self, + pixel_values: List[torch.Tensor], + patch_attn_mask: Optional[torch.Tensor] = None, + tgt_sizes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + res = [] + dtype = self.vpm.pos_embed.data.dtype + for pixel_value in pixel_values: + H, W = pixel_value[0].shape[-2:] + tgt_size = ( + math.ceil(H / self.vpm.patch_embed.patch_size[0]), + math.ceil(W / self.vpm.patch_embed.patch_size[0]), + ) + vision_embedding = self.vpm.forward_features( + pixel_value.unsqueeze(0).type(dtype)) + if (hasattr(self.vpm, "num_prefix_tokens") + and self.vpm.num_prefix_tokens > 0): + vision_embedding = vision_embedding[:, self.vpm. + num_prefix_tokens:] + res.append(self.resampler(vision_embedding, tgt_size)) + return torch.vstack(res) + + def get_vision_hidden_states(self, + data: MiniCPMVImageInputs) -> torch.Tensor: + pixel_values = data["data"] + + return self.get_vision_embedding(pixel_values) + + def is_default_weight_loading(self, name: str) -> bool: + return "resampler" in name or "vpm" in name + + +class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + # LoRA specific attributes + supported_lora_modules = [ + # vision encoder + "fc1", + "fc2", + "out_proj", + # language model + "qkv_proj", # same name with vision encoder + "o_proj", + "gate_up_proj", + "down_proj", + # resampler + "kv_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__(config, multimodal_config, cache_config, quant_config) + assert self.version == (2, 5) + + def init_llm( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> nn.Module: + return LLMWrapper(LlamaModel(config, + cache_config=cache_config, + quant_config=quant_config), + name="model") + + def init_vision_module(self) -> nn.Module: + model = Idefics2VisionTransformer(self.config.vision_config) + if self.config.drop_vision_last_layer: + model.encoder.layers = model.encoder.layers[:-1] + return model + + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module: + with set_default_torch_dtype(torch.float16): + resampler = Resampler2_5( + num_queries=self.config.query_num, + embed_dim=embed_dim, + num_heads=embed_dim // 128, + kv_dim=vision_dim, + ) + return resampler + + def get_vision_embedding( + self, + pixel_values: List[torch.Tensor], + patch_attn_mask: Optional[torch.Tensor] = None, + tgt_sizes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + vision_embedding = self.vpm(pixel_values, + patch_attention_mask=patch_attn_mask) + vision_embedding = self.resampler(vision_embedding, tgt_sizes) + return vision_embedding + + def get_vision_hidden_states(self, + data: MiniCPMVImageInputs) -> torch.Tensor: + pixel_values = data["data"] + tgt_sizes = data["tgt_sizes"] + + device = self.vpm.embeddings.position_embedding.weight.device + dtype = self.vpm.embeddings.position_embedding.weight.dtype + all_pixel_values_lst = [ + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values + ] + + max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item() + assert isinstance(max_patches, int) + + all_pixel_values = torch.nn.utils.rnn.pad_sequence( + all_pixel_values_lst, batch_first=True, padding_value=0.0) + B, L, _ = all_pixel_values.shape + all_pixel_values = all_pixel_values.permute(0, 2, + 1).reshape(B, 3, -1, L) + + patch_attn_mask = torch.zeros((B, 1, max_patches), + dtype=torch.bool, + device=device) + for i in range(B): + patch_attn_mask[i, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True + + return self.get_vision_embedding(all_pixel_values.type(dtype), + patch_attn_mask, tgt_sizes) + + def is_default_weight_loading(self, name: str) -> bool: + return "resampler" in name + + +class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + # LoRA specific attributes + supported_lora_modules = [ + # vision encoder + "fc1", + "fc2", + "out_proj", + # language model + "qkv_proj", # same name with vision encoder + "o_proj", + "gate_up_proj", + "down_proj", + # resampler + "kv_proj", + ] + + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__(config, multimodal_config, cache_config, quant_config) + assert self.version == (2, 6) + + def init_llm( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> nn.Module: + + return LLMWrapper(Qwen2Model(config, + cache_config=cache_config, + quant_config=quant_config), + name="model") + + def init_vision_module(self) -> nn.Module: + + model = Idefics2VisionTransformer(self.config.vision_config) + if self.config.drop_vision_last_layer: + model.encoder.layers = model.encoder.layers[:-1] + return model + + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module: + with set_default_torch_dtype(torch.float16): + # The resampler in 2.6 remains consistent with the one in 2.5. + resampler = Resampler2_5( + num_queries=self.config.query_num, + embed_dim=embed_dim, + num_heads=embed_dim // 128, + kv_dim=vision_dim, + ) + return resampler + + def get_vision_embedding( + self, + pixel_values: List[torch.Tensor], + patch_attn_mask: Optional[torch.Tensor] = None, + tgt_sizes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + vision_embedding = self.vpm( + pixel_values, + patch_attention_mask=patch_attn_mask, + tgt_sizes=tgt_sizes, + ) + return vision_embedding + + def get_vision_hidden_states(self, + data: MiniCPMVImageInputs) -> torch.Tensor: + pixel_values = data["data"] + tgt_sizes = data["tgt_sizes"] + + device = self.vpm.embeddings.position_embedding.weight.device + dtype = self.vpm.embeddings.position_embedding.weight.dtype + all_pixel_values_lst = [ + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values + ] + + max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item() + assert isinstance(max_patches, int) + + all_pixel_values = torch.nn.utils.rnn.pad_sequence( + all_pixel_values_lst, batch_first=True, padding_value=0.0) + B, L, _ = all_pixel_values.shape + all_pixel_values = all_pixel_values.permute(0, 2, + 1).reshape(B, 3, -1, L) + + patch_attn_mask = torch.zeros((B, 1, max_patches), + dtype=torch.bool, + device=device) + for i in range(B): + patch_attn_mask[i, 0, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True + vision_embedding = self.vpm( + all_pixel_values.type(dtype), + patch_attention_mask=patch_attn_mask, + tgt_sizes=tgt_sizes, + ) + + return self.resampler(vision_embedding, tgt_sizes) + + def is_default_weight_loading(self, name: str) -> bool: + return "resampler" in name + + +_SUPPORT_VERSION = { + (2, 0): MiniCPMV2_0, + (2, 5): MiniCPMV2_5, + (2, 6): MiniCPMV2_6 +} + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_minicpmv) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv) +@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv) +class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA): + """ + Different versions of MiniCPMV use different visual encoders and LLMs, + which is not conducive to the current integration logic of LoRA and + bitsandbytes in vLLM. Therefore, it is necessary to separate them. + """ + # Ensure that the LoRA support check passes when the class is not + # initialized, but set all these attributes to empty. + packed_modules_mapping = {} + supported_lora_modules = [] + embedding_modules = {} + embedding_padding_modules = [] + + def __new__(cls, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None): + if not hasattr(config, "version"): + if config.hidden_size == 2304 and config.query_num == 64: + version = (2, 0) + else: + version = (2, 5) + else: + version = str(config.version).split(".") + version = tuple([int(x) for x in version]) + # Dispatch class based on version + instance_class = _SUPPORT_VERSION.get(version) + if instance_class is None: + raise ValueError( + "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6") + return instance_class(config, multimodal_config, cache_config, + quant_config) diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py new file mode 100644 index 00000000..ff5d03f3 --- /dev/null +++ b/vllm/model_executor/models/mixtral.py @@ -0,0 +1,493 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Mixtral model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import MixtralConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import CompressedTensorsConfig, CompressionFormat + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class MixtralMoE(nn.Module): + """A tensor-parallel MoE implementation for Mixtral that shards each expert + across all ranks. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__(self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + prefix: str = ""): + super().__init__() + self.hidden_size = hidden_size + + # Gate always runs at half / full precision for now. + + self.gate = ReplicatedLinear(hidden_size, + num_experts, + bias=False, + params_dtype=params_dtype, + quant_config=None, + prefix=f"{prefix}.gate") + + self.experts = FusedMoE(num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + params_dtype=params_dtype, + reduce_results=True, + renormalize=True, + quant_config=quant_config, + tp_size=tp_size, + prefix=f"{prefix}.experts") + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states, router_logits) + return final_hidden_states.view(orig_shape) + + +class MixtralAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=int(self.rope_theta), + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class MixtralDecoderLayer(nn.Module): + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 10000) + self.self_attn = MixtralAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn") + self.block_sparse_moe = MixtralMoE( + num_experts=config.num_local_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.block_sparse_moe") + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + if (isinstance(quant_config, CompressedTensorsConfig) + and quant_config.quant_format == CompressionFormat.int_quantized.value + and quant_config.target_scheme_map['Linear']['input_activations'].num_bits == 8 + and quant_config.target_scheme_map['Linear']['weights'].num_bits == 8): + self.use_int_w8a8 = True + else: + self.use_int_w8a8 = False + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + from ixformer.contrib.vllm.layers import mixtral_decoder_layer_forward + if self.use_int_w8a8: + return mixtral_decoder_layer_forward( + self, + positions, + hidden_states, + kv_cache, + attn_metadata, + residual + ) + + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.block_sparse_moe(hidden_states) + return hidden_states, residual + + +class MixtralModel(nn.Module): + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: MixtralDecoderLayer( + config, cache_config, quant_config=quant_config, prefix=prefix + ), + prefix=f"{prefix}.layers") + + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + fall_back_to_pt_during_load = False + + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", "o_proj", "embed_tokens", "lm_head", "w1", "w2", "w3", + "gate" + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = MixtralModel(config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model") + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="w1", + ckpt_down_proj_name="w2", + ckpt_up_proj_name="w3", + num_experts=self.config.num_local_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/mixtral_quant.py b/vllm/model_executor/models/mixtral_quant.py new file mode 100644 index 00000000..63e2c60a --- /dev/null +++ b/vllm/model_executor/models/mixtral_quant.py @@ -0,0 +1,441 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Mixtral model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from transformers import MixtralConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class MixtralMLP(nn.Module): + + def __init__( + self, + num_experts: int, + hidden_size: int, + intermediate_size: int, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.num_experts = num_experts + self.ffn_dim = intermediate_size + self.hidden_dim = hidden_size + + self.w1 = ReplicatedLinear(self.hidden_dim, + self.ffn_dim, + bias=False, + quant_config=quant_config) + self.w2 = ReplicatedLinear(self.ffn_dim, + self.hidden_dim, + bias=False, + quant_config=quant_config) + self.w3 = ReplicatedLinear(self.hidden_dim, + self.ffn_dim, + bias=False, + quant_config=quant_config) + + # TODO: Use vllm's SiluAndMul + self.act_fn = nn.SiLU() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + w1_out, _ = self.w1(hidden_states) + w1_out = self.act_fn(w1_out) + w3_out, _ = self.w3(hidden_states) + current_hidden_states = w1_out * w3_out + current_hidden_states, _ = self.w2(current_hidden_states) + return current_hidden_states + + +class MixtralMoE(nn.Module): + + def __init__( + self, + config: MixtralConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + self.num_total_experts = config.num_local_experts + self.top_k = config.num_experts_per_tok + if self.tp_size > self.num_total_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {self.num_total_experts}.") + # Split experts equally between ranks + self.expert_indicies = np.array_split(range( + self.num_total_experts), self.tp_size)[self.rank].tolist() + if not self.expert_indicies: + raise ValueError( + f"Rank {self.rank} has no experts assigned to it.") + + self.experts = nn.ModuleList([ + MixtralMLP(self.num_total_experts, + config.hidden_size, + config.intermediate_size, + quant_config=quant_config) + if idx in self.expert_indicies else None + for idx in range(self.num_total_experts) + ]) + self.gate = ReplicatedLinear(config.hidden_size, + self.num_total_experts, + bias=False, + quant_config=None) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, + self.top_k, + dim=-1) + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + + final_hidden_states = None + for expert_idx in self.expert_indicies: + expert_layer = self.experts[expert_idx] + expert_mask = (selected_experts == expert_idx) + expert_weights = (routing_weights * expert_mask).sum(dim=-1, + keepdim=True) + + current_hidden_states = expert_layer(hidden_states).mul_( + expert_weights) + if final_hidden_states is None: + final_hidden_states = current_hidden_states + else: + final_hidden_states.add_(current_hidden_states) + + return tensor_model_parallel_all_reduce(final_hidden_states).view( + num_tokens, hidden_dim) + + +class MixtralAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + quant_config: Optional[QuantizationConfig] = None, + cache_config: Optional[CacheConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=int(self.rope_theta), + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class MixtralDecoderLayer(nn.Module): + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 10000) + self.self_attn = MixtralAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config) + self.block_sparse_moe = MixtralMoE(config=config, + quant_config=quant_config) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.block_sparse_moe(hidden_states) + return hidden_states, residual + + +class MixtralModel(nn.Module): + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: MixtralDecoderLayer( + config, cache_config, quant_config=quant_config), + prefix=f"{prefix}.layers") + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class MixtralForCausalLM(nn.Module, SupportsPP): + fall_back_to_pt_during_load = False + + def __init__( + self, + config: MixtralConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = MixtralModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip experts that are not assigned to this worker. + if ("block_sparse_moe.experts." in name + and name not in params_dict): + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/mllama.py b/vllm/model_executor/models/mllama.py new file mode 100644 index 00000000..034a0186 --- /dev/null +++ b/vllm/model_executor/models/mllama.py @@ -0,0 +1,1151 @@ +# coding=utf-8 +# Copyright 2024 the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Mllama model.""" +import math +from array import array +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers.models.mllama.configuration_mllama as config_mllama +from PIL import Image +from torch import nn +from transformers.modeling_outputs import (BaseModelOutput, + CausalLMOutputWithPast) +from transformers.models.mllama.image_processing_mllama import ( + get_optimal_tiled_canvas) + +import vllm.distributed.parallel_state as ps +from vllm.attention import Attention, AttentionMetadata, AttentionType +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData + +from .clip import CLIPMLP +from .interfaces import SupportsMultiModal +from .llama import LlamaDecoderLayer, LlamaMLP + +logger = init_logger(__name__) +MLLAMA_IMAGE_TOKEN_ID = 128256 +MLLAMA_IMAGE_TOKEN = "<|image|>" + + +class MllamaImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: """ + """(batch_size, max_num_image, max_num_chunk, num_channel, height, width)""" + aspect_ratio_ids: torch.Tensor + """Shape: `(batch_size, max_num_image)`""" + aspect_ratio_mask: torch.Tensor + """Shape: `(batch_size, max_num_image, max_num_tiles)`""" + + +# TODO: support LlamaImageEmbeddingInputs + + +def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs): + # move encoder_prompt to prompt + if llm_inputs.get("prompt") is None: + llm_inputs["prompt"] = llm_inputs["encoder_prompt"] + llm_inputs["prompt_token_ids"] = llm_inputs["encoder_prompt_token_ids"] + + # process multi-modal data + assert "decoder_multi_modal_data" not in llm_inputs, \ + "multi-modal data should be put in encoder message of mllama" + multi_modal_data = llm_inputs.get("encoder_multi_modal_data") + + if multi_modal_data is None or "image" not in multi_modal_data \ + or multi_modal_data["image"] is None: + # text-only + llm_inputs["encoder_prompt"] = "" + llm_inputs["encoder_prompt_token_ids"] = [] + llm_inputs["encoder_multi_modal_data"] = {} + return llm_inputs + + # get num_tiles + if isinstance(multi_modal_data['image'], Image.Image): + multi_modal_data['image'] = [multi_modal_data['image']] + hf_config = ctx.model_config.hf_config + num_tiles = 0 + for image in multi_modal_data["image"]: + width, height = image.size + tile_size = hf_config.vision_config.image_size + canvas_height, canvas_width = get_optimal_tiled_canvas( + image_height=height, + image_width=width, + max_image_tiles=hf_config.vision_config.max_num_tiles, + tile_size=tile_size, + ) + num_tiles_height = canvas_height // tile_size + num_tiles_width = canvas_width // tile_size + num_tiles += num_tiles_height * num_tiles_width + + # set encoder prompt based on num_tiles + assert hf_config.vision_config.image_size % 14 == 0, \ + "chunk size should be multiple of 14" + token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1 + num_tokens = num_tiles * token_per_chunk + llm_inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens + llm_inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID + ] * num_tokens + + return llm_inputs + + +def get_max_mllama_image_tokens(ctx: InputContext) -> int: + hf_config = ctx.model_config.hf_config + token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1 + return hf_config.vision_config.max_num_tiles * token_per_chunk + + +def dummy_decoder_seq_data(seq_len: int, num_images: int): + # <|image|> * num_images + 0 * (seq_len - num_images) + assert seq_len >= num_images, \ + "seq_len should be greater than or equal to num_images" + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + [MLLAMA_IMAGE_TOKEN_ID]) * num_images + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * (seq_len - num_images) + return SequenceData(token_ids) + + +def dummy_encoder_seq_data(ctx: InputContext, num_images: int): + num_tokens = get_max_mllama_image_tokens(ctx) * num_images + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + [MLLAMA_IMAGE_TOKEN_ID]) * num_tokens + return SequenceData(token_ids) + + +def dummy_image(num_images: int, ): + width = height = 1024 + image = Image.new("RGB", (width, height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + num_images = mm_counts["image"] + return dummy_decoder_seq_data(seq_len, num_images), None + + +def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + num_images = mm_counts["image"] + return dummy_encoder_seq_data(ctx, num_images), dummy_image(num_images) + + +def _prepare_aspect_ratio_attention_mask( + aspect_ratio_mask: torch.Tensor, + num_patches: int, + target_length: int, + dtype: torch.dtype, +) -> torch.Tensor: + # Expand aspect ratio mask to target_length + batch_size, max_num_tiles = aspect_ratio_mask.shape + attention_mask = aspect_ratio_mask.view(batch_size, max_num_tiles, 1, + 1).to(dtype) + attention_mask = attention_mask.repeat(1, 1, target_length, 1) + + # Mask padding patches + pad_patches = target_length - num_patches + attention_mask[:, :, -pad_patches:] = 0 + + # Invert the mask (0 -> 1, 1 -> 0) + attention_mask = 1 - attention_mask + + # Reshape to 2D and create 4D attention mask + # (batch_size, 1, max_num_tiles*target_length, max_num_tiles*target_length) + attention_mask = attention_mask.reshape(batch_size, + max_num_tiles * target_length, 1) + attention_mask = attention_mask @ attention_mask.transpose( + -1, -2) * torch.finfo(dtype).min + attention_mask = attention_mask.unsqueeze(1) + + return attention_mask + + +class ColumnParallelConv2dPatch(torch.nn.Module): + """Conv2D Patching layer with model parallelism. + Column parallel over unfolded input. + Arguments: + in_channels: Input channels. + out_channels: Output channels. + kernel_size: Size of convolution kernel. + stride (default 1): Stride for convolution. + bias (default False): Use bias in Conv2d. + Input: (bsz, in_channels, width, height) + Output: (bsz, num_tokens, out_channels) + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int]], + stride: Union[int, Tuple[int, int]], + bias: bool = False, + ) -> None: + super().__init__() + if isinstance(kernel_size, int): + kernel_size = (kernel_size, kernel_size) + self._unfold = torch.nn.Unfold(kernel_size=kernel_size, stride=stride) + self._linear = ColumnParallelLinear( + in_channels * kernel_size[0] * kernel_size[1], + out_channels, + bias=bias, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self._unfold(x) + x = x.permute(0, 2, 1).contiguous() + x, _ = self._linear(x) + return x + + +class MllamaPrecomputedAspectRatioEmbedding(nn.Module): + + def __init__(self, + config: config_mllama.MllamaVisionConfig, + is_gated: bool = True): + super().__init__() + self.max_num_tiles = config.max_num_tiles + self.hidden_size = config.hidden_size + self.max_aspect_ratio_id = config.max_aspect_ratio_id + self.is_gated = is_gated + + self.embedding = nn.Embedding(self.max_aspect_ratio_id + 1, + self.max_num_tiles * self.hidden_size) + if is_gated: + self.gate = nn.Parameter(torch.zeros(1)) + + def forward(self, hidden_state: torch.Tensor, + aspect_ratio_ids: torch.Tensor) -> torch.Tensor: + embeddings = self.embedding(aspect_ratio_ids) + embeddings = embeddings.reshape(-1, self.max_num_tiles, 1, + self.hidden_size) + + if self.is_gated: + embeddings = embeddings * self.gate.tanh() + + hidden_state = hidden_state + embeddings + return hidden_state + + +class MllamaPrecomputedPositionEmbedding(nn.Module): + + def __init__(self, config: config_mllama.MllamaVisionConfig): + super().__init__() + self.max_num_tiles = config.max_num_tiles + self.max_aspect_ratio_id = config.max_aspect_ratio_id + self.num_patches = (config.image_size // config.patch_size)**2 + 1 + self.hidden_size = config.hidden_size + self.scale = config.hidden_size**-0.5 + + self.gate = nn.Parameter(torch.zeros(1)) + + # position embedding + position_embedding = torch.randn(self.num_patches, self.hidden_size) + self.embedding = nn.Parameter(self.scale * position_embedding) + + # tile position embedding + self.tile_embedding = nn.Embedding( + self.max_aspect_ratio_id + 1, + self.max_num_tiles * self.num_patches * self.hidden_size) + + def forward(self, hidden_state: torch.Tensor, + aspect_ratio_ids: torch.Tensor) -> torch.Tensor: + # position embeddings + gated_position_embedding = (1 - self.gate.tanh()) * self.embedding + hidden_state = hidden_state + gated_position_embedding.view( + 1, 1, self.num_patches, self.hidden_size) + + # precomputed tile position embeddings + tile_position_embedding = self.tile_embedding(aspect_ratio_ids) + batch_size = hidden_state.shape[0] + tile_position_embedding = tile_position_embedding.reshape( + batch_size, self.max_num_tiles, self.num_patches, self.hidden_size) + gated_tile_position_embedding = self.gate.tanh( + ) * tile_position_embedding + hidden_state = hidden_state + gated_tile_position_embedding + + return hidden_state + + +# TODO: support other attention backends for attention in vision model +class MllamaVisionSdpaAttention(nn.Module): + + def __init__(self, config: config_mllama.MllamaVisionConfig): + super().__init__() + + model_parallel_size = get_tensor_model_parallel_world_size() + self.embed_dim = config.hidden_size + self.num_heads = config.attention_heads + self.head_dim = config.hidden_size // config.attention_heads + self.num_local_heads = self.num_heads // model_parallel_size + self.q_size = self.num_local_heads * self.head_dim + self.kv_size = self.num_local_heads * self.head_dim + self.scale = 1 / math.sqrt(self.head_dim) + self.qkv_proj = QKVParallelLinear( + self.embed_dim, + self.head_dim, + self.num_heads, + bias=False, + ) + self.o_proj = RowParallelLinear( + self.num_heads * self.head_dim, + self.embed_dim, + bias=False, + input_is_parallel=True, + ) + + def forward( + self, + hidden_state: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_state) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q = q.view(q.shape[0], q.shape[1], self.num_local_heads, + self.head_dim).transpose(1, 2) + k = k.view(k.shape[0], k.shape[1], self.num_local_heads, + self.head_dim).transpose(1, 2) + v = v.view(v.shape[0], v.shape[1], self.num_local_heads, + self.head_dim).transpose(1, 2) + + # TODO: remove padding in image encoder + attn_output = torch.empty_like(q) + for i in range(q.shape[0]): + attn = q[i] @ (k[i] * self.scale).permute(0,2,1) + if attention_mask is not None: + attn = attn + attention_mask[i] + attn = torch.softmax(attn, dim=-1) + + output = attn @ v[i] + attn_output[i] = output + # attn_output = F.scaled_dot_product_attention(q, + # k, + # v, + # attn_mask=attention_mask, + # dropout_p=0.0) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(attn_output.shape[0], + attn_output.shape[1], -1) + output, _ = self.o_proj(attn_output) + return output + + +class MllamaVisionEncoderLayer(nn.Module): + + def __init__(self, + config: config_mllama.MllamaVisionConfig, + is_gated: bool = False): + super().__init__() + + self.hidden_size = config.hidden_size + self.num_attention_heads = config.attention_heads + self.is_gated = is_gated + self.intermediate_size = config.intermediate_size + + self.self_attn = MllamaVisionSdpaAttention(config) + self.mlp = CLIPMLP(config) + + self.input_layernorm = nn.LayerNorm(self.hidden_size, + eps=config.norm_eps) + self.post_attention_layernorm = nn.LayerNorm(self.hidden_size, + eps=config.norm_eps) + + # there used to be an if else here, no code path + if is_gated: + self.gate_attn = nn.Parameter(torch.ones(1) * math.pi / 4) + self.gate_ffn = nn.Parameter(torch.ones(1) * math.pi / 4) + + def forward( + self, + hidden_state: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ): + # Self Attention + residual = hidden_state + hidden_state = self.input_layernorm(hidden_state) + hidden_state = self.self_attn(hidden_state, + attention_mask=attention_mask) + gate_attn = 1 if not self.is_gated else self.gate_attn.tanh() + hidden_state = residual + gate_attn * hidden_state + + # Feed forward + residual = hidden_state + hidden_state = self.post_attention_layernorm(hidden_state) + hidden_state = self.mlp(hidden_state) + gate_ffn = 1 if not self.is_gated else self.gate_ffn.tanh() + hidden_state = residual + gate_ffn * hidden_state + + return hidden_state + + +class MllamaVisionEncoder(nn.Module): + + def __init__(self, + config: config_mllama.MllamaVisionConfig, + num_layers=32, + is_gated=False, + output_hidden_states=None): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + MllamaVisionEncoderLayer(config, is_gated) + for _ in range(num_layers) + ]) + self.output_hidden_states = output_hidden_states or [] + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> Union[Tuple, BaseModelOutput]: + encoder_states = () + + for i, encoder_layer in enumerate(self.layers): + if i in self.output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + hidden_states = encoder_layer( + hidden_states, + attention_mask, + ) + + if len(self.layers) - 1 in self.output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + + return hidden_states, encoder_states + + +class MllamaVisionModel(nn.Module): + + def __init__(self, config: config_mllama.MllamaVisionConfig): + super().__init__() + self.image_size = config.image_size + self.patch_size = config.patch_size + self.max_num_tiles = config.max_num_tiles + self.hidden_size = config.hidden_size + self.in_channels = config.num_channels + self.intermediate_layers_indices = config.intermediate_layers_indices + + self.num_patches = (self.image_size // self.patch_size)**2 + 1 + self.scale = config.hidden_size**-0.5 + + self.patch_embedding = ColumnParallelConv2dPatch( + in_channels=config.num_channels, + out_channels=self.hidden_size, + kernel_size=self.patch_size, + stride=self.patch_size, + bias=False, + ) + + self.class_embedding = nn.Parameter(self.scale * + torch.randn(self.hidden_size)) + self.gated_positional_embedding = MllamaPrecomputedPositionEmbedding( + config) + + self.pre_tile_positional_embedding = \ + MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True) + self.post_tile_positional_embedding = \ + MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True) + + # layer norms + self.layernorm_pre = nn.LayerNorm(self.hidden_size) + self.layernorm_post = nn.LayerNorm(self.hidden_size) + + # encoders + self.transformer = MllamaVisionEncoder( + config, + config.num_hidden_layers, + is_gated=False, + output_hidden_states=config.intermediate_layers_indices) + self.global_transformer = MllamaVisionEncoder(config, + config.num_global_layers, + is_gated=True) + + def apply_class_embedding(self, + hidden_state: torch.Tensor) -> torch.Tensor: + batch_size, _, hidden_size = hidden_state.shape + class_embedding = self.class_embedding.expand(batch_size, 1, + hidden_size) + hidden_state = torch.cat([class_embedding, hidden_state], dim=1) + return hidden_state + + def forward(self, pixel_values: torch.Tensor, + aspect_ratio_ids: torch.Tensor, + aspect_ratio_mask: torch.Tensor) -> torch.Tensor: + batch_size, num_concurrent_media, num_tiles, num_channels, \ + height, width = pixel_values.shape + + pixel_values = pixel_values.reshape( + batch_size * num_concurrent_media * num_tiles, num_channels, + height, width) + aspect_ratio_ids = aspect_ratio_ids.reshape( + batch_size * num_concurrent_media, -1) + + # patch embedding + patch_embeds = self.patch_embedding( + pixel_values.to(self.layernorm_pre.weight.dtype)) + hidden_state = patch_embeds + hidden_state = ps.get_tp_group().all_gather(hidden_state) + + # tile embeddings + _, num_patches, dim = hidden_state.shape + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media, + num_tiles, -1, dim) + hidden_state = self.pre_tile_positional_embedding( + hidden_state, aspect_ratio_ids) + + # apply cls token + hidden_state = hidden_state.reshape( + batch_size * num_concurrent_media * num_tiles, num_patches, dim) + hidden_state = self.apply_class_embedding(hidden_state) + num_patches += 1 + + # apply position embeddings + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media, + num_tiles, num_patches, dim) + hidden_state = self.gated_positional_embedding(hidden_state, + aspect_ratio_ids) + + # apply encoder + hidden_state = self.layernorm_pre(hidden_state) + + # Compute the number of tokens to pad + num_padding_patches = (8 - (hidden_state.shape[-2] % 8)) % 8 + # Compute padding tuple for pad function + padding = ( + 0, 0, 0, num_padding_patches + ) # (pad_left, pad_right, pad_left for dim -2, pad_right for dim -2) + # Pad the tensor + hidden_state = F.pad(hidden_state, padding, mode="constant", value=0) + slice_index = -num_padding_patches if num_padding_patches > 0 else None + + attention_mask = aspect_ratio_mask.reshape( + batch_size * num_concurrent_media, -1) + attention_mask = _prepare_aspect_ratio_attention_mask( + aspect_ratio_mask=attention_mask, + num_patches=self.num_patches, + target_length=hidden_state.shape[2], + dtype=self.layernorm_pre.weight.dtype, + ) + + hidden_state = hidden_state.view(batch_size * num_concurrent_media, -1, + dim) + output = self.transformer( + hidden_state, + attention_mask=attention_mask, + ) + hidden_state, intermediate_hidden_states = output[0], output[1] + intermediate_hidden_states = torch.stack(intermediate_hidden_states, + dim=-1) + + # apply global encoder + hidden_state = self.layernorm_post(hidden_state) + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media, + num_tiles, + num_patches + num_padding_patches, + dim) + hidden_state = self.post_tile_positional_embedding( + hidden_state, aspect_ratio_ids) + hidden_state = hidden_state.reshape( + batch_size * num_concurrent_media, + num_tiles * (num_patches + num_padding_patches), dim) + hidden_state = self.global_transformer( + hidden_state, attention_mask=attention_mask)[0] + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media, + num_tiles, + num_patches + num_padding_patches, + dim) + hidden_state = hidden_state[:, :, :slice_index] + + # adding intermediate layer outputs + hidden_state = hidden_state.reshape(batch_size, num_concurrent_media, + num_tiles, num_patches, dim) + intermediate_hidden_states = intermediate_hidden_states.reshape( + batch_size * num_concurrent_media, num_tiles, + num_patches + num_padding_patches, -1) + intermediate_hidden_states = intermediate_hidden_states[:, :, : + slice_index] + intermediate_hidden_states = intermediate_hidden_states.reshape( + batch_size, num_concurrent_media, num_tiles, num_patches, -1) + hidden_state = torch.cat([hidden_state, intermediate_hidden_states], + dim=-1) + return hidden_state + + +class MllamaTextRMSNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-6): + """ + MllamaTextRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class MllamaTextCrossAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: Optional[config_mllama.MllamaTextConfig] = None, + layer_idx: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.model_parallel_size = get_tensor_model_parallel_world_size() + self.num_heads = self.config.num_attention_heads + self.num_local_heads = self.num_heads // self.model_parallel_size + self.num_key_value_heads = self.config.num_key_value_heads + self.num_local_key_value_heads = \ + self.num_key_value_heads // self.model_parallel_size + self.dropout = config.dropout + self.hidden_size = config.hidden_size + self.head_dim = config.hidden_size // self.num_heads + self.layer_idx = layer_idx + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.q_local_size = self.num_local_heads * self.head_dim + self.kv_local_size = self.num_local_key_value_heads * self.head_dim + + # TODO: change to Q/KV separate linear after #7448 is merged + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.num_heads, + self.num_key_value_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.num_heads * self.head_dim, + self.hidden_size, + bias=False, + input_is_parallel=True, + quant_config=quant_config, + ) + # vllm.model_executor.layers.layernorm.RMSNorm has precision issue, + # use huggingface's instead + self.q_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.scaling = self.head_dim**-0.5 + + self.attn = Attention( + self.num_local_heads, + self.head_dim, + self.scaling, + self.num_local_key_value_heads, + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor], + cross_attention_states: Optional[torch.Tensor], + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv_dec, _ = self.qkv_proj(hidden_states) + q, _, _ = qkv_dec.split( + [self.q_local_size, self.kv_local_size, self.kv_local_size], + dim=-1) + if cross_attention_states is None: + k = None + v = None + else: + qkv_enc, _ = self.qkv_proj(cross_attention_states) + _, k, v = qkv_enc.split( + [self.q_local_size, self.kv_local_size, self.kv_local_size], + dim=-1) + k = k.view(-1, self.num_local_key_value_heads, self.head_dim) + v = v.view(-1, self.num_local_key_value_heads, self.head_dim) + k = self.k_norm(k) + q = q.view(-1, self.num_local_heads, self.head_dim) + q = self.q_norm(q) + + output = self.attn(q, + k, + v, + kv_cache, + attn_metadata, + attn_type=AttentionType.ENCODER_DECODER) + out, _ = self.o_proj(output) + return out + + +class MllamaCrossAttentionDecoderLayer(torch.nn.Module): + """Cross-attention transformer block with tanh-gated attention + and feedforward.""" + + def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int, + quant_config: Optional[QuantizationConfig]) \ + -> None: + super().__init__() + self.layer_idx = layer_idx + self.cross_attn = MllamaTextCrossAttention( + config=config, + layer_idx=layer_idx, + quant_config=quant_config, + ) + + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.cross_attn_attn_gate = torch.nn.Parameter(torch.zeros(1)) + + self.mlp = LlamaMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.cross_attn_mlp_gate = torch.nn.Parameter(torch.zeros(1)) + + def forward( + self, + hidden_states: torch.Tensor, + cross_attention_states: torch.Tensor, + cross_attention_mask: torch.Tensor, + full_text_row_masked_out_mask: torch.Tensor, + kv_cache: List[torch.Tensor], + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + hidden_states = self.cross_attn( + hidden_states=hidden_states, + attention_mask=cross_attention_mask, + cross_attention_states=cross_attention_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = full_text_row_masked_out_mask * hidden_states + hidden_states = residual + self.cross_attn_attn_gate.tanh( + ) * hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = full_text_row_masked_out_mask * hidden_states + hidden_states = residual + self.cross_attn_mlp_gate.tanh( + ) * hidden_states + return hidden_states + + +class MllamaTextModel(nn.Module): + config_class = config_mllama.MllamaTextConfig + base_model_prefix = "model" + + def __init__(self, config: config_mllama.MllamaTextConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig]): + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8, + config.hidden_size) + self.cross_attention_layers = config.cross_attention_layers + + layers = [] + for layer_idx in range(config.num_hidden_layers): + if layer_idx in self.cross_attention_layers: + layers.append( + MllamaCrossAttentionDecoderLayer( + config, layer_idx, quant_config=quant_config)) + else: + # TODO: force LlamaDecoderLayer to config.attention_bias=False + layers.append( + LlamaDecoderLayer(config, + cache_config=cache_config, + quant_config=quant_config)) + + self.layers = nn.ModuleList(layers) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.LongTensor, + positions: Optional[torch.LongTensor], + cross_attention_states: Optional[torch.LongTensor], + cross_attention_mask: Optional[torch.LongTensor], + full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor, + torch.Tensor]], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + skip_cross_attention: bool, + ) -> torch.Tensor: + inputs_embeds = self.embed_tokens(input_ids) + hidden_states = inputs_embeds + + for idx, decoder_layer in enumerate(self.layers): + if isinstance(decoder_layer, MllamaCrossAttentionDecoderLayer): + if not skip_cross_attention: + hidden_states = decoder_layer( + hidden_states=hidden_states, + cross_attention_states=cross_attention_states, + cross_attention_mask=cross_attention_mask, + full_text_row_masked_out_mask= + full_text_row_masked_out_mask, + kv_cache=kv_caches[idx], + attn_metadata=attn_metadata, + ) + elif isinstance(decoder_layer, LlamaDecoderLayer): + hidden_states, residual = decoder_layer( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_caches[idx], + attn_metadata=attn_metadata, + residual=None, + ) + hidden_states = hidden_states + residual + else: + raise ValueError( + f"Unknown decoder layer type {type(decoder_layer)}") + hidden_states = self.norm(hidden_states) + return hidden_states + + +class MllamaForCausalLM(nn.Module): + config_class = config_mllama.MllamaTextConfig + base_model_prefix = "language_model" + _no_split_modules = [ + "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer" + ] + + def __init__(self, config: config_mllama.MllamaTextConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig]): + super().__init__() + self.vocab_size = config.vocab_size + self.model = MllamaTextModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + quant_config=quant_config, + ) + + def forward( + self, + input_ids: torch.LongTensor, + positions: Optional[torch.LongTensor], + cross_attention_states: Optional[torch.LongTensor], + cross_attention_mask: Optional[torch.LongTensor], + full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor, + torch.Tensor]], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + skip_cross_attention: bool, + ) -> torch.Tensor: + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + cross_attention_states=cross_attention_states, + cross_attention_mask=cross_attention_mask, + full_text_row_masked_out_mask=full_text_row_masked_out_mask, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + skip_cross_attention=skip_cross_attention, + ) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_mllama_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_decoder_data_for_mllama) +@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama) +@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama) +class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal): + + def __init__(self, + config: config_mllama.MllamaConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.vocab_size = config.text_config.vocab_size + self.hidden_size = config.text_config.hidden_size + self.max_num_tiles = config.vision_config.max_num_tiles + self.vision_output_dim = config.vision_config.vision_output_dim + self.pad_token_id = \ + config.pad_token_id if config.pad_token_id is not None else -1 + self.image_size = config.vision_config.image_size + + self.vision_model = MllamaVisionModel(config.vision_config) + self.language_model = MllamaForCausalLM( + config.text_config, + cache_config=cache_config, + quant_config=quant_config, + ) + self.multi_modal_projector = nn.Linear( + config.vision_config.vision_output_dim, + config.text_config.hidden_size, + bias=True, + ) + self.logits_processor = LogitsProcessor(config.output_hidden_states, + config.text_config.vocab_size) + self.sampler = Sampler() + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.language_model.lm_head, + hidden_states, sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def _parse_and_validate_image_input(self, **kwargs: object): + # tensor with the same shape will be batched together by + # MultiModalInputs.batch, so pixel_values here can be: + # - List[List[torch.Tensor]]: + # with shape (num_tiles, 3, image_res, image_res) + # - List[torch.Tensor]: + # with shape (num_image, num_tiles, 3, image_res, image_res) + # - torch.Tensor: + # with shape (bs, num_image, num_tiles, 3, image_res, image_res) + pixel_values: Optional[Union[List[List[torch.Tensor]], + List[torch.Tensor], + torch.Tensor]] = kwargs.pop( + "pixel_values", None) + image_embeds: Optional[Union[List[List[torch.Tensor]], + List[torch.Tensor], + torch.Tensor]] = kwargs.pop( + "image_embeds", None) + aspect_ratio_ids: Optional[Union[List[List[torch.Tensor]], + List[torch.Tensor], + torch.Tensor]] = kwargs.pop( + "aspect_ratio_ids", None) + aspect_ratio_mask: Optional[Union[List[List[torch.Tensor]], + List[torch.Tensor], + torch.Tensor]] = kwargs.pop( + "aspect_ratio_mask", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None and image_embeds is not None: + raise ValueError( + "Both pixel values and image embeds are provided.") + + if pixel_values is not None: + assert aspect_ratio_ids is not None + assert aspect_ratio_mask is not None + max_num_images = max([len(x[0]) for x in pixel_values]) + if max_num_images == 0: + raise ValueError("No images provided.") + max_num_tiles = max( + max([len(x) for x in y[0]]) for y in pixel_values) + device = self.multi_modal_projector.weight.device + bsz = len(pixel_values) + out_num_tiles = [] + out_images = torch.zeros( + bsz, + max_num_images, + max_num_tiles, + 3, + self.image_size, + self.image_size, + dtype=torch.float32, + device=device, + ) + out_ar_ids = torch.ones(bsz, + max_num_images, + dtype=torch.int64, + device=device) + out_ar_mask = torch.zeros(bsz, + max_num_images, + max_num_tiles, + dtype=torch.int64, + device=device) + for b in range(len(pixel_values)): + _num_tiles = [] + for i in range(len(pixel_values[b][0])): + img = pixel_values[b][0][i] + out_images[b, i, :img.shape[0]] = img + out_ar_ids[b, i] = aspect_ratio_ids[b][0][i] + out_ar_mask[b, i] = aspect_ratio_mask[b][0][i] + _num_tiles.append(img.shape[0]) + out_num_tiles.append(_num_tiles) + + return MllamaImagePixelInputs( + type="pixel_values", + data=out_images, + aspect_ratio_ids=out_ar_ids, + aspect_ratio_mask=out_ar_mask, + ) + + if image_embeds is not None: + raise NotImplementedError + + raise AssertionError("This line should be unreachable.") + + def flat_encoder_result(self, cross_attention_states: torch.Tensor, + attn_metadata: AttentionMetadata): + + cross_attention_states_flat = torch.zeros( + sum(attn_metadata.encoder_seq_lens), + cross_attention_states.shape[-1], + device=cross_attention_states.device, + dtype=cross_attention_states.dtype) + start_pos = 0 + for seq_len, vision_token_in_batch in zip( + attn_metadata.encoder_seq_lens, cross_attention_states): + end_pos = start_pos + seq_len + cross_attention_states_flat[ + start_pos:end_pos] = vision_token_in_batch[:seq_len] + start_pos = end_pos + cross_attention_states = cross_attention_states_flat + + full_text_row_masked_out_mask = torch.ones( + (attn_metadata.num_prefill_tokens, 1), dtype=torch.bool) + start_pos = 0 + for seq_len, encoder_seq_len in zip( + attn_metadata.seq_lens_tensor.cpu(), + attn_metadata.encoder_seq_lens): + if encoder_seq_len == 0: + full_text_row_masked_out_mask[start_pos:start_pos + + seq_len] = False + start_pos += seq_len + full_text_row_masked_out_mask = full_text_row_masked_out_mask.to( + cross_attention_states.device) + + return cross_attention_states, full_text_row_masked_out_mask + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + **kwargs: object, + ) -> Union[Tuple, CausalLMOutputWithPast]: + if attn_metadata.num_prefill_tokens > 0 and \ + attn_metadata.num_decode_tokens > 0: + raise ValueError("Chunk prefill not supported") + image_inputs = self._parse_and_validate_image_input(**kwargs) + if image_inputs is None: + cross_attention_mask = None + full_text_row_masked_out_mask = ( + attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to( + input_ids.device) + cross_attention_states = None + skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0 + else: + # NOTE: llama's reference implementation runs vision model on CPU + pixel_values = image_inputs['data'] + aspect_ratio_ids = image_inputs['aspect_ratio_ids'] + aspect_ratio_mask = image_inputs['aspect_ratio_mask'] + cross_attention_states = self.vision_model(pixel_values, + aspect_ratio_ids, + aspect_ratio_mask) + cross_attention_states = self.multi_modal_projector( + cross_attention_states) + + bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape) + cross_attention_states = cross_attention_states.view( + bsz, -1, image_token_dim) + + cross_attention_states, full_text_row_masked_out_mask = \ + self.flat_encoder_result(cross_attention_states, attn_metadata) + skip_cross_attention = False + # TODO: support multi-image by this mask + cross_attention_mask = None + + outputs = self.language_model( + input_ids=input_ids, + positions=positions, + cross_attention_states=cross_attention_states, + cross_attention_mask=cross_attention_mask, + full_text_row_masked_out_mask=full_text_row_masked_out_mask, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + skip_cross_attention=skip_cross_attention, + ) + + return outputs + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + updated_params = set() + for name, loaded_weight in weights: + if 'patch_embedding.weight' in name: + name = name.replace('patch_embedding.weight', + 'patch_embedding._linear.weight') + loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1) + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + param = params_dict[name] + updated_params.add(name) + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict.pop(name) + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/mlp_speculator.py b/vllm/model_executor/models/mlp_speculator.py new file mode 100644 index 00000000..42ccd012 --- /dev/null +++ b/vllm/model_executor/models/mlp_speculator.py @@ -0,0 +1,197 @@ +import math +from typing import Iterable, List, Tuple + +import torch +import torch.nn as nn + +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.transformers_utils.configs import MLPSpeculatorConfig + +SQRT2 = 2**0.5 + + +class MLPSpeculatorLayerNorm(nn.Module): + """ + A L2 normalization implementation + ... + Args + ---- + normalized_shape : int + Dimensionality of input data (size of final tensor axis) + eps : float + Safety term to prevent division by zero. Make sure the chosen value + fits in the range of your encoding scheme + (i.e. fp16 requires eps >= 6e-8). + elementwise_scale_and_shift : bool + Include a learned scaling and shift term after normalization. + """ + + def __init__( + self, + normalized_shape, + eps=1e-06, + elementwise_scale_and_shift=True, + ): + super(MLPSpeculatorLayerNorm, self).__init__() + self.elementwise_scale_and_shift = elementwise_scale_and_shift + if self.elementwise_scale_and_shift: + self.weight = nn.Parameter(torch.empty(normalized_shape)) + self.bias = nn.Parameter(torch.empty(normalized_shape)) + self.eps = eps + + def forward(self, x): + xf = x + xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps) + x = xf.type_as(x) + if self.elementwise_scale_and_shift: + x = self.weight * x + x = x + self.bias + return x + + +class MLPSpeculator(nn.Module): + """ + An implementation of the speculative models introduced in + "Accelerating Production LLMs with Combined Token/Embedding + Speculators" + https://arxiv.org/pdf/2404.19124 + + Trained speculators of this type are available on HF hub at: + https://huggingface.co/ibm-fms and https://huggingface.co/ibm-granite + """ + + def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None: + super().__init__() + self.n_predict = config.n_predict + self.vocab_size = config.vocab_size + self.emb_dim = config.emb_dim + self.inner_dim = config.inner_dim if config.inner_dim != 0 \ + else config.emb_dim + + self.max_speculative_tokens = config.num_lookahead_tokens + + self.tie_weights = config.tie_weights + self.scale_input = config.scale_input + + if self.tie_weights: + assert ( + self.n_predict > + 1), "You cannot tie weights between stages when only 1 exists" + embedding = VocabParallelEmbedding( + config.vocab_size, + self.inner_dim, + org_num_embeddings=config.vocab_size) + self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens) + + # the initial projection from the base model may + # have a different size, so that stays separate. + proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False) + proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False) + self.proj = nn.ModuleList([proj_first] + [proj_tied] * + (self.max_speculative_tokens - 1)) + + head = ParallelLMHead(self.vocab_size, self.inner_dim, bias=False) + self.head = nn.ModuleList([head] * self.max_speculative_tokens) + + ln = MLPSpeculatorLayerNorm(self.inner_dim, + elementwise_scale_and_shift=True) + self.ln = nn.ModuleList([ln] * self.max_speculative_tokens) + + else: + self.emb = nn.ModuleList([ + VocabParallelEmbedding(config.vocab_size, + self.inner_dim, + org_num_embeddings=config.vocab_size) + for _ in range(self.max_speculative_tokens) + ]) + + self.proj = nn.ModuleList([ + nn.Linear((self.emb_dim if i == 0 else self.inner_dim), + self.inner_dim, + bias=False) + for i in range(self.max_speculative_tokens) + ]) + + self.head = nn.ModuleList([ + ParallelLMHead(self.vocab_size, self.inner_dim, bias=False) + for _ in range(self.max_speculative_tokens) + ]) + self.ln = nn.ModuleList([ + MLPSpeculatorLayerNorm(self.inner_dim, + elementwise_scale_and_shift=True) + for _ in range(self.max_speculative_tokens) + ]) + if self.scale_input: + self.ln0 = MLPSpeculatorLayerNorm( + self.emb_dim, elementwise_scale_and_shift=False) + + self.state_weight = 0.5**(0.5 / config.n_predict) + self.emb_weight = math.sqrt( + (1 - self.state_weight**2) * (self.inner_dim / 2)) + self.activation = nn.GELU() + self.config = config + self.logits_processor = LogitsProcessor(config.vocab_size, + config.vocab_size, 1.0) + self.sampler = Sampler() + + def generate_proposals( + self, + input_ids: torch.Tensor, + previous_hidden_states: torch.Tensor, + num_predict_tokens: int, + sampling_metadata: SamplingMetadata, + ) -> List[SamplerOutput]: + if num_predict_tokens > self.max_speculative_tokens: + raise ValueError(f"Max speculative tokens for model is " + f"{self.max_speculative_tokens}, but " + f"{num_predict_tokens} were requested") + + # b x 1 x d + previous_hidden_states = previous_hidden_states.unsqueeze(1) + + if self.scale_input: + previous_hidden_states = self.ln0(previous_hidden_states) / SQRT2 + + # b x 1 + last_tokens = input_ids.unsqueeze(1) + + next_tokens = [] + + for head_index in range(num_predict_tokens): + + # Project and predict + z = self.emb[head_index](last_tokens) # b k d + states = self.proj[head_index](previous_hidden_states) + + # Weighted add of state_weight*state and emb_weight*z + # Let subsequent LN take care of denominator + # state_weight is close to 1, so shouldn't be any precision issues + states.add_(z, alpha=self.emb_weight / self.state_weight) + + states = self.activation(self.ln[head_index](states)) # b k d + previous_hidden_states = states + # TODO: not yet supporting top_k_tokens_per_head + states = states.flatten(0, 1) + + logits = self.logits_processor(self.head[head_index], states, + sampling_metadata) + + output = self.sampler(logits, sampling_metadata) + last_tokens = output.sampled_token_ids + next_tokens.append(output) + + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + param = params_dict.get(name.replace("speculator.", "")) + if param is not None: + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/module_mapping.py b/vllm/model_executor/models/module_mapping.py new file mode 100644 index 00000000..a9102a60 --- /dev/null +++ b/vllm/model_executor/models/module_mapping.py @@ -0,0 +1,69 @@ +# Adapted from +# https://github.com/modelscope/ms-swift/blob/v2.4.2/swift/utils/module_mapping.py + +from dataclasses import dataclass, field +from typing import List, Union + + +@dataclass +class ModelKeys: + model_type: str = None + + module_list: str = None + + embedding: str = None + + mlp: str = None + + down_proj: str = None + + attention: str = None + + o_proj: str = None + + q_proj: str = None + + k_proj: str = None + + v_proj: str = None + + qkv_proj: str = None + + qk_proj: str = None + + qa_proj: str = None + + qb_proj: str = None + + kva_proj: str = None + + kvb_proj: str = None + + output: str = None + + +@dataclass +class MultiModelKeys(ModelKeys): + language_model: List[str] = field(default_factory=list) + connector: List[str] = field(default_factory=list) + # vision tower and audio tower + tower_model: List[str] = field(default_factory=list) + generator: List[str] = field(default_factory=list) + + @staticmethod + def from_string_field(language_model: Union[str, List[str]] = None, + connector: Union[str, List[str]] = None, + tower_model: Union[str, List[str]] = None, + generator: Union[str, List[str]] = None, + **kwargs) -> 'MultiModelKeys': + + def to_list(value): + if value is None: + return [] + return [value] if isinstance(value, str) else list(value) + + return MultiModelKeys(language_model=to_list(language_model), + connector=to_list(connector), + tower_model=to_list(tower_model), + generator=to_list(generator), + **kwargs) diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py new file mode 100644 index 00000000..2ff4e046 --- /dev/null +++ b/vllm/model_executor/models/molmo.py @@ -0,0 +1,1290 @@ +import logging +import math +import regex as re +from array import array +from dataclasses import dataclass +from functools import lru_cache, partial +from typing import (Any, Iterable, List, Mapping, Optional, Tuple, TypedDict, + Union) + +import torch +from einops import rearrange +from PIL import Image +from torch import nn +from torch.nn import functional as F +from transformers import PretrainedConfig + +import vllm.envs as envs +from vllm.attention import Attention, AttentionMetadata +from vllm.attention.selector import (_Backend, backend_name_to_enum, + get_global_forced_attn_backend) +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather) +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.interfaces import SupportsMultiModal +from vllm.model_executor.models.utils import make_layers +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs +from vllm.platforms import current_platform +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors, + SequenceData) +from vllm.transformers_utils.processor import get_processor + +log = logging.getLogger(__name__) + +# TODO: hard-coded for now. Consider making it configurable. +VIT_LAYERS = [-2, -9] +NUM_PREFIX_TOKENS = 1 +ADDITIONAL_VOCAB_SIZE = 128 + + +class MolmoImageInputs(TypedDict): + images: torch.Tensor + """Shape: + `(batch_size, num_crops, num_patch, patch_dim)` + """ + + image_input_idx: torch.Tensor + """Shape: + `(batch_size, num_crops, num_patch)` + """ + + seq_len: torch.Tensor + """Shape: + `(batch_size, )` + """ + + image_masks: Optional[torch.Tensor] + """Shape: + `(batch_size, num_crops, num_patch)` + """ + + +@dataclass +class VisionBackboneConfig: + image_default_input_size: Tuple[int, int] = (336, 336) + image_patch_size: int = 14 + image_pos_patch_size: int = 14 + image_emb_dim: int = 1024 + image_num_heads: int = 16 + image_num_key_value_heads: int = 16 + image_num_layers: int = 23 + image_mlp_dim: int = 4096 + image_mlp_activations: str = "quick_gelu" + image_num_pos: int = 577 + image_norm_eps: float = 1e-5 + + def __post_init__(self): + self.image_default_input_size = tuple( + self.image_default_input_size) # type: ignore[assignment] + + @property + def image_num_patch(self): + h, w = self.image_default_input_size + return h // self.image_patch_size, w // self.image_patch_size + + +class ViTMLP(nn.Module): + """MLP used in Vision Transformer.""" + + def __init__( + self, + config: VisionBackboneConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.w1 = ColumnParallelLinear( + config.image_emb_dim, + config.image_mlp_dim, + bias=True, + quant_config=quant_config, + ) + # Activation function. + assert config.image_mlp_activations == "quick_gelu" + self.act = QuickGELU() + self.w2 = RowParallelLinear( + config.image_mlp_dim, + config.image_emb_dim, + bias=True, + quant_config=quant_config, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.w1(x) + x = self.act(x) + x, _ = self.w2(x) + return x + + +class MultiHeadDotProductAttention(nn.Module): + """Multi-head attention used in Vision Transformer.""" + + def __init__( + self, + config: VisionBackboneConfig, + use_bias: bool = True, + nlayers: int = 1, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.hidden_size = config.image_emb_dim + self.total_num_heads = config.image_num_heads + tp_size = get_tensor_model_parallel_world_size() + + assert self.hidden_size % self.total_num_heads == 0 + assert self.total_num_heads % tp_size == 0 + + self.num_heads = self.total_num_heads // tp_size + self.head_dim = self.hidden_size // self.total_num_heads + + self.total_num_kv_heads = config.image_num_key_value_heads + if self.total_num_kv_heads >= tp_size: + assert self.total_num_kv_heads % tp_size == 0 + else: + assert tp_size % self.total_num_kv_heads == 0 + + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + + self.wq = ColumnParallelLinear( + nlayers * self.hidden_size, + self.total_num_heads * self.head_dim, + bias=use_bias, + quant_config=quant_config, + ) + self.wk = ColumnParallelLinear( + nlayers * self.hidden_size, + self.total_num_kv_heads * self.head_dim, + bias=use_bias, + quant_config=quant_config, + ) + self.wv = ColumnParallelLinear( + nlayers * self.hidden_size, + self.total_num_kv_heads * self.head_dim, + bias=use_bias, + quant_config=quant_config, + ) + self.wo = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=use_bias, + quant_config=quant_config, + ) + + # Detect attention implementation. + selected_backend: Optional[_Backend] = get_global_forced_attn_backend() + if selected_backend is None: + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND + if backend_by_env_var is not None: + selected_backend = backend_name_to_enum(backend_by_env_var) + if selected_backend is None: + # For Volta and Turing GPUs, use xformers instead. + device_available = current_platform.get_device_capability()[0] >= 8 + if device_available: + from transformers.utils import is_flash_attn_2_available + if is_flash_attn_2_available(): + self._use_flash_attn = True + else: + log.warning( + "Current Molmo implementation has a bug with " + "`vllm-flash-attn` inside vision module, so we use " + "xformers backend instead. You can run `pip install " + "flash-attn to use flash-attention backend.") + self._use_flash_attn = False + else: + self._use_flash_attn = False + else: + if selected_backend == _Backend.FLASH_ATTN: + self._use_flash_attn = True + elif selected_backend == _Backend.XFORMERS: + self._use_flash_attn = False + else: + raise RuntimeError( + f"Molmo does not support {selected_backend} backend now.") + + def forward(self, + inputs_q: torch.Tensor, + inputs_kv: Optional[torch.Tensor] = None) -> torch.Tensor: + + if inputs_kv is not None: + inputs_k = inputs_kv + inputs_v = inputs_kv + else: + inputs_k = inputs_q + inputs_v = inputs_q + + xq, _ = self.wq(inputs_q) + xk, _ = self.wk(inputs_k) + xv, _ = self.wv(inputs_v) + q_shape = xq.size()[:-1] + (self.num_heads, self.head_dim) + kv_shape = xk.size()[:-1] + (self.num_kv_heads, self.head_dim) + xq = xq.view(*q_shape) + xk = xk.view(*kv_shape) + xv = xv.view(*kv_shape) + + if self._use_flash_attn: + from flash_attn import flash_attn_func + output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False) + else: + from xformers import ops as xops + output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0) + + output = rearrange(output, "b s h d -> b s (h d)").contiguous() + output, _ = self.wo(output) + + return output + + +class ResidualAttentionBlock(nn.Module): + """Residual attention block used in Vision Transformer.""" + + def __init__( + self, + config: VisionBackboneConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.attention = MultiHeadDotProductAttention( + config, quant_config=quant_config) + self.feed_forward = ViTMLP(config, quant_config) + self.attention_norm = nn.LayerNorm( + config.image_emb_dim, + eps=config.image_norm_eps, + ) + self.ffn_norm = nn.LayerNorm( + config.image_emb_dim, + eps=config.image_norm_eps, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + self.attention(self.attention_norm(x)) + x = x + self.feed_forward(self.ffn_norm(x)) + return x + + +class BlockCollection(nn.Module): + """Collection of residual attention blocks used in Vision Transformer.""" + + def __init__( + self, + config: VisionBackboneConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.resblocks = nn.ModuleList([ + ResidualAttentionBlock(config, quant_config) + for _ in range(config.image_num_layers) + ]) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + hidden_states = [] + for r in self.resblocks: + x = r(x) + hidden_states.append(x) + return hidden_states + + +def _expand_token(token: torch.Tensor, batch_size: int) -> torch.Tensor: + return token.view(1, 1, -1).expand(batch_size, -1, -1) + + +class VisionTransformer(nn.Module): + """Vision Transformer used in Vision Backbone.""" + + def __init__( + self, + config: VisionBackboneConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + scale = config.image_emb_dim**-0.5 + self.patch_num = config.image_num_patch + self.class_embedding = nn.Parameter( + torch.randn(config.image_emb_dim) * scale) + self.num_prefix_tokens: int = NUM_PREFIX_TOKENS + self.positional_embedding = nn.Parameter( + torch.randn(config.image_num_pos, config.image_emb_dim) * scale) + image_patch_size = config.image_patch_size + self.patch_embedding = nn.Linear( + image_patch_size * image_patch_size * 3, + config.image_emb_dim, + bias=False, + ) + self.pre_ln = nn.LayerNorm(config.image_emb_dim, + eps=config.image_norm_eps) + self.transformer = BlockCollection(config, quant_config) + + def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor: + cls_emb = self.positional_embedding[0:1] + pos_emb = self.positional_embedding[1:] + + pos_emb = pos_emb.reshape( + (int(math.sqrt(pos_emb.shape[0])), + int(math.sqrt(pos_emb.shape[0])), pos_emb.shape[1])) + + (patch_num_0, patch_num_1) = patch_num + + if pos_emb.shape[0] != patch_num_0 or pos_emb.shape[1] != patch_num_1: + # from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + pos_emb = pos_emb.unsqueeze(0).permute(0, 3, 1, 2) + pos_emb = F.interpolate( + pos_emb, + size=(patch_num_0, patch_num_1), + mode="bicubic", + align_corners=False, + antialias=True, + ) + pos_emb = pos_emb.permute(0, 2, 3, 1).squeeze(0) + + pos_emb = pos_emb.reshape(-1, pos_emb.shape[-1]) + x = x + torch.cat([cls_emb[None, :, :], pos_emb[None, :, :]], + dim=1).to(x.dtype) + return x + + def forward(self, + x: torch.Tensor, + patch_num: int = None) -> List[torch.Tensor]: + """ + : param x: (batch_size, num_patch, n_pixels) + """ + if patch_num is None: + patch_num = self.patch_num + B, N, D = x.shape + + x = self.patch_embedding(x) + + # class embeddings and positional embeddings + x = torch.cat( + [_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x], + dim=1) + x = self.add_pos_emb(x, patch_num) + + x = self.pre_ln(x) + + hidden_states = self.transformer(x) + return hidden_states + + +class MolmoAttention(nn.Module): + """Molmo's LLM attention.""" + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + + assert self.hidden_size % self.total_num_heads == 0 + assert self.total_num_heads % self.tp_size == 0 + + self.num_heads = self.total_num_heads // self.tp_size + self.total_num_kv_heads = config.num_key_value_heads \ + or self.total_num_heads + if self.total_num_kv_heads >= self.tp_size: + assert self.total_num_kv_heads % self.tp_size == 0 + else: + assert self.tp_size % self.total_num_kv_heads == 0 + + self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size) + self.head_dim = self.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + # Attention input projection. Projects x -> (q, k, v) + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.qkv_bias, + quant_config=quant_config, + ) + + self.tp_rank: Optional[int] = None + self.k_norm: Optional[nn.Module] = None + self.q_norm: Optional[nn.Module] = None + if config.attention_layer_norm: + self.tp_rank = get_tensor_model_parallel_rank() + self.k_norm = RMSNorm(self.total_num_kv_heads * self.head_dim, + eps=config.layer_norm_eps) + self.q_norm = RMSNorm(config.hidden_size, + eps=config.layer_norm_eps) + + # Rotary embeddings. + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + ) + self.scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + # Attention output projection. + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + ) + + def _apply_qk_norm(self, q: torch.Tensor, + k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + if self.tp_size > 1: + q = tensor_model_parallel_all_gather(q.contiguous()) + k = tensor_model_parallel_all_gather(k.contiguous()) + q = self.q_norm.forward_native(q) + k = self.k_norm.forward_native(k) + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + return q, k + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.q_norm is not None and self.k_norm is not None: + q, k = self._apply_qk_norm(q, k) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class MolmoMLP(nn.Module): + """Molmo's LLM mlp.""" + + def __init__( + self, + config: PretrainedConfig, + input_dim: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size // 2 + + # Feed-forward input projection. + self.gate_up_proj = MergedColumnParallelLinear( + input_dim or self.hidden_size, + [self.intermediate_size] * 2, + bias=False, + quant_config=quant_config, + ) + + # Activation function. + self.act_fn = SiluAndMul() + + # Feed-forward output projection. + self.down_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=False, + quant_config=quant_config, + ) + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class MolmoDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + # Attention block. + self.self_attn = MolmoAttention(config, cache_config, quant_config) + + # MLP block. + self.mlp = MolmoMLP(config, quant_config=quant_config) + + # LayerNorm + assert config.layer_norm_type == "rms" + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.layer_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class MolmoDecoderNormAfterLayer(MolmoDecoderLayer): + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]: + # Self Attention + residual = hidden_states + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + hidden_states = self.input_layernorm(hidden_states) + hidden_states = hidden_states + residual + residual = hidden_states + + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = hidden_states + residual + residual = None + return hidden_states, residual + + +class MolmoVisionBackbone(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + vision_config: VisionBackboneConfig, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.vit_layers = VIT_LAYERS + self.image_num_patch = vision_config.image_num_patch + self.llm_patches_per_crop = ( + (self.image_num_patch[0] + 1) // 2, + (self.image_num_patch[1] + 1) // 2, + ) + self.image_vit = VisionTransformer(vision_config, + quant_config=quant_config) + self.num_prefix_tokens = self.image_vit.num_prefix_tokens + assert self.num_prefix_tokens in { + 0, 1 + }, "Only 0 or 1 prefix tokens are supported" + self.image_pooling_2d = MultiHeadDotProductAttention( + vision_config, + nlayers=len(self.vit_layers), + quant_config=quant_config) + self.image_projector = MolmoMLP( + config, + input_dim=vision_config.image_emb_dim, + quant_config=quant_config, + ) + + image_dim = vision_config.image_emb_dim * len(self.vit_layers) + self.pad_embed = nn.Parameter(torch.zeros((2, image_dim))) + + @property + def dtype(self) -> torch.dtype: + return self.image_vit.patch_embedding.weight.dtype + + @property + def device(self) -> torch.device: + return self.image_vit.patch_embedding.weight.device + + def encode_image(self, images: torch.Tensor) -> torch.Tensor: + """ + : param images: (batch_size, num_crops, num_patch, n_pixels) + """ + B, T, N, D = images.shape + + mask = ~torch.all( + images.view(B * T, N, D) == -1, dim=(1, 2), keepdim=True) + + images = images.view(B * T, N, D) + image_features = self.image_vit(images) + + if self.vit_layers is not None: + features = [] + for layer in self.vit_layers: + features.append(image_features[layer]) + image_features = torch.cat(features, dim=-1) + else: + image_features = image_features[-1] + + if self.num_prefix_tokens > 0: + image_features = image_features[:, 1:] + + image_features = image_features * mask + image_features = image_features.view(B, T, N, -1) + + return image_features + + def forward( + self, images: torch.Tensor, image_masks: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + + # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim) # noqa: E501 + batch_size, num_image = images.shape[:2] + images = images.to(device=self.device, dtype=self.dtype) + image_features = self.encode_image(images) + + og_dtype = image_features.dtype + assert image_masks is not None + pad_embed = self.pad_embed[:, None, None, None, :] + all_pad = image_masks == 0 + partial_pad = torch.logical_and( + image_masks < 1, + torch.logical_not(all_pad)).to(dtype=torch.float32) + all_pad = all_pad.to(dtype=torch.float32) + image_features = image_features + pad_embed[0] * torch.unsqueeze( + all_pad, -1) + image_features = image_features + pad_embed[1] * torch.unsqueeze( + partial_pad, -1) + + image_features = image_features.to(og_dtype) + + image_features = image_features.reshape( + (batch_size, num_image) + self.image_num_patch + (-1, ), ) + + if self.image_num_patch[0] % 2 == 1: + # Pad so we can still pool 2x2 patches + image_features = F.pad( + image_features, + (0, 0, 0, 1, 0, 1, 0, 0, 0, 0), + ) + + # image pooling + image_features = rearrange( + image_features, + 'b n (h dh) (w dw) c -> (b n h w) (dh dw) c', + dh=2, + dw=2, + ) + + query = image_features.mean(-2, keepdim=True) + image_features = self.image_pooling_2d(query, image_features) + + h, w = self.llm_patches_per_crop + image_features = image_features.view(batch_size, num_image, h * w, -1) + + image_features = self.image_projector(image_features) + + # image_features: (batch_size, num_image, num_patch, d_model) + return image_features + + +class MolmoModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + + self.embedding_size = config.embedding_size or config.vocab_size + self.embedding_size += ADDITIONAL_VOCAB_SIZE + self.embed_tokens = VocabParallelEmbedding( + self.embedding_size, + config.hidden_size, + quant_config=quant_config, + ) + + decoder_layer = MolmoDecoderNormAfterLayer if config.norm_after \ + else MolmoDecoderLayer + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: decoder_layer(config, cache_config, quant_config), + prefix=f"{prefix}.layers", + ) + + assert config.layer_norm_type == "rms" + self.norm = RMSNorm(config.hidden_size, config.layer_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + # Apply blocks one-by-one. + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + if residual is not None: + hidden_states, _ = self.norm(hidden_states, residual) + else: + hidden_states = self.norm(hidden_states) + return hidden_states + + +cached_get_processor = lru_cache(get_processor) + + +def get_num_patches(num_tiles: int, crop_patches: int, left_margin: int, + right_margin: int, pooling_size: int) -> int: + crop_window_patches = crop_patches - (left_margin + right_margin) + if num_tiles > 1: + left_crop_window_patches = (crop_window_patches + left_margin + + pooling_size - + 1) // pooling_size * pooling_size + middle_crop_window_patches = (crop_window_patches + pooling_size - + 1) // pooling_size * pooling_size + right_crop_window_patches = (crop_window_patches + right_margin + + pooling_size - + 1) // pooling_size * pooling_size + return left_crop_window_patches + ( + num_tiles - + 2) * middle_crop_window_patches + right_crop_window_patches + else: + single_crop_window_patches = (crop_patches + pooling_size - + 1) // pooling_size * pooling_size + return single_crop_window_patches + + +def get_tokens(tiling_h: int, tiling_w: int, crop_patches: int, + left_margin: int, right_margin: int, pooling_size: int) -> int: + h = get_num_patches(tiling_h, crop_patches, left_margin, right_margin, + pooling_size) + w = get_num_patches(tiling_w, crop_patches, left_margin, right_margin, + pooling_size) + per_row = w // pooling_size + 1 + joint = per_row * (h // pooling_size) + 2 + image_token_length = (crop_patches + pooling_size - 1) // pooling_size + resize = (image_token_length + 1) * image_token_length + 2 + return resize + joint + + +def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int, + right_margin: int, pooling_size: int) -> int: + tilings = [] + for i in range(1, max_crops + 1): + for j in range(1, max_crops + 1): + if i * j <= max_crops: + tilings.append((i, j)) + tokens = [ + get_tokens(tilings[i][0], tilings[i][1], crop_patches, left_margin, + right_margin, pooling_size) for i in range(len(tilings)) + ] + return max(tokens) + + +def get_max_molmo_image_tokens(ctx: InputContext) -> int: + processor = cached_get_processor(ctx.model_config.model, + trust_remote_code=True, + revision=ctx.model_config.code_revision) + image_processor = processor.image_processor + max_llm_image_tokens = get_max_tokens( + image_processor.max_crops, + image_processor.base_image_input_size[0] // + image_processor.image_patch_size, + image_processor.overlap_margins[0], + image_processor.overlap_margins[1], + 2, + ) + return max_llm_image_tokens + + +# NOTE: preprocessing for the image data has been included in the +# 'input_processor_for_molmo' function +def image_input_mapper_for_molmo( + ctx: InputContext, + data: object, +): + return MultiModalInputs(data) + + +def dummy_data_for_molmo(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + processor = cached_get_processor(ctx.model_config.model, + trust_remote_code=True, + revision=ctx.model_config.code_revision) + image_processor = processor.image_processor + + base_image_input_d = image_processor.image_patch_size + left_margin, right_margin = image_processor.overlap_margins + max_crops = image_processor.max_crops + + # Assume: prompt_token_ids always starts with bos_token_id followed image tokens # noqa: E501 + max_llm_image_tokens = get_max_molmo_image_tokens(ctx) + if seq_len - max_llm_image_tokens - 1 < 0: + raise RuntimeError( + f"Molmo cannot process {max_crops} crops in a prompt, " + "please increase max_model_len or reduce number of crops") + + # The vertical image has the maximum number of image tokens due to column tokens. # noqa: E501 + tiling = (max_crops, 1) + total_margin_pixels = base_image_input_d * (right_margin + left_margin) + crop_patches = image_processor.base_image_input_size[ + 0] // base_image_input_d + crop_window_patches = crop_patches - (right_margin + left_margin) + crop_window_size = crop_window_patches * base_image_input_d + + h = crop_window_size * tiling[0] + total_margin_pixels + w = crop_window_size * tiling[1] + total_margin_pixels + + dummy_image = Image.new("RGB", (w, h), color="red") + + out = processor.process("dummy prompt", dummy_image) + + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + out["input_ids"][:1 + max_llm_image_tokens]) + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, + [0]) * (seq_len - max_llm_image_tokens - 1) + dummy_seqdata = SequenceData(token_ids) + dummy_imgdata = { + "images": out["images"], + "image_input_idx": out["image_input_idx"], + } + if "image_masks" in out: + dummy_imgdata["image_masks"] = out["image_masks"] + dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long) + return dummy_seqdata, {"image": dummy_imgdata} + + +def pad_images( + max_total_crops: int, + images: torch.Tensor, + image_input_idx: torch.Tensor, + image_masks: Optional[torch.Tensor] = None, +): + n = max_total_crops - images.shape[0] + images = F.pad(images, (0, 0, 0, 0, 0, n), value=-1) + image_input_idx = F.pad(image_input_idx, (0, 0, 0, n), value=-1) + if image_masks is not None: + image_masks = F.pad(image_masks, (0, 0, 0, n), value=-1) + return images, image_input_idx, image_masks + + +def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs): + prompt = llm_inputs["prompt"] + multi_modal_data = llm_inputs.get("multi_modal_data") + image = multi_modal_data.get("image") + processor = cached_get_processor(ctx.model_config.model, + trust_remote_code=True, + revision=ctx.model_config.code_revision) + + # NOTE: message formatting for raw text prompt is only applied for + # offline inference; for online inference, the prompt is always in + # instruction format and tokenized. + if prompt is not None and re.match(r"^User:[\s\S]*?(Assistant:)*$", + prompt): + out = processor.process(prompt, image, message_format="none") + elif prompt is not None: + out = processor.process(prompt, image) + else: + out = processor.process(None, + image, + tokens=llm_inputs["prompt_token_ids"]) + + image_processor = processor.image_processor + max_total_crops = 1 + image_processor.max_crops + if image is not None: + images, image_input_idx, image_masks = pad_images( + max_total_crops, + out["images"], + out["image_input_idx"], + out.get("image_masks"), + ) + else: + base_image_input_size = image_processor.base_image_input_size + image_patch_size = image_processor.image_patch_size + image_num_patch = ( + base_image_input_size[0] // image_patch_size, + base_image_input_size[1] // image_patch_size, + ) + n_pixels = image_patch_size * image_patch_size * 3 + n_patches = image_num_patch[0] * image_num_patch[1] + + image_length_w = image_processor.image_token_length_w + image_length_h = image_processor.image_token_length_h + tokens_per_image = image_length_w * image_length_h + images = torch.full( + (max_total_crops, n_patches, n_pixels), + -1, + dtype=torch.float32, + ) + image_input_idx = torch.full( + (max_total_crops, tokens_per_image), + -1, + dtype=torch.int32, + ) + if image_processor.image_padding_mask: + image_masks = torch.full( + (max_total_crops, n_patches), + -1, + dtype=torch.float32, + ) + + image_data = dict( + images=images, + image_input_idx=image_input_idx, + ) + if image_masks is not None: + image_data["image_masks"] = image_masks + + image_data["seq_len"] = torch.tensor(len(out["input_ids"]), + dtype=torch.long) + + multi_modal_data = dict(image=image_data) + + return LLMInputs( + prompt_token_ids=out["input_ids"], + prompt=llm_inputs["prompt"], + multi_modal_data=multi_modal_data, + ) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(image_input_mapper_for_molmo) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo) +@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo) +class MolmoForCausalLM(nn.Module, SupportsMultiModal): + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: Optional[MultiModalConfig] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[Mapping[str, Any]] = None, + ) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + vision_config = VisionBackboneConfig() + self.vision_backbone = MolmoVisionBackbone(config, vision_config, + quant_config) + self.model = MolmoModel(config, cache_config, quant_config) + + if self.config.weight_tying: + self.lm_head = self.model.transformer.wte + else: + self.lm_head = ParallelLMHead( + config.embedding_size or config.vocab_size, + config.hidden_size, + quant_config=quant_config, + ) + + self.logits_processor = LogitsProcessor(config.embedding_size + or config.vocab_size) + self.sampler = Sampler() + + def _parse_and_validate_image_input( + self, + **kwargs: object, + ) -> Optional[MolmoImageInputs]: + images = kwargs.pop("images", None) + image_masks = kwargs.pop("image_masks", None) + if images is None: + return None + + image_input_idx = kwargs.pop("image_input_idx", None) + seq_len = kwargs.pop("seq_len", None) + if image_input_idx is None: + raise ValueError("image_input_idx is required for Molmo model.") + if seq_len is None: + raise ValueError("seq_len is required for Molmo model.") + if not isinstance(seq_len, torch.Tensor): + seq_len = torch.tensor(seq_len) + + return MolmoImageInputs( + images=images, + image_input_idx=image_input_idx, + seq_len=seq_len, + image_masks=image_masks, + ) + + def _process_image_input( + self, + image_input: MolmoImageInputs, + ) -> torch.Tensor: + + image_features = self.vision_backbone( + images=image_input["images"], + image_masks=image_input["image_masks"], + ) + + return image_features + + def _merge_multimodal_embeddings( + self, + inputs_embeds: torch.Tensor, + image_features: torch.Tensor, + image_input_idx: torch.Tensor, + seq_len: Union[torch.Tensor, List[torch.Tensor]], + ) -> torch.Tensor: + batch_size, num_image, num_patch = image_features.shape[:3] + assert image_input_idx.shape == (batch_size, num_image, num_patch) + + image_features = image_features.to(inputs_embeds.device) + seq_len = seq_len.to(inputs_embeds.device) + + # insert the image feature into the embedding. + image_features = image_features.view(batch_size, num_image * num_patch, + -1) + image_input_idx = image_input_idx.view(batch_size, + num_image * num_patch) + + valid = image_input_idx >= 0 + image_features = image_features * valid[:, :, None].to( + image_features.dtype) + image_features = image_features.view( + batch_size * num_image * num_patch, -1).contiguous() + + image_input_idx = image_input_idx * valid.to(image_input_idx.dtype) + offset = torch.cat( + [seq_len.new_zeros( + (1)), seq_len.cumsum(dim=0)[:-1]], dim=0)[:, None] + image_input_idx = image_input_idx + offset.to(image_input_idx.dtype) + image_input_idx = image_input_idx.flatten()[:, None] + mat = image_input_idx == torch.arange( + seq_len.sum().item(), device=inputs_embeds.device)[None, :] + mat = mat.to(image_features.dtype) + + inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md', + image_features, mat) + + return inputs_embeds + + def forward( + self, + input_ids: torch.LongTensor, + positions: torch.LongTensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + **kwargs: object, + ) -> SamplerOutput: + + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + inputs_embeds = self.model.embed_tokens(input_ids) + image_features = self._process_image_input(image_input) + + inputs_embeds = self._merge_multimodal_embeddings( + inputs_embeds, + image_features, + image_input["image_input_idx"], + image_input["seq_len"], + ) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + inputs_embeds=inputs_embeds, + ) + + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + params_mapping = [ + ("model.transformer.ln_f.weight", "model.norm.weight"), + ("attn_out", "self_attn.o_proj"), + ("att_proj", "self_attn.qkv_proj"), + ("q_norm", "self_attn.q_norm"), + ("k_norm", "self_attn.k_norm"), + ("attn_norm", "input_layernorm"), + ("ff_norm", "post_attention_layernorm"), + ] + + params_dict = dict(self.named_parameters(remove_duplicate=False)) + + embedding_weight = dict() + projector_weight = dict() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + + if "wte.embedding" in name: + embedding_weight["embedding"] = loaded_weight + continue + + if "wte.new_embedding" in name: + embedding_weight["new_embedding"] = loaded_weight + continue + + if "vision_backbone" in name: + if name.startswith("model"): + name = name[len("model."):] + if 'image_projector' in name: + if 'w1' in name: + projector_weight['gate_proj'] = loaded_weight + elif 'w3' in name: + projector_weight['up_proj'] = loaded_weight + elif 'w2' in name: + projector_weight['down_proj'] = loaded_weight + else: + raise ValueError( + f"Unexpected projector weight: {name}") + continue + else: + if "transformer.blocks" in name: + name = name.replace("transformer.blocks", "layers") + + if "ff_proj" in name: + name = name.replace("ff_proj", "mlp.gate_up_proj") + assert 'weight' in name + up_weight, gate_weight = loaded_weight.chunk(2, dim=0) + loaded_weight = torch.cat([gate_weight, up_weight], dim=0) + + elif "ff_out" in name: + if "layers" in name: + name = name.replace("ff_out", "mlp.down_proj") + else: + # lm head + name = name.replace("model.transformer.ff_out", + "lm_head") + + else: + for (param_name, weight_name) in params_mapping: + if param_name in name: + name = name.replace(param_name, weight_name) + break + + try: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + except KeyError: + raise ValueError(f"Unexpected weight: {name}") from None + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + gate_up_proj_weight = torch.cat( + [projector_weight["gate_proj"], projector_weight["up_proj"]], + dim=0) + name = "vision_backbone.image_projector.gate_up_proj.weight" + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, gate_up_proj_weight) + + down_proj_weight = projector_weight["down_proj"] + name = "vision_backbone.image_projector.down_proj.weight" + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, down_proj_weight) + + embedding_weight = torch.cat( + [embedding_weight["embedding"], embedding_weight["new_embedding"]], + dim=0) + name = "model.embed_tokens.weight" + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, embedding_weight) diff --git a/vllm/model_executor/models/mpt.py b/vllm/model_executor/models/mpt.py new file mode 100644 index 00000000..e3d3937b --- /dev/null +++ b/vllm/model_executor/models/mpt.py @@ -0,0 +1,327 @@ +# coding=utf-8 +# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.mpt import MPTConfig + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +def _get_alibi_slopes( + total_num_heads: int, + alibi_bias_max: int, +) -> torch.Tensor: + next_power_of_2 = 2**math.ceil(math.log2(total_num_heads)) + m = torch.arange(1, next_power_of_2 + 1, dtype=torch.float32) + m = m.mul(alibi_bias_max / next_power_of_2) + slopes = 1.0 / torch.pow(2, m) + if next_power_of_2 != total_num_heads: + slopes = torch.concat([slopes[1::2], slopes[::2]])[:total_num_heads] + return slopes + + +class MPTAttention(nn.Module): + + def __init__( + self, + config: MPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.d_model = config.d_model + self.total_num_heads = config.n_heads + self.head_dim = self.d_model // self.total_num_heads + self.clip_qkv = config.attn_config["clip_qkv"] + self.qk_ln = config.attn_config["qk_ln"] + self.alibi_bias_max = config.attn_config["alibi_bias_max"] + if "kv_n_heads" in config.attn_config: + self.total_num_kv_heads = config.attn_config['kv_n_heads'] + else: + self.total_num_kv_heads = self.total_num_heads + assert not config.attn_config["prefix_lm"] + assert config.attn_config["alibi"] + + # pylint: disable=invalid-name + self.Wqkv = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=not config.no_bias, + quant_config=quant_config, + ) + if self.qk_ln: + self.q_ln = nn.LayerNorm(self.d_model) + self.k_ln = nn.LayerNorm(self.d_model) + self.out_proj = RowParallelLinear( + self.d_model, + self.d_model, + bias=not config.no_bias, + quant_config=quant_config, + ) + + tp_world_size = get_tensor_model_parallel_world_size() + assert self.total_num_heads % tp_world_size == 0 + self.num_heads = self.total_num_heads // tp_world_size + + if self.total_num_kv_heads >= tp_world_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_world_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_world_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + # Create the alibi slopes and slice them. + tp_rank = get_tensor_model_parallel_rank() + head_start = tp_rank * self.num_heads + head_end = (tp_rank + 1) * self.num_heads + alibi_slopes = _get_alibi_slopes(self.total_num_heads, + self.alibi_bias_max) + alibi_slopes = alibi_slopes[head_start:head_end].tolist() + + self.head_dim = self.d_model // self.total_num_heads + scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + scaling, + alibi_slopes=alibi_slopes, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + del position_ids # unused. + qkv, _ = self.Wqkv(hidden_states) + if self.clip_qkv is not None: + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.qk_ln: + q = self.q_ln(q) + k = self.k_ln(k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.out_proj(attn_output) + return output + + +class MPTMLP(nn.Module): + + def __init__( + self, + config: MPTConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.d_model + expansion_ratio = config.expansion_ratio + intermediate_size = expansion_ratio * hidden_size + self.up_proj = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=not config.no_bias, + quant_config=quant_config, + ) + self.act = get_act_fn("gelu", quant_config, intermediate_size) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=not config.no_bias, + quant_config=quant_config, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.up_proj(x) + x = self.act(x) + x, _ = self.down_proj(x) + return x + + +class MPTBlock(nn.Module): + + def __init__( + self, + config: MPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + hidden_size = config.d_model + self.norm_1 = nn.LayerNorm(hidden_size) + self.attn = MPTAttention(config, cache_config, quant_config) + self.norm_2 = nn.LayerNorm(hidden_size) + self.ffn = MPTMLP(config, quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + x = self.norm_1(hidden_states) + x = self.attn( + position_ids=position_ids, + hidden_states=x, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = hidden_states + x + x = self.norm_2(hidden_states) + x = self.ffn(x) + hidden_states = hidden_states + x + return hidden_states + + +class MPTModel(nn.Module): + + def __init__( + self, + config: MPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + assert config.embedding_fraction == 1.0 + assert config.norm_type == "low_precision_layernorm" + + self.wte = VocabParallelEmbedding( + config.vocab_size, + config.d_model, + ) + self.start_layer, self.end_layer, self.blocks = make_layers( + config.n_layers, + lambda prefix: MPTBlock(config, cache_config, quant_config), + prefix=f"{prefix}.blocks") + self.norm_f = nn.LayerNorm(config.d_model) + if config.no_bias: + for module in self.modules(): + if hasattr(module, "bias") and isinstance( + module.bias, nn.Parameter): + # Remove the bias term in Linear and LayerNorm. + module.register_parameter("bias", None) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.d_model)) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.wte(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + for i in range(self.start_layer, self.end_layer): + block = self.blocks[i] + hidden_states = block( + position_ids, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.norm_f(hidden_states) + return hidden_states + + +class MPTForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: MPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + assert config.tie_word_embeddings + self.quant_config = quant_config + + self.transformer = MPTModel(config, cache_config, quant_config) + self.lm_head = self.transformer.wte + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/nemotron.py b/vllm/model_executor/models/nemotron.py new file mode 100644 index 00000000..14515e16 --- /dev/null +++ b/vllm/model_executor/models/nemotron.py @@ -0,0 +1,525 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Nemotron model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs import NemotronConfig + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +# The architecture is pretty similar to Llama, with these changes: +# - There is no gate_proj, just up_proj +# - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm +# - Squared ReLU instead of SwiGLU +# - Adds a partial_rotary_factor to RoPE + + +def _cast_if_autocast_enabled(*args): + if not torch.is_autocast_enabled(): + return args + else: + return torch.cuda.amp.autocast_mode._cast( + args, torch.get_autocast_gpu_dtype()) + + +class NemotronLayerNorm1P(nn.LayerNorm): + + def __init__(self, + normalized_shape: Union[int, List[int], torch.Size], + eps: float = 1e-5, + elementwise_affine: bool = True, + bias: bool = True, + device=None, + dtype=None): + super().__init__(normalized_shape, eps, elementwise_affine, bias, + device, dtype) + + def forward( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if residual is not None: + x = x + residual + residual = x + args = _cast_if_autocast_enabled(x, self.normalized_shape, + self.weight + 1, self.bias, self.eps) + with torch.cuda.amp.autocast(enabled=False): + x = torch.nn.functional.layer_norm(*args) + return x if residual is None else (x, residual) + + +class NemotronMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.up_proj = ColumnParallelLinear(input_size=hidden_size, + output_size=intermediate_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.up_proj") + self.down_proj = RowParallelLinear(input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj") + self.act_fn = get_act_fn(hidden_act) + + def forward(self, x): + up, _ = self.up_proj(x) + x = self.act_fn(up) + x, _ = self.down_proj(x) + return x + + +class NemotronAttention(nn.Module): + + def __init__( + self, + config: NemotronConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", + self.hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.partial_rotary_factor = config.partial_rotary_factor + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + partial_rotary_factor=self.partial_rotary_factor, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class NemotronDecoderLayer(nn.Module): + + def __init__( + self, + config: NemotronConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + self.self_attn = NemotronAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + cache_config=cache_config, + prefix=f"{prefix}.self_attn", + ) + self.mlp = NemotronMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = NemotronLayerNorm1P(config.hidden_size, + eps=config.norm_eps) + self.post_attention_layernorm = NemotronLayerNorm1P( + config.hidden_size, eps=config.norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class NemotronModel(nn.Module): + + def __init__( + self, + config: NemotronConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: NemotronDecoderLayer(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix), + prefix=f"{prefix}.layers") + if get_pp_group().is_last_rank: + self.norm = NemotronLayerNorm1P(config.hidden_size, + eps=config.norm_eps) + else: + self.norm = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", "o_proj", "up_proj", "down_proj", "embed_tokens", "lm_head" + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + } + + def __init__( + self, + config: NemotronConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + assert isinstance(config, NemotronConfig) + + self.config = config + self.lora_config = lora_config + + self.model = NemotronModel(config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model") + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + self.sampler = Sampler() + else: + self.lm_head = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return model_output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/nvlm_d.py b/vllm/model_executor/models/nvlm_d.py new file mode 100644 index 00000000..a52e3cb6 --- /dev/null +++ b/vllm/model_executor/models/nvlm_d.py @@ -0,0 +1,64 @@ +# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py +# -------------------------------------------------------- +# NVLM-D +# Copyright (c) 2024 NVIDIA +# Licensed under Apache 2.0 License [see LICENSE for details] +# -------------------------------------------------------- +import torch.nn as nn +from transformers import PretrainedConfig + +from vllm.inputs import INPUT_REGISTRY +from vllm.multimodal import MULTIMODAL_REGISTRY + +from .intern_vit import InternVisionModel +from .internvl import (InternVLChatModel, InternVLInputPipeline, + get_max_internvl_image_tokens) + +IMG_START = '<|vision_start|>' +IMG_END = '<|vision_end|>' +IMG_CONTEXT = '<|vision_pad|>' + + +class NVLMInputPipeline(InternVLInputPipeline): + + def _create_image_prompt(self, feature_size: int, num_patches: int) -> str: + tile_pos_identifiers = ([f"" + for i in range(1, num_patches)] + + [""]) + context_size = feature_size // num_patches + + return '' + ''.join( + tile_pos_identifier + self.img_context_token * context_size + for tile_pos_identifier in tile_pos_identifiers) + '' + + +input_pipeline = NVLMInputPipeline(IMG_START, IMG_END, IMG_CONTEXT) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens) +@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data) +@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor) +class NVLM_D_Model(InternVLChatModel): + + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential: + vit_hidden_size = config.vision_config.hidden_size + llm_intermediate_size = config.text_config.intermediate_size + llm_hidden_size = config.text_config.hidden_size + + return nn.Sequential( + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2), + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2, + llm_intermediate_size, + bias=False), + nn.GELU(), + nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False), + ) + + def _init_vision_model(self, config: PretrainedConfig, + num_hidden_layers: int): + # We added additional dummy heads to the original num of heads to make + # the number of heads divisible by 8. + return InternVisionModel(config.vision_config, + num_hidden_layers_override=num_hidden_layers, + num_dummy_heads=7) diff --git a/vllm/model_executor/models/olmo.py b/vllm/model_executor/models/olmo.py new file mode 100644 index 00000000..5ca7c66f --- /dev/null +++ b/vllm/model_executor/models/olmo.py @@ -0,0 +1,394 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.40.1/src/transformers/models/olmo/modeling_olmo.py +# Copyright 2024 The vLLM team. +# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only OLMo model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import OlmoConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class OlmoAttention(nn.Module): + """ + This is the attention block where the output is computed as + ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))`` + (plus another skip connection). + """ + + def __init__( + self, + config: OlmoConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + self.total_num_heads = config.num_attention_heads + + assert self.hidden_size % self.total_num_heads == 0 + assert self.total_num_heads % tensor_model_parallel_world_size == 0 + + self.num_heads = (self.total_num_heads // + tensor_model_parallel_world_size) + self.head_dim = self.hidden_size // self.total_num_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.clip_qkv = config.clip_qkv + + # Attention input projection. Projects x -> (q, k, v) + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + bias=config.attention_bias, + quant_config=quant_config, + ) + + # Rotary embeddings. + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + ) + self.scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scaling, + cache_config=cache_config, + quant_config=quant_config) + + # Attention output projection. + self.o_proj = RowParallelLinear( + self.hidden_size, + self.hidden_size, + bias=config.attention_bias, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + if self.clip_qkv is not None: + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv) + q, k, v = qkv.chunk(chunks=3, dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class OlmoMLP(nn.Module): + """ + This is the MLP block where the output is computed as + ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))`` + (plus another skip connection). + """ + + def __init__( + self, + config: OlmoConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + + # Feed-forward input projection. + self.gate_up_proj = MergedColumnParallelLinear( + self.hidden_size, + [self.intermediate_size] * 2, + bias=False, + quant_config=quant_config, + ) + + # Activation function. + self.act_fn = SiluAndMul() + + # Feed-forward output projection. + self.down_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=False, + quant_config=quant_config, + ) + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class OlmoDecoderLayer(nn.Module): + """ + This is a typical transformer block where the output is + computed as ``MLP(LN(x + Attention(LN(x))))`` + (plus another skip connection). + """ + + def __init__(self, + config: OlmoConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + # Attention block. + self.self_attn = OlmoAttention(config, cache_config, quant_config) + + # MLP block. + self.mlp = OlmoMLP(config, quant_config) + + # LayerNorm + self.input_layernorm = nn.LayerNorm(config.hidden_size, + elementwise_affine=False, + bias=False) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + elementwise_affine=False, + bias=False) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]: + # Attention block. + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn(positions, hidden_states, kv_cache, + attn_metadata) + hidden_states = hidden_states + residual + + # MLP block. + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +class OlmoModel(nn.Module): + + def __init__(self, + config: OlmoConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + self.config = config + + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: OlmoDecoderLayer(config, cache_config, quant_config + ), + prefix=f"{prefix}.layers") + self.norm = nn.LayerNorm(config.hidden_size, + elementwise_affine=False, + bias=False) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + """ + :param input_ids: A tensor of shape `(batch_size, seq_len)`. + """ + if get_pp_group().is_first_rank: + # Get embeddings of input. + # shape: (batch_size, seq_len, d_model) + inputs_embeds = self.embed_tokens(input_ids) + + # embed positions + hidden_states = inputs_embeds + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + # Apply blocks one-by-one. + for i in range(self.start_layer, self.end_layer): + # shape: (batch_size, seq_len, d_model) + hidden_states = self.layers[i]( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + # Apply final layer norm. + # shape: (batch_size, seq_len or 1, d_model) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class OlmoForCausalLM(nn.Module, SupportsPP): + """ + Extremely barebones HF model wrapper. + """ + + def __init__(self, + config: OlmoConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + self.model = OlmoModel(config, cache_config, quant_config) + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.unpadded_vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=quant_config, + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + # With tie_word_embeddings, we can skip lm_head.weight + # The weight might appear unnecessarily in the files if the model is + # processed with quantization, LoRA, fine-tuning, etc. + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/olmoe.py b/vllm/model_executor/models/olmoe.py new file mode 100644 index 00000000..a1ba80e0 --- /dev/null +++ b/vllm/model_executor/models/olmoe.py @@ -0,0 +1,445 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only OLMoE model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import print_warning_once + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class OlmoeMoE(nn.Module): + """A tensor-parallel MoE implementation for Olmoe that shards each expert + across all ranks. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__(self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + prefix: str = ""): + super().__init__() + self.hidden_size = hidden_size + + # Gate always runs at half / full precision for now. + self.gate = ReplicatedLinear(hidden_size, + num_experts, + bias=False, + quant_config=None) + + self.experts = FusedMoE(num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + reduce_results=True, + renormalize=False, + quant_config=quant_config, + tp_size=tp_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_dim = hidden_states.shape[-1] + hidden_states = hidden_states.view(-1, hidden_dim) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states=hidden_states, + router_logits=router_logits) + return final_hidden_states.view(orig_shape) + + +class OlmoeAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 4096, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.q_norm = RMSNorm(hidden_size, eps=1e-5) + self.k_norm = RMSNorm(hidden_size, eps=1e-5) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.q_norm(q.contiguous()), self.k_norm(k.contiguous()) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class OlmoeDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 4096) + + self.self_attn = OlmoeAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + + self.mlp = OlmoeMoE( + num_experts=config.num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5) + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class OlmoeModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: OlmoeDecoderLayer(config, int( + prefix.split(".")[-1]), cache_config, quant_config), + prefix=f"{prefix}.layers") + self.norm = RMSNorm(config.hidden_size, eps=1e-5) + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class OlmoeForCausalLM(nn.Module, SupportsPP): + + fall_back_to_pt_during_load = False + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = OlmoeModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if "mlp.experts" in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Remapping the name of FP8 kv-scale. + if name.endswith("kv_scale"): + remapped_kv_scale_name = name.replace( + ".kv_scale", ".attn.kv_scale") + if remapped_kv_scale_name not in params_dict: + print_warning_once( + "Found kv scale in the checkpoint " + f"(e.g. {name}), but not found the expected " + f"name in the model " + f"(e.g. {remapped_kv_scale_name}). " + "kv-scale is not loaded.") + continue + else: + name = remapped_kv_scale_name + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/opt.py b/vllm/model_executor/models/opt.py new file mode 100644 index 00000000..3bcdb0d8 --- /dev/null +++ b/vllm/model_executor/models/opt.py @@ -0,0 +1,416 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py +# Copyright 2023 The vLLM team. +# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights +# reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only OPT model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import OPTConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class OPTLearnedPositionalEmbedding(nn.Embedding): + + def __init__(self, num_embeddings: int, embedding_dim: int): + # OPT is set up so that if padding_idx is specified then offset the + # embedding ids by 2 and adjust num_embeddings appropriately. Other + # models don't have this hack + self.offset = 2 + super().__init__(num_embeddings + self.offset, embedding_dim) + + def forward(self, positions: torch.Tensor): + return super().forward(positions + self.offset) + + +class OPTAttention(nn.Module): + + def __init__( + self, + embed_dim: int, + num_heads: int, + bias: bool = True, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.embed_dim = embed_dim + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + total_num_heads = num_heads + assert num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = total_num_heads // tensor_model_parallel_world_size + self.head_dim = embed_dim // total_num_heads + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + embed_dim, + self.head_dim, + total_num_heads, + bias=bias, + quant_config=quant_config, + ) + self.out_proj = RowParallelLinear( + embed_dim, + embed_dim, + bias=bias, + quant_config=quant_config, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.out_proj(attn_output) + return output + + +class OPTDecoderLayer(nn.Module): + + def __init__( + self, + config: OPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.self_attn = OPTAttention( + embed_dim=self.embed_dim, + num_heads=config.num_attention_heads, + bias=config.enable_bias, + cache_config=cache_config, + quant_config=quant_config, + ) + self.do_layer_norm_before = config.do_layer_norm_before + + self.self_attn_layer_norm = nn.LayerNorm( + self.embed_dim, + elementwise_affine=config.layer_norm_elementwise_affine) + self.fc1 = ColumnParallelLinear( + self.embed_dim, + config.ffn_dim, + bias=config.enable_bias, + quant_config=quant_config, + ) + self.activation_fn = get_act_fn(config.activation_function, + quant_config, config.ffn_dim) + self.fc2 = RowParallelLinear( + config.ffn_dim, + self.embed_dim, + bias=config.enable_bias, + quant_config=quant_config, + ) + self.final_layer_norm = nn.LayerNorm( + self.embed_dim, + elementwise_affine=config.layer_norm_elementwise_affine) + + def forward( + self, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # Self Attention + residual = hidden_states + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention + if self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states = self.self_attn(hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata) + hidden_states = residual + hidden_states + # 350m applies layer norm AFTER attention + if not self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Fully Connected + residual = hidden_states + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention + if self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + hidden_states = residual + hidden_states + # 350m applies layer norm AFTER attention + if not self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) + return hidden_states + + +class OPTDecoder(nn.Module): + + def __init__( + self, + config: OPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_position_embeddings + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.word_embed_proj_dim, + ) + # Positional embeddings are replicated (not sharded). + self.embed_positions = OPTLearnedPositionalEmbedding( + config.max_position_embeddings, config.hidden_size) + + # Project out & in will be replicated if they exist. + if config.word_embed_proj_dim != config.hidden_size: + self.project_out = ReplicatedLinear(config.hidden_size, + config.word_embed_proj_dim, + bias=False, + quant_config=quant_config) + else: + self.project_out = None + + if config.word_embed_proj_dim != config.hidden_size: + self.project_in = ReplicatedLinear(config.word_embed_proj_dim, + config.hidden_size, + bias=False, + quant_config=quant_config) + else: + self.project_in = None + + # Note that the only purpose of `config._remove_final_layer_norm` is to + # keep backward compatibility with checkpoints that have been fine-tuned + # before transformers v4.20.1 + # see https://github.com/facebookresearch/metaseq/pull/164 + if config.do_layer_norm_before and not config._remove_final_layer_norm: + self.final_layer_norm = nn.LayerNorm( + config.hidden_size, + elementwise_affine=config.layer_norm_elementwise_affine) + else: + self.final_layer_norm = None + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: OPTDecoderLayer(config, cache_config, quant_config), + prefix=f"{prefix}.layers") + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings(input_ids) + pos_embeds = self.embed_positions(positions) + if self.project_in is not None: + inputs_embeds, _ = self.project_in(inputs_embeds) + hidden_states = inputs_embeds + pos_embeds + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer(hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + if self.final_layer_norm is not None: + hidden_states = self.final_layer_norm(hidden_states) + if self.project_out is not None: + hidden_states, _ = self.project_out(hidden_states) + return hidden_states + + +class OPTModel(nn.Module): + + def __init__( + self, + config: OPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.decoder = OPTDecoder(config, cache_config, quant_config) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.decoder.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + return self.decoder(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + +class OPTForCausalLM(nn.Module, SupportsPP): + + # BitandBytes specific attributes + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + } + default_bitsandbytes_target_modules = [ + ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2." + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".out_proj.", ".fc2."] + + def __init__( + self, + config: OPTConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = OPTModel(config, cache_config, quant_config) + if self.config.tie_word_embeddings: + self.lm_head = self.model.decoder.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.word_embed_proj_dim) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "lm_head.weight" in name and self.config.tie_word_embeddings: + continue + if name.startswith("decoder."): + name = "model." + name + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/orion.py b/vllm/model_executor/models/orion.py new file mode 100644 index 00000000..0913193f --- /dev/null +++ b/vllm/model_executor/models/orion.py @@ -0,0 +1,363 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/modeling_orion.py +# Copyright (c) OrionStar Inc. +# LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE +"""Inference-only Orion-14B model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class OrionMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class OrionAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class OrionDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = OrionAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + self.mlp = OrionMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states, None + + +class OrionModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: OrionDecoderLayer( + config, + cache_config, + quant_config, + ), + prefix=f"{prefix}.layers") + self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class OrionForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = OrionModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py new file mode 100644 index 00000000..99d000ea --- /dev/null +++ b/vllm/model_executor/models/paligemma.py @@ -0,0 +1,295 @@ +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union) + +import torch +from torch import nn +from transformers import PaliGemmaConfig + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.models.gemma import GemmaForCausalLM +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsMultiModal, SupportsPP +from .siglip import (SiglipVisionModel, dummy_image_for_siglip, + dummy_seq_data_for_siglip, get_max_siglip_image_tokens) +from .utils import AutoWeightsLoader, merge_multimodal_embeddings + +logger = init_logger(__name__) + + +class PaliGemmaImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: `(batch_size * num_images, num_channels, height, width)`""" + + +class PaliGemmaImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +PaliGemmaImageInputs = Union[PaliGemmaImagePixelInputs, + PaliGemmaImageEmbeddingInputs] + + +def get_max_paligemma_image_tokens(ctx: InputContext): + hf_config = ctx.get_hf_config(PaliGemmaConfig) + vision_config = hf_config.vision_config + + return get_max_siglip_image_tokens(vision_config) + + +def dummy_data_for_paligemma(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + hf_config = ctx.get_hf_config(PaliGemmaConfig) + vision_config = hf_config.vision_config + num_images = mm_counts["image"] + + seq_data = dummy_seq_data_for_siglip( + vision_config, + seq_len, + num_images, + image_token_id=hf_config.image_token_index, + ) + + mm_data = dummy_image_for_siglip(vision_config, num_images) + return seq_data, mm_data + + +def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs): + + """ + The correct prompt format needs to be: + '' * image_feature_size + '' + prompt + '\n' + + See https://github.com/huggingface/transformers/blob/25245ec26dc29bcf6102e1b4ddd0dfd02e720cf5/src/transformers/models/paligemma/processing_paligemma.py#L55 + """ # noqa + + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_config(PaliGemmaConfig) + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + image_feature_size = hf_config.text_config.num_image_tokens + image_token_str = tokenizer.decode(hf_config.image_token_index) + bos_token = tokenizer.decode(hf_config.bos_token_id) + image_token_str_pad = image_token_str * image_feature_size + image_token_ids_pad = [hf_config.image_token_index] * image_feature_size + + orig_prompt = llm_inputs.get("prompt") + orig_prompt_ids = llm_inputs.get("prompt_token_ids") + + if orig_prompt is not None and image_token_str in orig_prompt: + logger.warning( + "The image token '%s' was detected in the prompt and " + "will be removed. Please follow the proper prompt format" + " documented on HuggingFace.", image_token_str) + orig_prompt = orig_prompt.replace(image_token_str, "") + orig_prompt_ids.remove(hf_config.image_token_index) + + new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n" + new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +class PaliGemmaMultiModalProjector(nn.Module): + + def __init__(self, vision_hidden_size: int, projection_dim: int): + super().__init__() + + self.linear = nn.Linear(vision_hidden_size, projection_dim, bias=True) + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + hidden_states = self.linear(image_features) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma) +@INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma) +class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: PaliGemmaConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + self.vision_tower = SiglipVisionModel(config.vision_config) + self.multi_modal_projector = PaliGemmaMultiModalProjector( + vision_hidden_size=config.vision_config.hidden_size, + projection_dim=config.vision_config.projection_dim) + + self.quant_config = quant_config + self.language_model = GemmaForCausalLM(config.text_config, + cache_config, quant_config) + logit_scale = getattr(config, "logit_scale", 1.0) + self.language_model.logits_processor.scale *= logit_scale + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @property + def sampler(self): + return self.language_model.sampler + + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor: + h = w = self.config.vision_config.image_size + expected_dims = (3, h, w) + actual_dims = tuple(data.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("batch_size", *map(str, expected_dims)) + raise ValueError( + f"The expected shape of pixel values is {expected_expr}. " + f"You supplied {tuple(data.shape)}.") + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[PaliGemmaImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, torch.Tensor): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + # Remove the N dimension until multiple images are supported. + pixel_values = pixel_values.squeeze(1) + + return PaliGemmaImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values(pixel_values), + ) + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + # Remove the N dimension until multiple images are supported. + image_embeds = image_embeds.squeeze(1) + + return PaliGemmaImageEmbeddingInputs( + type="image_embeds", + data=image_embeds, + ) + + raise AssertionError("This line should be unreachable.") + + def _image_pixels_to_features( + self, + vision_tower: SiglipVisionModel, + pixel_values: torch.Tensor, + ) -> torch.Tensor: + + target_dtype = vision_tower.get_input_embeddings().weight.dtype + image_features = vision_tower(pixel_values.to(dtype=target_dtype)) + + return image_features + + def _process_image_input( + self, + image_input: PaliGemmaImageInputs, + ) -> torch.Tensor: + + if image_input["type"] == "image_embeds": + return image_input["data"] + + assert self.vision_tower is not None + pixel_values = image_input["data"] + image_features = self._image_pixels_to_features( + self.vision_tower, + pixel_values, + ) + + return self.multi_modal_projector(image_features) + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object) -> Union[SamplerOutput, IntermediateTensors]: + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + parsed_image_input = self._parse_and_validate_image_input(**kwargs) + + if parsed_image_input is not None: + vision_embeddings = self._process_image_input( + parsed_image_input) + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa + vision_embeddings = vision_embeddings * ( + self.config.hidden_size**-0.5) + + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.config.image_token_index) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/persimmon.py b/vllm/model_executor/models/persimmon.py new file mode 100644 index 00000000..b625d19f --- /dev/null +++ b/vllm/model_executor/models/persimmon.py @@ -0,0 +1,354 @@ +# coding=utf-8 +# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py +# Copyright 2023 The vLLM team. +# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only persimmon model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PersimmonConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class PersimmonMLP(nn.Module): + + def __init__(self, + config: PersimmonConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size, + config.intermediate_size, + quant_config=quant_config) + self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, + config.hidden_size, + quant_config=quant_config) + self.act = get_act_fn(config.hidden_act, quant_config) + + def forward(self, hidden_states) -> torch.Tensor: + hidden_states, _ = self.dense_h_to_4h(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.dense_4h_to_h(hidden_states) + return hidden_states + + +class PersimmonAttention(nn.Module): + + def __init__(self, + config: PersimmonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + tensor_parallel_world_size = get_tensor_model_parallel_world_size() + + self.hidden_size = config.hidden_size + self.total_num_heads = config.num_attention_heads + self.num_heads = self.total_num_heads // tensor_parallel_world_size + self.head_dim = self.hidden_size // self.total_num_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.partial_rotary_factor = config.partial_rotary_factor + self.is_causal = True + + assert (self.head_dim * self.total_num_heads) == self.hidden_size + assert self.total_num_heads % tensor_parallel_world_size == 0 + + self.query_key_value = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + bias=True, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=True, + quant_config=quant_config, + ) + self.is_qk_layernorm = config.qk_layernorm + + if self.is_qk_layernorm: + self.q_layernorm = nn.LayerNorm(self.head_dim) + self.k_layernorm = nn.LayerNorm(self.head_dim) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=int(self.partial_rotary_factor * self.head_dim), + max_position=self.max_position_embeddings, + base=self.rope_theta, + ) + self.scaling = self.head_dim**-0.5 + self.attn = Attention(self.num_heads, + self.head_dim, + scale=self.scaling, + cache_config=cache_config, + quant_config=quant_config) + + def _split_heads(self, x: torch.Tensor) -> torch.Tensor: + # [seq_length, hidden_size] -> [seq_length, num_heads, head_dim] + seq_length = x.shape[0] + return x.view(seq_length, self.num_heads, self.head_dim) + + def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: + # [seq_length, num_heads, head_dim] -> [seq_length, hidden_size] + seq_length = x.shape[0] + return x.view(seq_length, self.num_heads * self.head_dim) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # [seq_length, 3 x hidden_size] + qkv, _ = self.query_key_value(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + + if self.is_qk_layernorm: + # [seq_length, num_heads, head_dim] + q = self._split_heads(q) + k = self._split_heads(k) + + q = self.q_layernorm(q) + k = self.k_layernorm(k) + + q = self._merge_heads(q) + k = self._merge_heads(k) + + q, k = self.rotary_emb(position_ids, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.dense(attn_output) + return output + + +class PersimmonDecoderLayer(nn.Module): + + def __init__(self, + config: PersimmonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = PersimmonAttention(config=config, + cache_config=cache_config, + quant_config=quant_config) + self.mlp = PersimmonMLP(config, quant_config=quant_config) + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states = self.self_attn( + position_ids=position_ids, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + + hidden_states = hidden_states + residual + + outputs = hidden_states + return outputs + + +class PersimmonModel(nn.Module): + + def __init__(self, + config: PersimmonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: PersimmonDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + self.final_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_tokens(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + hidden_states = self.layers[i]( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states + + +class PersimmonForCausalLM(nn.Module, SupportsPP): + + def __init__(self, + config: PersimmonConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + self.vocab_size = config.vocab_size + self.model = PersimmonModel(config, + cache_config=cache_config, + quant_config=quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + bias=False) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ): + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + + if "query_key_value" in name: + # copy from vllm/model_executor/models/bloom.py + # NOTE: Persimmon's fused QKV's output_dim has the shape of + # (num_heads * 3 * head_size), while the + # required shape is (3 * num_heads * head_size). + # Thus, we need weight conversion. + output_dim = getattr(param, "output_dim", None) + num_heads = self.config.num_attention_heads + if output_dim is not None: + loaded_weight_shape = loaded_weight.shape + loaded_weight = loaded_weight.view( + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) + + loaded_weight_shape[output_dim + 1:]) + loaded_weight = loaded_weight.transpose( + output_dim, output_dim + 1) + loaded_weight = loaded_weight.reshape(loaded_weight_shape) + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/phi.py b/vllm/model_executor/models/phi.py new file mode 100644 index 00000000..0918f21a --- /dev/null +++ b/vllm/model_executor/models/phi.py @@ -0,0 +1,374 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py +# Copyright 2023 The vLLM team. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +# +# BSD 3-Clause License +# +# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Inference-only Phi-1.5 model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PhiConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class PhiAttention(nn.Module): + + def __init__(self, + config: PhiConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.total_num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.total_num_heads + + tensor_model_parallel_world_size = ( + get_tensor_model_parallel_world_size()) + assert self.total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = (self.total_num_heads // + tensor_model_parallel_world_size) + + # pylint: disable=C0103 + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_size, + self.total_num_heads, + bias=True, + quant_config=quant_config, + ) + self.dense = RowParallelLinear( + self.hidden_size, + self.hidden_size, + quant_config=quant_config, + ) + + scaling = self.head_size**-0.5 + rotary_dim = int(config.partial_rotary_factor * + (config.hidden_size // config.num_attention_heads)) + assert rotary_dim % 2 == 0 + + # pylint: disable=C0301 + # Refer to: + # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518 + rope_theta = 10000 + max_position_embeddings = getattr(config, "n_positions", 2048) + self.rotary_emb = get_rope( + self.head_size, + rotary_dim=rotary_dim, + max_position=max_position_embeddings, + base=rope_theta, + ) + self.attn = Attention(self.num_heads, + self.head_size, + scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + q, k = self.rotary_emb(position_ids, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.dense(attn_output) + return output + + +class PhiMLP(nn.Module): + + def __init__(self, + config: PhiConfig, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + + n_inner = getattr(config, "n_inner", None) + n_inner = n_inner if n_inner is not None else 4 * config.hidden_size + + self.fc1 = ColumnParallelLinear( + config.hidden_size, + n_inner, + quant_config=quant_config, + ) + self.fc2 = RowParallelLinear( + n_inner, + config.hidden_size, + quant_config=quant_config, + ) + self.act = get_act_fn(config.hidden_act, quant_config, n_inner) + + def forward(self, hidden_states): + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + return hidden_states + + +class PhiLayer(nn.Module): + + def __init__(self, + config: PhiConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.self_attn = PhiAttention(config, cache_config, quant_config) + self.mlp = PhiMLP(config, quant_config) + + def forward( + self, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + attn_outputs = self.self_attn( + position_ids=position_ids, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + feed_forward_hidden_states = self.mlp(hidden_states) + hidden_states = attn_outputs + feed_forward_hidden_states + residual + return hidden_states + + +class PhiModel(nn.Module): + + def __init__(self, + config: PhiConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + self.config = config + self.quant_config = quant_config + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: PhiLayer(config, cache_config, quant_config), + prefix=f"{prefix}.layers") + self.final_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ] + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "dense", + "fc1", + "fc2", + ] + + # BitandBytes specific attributes + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + } + default_bitsandbytes_target_modules = [ + ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense." + ] + # in TP, these weights are partitioned along the column dimension (dim=-1) + column_parallel_weights_modules = [".fc2.", ".dense."] + + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: PhiConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__() + + self.config = config + # lm_head use bias, cannot share word embeddings + assert not config.tie_word_embeddings + self.lora_config = lora_config + + self.quant_config = quant_config + + self.model = PhiModel(config, cache_config, quant_config) + + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + bias=True, + quant_config=quant_config) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata, self.lm_head.bias) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v") + ] + params_dict = dict(self.named_parameters()) + + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # pylint: disable=E1136 + + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/phi3.py b/vllm/model_executor/models/phi3.py new file mode 100644 index 00000000..02b2ff01 --- /dev/null +++ b/vllm/model_executor/models/phi3.py @@ -0,0 +1,17 @@ +# coding=utf-8 +# Adapted from llama.py +"""Inference-only Phi3 model code inherit from Llama.py""" + +from vllm.model_executor.models.llama import LlamaForCausalLM + + +class Phi3ForCausalLM(LlamaForCausalLM): + + packed_modules_mapping = { + "qkv_proj": [ + "qkv_proj", + ], + "gate_up_proj": [ + "gate_up_proj", + ], + } diff --git a/vllm/model_executor/models/phi3_small.py b/vllm/model_executor/models/phi3_small.py new file mode 100644 index 00000000..c9af975a --- /dev/null +++ b/vllm/model_executor/models/phi3_small.py @@ -0,0 +1,473 @@ +import math +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers.configuration_utils import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +def load_column_parallel_weight(param: torch.nn.Parameter, + loaded_weight: torch.Tensor): + tp = get_tensor_model_parallel_world_size() + rk = get_tensor_model_parallel_rank() + assert param.size(0) * tp == loaded_weight.size(0) + s = rk * param.size(0) + e = (rk + 1) * param.size(0) + loaded_weight = loaded_weight[s:e] + assert param.shape == loaded_weight.shape + param.data.copy_(loaded_weight) + + +class HeadMajorQKVParallelLinear(QKVParallelLinear): + + def weight_loader(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor): + return load_column_parallel_weight(param, loaded_weight) + + +class HeadMajorColumnParallelLinear(MergedColumnParallelLinear): + + def weight_loader(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor): + return load_column_parallel_weight(param, loaded_weight) + + +# @torch.jit.script +def quick_gelu(x): + return x * torch.sigmoid(1.702 * x) + + +# @torch.jit.script +def gegelu(input, limit: Optional[float] = None): + a_gelu, a_linear = input[..., ::2], input[..., 1::2] + if limit is not None: + a_gelu = torch.where(torch.isinf(a_gelu), a_gelu, + a_gelu.clamp(min=None, max=limit)) + a_linear = torch.where( + torch.isinf(a_linear), + a_linear, + a_linear.clamp(min=-limit, max=limit), + ) + out_gelu = quick_gelu(a_gelu) + return out_gelu * (a_linear + 1) + + +class Phi3SmallMLP(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + assert (self.config.hidden_act == "gegelu" + ), "Only `gegelu` is supported for the 4.7 series of models .." + self.hidden_size = config.hidden_size + self.gegelu_limit = config.gegelu_limit + self.intermediate_size = config.intermediate_size + + self.up_proj = HeadMajorColumnParallelLinear( + self.hidden_size, + 2 * [self.intermediate_size], + bias=True, + quant_config=quant_config, + ) + self.down_proj = RowParallelLinear( + self.intermediate_size, + self.hidden_size, + bias=True, + quant_config=quant_config, + ) + + def forward(self, x): + gate_up, _ = self.up_proj(x) + x = gegelu(gate_up) + x, _ = self.down_proj(x) + return x + + +class Phi3SmallSelfAttention(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.config = config + self.sparse_block_size = config.blocksparse_block_size + self.homo_heads = config.blocksparse_homo_head_pattern + self.local_blocks = config.blocksparse_num_local_blocks + self.vert_stride = config.blocksparse_vert_stride + + assert (config.blocksparse_block_size == + config.blocksparse_triton_kernel_block_size) + + self.hidden_size = config.hidden_size + # Number of Query Heads + self.num_heads = config.num_attention_heads + + self.head_dim = self.hidden_size // self.num_heads + self.tp_size = get_tensor_model_parallel_world_size() + # Number of total Key Value Heads before tensor parallel + self.num_key_value_heads = config.num_key_value_heads + self.num_q_per_kv = self.num_heads // self.num_key_value_heads + if self.tp_size > 1: + assert self.num_key_value_heads % self.tp_size == 0 + self.num_kv_heads_per_partion = max( + 1, self.num_key_value_heads // self.tp_size) + self.num_heads_per_partition = self.num_heads // self.tp_size + + self.max_position_embeddings = config.max_position_embeddings + self.rope_embedding_base = config.rope_embedding_base + self.rope_position_scale = config.rope_position_scale + self.is_causal = True + + norm_factor = None + if config.mup_use_scaling: + norm_factor = self.head_dim / config.mup_attn_multiplier + else: + norm_factor = math.sqrt(self.head_dim) + self.scale = 1 / norm_factor + + self.query_key_value = HeadMajorQKVParallelLinear( + self.hidden_size, + self.head_dim, + self.num_heads, + self.num_key_value_heads, + bias=True, + quant_config=quant_config, + ) + + self.dense = RowParallelLinear(self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config) + + if getattr(self.config, "rope_scaling", None) is not None: + rope_scaling = self.config.rope_scaling + for key in rope_scaling: + if isinstance(rope_scaling[key], list): + rope_scaling[key] = tuple(rope_scaling[key]) + + if "factor" not in rope_scaling: + rope_scaling["factor"] = self.rope_position_scale + else: + rope_scaling = { + "type": "linear", + "factor": self.rope_position_scale, + } + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_embedding_base, + rope_scaling=rope_scaling, + ) + + # blocksparse params + self.blocksparse_block_size = config.blocksparse_block_size + self.blocksparse_num_local_blocks = config.blocksparse_num_local_blocks + self.blocksparse_vert_stride = config.blocksparse_vert_stride + + use_dense_attn = (getattr(self.config, + "dense_attention_every_n_layers", None) + and (self.layer_idx + 1) % + self.config.dense_attention_every_n_layers == 0) + + bs_params = None + if not use_dense_attn: + bs_params = { + 'max_seqlen': self.max_position_embeddings, + 'num_heads': self.num_heads_per_partition, + "num_kv_heads": self.num_kv_heads_per_partion, + "block_size": self.sparse_block_size, + "local_blocks": self.local_blocks, + "vert_stride": self.vert_stride, + "homo_head": self.homo_heads + } + + self.attn = Attention( + self.num_heads_per_partition, + self.head_dim, + self.scale, + num_kv_heads=self.num_kv_heads_per_partion, + cache_config=cache_config, + quant_config=quant_config, + blocksparse_params=bs_params, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], + Optional[Tuple[torch.Tensor]]]: + qkv, _ = self.query_key_value(hidden_states) + + qkv = qkv.view(qkv.shape[:-1] + + (-1, (self.num_q_per_kv + 2), self.head_dim)) + q, k, v = qkv.split([self.num_q_per_kv, 1, 1], dim=-2) + + # NOTE: this is required by RotaryEmbed, which indeed does not have to + # TODO: allow 3D QK for rotary forward + q = q.reshape(-1, self.head_dim * self.num_heads_per_partition) + k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partion) + v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partion) + + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata=attn_metadata) + output, _ = self.dense(attn_output) + + return output + + +class Phi3SmallDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Phi3SmallSelfAttention(config, + layer_idx, + cache_config=cache_config, + quant_config=quant_config) + self.mlp = Phi3SmallMLP(config, quant_config) + + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_epsilon) + self.post_attention_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_epsilon) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +class Phi3SmallModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.mup_embedding_multiplier = config.mup_embedding_multiplier + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Phi3SmallDecoderLayer(config, + int(prefix.split('.')[-1]), + cache_config, quant_config), + prefix=f"{prefix}.layers") + + self.final_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def forward( + self, + input_ids: torch.LongTensor, + positions: Optional[torch.LongTensor], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + if (self.mup_embedding_multiplier is not None + and self.mup_embedding_multiplier > 0.0): + hidden_states = hidden_states * self.mup_embedding_multiplier + else: + assert intermediate_tensors + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states + + +class Phi3SmallForCausalLM(nn.Module, SupportsPP): + _tied_weights_keys = ["lm_head.weight"] + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = Phi3SmallModel(config, cache_config, quant_config) + self.vocab_size = config.vocab_size + self.mup_width_multiplier = config.mup_width_multiplier + self.lm_head = ParallelLMHead( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + quant_config=quant_config, + ) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + # tokens in tiktoken but not used + if hasattr(config, 'dummy_token_indices'): + device = self.lm_head.weight.device + self.register_buffer('dummy_token_indices', + torch.LongTensor( + config.dummy_token_indices).to(device), + persistent=False) + else: + self.dummy_token_indices = None + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, value): + self.lm_head = value + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + if self.dummy_token_indices is not None and logits is not None: + logits.index_fill_(-1, self.dummy_token_indices, -torch.inf) + return logits + + def forward( + self, + input_ids: torch.LongTensor, + positions: Optional[torch.LongTensor], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + output_hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + ) + output_hidden_states = output_hidden_states + return output_hidden_states + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + + next_tokens = self.sampler(logits / self.mup_width_multiplier, + sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py new file mode 100644 index 00000000..523375f8 --- /dev/null +++ b/vllm/model_executor/models/phi3v.py @@ -0,0 +1,694 @@ +# coding=utf-8 +# Copyright 2024 The vLLM team. +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import itertools +import regex as re +from functools import cached_property, lru_cache +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, + Tuple, TypedDict, Union) + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from transformers import CLIPVisionConfig, PretrainedConfig + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, ModelConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.models.clip import CLIPVisionModel +from vllm.model_executor.models.llama import LlamaForCausalLM +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token +from vllm.sequence import IntermediateTensors +from vllm.utils import is_list_of + +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, + merge_multimodal_embeddings) + +logger = init_logger(__name__) + +# Cannot find the following 2 numbers from hf config. +_IMAGE_TOKEN_ID = 32044 + +# Result in the max possible feature size (h:w = 16:1) +MAX_IMAGE_FEATURE_SIZE_HEIGHT = 8000 +MAX_IMAGE_FEATURE_SIZE_WIDTH = 50 + +CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0, + hidden_act="quick_gelu", + hidden_size=1024, + image_size=336, + intermediate_size=4096, + num_attention_heads=16, + num_channels=3, + num_hidden_layers=24, + patch_size=14, + projection_dim=768) + + +def _init_img_processor(hf_config: PretrainedConfig): + clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG + layer_idx = hf_config.img_processor.get('layer_idx', -2) + + # Initialize the CLIP only up to the required feature layer + if layer_idx < 0: + num_hidden_layers = clip_config.num_hidden_layers + \ + layer_idx + 1 + else: + num_hidden_layers = layer_idx + 1 + + img_processor = CLIPVisionModel( + clip_config, num_hidden_layers_override=num_hidden_layers) + + return img_processor + + +class Phi3VImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: Union[torch.Tensor, List[torch.Tensor]] + """ + Shape: + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)` + + Note that `num_patches` may be different per batch and image, + in which case the data is passed as a list instead of a batched tensor. + """ + + image_sizes: torch.Tensor + """ + Shape: `(batch_size * num_images, 2)` + + This should be in `(height, width)` format. + """ + + +class Phi3VImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: Union[torch.Tensor, List[torch.Tensor]] + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +Phi3VImageInputs = Union[Phi3VImagePixelInputs, Phi3VImageEmbeddingInputs] + + +class Phi3ImageEmbeddingBase(nn.Module): + + def __init__(self) -> None: + super().__init__() + self.layer_idx: int + self.type_feature: str + self.img_processor: CLIPVisionModel + + def get_img_features(self, + img_embeds: torch.FloatTensor) -> torch.FloatTensor: + TYPE_FEATURE = self.type_feature + + # NOTE: we skip the step to select the vision feature layer since + # this is already done inside the img_processor + img_feature = self.img_processor(img_embeds) + + if TYPE_FEATURE == "patch": + patch_feature = img_feature[:, 1:] + return patch_feature + + if TYPE_FEATURE == "cls_patch": + return img_feature + + raise NotImplementedError + + +# adapted from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_embedding_phi3_v.py +class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase): + """Phi3 Image embedding with HD transform.""" + + def __init__(self, config: PretrainedConfig) -> None: + super().__init__() + + # n_embed or hidden_size + hidden_size = config.n_embd if hasattr( + config, 'n_embd') else config.hidden_size + + self.img_processor = _init_img_processor(config) + + image_dim_out = config.img_processor['image_dim_out'] + self.num_img_tokens = config.img_processor['num_img_tokens'] + + self.image_dim_out = image_dim_out + + # global_gn and sub_gn for hd transform, serves as line separator + self.use_hd_transform = config.embd_layer.get('use_hd_transform', + False) + self.with_learnable_separator = config.embd_layer.get( + 'with_learnable_separator', False) + self.hd_transform_order = config.embd_layer.get( + 'hd_transform_order', 'glb_sub') + # with_hd_transform and with_learnable_separator should have same value + assert self.use_hd_transform and self.with_learnable_separator + + # 1024 * 4, merge spatial to channel dimension + self.glb_GN = nn.Parameter(torch.empty([1, 1, self.image_dim_out * 4])) + self.sub_GN = nn.Parameter( + torch.empty([1, 1, 1, self.image_dim_out * 4])) + + dim_projection = hidden_size + depth = 2 + layers = [nn.Linear(image_dim_out * 4, dim_projection)] + for _ in range(1, depth): + layers.extend( + [nn.GELU(), + nn.Linear(dim_projection, dim_projection)]) + self.img_projection = nn.Sequential(*layers) + + self.type_feature = config.img_processor.get('type_feature', 'patch') + + def forward(self, pixel_values: torch.FloatTensor, + image_sizes: torch.Tensor) -> torch.FloatTensor: + """ + process image and return vision embeddings. + + pixel_values: (num_images, num_crops, c, h, w) + output: (num_images, num_img_tokens, hidden_size) + """ + num_images, num_crops, c, h, w = pixel_values.shape + pixel_values = pixel_values.flatten(0, 1) + img_features = self.get_img_features(pixel_values) + img_features = img_features.reshape(num_images, num_crops, -1, + self.image_dim_out) + image_features_proj = self.hd_feature_transform( + img_features, image_sizes) + return image_features_proj + + def hd_feature_transform(self, image_features, image_sizes): + """ + image_features: (num_images, num_crops+1, 24*24, 1024) + """ + assert ( + self.hd_transform_order == 'sub_glb' + ), f'hd_transform_order `{self.hd_transform_order}` not implemented' + if isinstance(self.img_projection, nn.Sequential): + target_device = self.img_projection[0].bias.device + target_dtype = self.img_projection[0].bias.dtype + else: # It's a single nn.Linear layer + target_device = self.img_projection.bias.device + target_dtype = self.img_projection.bias.dtype + + global_image_features = image_features[:, + 0] # (num_images, 24*24, 1024) + # global feature can be viewed as a special HD case with num_crops 1x1 + global_image_features_hd = self.reshape_hd_patches_2x2merge( + global_image_features, 1, 1) + global_image_features_hd_newline = self.add_image_newline( + global_image_features_hd) + + batch_image_features_proj = [] + # need a for loop to process each image because of different image sizes + # (patch arrangement is different for each image) + for i, img_size in enumerate(image_sizes): + h, w = img_size + h_crop = h // 336 + w_crop = w // 336 + num_crops = h_crop * w_crop + + # NOTE: real num_crops is padded + # (num_crops, 24*24, 1024) + sub_image_features = image_features[i, 1:1 + num_crops] + sub_image_features_hd = self.reshape_hd_patches_2x2merge( + sub_image_features, h_crop, w_crop) + sub_image_features_hd_newline = self.add_image_newline( + sub_image_features_hd) + + # [sub features, separator, global features] + image_embeddings = torch.cat([ + sub_image_features_hd_newline.squeeze( + 0), # (h_crop*12*(w_crop*12+1), 4096) + self.glb_GN.squeeze(0), + global_image_features_hd_newline[i], + ]) + img_proj = self.img_projection( + image_embeddings.to(target_device, target_dtype)) + batch_image_features_proj.append(img_proj) + + return batch_image_features_proj + + def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): + """ + image_features: (num_images*num_crops, 24*24, 1024) + output: (num_images, h_crop*12, w_crop*12, 4096) + where h_crop*w_crop == num_crops + """ + N, L, C = image_features.shape + assert L == 576 and C == 1024 and N % (h_crop * w_crop) == 0 + num_images = N // (h_crop * w_crop) + H = int(L**0.5) + image_features_hd = ( + image_features.reshape(N, H, H, C) # N, 24, 24, 1024 + .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024 + .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024 + .reshape(N, -1, 4 * C) # N, 144, 4096 + .reshape(num_images, h_crop, w_crop, H // 2, H // 2, + -1) # n_img, h_crop, w_crop, 12, 12, 4096 + .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096 + .reshape(num_images, h_crop * H // 2, w_crop * H // 2, + 4 * C) # n_img, h_crop*12, w_crop*12, 4096 + ) + return image_features_hd + + def add_image_newline(self, image_features_hd): + """ + image_features_hd: (num_images, h_crop*12, w_crop*12, 4096) + output: (num_images, (h_crop*12) * (w_crop*12+1), 4096) + """ + num_images, h, w, hid_dim = image_features_hd.shape + # add the newline token to the HD image feature patches + newline_embeddings = self.sub_GN.expand(num_images, h, -1, + -1) # (n_img, h, 1, hid_dim) + image_features_hd_newline = torch.cat( + [image_features_hd, newline_embeddings], + dim=2).reshape(num_images, -1, hid_dim) + return image_features_hd_newline + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loader.load_weights(weights) + + +# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57 +def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336): + target_height = int(np.ceil(height / padding_unit) * padding_unit) + top_padding = int((target_height - height) / 2) + bottom_padding = target_height - height - top_padding + padded_width = width + padded_height = height + top_padding + bottom_padding + return padded_width, padded_height + + +# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L90 +def _calc_hd_transform_size(*, width: int, height: int, hd_num: int): + transposed = False + if width < height: + width, height = height, width + transposed = True + + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= hd_num: + scale += 1 + scale -= 1 + + new_width = int(scale * 336) + new_height = int(new_width / ratio) + + padded_width, padded_height = _calc_padded_size(width=new_width, + height=new_height) + + if transposed: + padded_width, padded_height = padded_height, padded_width + + return padded_width, padded_height + + +# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L181 +def get_phi3v_image_feature_size( + hf_config: Dict[str, Any], + *, + input_height: int, + input_width: int, + num_crops: int, +) -> int: + if num_crops is None: + num_crops = hf_config.get("num_crops", 16) + new_width, new_height = _calc_hd_transform_size(width=input_width, + height=input_height, + hd_num=num_crops) + + return (new_height // 336 * new_width // 336 + 1) * 144 + 1 \ + + (new_height // 336 + 1) * 12 + + +def get_max_phi3v_image_tokens(ctx: InputContext, + *, + num_crops: Optional[int] = None): + + return get_phi3v_image_feature_size( + ctx.get_hf_image_processor_config(), + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH, + num_crops=num_crops, + ) + + +def dummy_data_for_phi3v(ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], + *, + num_crops: Optional[int] = None): + num_images = mm_counts["image"] + + image_feature_size = get_max_phi3v_image_tokens(ctx, num_crops=num_crops) + + seq_data = dummy_seq_data_for_clip( + CLIP_VIT_LARGE_PATCH14_336_CONFIG, + seq_len, + num_images, + image_token_id=_IMAGE_TOKEN_ID, + image_feature_size_override=image_feature_size, + ) + mm_data = dummy_image_for_clip( + CLIP_VIT_LARGE_PATCH14_336_CONFIG, + num_images, + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH, + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT, + ) + + return seq_data, mm_data + + +# Reserve this function to also handle placeholders for additional images +# [ref: PR #5820] +@lru_cache +def _get_image_placeholder_token_ids(model_config: ModelConfig, + idx: int) -> List[int]: + assert idx > 0 + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + # We need to get the token for "<", not "▁<" + # https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/raw/main/tokenizer.json + a_token_id, = tokenizer.encode("a", add_special_tokens=False) + a_token_id_, *image_placeholder_token_ids = tokenizer.encode( + f"a<|image_{idx}|>", add_special_tokens=False) + assert a_token_id == a_token_id_ + + return image_placeholder_token_ids + + +def input_processor_for_phi3v(ctx: InputContext, + llm_inputs: LLMInputs, + *, + num_crops: Optional[int] = None): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + model_config = ctx.model_config + hf_config = ctx.get_hf_image_processor_config() + + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + w, h = image_data.size + image_feature_size = [ + get_phi3v_image_feature_size(hf_config, + input_width=w, + input_height=h, + num_crops=num_crops) + ] + image_data = [image_data] + elif is_list_of(image_data, Image.Image): + image_feature_size = [] + for image in image_data: + w, h = image.size + image_feature_size.append( + get_phi3v_image_feature_size(hf_config, + input_width=w, + input_height=h, + num_crops=num_crops)) + elif isinstance(image_data, torch.Tensor): + image_feature_size = [image_data.shape[0]] + image_data = [image_data] + elif is_list_of(image_data, torch.Tensor): + image_feature_size = [item.shape[0] for item in image_data] + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + prompt = llm_inputs.get("prompt") + if prompt is None: + # for async server request, we assume prompt and its token_ids is always + # in correct format. And num_image_tags == len(image_data) always True. + image_idx = range(1, len(image_data) + 1) + new_prompt = None + else: + image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt))) + if prompt.count("<|image|>") > 0: + logger.warning("Please follow the prompt format that is " + "documented on HuggingFace which does not involve " + "repeating <|image|> tokens.") + elif (num_image_tags := len(image_idx)) > 1: + assert num_image_tags == len( + image_data), "The count of image_placeholder not match image's" + new_prompt = prompt + + prompt_token_ids = llm_inputs["prompt_token_ids"].copy() + + # masked place_holder with image token id + for idx in image_idx: + image_token_ids = _get_image_placeholder_token_ids(model_config, + idx=idx) + for i in range(len(prompt_token_ids) - len(image_token_ids) + 1): + if prompt_token_ids[i:i + len(image_token_ids)] == image_token_ids: + prompt_token_ids[i:i + len(image_token_ids)] = [ + _IMAGE_TOKEN_ID + ] * len(image_token_ids) + break + + # merge consecutive tag ids + merged_token_ids: List[int] = [] + for is_placeholder, token_ids in itertools.groupby( + prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID): + if is_placeholder: + merged_token_ids.append(_IMAGE_TOKEN_ID) + else: + merged_token_ids.extend(list(token_ids)) + + # TODO: Move this to utils or integrate with clip. + new_token_ids: List[int] = [] + placeholder_idx = 0 + while merged_token_ids: + token_id = merged_token_ids.pop(0) + if token_id == _IMAGE_TOKEN_ID: + new_token_ids.extend( + repeat_and_pad_token( + _IMAGE_TOKEN_ID, + repeat_count=image_feature_size[placeholder_idx], + )) + placeholder_idx += 1 + else: + new_token_ids.append(token_id) + + # NOTE: Create a defensive copy of the original inputs + llm_inputs = LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + return llm_inputs + + +@MULTIMODAL_REGISTRY.register_image_input_mapper() +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v) +@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v) +class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + self.image_token_id = _IMAGE_TOKEN_ID + + # TODO: Optionally initializes this for supporting embeddings. + self.vision_embed_tokens = Phi3HDImageEmbedding(config) + + self.language_model = LlamaForCausalLM(config, cache_config, + quant_config) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor: + expected_dims = (2, ) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape) + + if actual_dims != expected_dims: + expected_expr = str(expected_dims) + raise ValueError( + f"The expected shape of image sizes per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _validate_pixel_values( + self, data: Union[torch.Tensor, List[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor]]: + + h = w = CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size + expected_dims = (3, h, w) + + def _validate_shape(d: torch.Tensor): + actual_dims = tuple(d.shape[1:]) + + if actual_dims != expected_dims: + expected_expr = ("num_patches", *map(str, expected_dims)) + raise ValueError( + "The expected shape of pixel values per image per batch " + f"is {expected_expr}. You supplied {tuple(d.shape)}.") + + for d in data: + _validate_shape(d) + + return data + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[Phi3VImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_sizes = kwargs.pop("image_sizes", None) + image_embeds = kwargs.pop("image_embeds", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of pixel values. " + f"Got type: {type(pixel_values)}") + + if not isinstance(image_sizes, (torch.Tensor, list)): + raise ValueError("Incorrect type of image sizes. " + f"Got type: {type(image_sizes)}") + + return Phi3VImagePixelInputs( + type="pixel_values", + data=self._validate_pixel_values(flatten_bn(pixel_values)), + image_sizes=self._validate_image_sizes( + flatten_bn(image_sizes, concat=True))) + + if image_embeds is not None: + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + + return Phi3VImageEmbeddingInputs( + type="image_embeds", + data=flatten_bn(image_embeds), + ) + + raise AssertionError("This line should be unreachable.") + + def _process_image_input( + self, + image_input: Phi3VImageInputs, + ) -> torch.Tensor: + + if image_input["type"] == "image_embeds": + image_data = image_input["data"] + if is_list_of(image_data, torch.Tensor): + # it's already a list of tensors + return image_data + if len(image_data.shape) == 3: + # 3D tensor + return list(torch.unbind(image_data, dim=0)) + raise ValueError( + "We expect batched 2D tensors;" + "this can be either a list of 2D tensors or a single 3D tensor." + ) + + assert self.vision_embed_tokens is not None + image_embeds = self.vision_embed_tokens(image_input["data"], + image_input["image_sizes"]) + + return image_embeds + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object): + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.image_token_id) + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "model.vision_embed_tokens.": "vision_embed_tokens.", + "lm_head.": "language_model.lm_head.", + "model.": "language_model.model.", + }) + + loader = AutoWeightsLoader(self) + loader.load_weights(weights, mapper=hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/phimoe.py b/vllm/model_executor/models/phimoe.py new file mode 100644 index 00000000..4310655e --- /dev/null +++ b/vllm/model_executor/models/phimoe.py @@ -0,0 +1,663 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only PhiMoE model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers.configuration_utils import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.linear import (QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class PhiMoEConfig(PretrainedConfig): + + model_type = "phimoe" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=4096 * 32, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + rope_theta=1e6, + sliding_window=None, + attention_dropout=0.0, + num_experts_per_tok=2, + num_local_experts=16, + output_router_logits=False, + router_aux_loss_coef=0.001, + router_jitter_noise=0.0, + attention_bias=False, + lm_head_bias=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.sliding_window = sliding_window + self.attention_bias = attention_bias + self.lm_head_bias = lm_head_bias + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.attention_dropout = attention_dropout + + self.num_experts_per_tok = num_experts_per_tok + self.num_local_experts = num_local_experts + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + self.router_jitter_noise = router_jitter_noise + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class mp(torch.autograd.Function): + + @staticmethod + def forward( + ctx, + scores: torch.Tensor, + multiplier: torch.Tensor, + selected_experts: torch.Tensor, + masked_gates: torch.Tensor, + mask_for_one: torch.Tensor, + ): + ctx.save_for_backward(multiplier, selected_experts, masked_gates) + return multiplier * mask_for_one + + @staticmethod + def backward( + ctx, + grad_at_output: torch.Tensor, + ): + multiplier, selected_experts, masked_gates = ctx.saved_tensors + + grad_at_output = grad_at_output * multiplier + + grad_at_scores_expaned = masked_gates * grad_at_output.mul(-1) + grad_at_scores_expaned.scatter_add_( + dim=-1, + index=selected_experts, + src=grad_at_output, + ) + + return ( + grad_at_scores_expaned, + None, + None, + None, + None, + ) + + +def sparsemixer(scores, jitter_eps=0.01): + ################ first expert ################ + + with torch.no_grad(): + # compute mask for sparsity + mask_logits_threshold, max_ind = scores.max(dim=-1, keepdim=True) + factor = scores.abs().clamp(min=mask_logits_threshold) + mask_logits_threshold = ( + (mask_logits_threshold - scores) / factor) > (2 * jitter_eps) + + # apply mask + masked_gates = scores.masked_fill(mask_logits_threshold, float("-inf")) + selected_experts = max_ind + + # compute scores for gradients + masked_gates = torch.softmax(masked_gates, dim=-1) + multiplier_o = masked_gates.gather(dim=-1, index=selected_experts) + + multiplier = multiplier_o + + # masked out first expert + masked_scores = torch.scatter( + scores, + -1, + selected_experts, + float("-inf"), + ) + with torch.no_grad(): + # compute mask for sparsity + mask_logits_threshold, max_ind = masked_scores.max(dim=-1, + keepdim=True) + factor = scores.abs().clamp(min=mask_logits_threshold) + mask_logits_threshold = ( + (mask_logits_threshold - scores) / factor) > (2 * jitter_eps) + + # apply mask + masked_gates_top2 = masked_scores.masked_fill(mask_logits_threshold, + float("-inf")) + selected_experts_top2 = max_ind + # compute scores for gradients + masked_gates_top2 = torch.softmax(masked_gates_top2, dim=-1) + multiplier_top2 = masked_gates_top2.gather(dim=-1, + index=selected_experts_top2) + + multiplier = torch.concat((multiplier, multiplier_top2), dim=-1) + selected_experts = torch.concat((selected_experts, selected_experts_top2), + dim=-1) + multiplier = multiplier.to(torch.float32) + selected_experts = selected_experts.to(torch.int32) + return ( + multiplier, + selected_experts, + ) + + +def phimoe_routing_function( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + assert hidden_states.shape[0] == gating_output.shape[0], ( + "Number of tokens mismatch") + assert topk == 2, "Only top-2 routing is supported" + assert renormalize is False, "Renormalization is not supported" + + topk_weights, topk_ids = sparsemixer(gating_output) + return topk_weights, topk_ids + + +class PhiMoE(nn.Module): + """A tensor-parallel MoE implementation for PhiMoE that shards each expert + across all ranks. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + ): + super().__init__() + self.hidden_size = hidden_size + + # Gate always runs at half / full precision for now. + self.gate = ReplicatedLinear( + hidden_size, + num_experts, + bias=False, + params_dtype=params_dtype, + quant_config=None, + ) + + self.experts = FusedMoE( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + params_dtype=params_dtype, + reduce_results=True, + renormalize=False, + quant_config=quant_config, + tp_size=tp_size, + custom_routing_function=phimoe_routing_function) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states, router_logits) + return final_hidden_states.view(orig_shape) + + +class PhiMoEAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + rope_scaling: Optional[dict] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=True, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=True, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=int(self.rope_theta), + is_neox_style=True, + rope_scaling=self.rope_scaling, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class PhiMoEDecoderLayer(nn.Module): + + def __init__( + self, + config: PhiMoEConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 10000) + self.self_attn = PhiMoEAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=config.rope_scaling, + ) + self.block_sparse_moe = PhiMoE( + num_experts=config.num_local_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + quant_config=quant_config, + ) + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.rms_norm_eps, + elementwise_affine=True) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.rms_norm_eps, + elementwise_affine=True) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + residual = hidden_states + + # Self Attention + hidden_states = self.input_layernorm(hidden_states) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = hidden_states + residual + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.block_sparse_moe(hidden_states) + + hidden_states = hidden_states + residual + return hidden_states, residual + + +class PhiMoEModel(nn.Module): + + def __init__( + self, + config: PhiMoEConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + lora_vocab = ((lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0) + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: PhiMoEDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers") + self.norm = nn.LayerNorm(config.hidden_size, + eps=config.rms_norm_eps, + elementwise_affine=True) + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states = self.norm(hidden_states) + return hidden_states + + +class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + fall_back_to_pt_during_load = False + + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "embed_tokens", + "lm_head", + "w1", + "w2", + "w3", + "gate", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: PhiMoEConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = PhiMoEModel(config, + cache_config, + quant_config, + lora_config=lora_config) + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=( + DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size), + quant_config=None, + bias=True, + ) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="w1", + ckpt_down_proj_name="w2", + ckpt_up_proj_name="w3", + num_experts=self.config.num_local_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader( + param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id, + ) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py new file mode 100644 index 00000000..c8957dca --- /dev/null +++ b/vllm/model_executor/models/pixtral.py @@ -0,0 +1,578 @@ +from dataclasses import dataclass, fields +from functools import cached_property +from itertools import tee +from typing import Iterable, List, Mapping, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mistral_common.protocol.instruct.messages import ImageChunk +from PIL import Image +from transformers import PretrainedConfig +from xformers.ops.fmha import memory_efficient_attention +from xformers.ops.fmha.attn_bias import BlockDiagonalMask + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.utils import merge_multimodal_embeddings +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import IntermediateTensors, SequenceData + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import init_vllm_registered_model + + +def get_max_pixtral_image_tokens(ctx: InputContext): + tokenizer = cached_get_tokenizer( + ctx.model_config.tokenizer, + tokenizer_mode=ctx.model_config.tokenizer_mode) + mm_encoder = tokenizer.instruct.mm_encoder + + max_image_size = mm_encoder.mm_config.max_image_size + image_patch_size = mm_encoder.mm_config.image_patch_size + + return ((max_image_size // image_patch_size)**2) + + +def dummy_data_for_pixtral(ctx: InputContext, seq_len: int, + mm_counts: Mapping[str, int]): + tokenizer = cached_get_tokenizer( + ctx.model_config.tokenizer, + tokenizer_mode=ctx.model_config.tokenizer_mode) + + mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder + patch_size = mm_encoder.mm_config.image_patch_size + image_token_id = mm_encoder.special_ids.img + + mm_config = ctx.model_config.multimodal_config + num_images = mm_config.limit_per_prompt.get("image", 1) + + # dummy size + size = 256 + image = Image.new("RGB", (size, size), color=0) + + image_feature_size = (size**2) // (patch_size**2) + + num_image_tokens = image_feature_size * num_images + seq_data = SequenceData.from_token_counts( + (image_token_id, num_image_tokens), + (0, seq_len - num_image_tokens), + ) + + mm_data = {"image": num_images * [image]} + return seq_data, mm_data + + +def input_mapper_for_pixtral(ctx: InputContext, + data: object) -> MultiModalInputs: + """Maps the input data to its MultiModalInputs (if any). + + Args: + ctx: Context of the loaded model. + data: data potentially containing image/image embeddings to be mapped + to pixel_values in .forward() for a visual QWenLMHeadModel model. + + Returns: + MultiModalInputs containing the stacked normalized images tensor or + image embeddings. + """ + # Early exit if we have provided an image to a language only Qwen model + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode) + + data_list = data if isinstance(data, list) else [data] + + images = [] + for image_data in data_list: + image = ImageChunk(image=image_data) + encoding = tokenizer.instruct.mm_encoder(image) + image = torch.from_numpy(encoding.image).to(device="cuda", + dtype=torch.float16) + images.append(image) + + return MultiModalInputs({"images": images}) + + +def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is not None and "image" in multi_modal_data: + tokenizer = cached_get_tokenizer( + ctx.model_config.tokenizer, + tokenizer_mode=ctx.model_config.tokenizer_mode) + + mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder + image_token_id = mm_encoder.special_ids.img + + if image_token_id not in llm_inputs['prompt_token_ids']: + raise ValueError( + (f"You've passed {llm_inputs=} without {image_token_id=}" + " Make sure to process your input via mistral_common's" + " tokenizer or pass a chat completion request. For more" + " For more info, see: " + "https://github.com/vllm-project/vllm/issues/8411.")) + + return llm_inputs + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_pixtral_image_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_pixtral) +@INPUT_REGISTRY.register_input_processor(input_processor_for_pixtral) +class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + self.config = config + self.multimodal_config = multimodal_config + + dataclass_fields = {field.name for field in fields(VisionEncoderArgs)} + vision_args = { + key: value + for key, value in self.config.vision_config.to_dict().items() + if key in dataclass_fields + } + + self.vision_args = VisionEncoderArgs(**vision_args) + + # init MistralForCausalLM + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + + self.vision_encoder = VisionTransformer(self.vision_args) + self.vision_language_adapter = VisionLanguageAdapter( + self.vision_args, dim=config.text_config.hidden_size) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for pixtral. + + TODO + + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + + if image_input is not None: + vision_embeddings = self._process_image_input(image_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, vision_embeddings, + self.vision_args.image_token_id) + + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model(input_ids, + positions, + kv_caches, + attn_metadata, + intermediate_tensors, + inputs_embeds=inputs_embeds) + + return hidden_states + + def _parse_and_validate_image_input( + self, + images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor], + torch.Tensor]] = None + ) -> Optional[List[torch.Tensor]]: + if images is None: + return None + + if isinstance(images, torch.Tensor): + # if passed as batch take all images + N, B, C, W, H = images.shape + images = images.reshape(N * B, C, W, H) + images = [images[i] for i in range(images.size(0))] + elif isinstance(images, list): + # if passed as list flatten lists of tensors + flatten_images = [] + for imgs_per_req in images: + imgs_per_req = [ + imgs_per_req[i] for i in range(imgs_per_req.size(0)) + ] if isinstance(imgs_per_req, torch.Tensor) else imgs_per_req + + flatten_images.extend(imgs_per_req) + + images = flatten_images + + return images + + def _process_image_input(self, + image_input: List[torch.Tensor]) -> torch.Tensor: + return self.vision_language_adapter(self.vision_encoder(image_input)) + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + def is_vision_encoder_weights(weight: Tuple[str, torch.Tensor]): + return weight[0].startswith("vision_encoder") + + def is_vision_lang_adapter_weights(weight: Tuple[str, torch.Tensor]): + return weight[0].startswith("vision_language_adapter") + + def is_vision_weights(weight: Tuple[str, torch.Tensor]): + return is_vision_encoder_weights( + weight) or is_vision_lang_adapter_weights(weight) + + llm_weights, vision_encoder_weights, vision_lang_adapter_weights = tee( + weights, 3) + + # llm + llm_weights = filter(lambda x: not is_vision_weights(x), llm_weights) + self.language_model.load_weights(llm_weights) + + # vision encoder + vision_encoder_weights = filter(is_vision_encoder_weights, + vision_encoder_weights) + vision_encoder_dict = dict(self.vision_encoder.named_parameters()) + for name, loaded_weight in vision_encoder_weights: + # cut 'vision_encoder.' + name = '.'.join(name.split(".")[1:]) + param = vision_encoder_dict[name] + + default_weight_loader(param, loaded_weight) + + # adapter + vision_lang_adapter_weights = filter(is_vision_lang_adapter_weights, + vision_lang_adapter_weights) + vision_lang_adpter_dict = dict( + self.vision_language_adapter.named_parameters()) + for name, loaded_weight in vision_lang_adapter_weights: + # cut 'vision_language_adapter.' + name = '.'.join(name.split(".")[1:]) + param = vision_lang_adpter_dict[name] + default_weight_loader(param, loaded_weight) + + +# Vision encoder +@dataclass +class VisionEncoderArgs: + hidden_size: int + num_channels: int + image_size: int + patch_size: int + intermediate_size: int + num_hidden_layers: int + num_attention_heads: int + rope_theta: float # for rope-2D + image_token_id: int + + +def _reshape_for_broadcast(freqs_cis: torch.Tensor, + x: torch.Tensor) -> torch.Tensor: + """ + freqs_cis: complex - (seq_len, head_dim / 2) + x: complex - (bsz, seq_len, head_dim / 2) + """ + ndim = x.ndim + assert ndim > 1 + assert freqs_cis.shape == (x.shape[1], x.shape[-1]), ( + freqs_cis.shape, + (x.shape[1], x.shape[-1]), + ) + shape = [ + d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape) + ] + return freqs_cis.view(*shape) + + +def precompute_freqs_cis_2d( + dim: int, + height: int, + width: int, + theta: float, +) -> torch.Tensor: + """ + freqs_cis: 2D complex tensor of shape (height, width, dim // 2) + to be indexed by (height, width) position tuples + """ + # (dim / 2) frequency bases + freqs = 1.0 / (theta**(torch.arange(0, dim, 2).float() / dim)) + + h = torch.arange(height, device=freqs.device) + w = torch.arange(width, device=freqs.device) + + freqs_h = torch.outer(h, freqs[::2]).float() + freqs_w = torch.outer(w, freqs[1::2]).float() + freqs_2d = torch.cat( + [ + freqs_h[:, None, :].repeat(1, width, 1), + freqs_w[None, :, :].repeat(height, 1, 1), + ], + dim=-1, + ) + return torch.polar(torch.ones_like(freqs_2d), freqs_2d) + + +def apply_rotary_emb_vit( + xq: torch.Tensor, + xk: torch.Tensor, + freqs_cis: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) + assert freqs_cis.dtype == torch.complex64 + freqs_cis = _reshape_for_broadcast(freqs_cis, xq_) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) + return xq_out.type_as(xq), xk_out.type_as(xk) + + +class FeedForward(nn.Module): + + def __init__(self, args: VisionEncoderArgs): + super().__init__() + assert args.intermediate_size is not None + self.w1 = nn.Linear(args.hidden_size, + args.intermediate_size, + bias=False) + self.w2 = nn.Linear(args.intermediate_size, + args.hidden_size, + bias=False) + self.w3 = nn.Linear(args.hidden_size, + args.intermediate_size, + bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class Attention(nn.Module): + + def __init__(self, args: VisionEncoderArgs): + super().__init__() + self.args = args + assert not args.hidden_size % args.num_attention_heads + self.n_heads = args.num_attention_heads + self.head_dim = args.hidden_size // args.num_attention_heads + + self.wq = nn.Linear(args.hidden_size, args.hidden_size, bias=False) + self.wk = nn.Linear(args.hidden_size, args.hidden_size, bias=False) + self.wv = nn.Linear(args.hidden_size, args.hidden_size, bias=False) + self.wo = nn.Linear(args.hidden_size, args.hidden_size, bias=False) + + def forward( + self, + x: torch.Tensor, + mask: BlockDiagonalMask, + freqs_cis: torch.Tensor, + ) -> torch.Tensor: + batch, patches, _ = x.shape + + q, k, v = self.wq(x), self.wk(x), self.wv(x) + q = q.reshape(batch, patches, self.n_heads, self.head_dim) + k = k.reshape(batch, patches, self.n_heads, self.head_dim) + v = v.reshape(batch, patches, self.n_heads, self.head_dim) + + q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis) + out = memory_efficient_attention(q, k, v, attn_bias=mask) + out = out.reshape(batch, patches, self.n_heads * self.head_dim) + return self.wo(out) + + +class TransformerBlock(nn.Module): + + def __init__(self, args: VisionEncoderArgs): + super().__init__() + self.attention = Attention(args) + self.feed_forward = FeedForward(args) + self.attention_norm = RMSNorm(args.hidden_size, eps=1e-5) + self.ffn_norm = RMSNorm(args.hidden_size, eps=1e-5) + + def forward( + self, + x: torch.Tensor, + mask: BlockDiagonalMask, + freqs_cis: torch.Tensor, + ) -> torch.Tensor: + r = self.attention.forward(self.attention_norm(x), + mask=mask, + freqs_cis=freqs_cis) + h = x + r + r = self.feed_forward.forward(self.ffn_norm(h)) + out = h + r + return out + + +class Transformer(nn.Module): + + def __init__(self, args: VisionEncoderArgs): + super().__init__() + self.layers = torch.nn.ModuleList() + for _ in range(args.num_hidden_layers): + self.layers.append(TransformerBlock(args)) + + def forward( + self, + x: torch.Tensor, + mask: BlockDiagonalMask, + freqs_cis: Optional[torch.Tensor], + ) -> torch.Tensor: + for layer in self.layers: + x = layer(x, mask=mask, freqs_cis=freqs_cis) + return x + + +def position_meshgrid(patch_embeds_list: List[torch.Tensor], ) -> torch.Tensor: + positions = torch.cat([ + torch.stack( + torch.meshgrid( + torch.arange(p.shape[-2]), + torch.arange(p.shape[-1]), + indexing="ij", + ), + dim=-1, + ).reshape(-1, 2) for p in patch_embeds_list + ]) + return positions + + +class VisionTransformer(nn.Module): + + def __init__(self, args: VisionEncoderArgs): + super().__init__() + self.args = args + self.patch_conv = nn.Conv2d( + in_channels=args.num_channels, + out_channels=args.hidden_size, + kernel_size=args.patch_size, + stride=args.patch_size, + bias=False, + ) + self.ln_pre = RMSNorm(args.hidden_size, eps=1e-5) + self.transformer = Transformer(args) + + head_dim = self.args.hidden_size // self.args.num_attention_heads + assert head_dim % 2 == 0, "ROPE requires even head_dim" + self._freqs_cis: Optional[torch.Tensor] = None + + @property + def max_patches_per_side(self) -> int: + return self.args.image_size // self.args.patch_size + + @property + def device(self) -> torch.device: + return next(self.parameters()).device + + @property + def dtype(self) -> torch.device: + return next(self.parameters()).dtype + + @property + def freqs_cis(self) -> torch.Tensor: + if self._freqs_cis is None: + self._freqs_cis = precompute_freqs_cis_2d( + dim=self.args.hidden_size // self.args.num_attention_heads, + height=self.max_patches_per_side, + width=self.max_patches_per_side, + theta=self.args.rope_theta, + ) + + if self._freqs_cis.device != self.device: + self._freqs_cis = self._freqs_cis.to(device=self.device) + + return self._freqs_cis + + def forward( + self, + images: List[torch.Tensor], + ) -> torch.Tensor: + """ + Args: + images: list of N_img images of variable sizes, + each of shape (C, H, W) + Returns: + image_features: tensor of token features for + all tokens of all images of shape (N_toks, D) + """ + # pass images through initial convolution independently + patch_embeds_list = [ + self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in images + ] + + # flatten to a single sequence + patch_embeds = torch.cat( + [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1) + patch_embeds = self.ln_pre(patch_embeds) + + # positional embeddings + positions = position_meshgrid(patch_embeds_list).to(self.device) + freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]] + + # pass through Transformer with a block diagonal mask delimiting images + mask = BlockDiagonalMask.from_seqlens( + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], ) + out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis) + + # remove batch dimension of the single sequence + return out.squeeze(0) + + +class VisionLanguageAdapter(nn.Module): + + def __init__(self, args: VisionEncoderArgs, dim: int): + super().__init__() + assert isinstance(args, VisionEncoderArgs) + self.w_in = nn.Linear( + args.hidden_size, + dim, + bias=True, + ) + self.gelu = nn.GELU() + self.w_out = nn.Linear(dim, dim, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w_out(self.gelu(self.w_in(x))) diff --git a/vllm/model_executor/models/qwen.py b/vllm/model_executor/models/qwen.py new file mode 100644 index 00000000..3c94173b --- /dev/null +++ b/vllm/model_executor/models/qwen.py @@ -0,0 +1,991 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py +# Copyright (c) Alibaba Cloud. +# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE +"""Inference-only QWen model compatible with HuggingFace weights.""" + +import math +import regex as re +from functools import partial +from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping, + Optional, Tuple, TypedDict, Union) + +import numpy as np +import torch +from PIL import Image +from torch import nn +from torchvision import transforms +from torchvision.transforms import InterpolationMode +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs +from vllm.multimodal.utils import cached_get_tokenizer +from vllm.sequence import IntermediateTensors, SequenceData +from vllm.utils import is_list_of + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (flatten_bn, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + +logger = init_logger(__name__) + +# NOTE: Qwen models have a few other special tags, e.g., ref, bbox, quad; +# for the time being, these tags are not considered as special at encoding +# time. This may change as VLLMs multimodal API changes in the future. +IMG_START = "" +IMG_END = "" +IMG_PAD = "" +# Image context is fixed at 256 for all images +MAX_QWEN_IMG_TOKENS = 256 +# Image normalization params +CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) +CLIP_STD = (0.26862954, 0.26130258, 0.27577711) + + +class QwenImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """ + Shape: `(batch_size * num_images, 3, image_size, image_size)` + + Note that image_size is the value in the vision config to which we resize + the image to in the normalization transform. Currently multi-image support + can only be leveraged by passing image embeddings directly. + """ + + +class QwenImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, 256, hidden_size)` + + `hidden_size` must match the hidden size of the language model backbone + and is stored in the visual config of the model if we have one. + """ + + +QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs] + + +class VisualAttention(nn.Module): + """self-attention layer class. + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + bias: bool = True, + kdim: Optional[int] = None, + vdim: Optional[int] = None, + ): + super().__init__() + self.embed_dim = embed_dim + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self._qkv_same_embed_dim = self.kdim == embed_dim \ + and self.vdim == embed_dim + + self.num_heads = num_heads + + # Per attention head and per partition values. + assert embed_dim % num_heads == 0 + self.hidden_size_per_attention_head = embed_dim // num_heads + self.num_attention_heads_per_partition = num_heads + self.hidden_size_per_partition = embed_dim + + # Strided linear layer. + assert self._qkv_same_embed_dim, \ + 'Visual Attention implementation only supports self-attention' + self.in_proj = nn.Linear(embed_dim, 3 * embed_dim) + self.out_proj = nn.Linear(embed_dim, embed_dim) + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + + def forward( + self, + x: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # query/key/value: [sq, b, h] + sq, b, _ = x.size() + mixed_x_layer = self.in_proj(x) + + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn] + new_tensor_shape = mixed_x_layer.size()[:-1] + \ + (self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head) + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) + + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] + query_layer, key_layer, value_layer = mixed_x_layer.split( + self.hidden_size_per_attention_head, dim=-1) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view( + sq, b * self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head).transpose(0, 1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view( + sq, b * self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head).transpose(0, 1) + + q_scaled = query_layer / self.norm_factor + if attn_mask is not None: + attention_probs = torch.baddbmm(attn_mask, q_scaled, + key_layer.transpose(-2, -1)) + else: + attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1)) + attention_probs = attention_probs.softmax(dim=-1) + + value_layer = value_layer.view( + sq, b * self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head).transpose(0, 1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer) + + # change view [b, np, sq, hn] + context_layer = context_layer.view( + b, self.num_attention_heads_per_partition, sq, + self.hidden_size_per_attention_head) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size_per_partition,) + context_layer = context_layer.view(*new_context_layer_shape) + + output = self.out_proj(context_layer) + + return output + + +class QwenVMLP(nn.Module): + """MLP for the visual component of the Qwen model.""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.c_fc = ColumnParallelLinear(hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config) + self.act_fn = get_act_fn("gelu", quant_config, intermediate_size) + self.c_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + ) + + def forward(self, x): + x, _ = self.c_fc(x) + x = self.act_fn(x) + x, _ = self.c_proj(x) + return x + + +class VisualAttentionBlock(nn.Module): + + def __init__( + self, + d_model: int, + n_head: int, + mlp_ratio: float = 4.0, + norm_layer: Callable = nn.LayerNorm, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.ln_1 = norm_layer(d_model) + self.ln_2 = norm_layer(d_model) + mlp_width = int(d_model * mlp_ratio) + self.attn = VisualAttention(d_model, n_head) + self.mlp = QwenVMLP( + hidden_size=d_model, + intermediate_size=mlp_width, + quant_config=quant_config, + ) + + def attention( + self, + x: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None + return self.attn(x, attn_mask=attn_mask) + + def forward( + self, + x: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + x = x + self.attention(self.ln_1(x), attn_mask=attn_mask) + x = x + self.mlp(self.ln_2(x)) + return x + + +class TransformerBlock(nn.Module): + + def __init__( + self, + width: int, + layers: int, + heads: int, + mlp_ratio: float = 4.0, + norm_layer: Callable = nn.LayerNorm, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.width = width + self.layers = layers + + self.resblocks = nn.ModuleList([ + VisualAttentionBlock(width, + heads, + mlp_ratio, + norm_layer=norm_layer, + quant_config=quant_config) + for _ in range(layers) + ]) + + def get_cast_dtype(self) -> torch.dtype: + return self.resblocks[0].mlp.c_fc.weight.dtype + + def get_cast_device(self) -> torch.device: + return self.resblocks[0].mlp.c_fc.weight.device + + def forward(self, + x: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + for r in self.resblocks: + x = r(x, attn_mask=attn_mask) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size: int, + patch_size: int, + width: int, + layers: int, + heads: int, + mlp_ratio: float, + n_queries: int = 256, + output_dim: int = 512, + image_start_id: int = 151857, + quant_config: Optional[QuantizationConfig] = None, + **kwargs): + super().__init__() + image_height, image_width = self.image_size = (image_size, image_size) + patch_height, patch_width = self.patch_size = (patch_size, patch_size) + self.grid_size = (image_height // patch_height, + image_width // patch_width) + self.output_dim = output_dim + self.conv1 = nn.Conv2d(in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + # class embeddings and positional embeddings + scale = width**-0.5 + self.positional_embedding = nn.Parameter(scale * + torch.randn(256, width)) + + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + self.ln_pre = norm_layer(width) + self.transformer = TransformerBlock(width, + layers, + heads, + mlp_ratio, + norm_layer=norm_layer, + quant_config=quant_config) + + self.attn_pool = Resampler2( + grid_size=int(math.sqrt(n_queries)), + embed_dim=output_dim, + num_heads=output_dim // 128, + kv_dim=width, + norm_layer=norm_layer, + adaptive=False, + do_post_projection=False, + ).to( + device=self.positional_embedding.device, + dtype=self.positional_embedding.dtype, + ) + + self.ln_post = norm_layer(output_dim) + self.proj = nn.Parameter( + (output_dim**-0.5) * torch.randn(output_dim, output_dim)) + self.image_start_id = image_start_id + self.image_end_id = image_start_id + 1 + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.to( + dtype=self.transformer.get_cast_dtype(), + device=self.transformer.get_cast_device(), + ) + + # to patches + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + x = x + get_abs_pos(self.positional_embedding, int(math.sqrt( + x.size(1)))) + + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.attn_pool(x) + x = self.ln_post(x) + x = x @ self.proj + + return x + + def get_image_positions(self, + input_ids: torch.Tensor) -> Optional[torch.Tensor]: + """Given the input IDs, extracts start/stop points corresponding to + images. + + args: + Returns: + Optional torch tensor corresponding to start/stop pairs of images. + """ + if torch.any(input_ids == self.image_start_id): + bos_pos = torch.where(input_ids == self.image_start_id) + eos_pos = torch.where(input_ids == self.image_end_id) + return torch.stack((bos_pos[0], eos_pos[0]), dim=1) + return None + + +class QWenMLP(nn.Module): + """MLP for the language component of the Qwen model, which contains a + MergedColumnParallelLinear merging 2 outputs via silu activation.""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str = "silu", + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.c_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.c_proj(x) + return x + + +class QWenAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + max_position_embeddings: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.hidden_size = hidden_size + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size( + ) + self.total_num_heads = num_heads + assert self.total_num_heads % tensor_model_parallel_world_size == 0 + self.num_heads = (self.total_num_heads // + tensor_model_parallel_world_size) + self.head_dim = hidden_size // self.total_num_heads + self.c_attn = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + bias=True, + quant_config=quant_config, + ) + self.c_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + self.scaling = self.head_dim**-0.5 + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.c_attn(hidden_states) + q, k, v = qkv.chunk(chunks=3, dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.c_proj(attn_output) + return output + + +class QWenBlock(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + self.attn = QWenAttention(config.hidden_size, + config.num_attention_heads, + config.max_position_embeddings, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + cache_config=cache_config, + quant_config=quant_config) + + self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + + self.mlp = QWenMLP(config.hidden_size, + config.intermediate_size // 2, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.ln_1(hidden_states) + else: + hidden_states, residual = self.ln_1(hidden_states, residual) + hidden_states = self.attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.ln_2(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class QWenModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.vocab_size = config.vocab_size + + self.wte = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.h = make_layers( + config.num_hidden_layers, + lambda prefix: QWenBlock(config, cache_config, quant_config), + prefix=f"{prefix}.h") + self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + self.visual = VisionTransformer(**config.visual, + quant_config=quant_config) if hasattr( + config, "visual") else None + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + pixel_values: Optional[QwenImageInputs], + ) -> Union[torch.Tensor, IntermediateTensors]: + img_pos = None + # If pixel / visual embeddings are provided, this is a visual model + if pixel_values is not None and self.visual is not None: + if pixel_values["type"] != "image_embeds": + image_embeds = self.visual(pixel_values["data"]) + else: + image_embeds = pixel_values["data"] + + # features should be of shape (# images, 256, hidden_dim) + img_pos = self.visual.get_image_positions(input_ids) + if isinstance( + img_pos, + np.ndarray) and img_pos.shape[0] != image_embeds.shape[0]: + raise ValueError( + f"Number of placeholders: {img_pos.shape[0]} " + f"does not match number of images {image_embeds.shape[0]}." + ) + + if get_pp_group().is_first_rank: + hidden_states = self.wte(input_ids) + # Merge the image embeddings into the hidden states if actually have + # visual features and the corresponding image tokens + if img_pos is not None: + for idx, (img_bos, img_eos) in enumerate(img_pos): + hidden_states[img_bos + 1:img_eos] = image_embeds[idx] + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.h[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.ln_f(hidden_states, residual) + return hidden_states + + +def get_image_text(image_num: int, padding: bool) -> str: + """Retrieves a placeholder text that when tokenized, will be expanded with + image pads. + + Args: + image_num: The number of the image that we want a text prompt for. + Images should be indexed starting at 1. + padding: Whether or not padding should be manually added. + + Returns: + Text placeholder prompt for the image being considered. + """ + image_start = f"Picture {image_num}: {IMG_START}" + image_end = f"{IMG_END}\n" + if not padding: + return f"{image_start}{image_end}" + return f"{image_start}{MAX_QWEN_IMG_TOKENS * IMG_PAD}{image_end}" + + +def input_processor_for_qwen(ctx: InputContext, + llm_inputs: LLMInputs) -> LLMInputs: + """Processes the inputs, which may or may not be multimodal. + Multimodal inputs will only be processed if the model has a "visual" + component in its model config, otherwise they'll be ignored. + + Args: + ctx: Context of the loaded model. + llm_inputs: LLM inputs which may have a multi_modal_data attribute. + + Returns: + If the model is language only or not multimodal inputs were provided, + returns llm_inputs unmodified. Otherwise, processes the multimodal + images / image embeddings and adds the fixed-length image placeholders. + """ + multi_modal_data = llm_inputs.get("multi_modal_data") + + # Only process images if we have multimodal data and a visual config + hf_config = ctx.get_hf_config() + if (multi_modal_data is None or "image" not in multi_modal_data + or not hasattr(hf_config, "visual")): + return llm_inputs + + prompt = llm_inputs.get("prompt") + prompt_token_ids = llm_inputs["prompt_token_ids"] + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + image_data = multi_modal_data["image"] + if isinstance(image_data, torch.Tensor): + num_dims = len(image_data.shape) + if num_dims < 2 or num_dims > 3: + raise ValueError( + f"Expected img embeds to be have 3 dimensions, got {num_dims}") + num_images = 1 if num_dims == 2 else image_data.shape[0] + elif isinstance(image_data, Image.Image): + num_images = 1 + elif is_list_of(image_data, Image.Image): + num_images = len(image_data) + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + + if prompt is None: + prompt = tokenizer.decode(prompt_token_ids) + + # Drops anything between / tags; encoding with the tokenizer + # will automatically add the image pads for the context. + new_prompt, num_matched_images = re.subn( + r"(Picture \d*: ).*?(<\/img>\n)", + r"\1\2", + prompt, + ) + + if num_matched_images != num_images: + logger.warning( + "Number of matched image placeholders %s doesn't match the number " + "of expected images %s; check your placeholder formatting.", + num_matched_images, num_images) + + new_prompt_token_ids = tokenizer.encode(new_prompt) + + return LLMInputs(prompt=new_prompt, + prompt_token_ids=new_prompt_token_ids, + multi_modal_data=multi_modal_data) + + +def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs: + """Maps the input data to its MultiModalInputs (if any). + + Args: + ctx: Context of the loaded model. + data: data potentially containing image/image embeddings to be mapped + to pixel_values in .forward() for a visual QWenLMHeadModel model. + + Returns: + MultiModalInputs containing the stacked normalized images tensor or + image embeddings. + """ + # Early exit if we have provided an image to a language only Qwen model + hf_config = ctx.get_hf_config() + if not hasattr(hf_config, "visual"): + logger.warning( + "Images were provided but this model has no visual config; " + "multimodal inputs will not be forwarded to the model.") + return MultiModalInputs() + + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + + image_pair_tok = tokenizer.encode(IMG_START + IMG_END, + add_special_tokens=False, + return_tensors="pt").squeeze() + image_start_id = image_pair_tok[0] + image_end_id = image_pair_tok[-1] + if (image_start_id + 1) != image_end_id: + raise ValueError( + f"Found image end ID {image_end_id}, but expected {IMG_START} + 1") + if len(image_pair_tok) != (MAX_QWEN_IMG_TOKENS + 2): + raise ValueError( + f"Expected image context length of {MAX_QWEN_IMG_TOKENS}, " + f"but got {image_pair_tok - 2}") + + hf_config = ctx.get_hf_config() + image_size = hf_config.visual["image_size"] + img_emb_size = hf_config.visual["output_dim"] + + if isinstance(data, torch.Tensor): + # It's expected that our values have already been processed + # by the visual transformer; shape is expected to be: + # (# images, 256, hidden_size) + if len(data.shape) == 2: + # Assume only one image embed was provided; unsqueeze the extra dim + data = data.unsqueeze(0) + if len(data.shape) != 3 or data.shape[ + 1] != MAX_QWEN_IMG_TOKENS or data.shape[2] != img_emb_size: + raise ValueError( + "Expected image embeds to be a tensor of shape" + f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but " + f"received shape [{data.shape}]") + pixel_values = data + else: + transform = build_normalization_transform(image_size) + if not isinstance(data, (list, tuple)): + data = [data] + transformed_images = [transform(datum) for datum in data] + pixel_values = torch.stack(transformed_images, dim=0) + return MultiModalInputs({"pixel_values": pixel_values}) + + +def build_normalization_transform(image_size: int) -> transforms.Compose: + """Builds a normalization transform which can be applied to one or + more input images from which we want to extract visual features. + + Args: + image_size: size of the image to be processed for visual embeddings. + + Returns: + Callable transform for normalizing and resizing one RGB image. + """ + return transforms.Compose([ + transforms.Resize((image_size, image_size), + interpolation=InterpolationMode.BICUBIC), + transforms.ToTensor(), + transforms.Normalize(mean=CLIP_MEAN, std=CLIP_STD), + ]) + + +def dummy_data_for_qwen( + ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], +) -> Tuple[SequenceData, Optional[Dict]]: + """Build dummy data for warming up Qwen models; this will only contain text + matching the defaults for VLLM unless the model has a visual config. + + Args: + ctx: Context of the loaded model. + seq_len: Number of tokens in the text sequence. + mm_counts: multimodal data counts. + + Returns: + Tuple containing sequential and multimodal data. + """ + hf_config = ctx.get_hf_config() + + # The presence of a visual config indicates this is a multimodal model. + # If we don't have it, the model is considered an LLM for warmup purposes. + if not hasattr(hf_config, "visual"): + seq_data = SequenceData.from_token_counts((0, seq_len)) + mm_data = None + return seq_data, mm_data + + # We have a visual component - use images to warm up + num_images = mm_counts["image"] + model_config = ctx.model_config + tokenizer = cached_get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code) + + # Build the image prompts with no imgpads; the tokenizer will add img pads + image_prompt = ''.join( + [get_image_text(idx, False) for idx in range(1, num_images + 1)]) + toks = tokenizer.encode(image_prompt, add_special_tokens=False) + + # Make sure we actually get the fixed context size per tok padding + num_pads = toks.count(tokenizer.encode(IMG_PAD)[0]) + if num_pads != (num_images * MAX_QWEN_IMG_TOKENS): + raise ValueError( + f"Tokenized dummy data should encode {MAX_QWEN_IMG_TOKENS} pads" + f" per image, but got {num_pads} pads for {num_images} image(s)" + " in total. Are you using a qwen tokenizer?") + + # Ensure the number of tokens is at minimum the sequence length provided + if len(toks) < seq_len: + toks += [0] * (seq_len - len(toks)) + + seq_data = SequenceData.from_seqs(toks) + + # Build the input images; width/height doesn't actually matter here since + # the data will get resized and the # of tokens per image is constant + image = Image.new("RGB", (224, 224), color=0) + mm_data = {"image": image if num_images == 1 else [image] * num_images} + return seq_data, mm_data + + +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen) +@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen) +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen) +class QWenLMHeadModel(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.multimodal_config = multimodal_config + self.quant_config = quant_config + self.transformer = QWenModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.transformer.wte.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.transformer.make_empty_intermediate_tensors) + + def _get_image_input_type( + self, + pixel_values: Optional[torch.Tensor]) -> Optional[QwenImageInputs]: + """Determines if the provided pixel_values are normalized pixel values + or image embeddings. + + Args: + pixel_values: Optional data to processed into visual embeddings. + + Returns: + None of the QwenImageInputs type used to determine whether or not + the visual transformer needs to process the pixel_values. + """ + if pixel_values is not None and self.transformer.visual is not None: + pixel_values = flatten_bn(pixel_values) + if len(pixel_values.shape) == 3 and pixel_values.shape[ + 1] == MAX_QWEN_IMG_TOKENS and pixel_values.shape[ + 2] == self.config.visual["output_dim"]: + return QwenImageEmbeddingInputs( + type="image_embeds", + data=pixel_values, + ) + else: + # If we have the wrong shape, assume we still need to process + return QwenImagePixelInputs( + type="pixel_values", + data=pixel_values, + ) + return None + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + pixel_values: Optional[torch.Tensor] = None + ) -> Union[torch.Tensor, IntermediateTensors]: + if intermediate_tensors is not None: + input_ids = None + pixel_values = None + else: + pixel_values = self._get_image_input_type(pixel_values) + + hidden_states = self.transformer(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors, + pixel_values) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "w2", 0), + ("gate_up_proj", "w1", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py new file mode 100644 index 00000000..08ec53d9 --- /dev/null +++ b/vllm/model_executor/models/qwen2.py @@ -0,0 +1,457 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen2 model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import Qwen2Config + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class Qwen2MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Qwen2Attention(nn.Module): + + def __init__(self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + rope_scaling: Optional[Tuple] = None) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=True, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=self.rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Qwen2DecoderLayer(nn.Module): + + def __init__( + self, + config: Qwen2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 1000000) + rope_scaling = getattr(config, "rope_scaling", None) + self.self_attn = Qwen2Attention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=rope_scaling) + self.mlp = Qwen2MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class Qwen2Model(nn.Module): + + def __init__( + self, + config: Qwen2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + decoder_layer_type: type[nn.Module] = Qwen2DecoderLayer + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + ) + else: + self.embed_tokens = PPMissingLayer() + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: decoder_layer_type(config=config, + cache_config=cache_config, + quant_config=quant_config), + prefix=f"{prefix}.layers", + ) + + # from ixformer.inference.overlap import create_vllm_llama_decoder_layer + from vllm.distributed import get_tp_group + + # for layer_idx in range(self.start_layer, self.end_layer): + # self.layers[layer_idx] = create_vllm_llama_decoder_layer( + # self.layers[layer_idx], + # model_id = id(self), + # layer_idx = layer_idx, + # enable_overlap = True, + # group = get_tp_group(), + # quant_config=quant_config, + # activation_dtype=torch.get_default_dtype() + # ) + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + +class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: Qwen2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + # TODO (@robertgshaw2): see if this can be moved out + if (cache_config.sliding_window is not None + and hasattr(config, "max_window_layers")): + raise ValueError("Sliding window for some but all layers is not " + "supported. This model uses sliding window " + "but `max_window_layers` = %s is less than " + "`num_hidden_layers` = %s. Please open an issue " + "to discuss this feature." % ( + config.max_window_layers, + config.num_hidden_layers, + )) + + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = Qwen2Model(config, cache_config, quant_config) + + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + loader.load_weights(weights) \ No newline at end of file diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py new file mode 100644 index 00000000..bae5e54c --- /dev/null +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -0,0 +1,1213 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights.""" +from functools import lru_cache, partial +from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional, + Tuple, Type, TypedDict, Union) + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from PIL import Image +from transformers.image_utils import (get_image_size, + infer_channel_dimension_format, + to_numpy_array) +from transformers.models.qwen2_vl.image_processing_qwen2_vl import (make_batched_videos, smart_resize) +from transformers.models.emu3.image_processing_emu3 import make_batched_images + +import vllm.envs as envs +from vllm.attention import AttentionMetadata +from vllm.attention.selector import (_Backend, backend_name_to_enum, + get_global_forced_attn_backend) +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import get_pp_group, parallel_state, tensor_model_parallel_all_gather +from vllm.distributed import utils as dist_utils +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict, + MultiModalInputs) +from vllm.multimodal.base import MultiModalData +from vllm.multimodal.image import cached_get_image_processor +from vllm.platforms import current_platform +from vllm.sequence import IntermediateTensors, SequenceData +from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( + Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig) +from vllm.transformers_utils.processor import get_processor +from vllm.utils import is_cpu + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory) + +logger = init_logger(__name__) + +# === Vision Inputs === # + + +class Qwen2_5_VLImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: + `(num_patches, num_channels * patch_size * patch_size)` + """ + + image_grid_thw: torch.Tensor + """Shape: `(num_images, 3)` + This should be in `(grid_t, grid_h, grid_w)` format. + """ + + +class Qwen2_5_VLImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + `hidden_size` must match the hidden size of language model backbone. + """ + + +Qwen2_5_VLImageInputs = Union[Qwen2_5_VLImagePixelInputs, + Qwen2_5_VLImageEmbeddingInputs] + + +class Qwen2_5_VLVideoInputs(TypedDict): + pixel_values_videos: torch.Tensor + """Shape: + `(num_patches, + num_channels * temporal_patch_size * patch_size * patch_size)` + """ + + video_grid_thw: torch.Tensor + """Shape: `(num_videos, 3)` + + This should be in `(grid_t, grid_h, grid_w)` format. + """ + + +# === Vision Encoder === # + + +class Qwen2_5_VisionMLP(nn.Module): + + def __init__( + self, + in_features: int, + hidden_features: int = None, + bias: bool = False, + act_layer: Callable[[torch.Tensor], torch.Tensor] = F.silu, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.gate_proj = ColumnParallelLinear(in_features, + hidden_features, + bias=bias, + quant_config=quant_config) + self.up_proj = ColumnParallelLinear(in_features, + hidden_features, + bias=bias, + quant_config=quant_config) + self.down_proj = RowParallelLinear(hidden_features, + in_features, + bias=bias, + quant_config=quant_config) + self.act = act_layer + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_gate, _ = self.gate_proj(x) + x_gate = self.act(x_gate) + x_up, _ = self.up_proj(x) + x_down, _ = self.down_proj(x_gate * x_up) + return x_down + + +def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor: + if not interleaved: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + else: + x1, x2 = x[..., ::2], x[..., 1::2] + return rearrange(torch.stack((-x2, x1), dim=-1), + "... d two -> ... (d two)", + two=2) + + +def apply_rotary_emb_torch(x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + interleaved: bool = False) -> torch.Tensor: + """ + x: (batch_size, seqlen, nheads, headdim) + cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2) + """ + ro_dim = cos.shape[-1] * 2 + assert ro_dim <= x.shape[-1] + cos = repeat( + cos, + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)") + sin = repeat( + sin, + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)") + return torch.cat( + [ + x[..., :ro_dim] * cos + + rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:] + ], + dim=-1, + ) + + +def apply_rotary_pos_emb_vision(t: torch.Tensor, + freqs: torch.Tensor) -> torch.Tensor: + t_ = t.float() + cos = freqs.cos() + sin = freqs.sin() + output = apply_rotary_emb_torch(t_, cos, sin).type_as(t) + return output + + +class Qwen2_5_VisionAttention(nn.Module): + + def __init__( + self, + embed_dim: Optional[int] = None, + num_heads: Optional[int] = None, + projection_size: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + # Per attention head and per partition values. + self.tp_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_attention_head = dist_utils.divide( + projection_size, num_heads) + self.num_attention_heads_per_partition = dist_utils.divide( + num_heads, self.tp_size) + + self.qkv = ColumnParallelLinear(input_size=embed_dim, + output_size=3 * projection_size, + quant_config=quant_config) + self.proj = RowParallelLinear(input_size=projection_size, + output_size=embed_dim, + quant_config=quant_config) + + # Detect attention implementation. + selected_backend: Optional[_Backend] = get_global_forced_attn_backend() + if selected_backend is None: + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND + if backend_by_env_var is not None: + selected_backend = backend_name_to_enum(backend_by_env_var) + if selected_backend is None: + # For Volta and Turing GPUs, use xformers instead. + device_available = current_platform.has_device_capability(80) + if device_available: + from transformers.utils import is_flash_attn_2_available + + if is_flash_attn_2_available(): + self._use_flash_attn = True + else: + logger.warning( + "Current Qwen2-VL implementation has a bug with " + "`vllm-flash-attn` inside vision module, so we use " + "xformers backend instead. You can run `pip install " + "flash-attn to use flash-attention backend.") + self._use_flash_attn = False + else: + self._use_flash_attn = False + else: + if selected_backend == _Backend.FLASH_ATTN: + self._use_flash_attn = True + elif selected_backend == _Backend.XFORMERS: + self._use_flash_attn = False + else: + raise RuntimeError( + f"Qwen2-5-VL does not support {selected_backend} backend now." + ) + def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]: + # [s, b, 3 * head * head_dim] + seq_len, bs, _ = qkv.shape + if self.tp_size > 1: + qkv = tensor_model_parallel_all_gather(qkv) + + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim] + q, k, v = qkv.chunk(3, dim=2) + + # 3 * [s, b, head * head_dim] + if self.tp_size > 1: + splitter = partial(dist_utils.split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + v = splitter(v)[self.tp_rank] + + # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim] + new_shape = (seq_len, bs, self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head) + q, k, v = (x.view(*new_shape) for x in (q, k, v)) + return q, k, v + + def forward( + self, + x: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor = None, + ) -> torch.Tensor: + # [s, b, c] --> [s, b, head * 3 * head_dim] + x, _ = self.qkv(x) + + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim] + q, k, v = self.split_qkv(x) + batch_size = q.shape[1] + + q, k, v = [ + rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v) + ] + if rotary_pos_emb is not None: + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb) + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb) + + from ixformer.contrib.xformers import ops as xops + from xformers.ops.fmha.attn_bias import (AttentionBias, + BlockDiagonalMask,) + + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, + kv_seqlen=None) + context_layer = xops.memory_efficient_attention_forward( + q, k, v, attn_bias=attn_bias, p=0, scale=None, op=xops.fmha.flash.FwOp()) + context_layer = rearrange(context_layer, + "b s h d -> s b (h d)").contiguous() + + output, _ = self.proj(context_layer) + return output + + +class Qwen2_5_VisionBlock(nn.Module): + + def __init__( + self, + dim: int, + num_heads: int, + mlp_hidden_dim: int, + act_layer: Callable[[torch.Tensor], torch.Tensor] = F.silu, + norm_layer: Type[nn.Module] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = partial(nn.LayerNorm, eps=1e-6) + self.norm1 = norm_layer(dim) + self.norm2 = norm_layer(dim) + + + self.attn = Qwen2_5_VisionAttention(embed_dim=dim, + num_heads=num_heads, + projection_size=dim, + quant_config=quant_config) + self.mlp = Qwen2_5_VisionMLP(in_features=dim, + hidden_features=mlp_hidden_dim, + bias=True, + act_layer=act_layer, + quant_config=quant_config) + + def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb) + x = x + self.mlp(self.norm2(x)) + return x + + +class Qwen2_5_VisionPatchEmbed(nn.Module): + + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 2, + in_chans: int = 3, + hidden_size: int = 1152, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.hidden_size = hidden_size + + kernel_size = [temporal_patch_size, patch_size, patch_size] + self.proj = nn.Conv3d(in_chans, + hidden_size, + kernel_size=kernel_size, + stride=kernel_size, + bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + L, C = x.shape + x = x.view(L, -1, self.temporal_patch_size, self.patch_size, + self.patch_size) + x = self.proj(x).view(L, self.hidden_size) + return x + + +class Qwen2_5_VisionPatchMerger(nn.Module): + + def __init__( + self, + d_model: int, + context_dim: int, + norm_layer: Type[nn.Module] = None, + spatial_merge_size: int = 2, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = context_dim * (spatial_merge_size**2) + if norm_layer is None: + norm_layer = partial(nn.LayerNorm, eps=1e-6) + self.ln_q = norm_layer(context_dim) + self.mlp = nn.ModuleList([ + ColumnParallelLinear(self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config), + nn.GELU(), + RowParallelLinear(self.hidden_size, + d_model, + bias=True, + quant_config=quant_config), + ]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.ln_q(x) + x = x.view(-1, self.hidden_size) + + mlp_fc1, mlp_act, mlp_fc2 = self.mlp + x_parallel, _ = mlp_fc1(x) + x_parallel = mlp_act(x_parallel) + out, _ = mlp_fc2(x_parallel) + return out + + +class Qwen2_5_VisionRotaryEmbedding(nn.Module): + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + inv_freq = 1.0 / (theta + **(torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self._seq_len_cached = 0 + self._freqs_cached = None + + def update_freqs_cache(self, seqlen: int) -> None: + if seqlen > self._seq_len_cached: + seqlen *= 2 + self._seq_len_cached = seqlen + self.inv_freq = 1.0 / (self.theta**(torch.arange( + 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device) + / self.dim)) + seq = torch.arange(seqlen, + device=self.inv_freq.device, + dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + self._freqs_cached = freqs + + def forward(self, seqlen: int) -> torch.Tensor: + self.update_freqs_cache(seqlen) + return self._freqs_cached[:seqlen] + + +class Qwen2_5_VisionTransformer(nn.Module): + + def __init__( + self, + vision_config: Qwen2_5_VLVisionConfig, + norm_eps: float = 1e-6, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + + patch_size: int = vision_config.patch_size + temporal_patch_size: int = vision_config.temporal_patch_size + spatial_merge_size: int = vision_config.spatial_merge_size + in_channels: int = vision_config.in_channels + hidden_size: int = vision_config.hidden_size + depth: int = vision_config.depth + num_heads: int = vision_config.num_heads + self.spatial_merge_size = spatial_merge_size + self.window_size = vision_config.window_size + self.patch_size = patch_size + self.spatial_merge_unit = self.spatial_merge_size**2 + self.fullatt_block_indexes = vision_config.fullatt_block_indexes + + self.patch_embed = Qwen2_5_VisionPatchEmbed( + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + in_chans=in_channels, + hidden_size=hidden_size, + ) + norm_layer = partial(RMSNorm, eps=norm_eps) + head_dim = hidden_size // num_heads + self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList([ + Qwen2_5_VisionBlock( + dim=hidden_size, + num_heads=num_heads, + mlp_hidden_dim=vision_config.intermediate_size, + norm_layer=norm_layer, + quant_config=quant_config, + ) for layer_idx in range(depth) + ]) + self.merger = Qwen2_5_VisionPatchMerger( + d_model=vision_config.out_hidden_size, + context_dim=hidden_size, + norm_layer=norm_layer, + spatial_merge_size=self.spatial_merge_size, + quant_config=quant_config, + ) + + @property + def dtype(self) -> torch.dtype: + return self.patch_embed.proj.weight.dtype + + @property + def device(self) -> torch.device: + return self.patch_embed.proj.weight.device + + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten() + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten() + pos_ids.append( + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + def get_window_index(self, grid_thw): + window_index: list = [] + cu_window_seqlens: list = [0] + window_index_id = 0 + vit_merger_window_size = (self.window_size // + self.spatial_merge_size // self.patch_size) + + for grid_t, grid_h, grid_w in grid_thw: + llm_grid_h = grid_h // self.spatial_merge_size + llm_grid_w = grid_w // self.spatial_merge_size + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape( + grid_t, llm_grid_h, llm_grid_w) + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size + index_padded = F.pad(index, (0, pad_w, 0, pad_h), 'constant', -100) + index_padded = index_padded.reshape(grid_t, num_windows_h, + vit_merger_window_size, + num_windows_w, + vit_merger_window_size) + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( + grid_t, num_windows_h * num_windows_w, vit_merger_window_size, + vit_merger_window_size) + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1) + index_padded = index_padded.reshape(-1) + index_new = index_padded[index_padded != -100] + window_index.append(index_new + window_index_id) + cu_seqlens_tmp = seqlens.cumsum( + 0) * self.spatial_merge_unit + cu_window_seqlens[-1] + cu_window_seqlens.extend(cu_seqlens_tmp.tolist()) + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item() + window_index = torch.cat(window_index, dim=0) + return window_index, cu_window_seqlens + + def forward( + self, + x: torch.Tensor, + grid_thw: torch.Tensor, + ) -> torch.Tensor: + # patchify + hidden_states = x.to(device=self.device, dtype=self.dtype) + hidden_states = self.patch_embed(hidden_states) + + # compute position embedding + rotary_pos_emb = self.rot_pos_emb(grid_thw) + + # windows attention + window_index, cu_window_seqlens = self.get_window_index(grid_thw) + cu_window_seqlens = torch.tensor( + cu_window_seqlens, + device=hidden_states.device, + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32) + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) + seq_len, _ = hidden_states.size() + hidden_states = hidden_states.reshape( + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + hidden_states = hidden_states[window_index, :, :] + hidden_states = hidden_states.reshape(seq_len, -1) + rotary_pos_emb = rotary_pos_emb.reshape( + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + rotary_pos_emb = rotary_pos_emb[window_index, :, :] + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) + # compute cu_seqlens + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], + grid_thw[:, 0]).cumsum( + dim=0, dtype=torch.int32) + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) + + # transformers + hidden_states = hidden_states.unsqueeze(1) + for layer_num, blk in enumerate(self.blocks): + if layer_num in self.fullatt_block_indexes: + cu_seqlens_now = cu_seqlens + else: + cu_seqlens_now = cu_window_seqlens + hidden_states = blk(hidden_states, + cu_seqlens=cu_seqlens_now, + rotary_pos_emb=rotary_pos_emb) + + # adapter + hidden_states = self.merger(hidden_states) + reverse_indices = torch.argsort(window_index) + hidden_states = hidden_states[reverse_indices, :] + return hidden_states + + +# === Vision input helpers === # + +cached_get_processor = lru_cache(get_processor) + + +def mm_input_mapper_for_qwen2_5_vl( + ctx: InputContext, + data: MultiModalData[object], + data_type_key: str, +) -> MultiModalInputs: + """Input mapper for Qwen2-VL.""" + if data_type_key == "image" and isinstance(data, dict): + return MultiModalInputs({ + "image_embeds": data.get("image_embeds"), + "image_grid_thw": data.get("image_grid_thw"), + }) + model_config = ctx.model_config + image_processor = cached_get_image_processor( + model_config.model, trust_remote_code=model_config.trust_remote_code) + if image_processor is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + + images = None + videos = None + if data_type_key == "image": + images = data + else: + assert data_type_key == "video" + videos = data + + try: + batch_data = image_processor \ + .preprocess(images=images, videos=videos, return_tensors="pt") \ + .data + except Exception: + logger.error("Failed to process image (%s)", data) + raise + + return MultiModalInputs(batch_data) + + +image_input_mapper_for_qwen2_5_vl = partial(mm_input_mapper_for_qwen2_5_vl, + data_type_key="image") +video_input_mapper_for_qwen2_5_vl = partial(mm_input_mapper_for_qwen2_5_vl, + data_type_key="video") + + +def _get_vision_info( + image_processor, + height: int, + width: int, + min_pixels: int, + max_pixels: int, + do_resize: bool = True, + data_type_key: str = "image", + mm_count: int = 1, +): + """Get information (resized height / width and number of vision tokens) + of input image / video frame.""" + + if do_resize: + resized_height, resized_width = smart_resize( + height=height, + width=width, + factor=image_processor.patch_size * image_processor.merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + else: + resized_height, resized_width = height, width + + if data_type_key == "image": + grid_t = mm_count + else: + assert data_type_key == "video" + grid_t = max(mm_count // image_processor.temporal_patch_size, 1) + + grid_h = resized_height // image_processor.patch_size + grid_w = resized_width // image_processor.patch_size + vision_tokens = grid_t * grid_h * grid_w + llm_num_vision_tokens = (vision_tokens // image_processor.merge_size // + image_processor.merge_size) + + return resized_height, resized_width, llm_num_vision_tokens + + +def _get_max_image_info( + image_processor, + data_type_key: str = "image", + mm_count: int = 1, +): + return _get_vision_info( + image_processor, + height=9999999, + width=9999999, + + # Limit min / max pixels. + min_pixels=max(image_processor.min_pixels, 28 * 28), + max_pixels=min(image_processor.max_pixels, 1280 * 28 * 28), + data_type_key=data_type_key, + mm_count=mm_count, + ) + + +def get_max_qwen2_5_vl_mm_tokens(ctx: InputContext, data_type_key: str) -> int: + image_processor = cached_get_image_processor(ctx.model_config.model) + max_resized_height, max_resized_width, max_llm_image_tokens = \ + _get_max_image_info(image_processor, data_type_key=data_type_key, + mm_count=1) + return max_llm_image_tokens + + +get_max_qwen2_5_vl_image_tokens = partial(get_max_qwen2_5_vl_mm_tokens, + data_type_key="image") +get_max_qwen2_5_vl_video_tokens = partial(get_max_qwen2_5_vl_mm_tokens, + data_type_key="video") + + +def dummy_data_for_qwen2_5_vl( + ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int] +) -> Tuple[SequenceData, Optional[MultiModalDataDict]]: + image_processor = cached_get_image_processor(ctx.model_config.model) + + num_images = mm_counts["image"] + max_resized_height, max_resized_width, max_llm_image_tokens = \ + _get_max_image_info(image_processor, data_type_key="image", + mm_count=num_images) + if seq_len - max_llm_image_tokens - 2 < 0: + raise RuntimeError( + f"Qwen2-VL cannot process {num_images} images in a prompt, " + "please increase max_model_len or reduce image limit by " + "--limit-mm-per-prompt.") + + # Check video counts. + num_videos = mm_counts["video"] + max_resized_height, max_resized_width, max_llm_video_tokens = \ + _get_max_image_info(image_processor, data_type_key="video", + mm_count=num_videos) + if seq_len - max_llm_video_tokens - 2 < 0: + raise RuntimeError( + f"Qwen2-VL cannot process {num_images} videos in a prompt, " + "please increase max_model_len or reduce video limit by " + "--limit-mm-per-prompt.") + + hf_config = ctx.get_hf_config(Qwen2_5_VLConfig) + + dummy_seqdata = SequenceData.from_token_counts( + (hf_config.vision_start_token_id, 1), + (hf_config.image_token_id, max_llm_image_tokens), + (hf_config.vision_end_token_id, 1), + (0, seq_len - max_llm_image_tokens - 2), + ) + + dummy_image = Image.new("RGB", (max_resized_width, max_resized_height), + color=0) + + return dummy_seqdata, { + "image": dummy_image if num_images == 1 else [dummy_image] * num_images + } + + +def _get_llm_num_vision_tokens( + mm_inputs: list, + data_type_key: str, + image_processor, +): + """Get number of vision tokens of multimodal inputs. + + This method is derived from `transformers.models.qwen2_vl. + image_processing_qwen2_vl.Qwen2_5_VLImageProcessor._preprocess`. + """ + image = to_numpy_array(mm_inputs[0]) + input_data_format = infer_channel_dimension_format(image) + height, width = get_image_size(image, channel_dim=input_data_format) + _, _, llm_num_vision_tokens = _get_vision_info( + image_processor, + height=height, + width=width, + min_pixels=image_processor.min_pixels, + max_pixels=image_processor.max_pixels, + do_resize=image_processor.do_resize, + data_type_key=data_type_key, + mm_count=len(mm_inputs), + ) + return llm_num_vision_tokens + + +def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable, + data_type_key: str, image_processor: Any, + prompt_token_ids: List[int]) -> List[int]: + """ + Expand pad tokens for multi-modal inputs (e.g., images or videos). + + Args: + inputs (list): The multi-modal inputs (e.g., images or videos). + token_id (int): The token ID used to represent the multi-modal input. + make_batched_fn (Callable): A function to batch the inputs. + data_type_key (str): The type of the multi-modal input. + image_processor (Any): The image processor used to process the inputs. + prompt_token_ids (List[int]): The list of token IDs in the prompt. + + Returns: + List[int]: The list of token IDs for the multi-modal inputs. + """ + indices = [ + idx for idx, token in enumerate(prompt_token_ids) if token == token_id + ] + inputs = make_batched_fn(inputs) + assert len(indices) == len(inputs) + + prompt_token_ids_with_data = [] + for cnt, data in enumerate(inputs): + num_tokens = _get_llm_num_vision_tokens( + [data] if data_type_key == "image" else data, + data_type_key=data_type_key, + image_processor=image_processor, + ) + if cnt == 0: + end_idx = indices[cnt] + non_data_tokens = prompt_token_ids[:end_idx] + else: + non_data_tokens = prompt_token_ids[indices[cnt - 1] + + 1:indices[cnt]] + prompt_token_ids_with_data.extend(non_data_tokens) + prompt_token_ids_with_data.extend(token_id for _ in range(num_tokens)) + prompt_token_ids_with_data.extend(prompt_token_ids[indices[-1] + 1:]) + return prompt_token_ids_with_data + + +def input_processor_for_qwen2_5_vl(ctx: InputContext, + llm_inputs: LLMInputs) -> LLMInputs: + multi_modal_data = llm_inputs.get("multi_modal_data", None) + if multi_modal_data is None: + return llm_inputs + + image_inputs = multi_modal_data.get("image", None) + video_inputs = multi_modal_data.get("video", None) + + processor = cached_get_processor(ctx.model_config.model) + image_processor = processor.image_processor + hf_config = ctx.get_hf_config(Qwen2_5_VLConfig) + + # To avoid redundant processing of vision objects (resize, rescale, etc.), + # we extract code of calculating number of vision tokens from + # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2_5_VLProcessor`. + # + # The following code is equivalent to: + # prompt = llm_inputs["prompt"] + # inputs = processor(text=[prompt], + # images=image_inputs, + # videos=video_inputs, + # padding=True, + # return_tensors="pt") + # prompt_token_ids = inputs["input_ids"][0].tolist() + + prompt_token_ids = llm_inputs.get("prompt_token_ids", None) + if prompt_token_ids is None: + prompt = llm_inputs["prompt"] + prompt_token_ids = processor.tokenizer( + prompt, + padding=True, + return_tensors=None, + )["input_ids"] + + # Expand image pad tokens. + + if image_inputs is not None: + if isinstance(image_inputs, dict): + prompt_token_ids_with_image = [] + image_indices = [ + idx for idx, token in enumerate(prompt_token_ids) + if token == hf_config.image_token_id + ] + image_cnt = len(image_indices) + embed_dim = image_inputs.get('image_embeds').size(0) + assert embed_dim % image_cnt == 0 + num_pad_tokens = embed_dim // image_cnt + for idx, token in enumerate(prompt_token_ids): + if idx in image_indices: + prompt_token_ids_with_image.extend([token] * + num_pad_tokens) + else: + prompt_token_ids_with_image.append(token) + prompt_token_ids = prompt_token_ids_with_image + else: + prompt_token_ids = _expand_pad_tokens(image_inputs, + hf_config.image_token_id, + make_batched_images, "image", + image_processor, + prompt_token_ids) + + if video_inputs is not None: + prompt_token_ids = _expand_pad_tokens(video_inputs, + hf_config.video_token_id, + make_batched_videos, "video", + image_processor, + prompt_token_ids) + + return LLMInputs( + prompt_token_ids=prompt_token_ids, + prompt=llm_inputs["prompt"], + multi_modal_data=multi_modal_data, + ) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper( + image_input_mapper_for_qwen2_5_vl) +@MULTIMODAL_REGISTRY.register_input_mapper("video", + video_input_mapper_for_qwen2_5_vl) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_5_vl_image_tokens) +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "video", get_max_qwen2_5_vl_video_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_5_vl) +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_5_vl) +class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: Qwen2_5_VLConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + assert not cache_config.enable_prefix_caching, \ + "Qwen2-VL currently does not support prefix caching" + + self.config = config + self.multimodal_config = multimodal_config + + self.visual = Qwen2_5_VisionTransformer( + config.vision_config, + norm_eps=getattr(config, "rms_norm_eps", 1e-6), + + # NOTE: Qwen2-5-VL vision encoder does not support any + # quantization method now. + quant_config=None, + ) + + self.model = Qwen2Model(config, cache_config, quant_config) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def _validate_and_reshape_mm_tensor(self, + mm_input: Union[torch.Tensor, + List[torch.Tensor]], + name: str) -> torch.Tensor: + if not isinstance(mm_input, (torch.Tensor, list)): + raise ValueError(f"Incorrect type of {name}. " + f"Got type: {type(mm_input)}") + if isinstance(mm_input, torch.Tensor): + if mm_input.ndim == 2: + return mm_input + if mm_input.ndim != 3: + raise ValueError(f"{name} should be 2D or batched 3D tensor. " + f"Got ndim: {mm_input.ndim}") + return torch.concat(list(mm_input)) + else: + return torch.concat(mm_input) + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[Qwen2_5_VLImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + image_grid_thw = kwargs.pop("image_grid_thw", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + pixel_values = self._validate_and_reshape_mm_tensor( + pixel_values, "image pixel values") + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw") + + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of image pixel values. " + f"Got type: {type(pixel_values)}") + + return Qwen2_5_VLImagePixelInputs(type="pixel_values", + data=pixel_values, + image_grid_thw=image_grid_thw) + + if image_embeds is not None: + image_embeds = self._validate_and_reshape_mm_tensor( + image_embeds, "image embeds") + + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + return Qwen2_5_VLImageEmbeddingInputs(type="image_embeds", + data=image_embeds) + + def _parse_and_validate_video_input( + self, **kwargs: object) -> Optional[Qwen2_5_VLVideoInputs]: + pixel_values_videos = kwargs.pop("pixel_values_videos", None) + video_grid_thw = kwargs.pop("video_grid_thw", None) + + if pixel_values_videos is None: + return None + + pixel_values_videos = self._validate_and_reshape_mm_tensor( + pixel_values_videos, "video pixel values") + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw") + + return Qwen2_5_VLVideoInputs( + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + ) + + def _process_image_input(self, + image_input: Qwen2_5_VLImageInputs) -> torch.Tensor: + if image_input["type"] == "image_embeds": + return image_input["data"].type(self.visual.dtype) + + pixel_values = image_input["data"].type(self.visual.dtype) + image_embeds = self.visual(pixel_values, + grid_thw=image_input["image_grid_thw"]) + return image_embeds + + def _process_video_input(self, + video_input: Qwen2_5_VLVideoInputs) -> torch.Tensor: + pixel_values_videos = video_input["pixel_values_videos"].type( + self.visual.dtype) + video_embeds = self.visual(pixel_values_videos, + grid_thw=video_input["video_grid_thw"]) + return video_embeds + + def _merge_multimodal_embeddings( + self, + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, + multimodal_embeddings: torch.Tensor, + placeholder_token_id: int, + ) -> torch.Tensor: + mask = (input_ids == placeholder_token_id) + inputs_embeds[mask, :] = multimodal_embeddings + return inputs_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for Qwen2-VL. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + positions: Flattened (concatenated) position ids corresponding to a + batch. + **NOTE**: If mrope is enabled (default setting for Qwen2-VL + opensource models), the shape will be `(3, seq_len)`, + otherwise it will be `(seq_len,). + pixel_values: Pixel values to be fed to a model. + `None` if no images are passed. + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM. + `None` if no images are passed. + pixel_values_videos: Pixel values of videos to be fed to a model. + `None` if no videos are passed. + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM. + `None` if no videos are passed. + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + video_input = self._parse_and_validate_video_input(**kwargs) + + if image_input is None and video_input is None: + inputs_embeds = None + else: + rope_scaling = getattr(self.config, "rope_scaling", {}) + if rope_scaling.get("type", None) == "mrope": + assert positions.ndim == 2 and positions.size(0) == 3, ( + "multimodal section rotary embedding requires " + f"(3, seq_len) positions, but got {positions.size()}") + + inputs_embeds = self.model.embed_tokens(input_ids) + + if image_input is not None: + image_embeds = self._process_image_input(image_input) + inputs_embeds = self._merge_multimodal_embeddings( + input_ids, + inputs_embeds, + image_embeds, + placeholder_token_id=self.config.image_token_id, + ) + + if video_input is not None: + video_embeds = self._process_video_input(video_input) + inputs_embeds = self._merge_multimodal_embeddings( + input_ids, + inputs_embeds, + video_embeds, + placeholder_token_id=self.config.video_token_id, + ) + + input_ids = None + + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "up_proj", 1), + ("gate_up_proj", "gate_proj", 0), + ] + + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + is_visual_gate_up = "visual" in name and ("gate_proj" in name or "up_proj" in name) + if is_visual_gate_up: + if name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + try: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + except KeyError: + raise ValueError(f"Unexpected weight: {name}") from None + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py new file mode 100644 index 00000000..d4475b7c --- /dev/null +++ b/vllm/model_executor/models/qwen2_moe.py @@ -0,0 +1,523 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen2MoE model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import (get_pp_group, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import print_warning_once + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class Qwen2MoeMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Qwen2MoeSparseMoeBlock(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + + if self.tp_size > config.num_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.num_experts}.") + + self.experts = FusedMoE(num_experts=config.num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + reduce_results=False, + renormalize=config.norm_topk_prob, + quant_config=quant_config) + + self.gate = ReplicatedLinear(config.hidden_size, + config.num_experts, + bias=False, + quant_config=None) + if config.shared_expert_intermediate_size > 0: + self.shared_expert = Qwen2MoeMLP( + hidden_size=config.hidden_size, + intermediate_size=config.shared_expert_intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + ) + else: + self.shared_expert = None + self.shared_expert_gate = torch.nn.Linear(config.hidden_size, + 1, + bias=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_dim = hidden_states.shape[-1] + hidden_states = hidden_states.view(-1, hidden_dim) + shared_output = None + if self.shared_expert is not None: + shared_output = self.shared_expert(hidden_states) + if self.shared_expert_gate is not None: + shared_output = F.sigmoid( + self.shared_expert_gate(hidden_states)) * shared_output + + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states=hidden_states, + router_logits=router_logits) + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states.view(orig_shape) + + +class Qwen2MoeAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=True, + quant_config=quant_config, + ) + + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Qwen2MoeDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = Qwen2MoeAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + ) + + # Note: Qwen/Qwen2-57B-A14B-Instruct does not have + # `mlp_only_layers` in the config. + mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else + config.mlp_only_layers) + if (layer_idx not in mlp_only_layers) and ( + config.num_experts > 0 and + (layer_idx + 1) % config.decoder_sparse_step == 0): + self.mlp = Qwen2MoeSparseMoeBlock(config=config, + quant_config=quant_config) + else: + self.mlp = Qwen2MoeMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class Qwen2MoeModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Qwen2MoeDecoderLayer(config=config, + layer_idx=int( + prefix.split(".")[-1]), + cache_config=cache_config, + quant_config=quant_config), + prefix=f"{prefix}.layers", + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class Qwen2MoeForCausalLM(nn.Module, SupportsPP): + + fall_back_to_pt_during_load = False + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = Qwen2MoeModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if "mlp.experts" in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Remapping the name of FP8 kv-scale. + if name.endswith("kv_scale"): + remapped_kv_scale_name = name.replace( + ".kv_scale", ".attn.kv_scale") + if remapped_kv_scale_name not in params_dict: + print_warning_once( + "Found kv scale in the checkpoint " + f"(e.g. {name}), but not found the expected " + f"name in the model " + f"(e.g. {remapped_kv_scale_name}). " + "kv-scale is not loaded.") + continue + else: + name = remapped_kv_scale_name + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/qwen2_rm.py b/vllm/model_executor/models/qwen2_rm.py new file mode 100644 index 00000000..2edcb944 --- /dev/null +++ b/vllm/model_executor/models/qwen2_rm.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +"""Inference-only Qwen2-RM model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import Qwen2Config + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.pooler import Pooler, PoolingType +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.pooling_metadata import PoolingMetadata +from vllm.sequence import IntermediateTensors, PoolerOutput + +from .interfaces import SupportsPP +from .qwen2 import Qwen2Model +from .utils import AutoWeightsLoader + + +class ReLU(nn.Module): + + def __init__(self): + super().__init__() + self.activation = nn.ReLU() + + def forward(self, input): + input, _ = input + return self.activation(input) + + +class Qwen2ForRewardModel(nn.Module, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + + def __init__( + self, + config: Qwen2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + # TODO (@robertgshaw2): see if this can be moved out + if (cache_config.sliding_window is not None + and hasattr(config, "max_window_layers")): + raise ValueError("Sliding window for some but all layers is not " + "supported. This model uses sliding window " + "but `max_window_layers` = %s is less than " + "`num_hidden_layers` = %s. Please open an issue " + "to discuss this feature." % ( + config.max_window_layers, + config.num_hidden_layers, + )) + + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = Qwen2Model(config, cache_config, quant_config) + + self.score = nn.Sequential( + ColumnParallelLinear(config.hidden_size, + config.hidden_size, + quant_config=quant_config), + ReLU(), + RowParallelLinear(config.hidden_size, 1, + quant_config=quant_config), + ) + self._pooler = Pooler(pooling_type=PoolingType.ALL, normalize=False) + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + logits, _ = self.score(hidden_states) + return logits + + def pooler( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> Optional[PoolerOutput]: + return self._pooler(hidden_states, pooling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self,ignore_unexpected_prefixes=["lm_head"]) + loader.load_weights(weights) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py new file mode 100644 index 00000000..4a39b3fb --- /dev/null +++ b/vllm/model_executor/models/qwen2_vl.py @@ -0,0 +1,1174 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen2-VL model compatible with HuggingFace weights.""" +from functools import lru_cache, partial +from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional, + Tuple, Type, TypedDict, Union) + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from PIL import Image +from transformers.image_utils import (get_image_size, + infer_channel_dimension_format, + to_numpy_array) +from transformers.models.qwen2_vl.image_processing_qwen2_vl import ( + make_batched_images, make_batched_videos, smart_resize) + +import vllm.envs as envs +from vllm.attention import AttentionMetadata +from vllm.attention.selector import (_Backend, backend_name_to_enum, + get_global_forced_attn_backend) +from vllm.config import CacheConfig, MultiModalConfig +from vllm.distributed import get_pp_group, parallel_state +from vllm.distributed import utils as dist_utils +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.activation import QuickGELU +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict, + MultiModalInputs) +from vllm.multimodal.base import MultiModalData +from vllm.multimodal.image import cached_get_image_processor +from vllm.platforms import current_platform +from vllm.sequence import IntermediateTensors, SequenceData +from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig, + Qwen2VLVisionConfig) +from vllm.transformers_utils.processor import get_processor +from vllm.utils import is_cpu + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory) + +logger = init_logger(__name__) + +# === Vision Inputs === # + + +class Qwen2VLImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + data: torch.Tensor + """Shape: + `(num_patches, num_channels * patch_size * patch_size)` + """ + + image_grid_thw: torch.Tensor + """Shape: `(num_images, 3)` + This should be in `(grid_t, grid_h, grid_w)` format. + """ + + +class Qwen2VLImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: torch.Tensor + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)` + `hidden_size` must match the hidden size of language model backbone. + """ + + +Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs, + Qwen2VLImageEmbeddingInputs] + + +class Qwen2VLVideoInputs(TypedDict): + pixel_values_videos: torch.Tensor + """Shape: + `(num_patches, + num_channels * temporal_patch_size * patch_size * patch_size)` + """ + + video_grid_thw: torch.Tensor + """Shape: `(num_videos, 3)` + + This should be in `(grid_t, grid_h, grid_w)` format. + """ + + +# === Vision Encoder === # + + +class Qwen2VisionMLP(nn.Module): + + def __init__( + self, + in_features: int, + hidden_features: int = None, + act_layer: Type[nn.Module] = QuickGELU, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.fc1 = ColumnParallelLinear(in_features, + hidden_features, + quant_config=quant_config) + self.act = act_layer() + self.fc2 = RowParallelLinear(hidden_features, + in_features, + quant_config=quant_config) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_parallel, _ = self.fc1(x) + x_parallel = self.act(x_parallel) + x, _ = self.fc2(x_parallel) + return x + + +def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor: + if not interleaved: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + else: + x1, x2 = x[..., ::2], x[..., 1::2] + return rearrange(torch.stack((-x2, x1), dim=-1), + "... d two -> ... (d two)", + two=2) + + +def apply_rotary_emb_torch(x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + interleaved: bool = False) -> torch.Tensor: + """ + x: (batch_size, seqlen, nheads, headdim) + cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2) + """ + ro_dim = cos.shape[-1] * 2 + assert ro_dim <= x.shape[-1] + cos = repeat( + cos, + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)") + sin = repeat( + sin, + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)") + return torch.cat( + [ + x[..., :ro_dim] * cos + + rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:] + ], + dim=-1, + ) + + +def apply_rotary_pos_emb_vision(t: torch.Tensor, + freqs: torch.Tensor) -> torch.Tensor: + t_ = t.float() + cos = freqs.cos() + sin = freqs.sin() + output = apply_rotary_emb_torch(t_, cos, sin).type_as(t) + return output + + +class Qwen2VisionAttention(nn.Module): + + def __init__( + self, + embed_dim: Optional[int] = None, + num_heads: Optional[int] = None, + projection_size: Optional[int] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + # Per attention head and per partition values. + world_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_attention_head = dist_utils.divide( + projection_size, num_heads) + self.num_attention_heads_per_partition = dist_utils.divide( + num_heads, world_size) + + self.qkv = ColumnParallelLinear(input_size=embed_dim, + output_size=3 * projection_size, + quant_config=quant_config) + self.proj = RowParallelLinear(input_size=projection_size, + output_size=embed_dim, + quant_config=quant_config) + + # Detect attention implementation. + selected_backend: Optional[_Backend] = get_global_forced_attn_backend() + if selected_backend is None: + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND + if backend_by_env_var is not None: + selected_backend = backend_name_to_enum(backend_by_env_var) + if selected_backend is None: + # For Volta and Turing GPUs, use xformers instead. + device_available = current_platform.has_device_capability(80) + if device_available: + from transformers.utils import is_flash_attn_2_available + + if is_flash_attn_2_available(): + self._use_flash_attn = True + else: + logger.warning( + "Current Qwen2-VL implementation has a bug with " + "`vllm-flash-attn` inside vision module, so we use " + "xformers backend instead. You can run `pip install " + "flash-attn to use flash-attention backend.") + self._use_flash_attn = False + else: + self._use_flash_attn = False + else: + if selected_backend == _Backend.FLASH_ATTN: + self._use_flash_attn = True + elif selected_backend == _Backend.XFORMERS: + self._use_flash_attn = False + else: + raise RuntimeError( + f"Qwen2-VL does not support {selected_backend} backend now." + ) + + def forward( + self, + x: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor = None, + ) -> torch.Tensor: + # [s, b, c] --> [s, b, head * 3 * head_dim] + x, _ = self.qkv(x) + + # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim] + new_x_shape = x.size()[:-1] + ( + self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head, + ) + x = x.view(*new_x_shape) + + # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim] + q, k, v = dist_utils.split_tensor_along_last_dim(x, 3) + batch_size = q.shape[1] + + q, k, v = [ + rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v) + ] + if rotary_pos_emb is not None: + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb) + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb) + + if self._use_flash_attn: + # from vllm_flash_attn.flash_attn_interface import ( + # flash_attn_varlen_func) + from flash_attn import flash_attn_varlen_func + + q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]] + + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + output = flash_attn_varlen_func(q, + k, + v, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + dropout_p=0, + causal=False) + + context_layer = rearrange(output, + "(b s) ... -> b s ...", + b=batch_size) + elif is_cpu(): + seq_length = q.size(1) + q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]] + attention_mask = torch.zeros([1, seq_length, seq_length], + device=q.device, + dtype=torch.bool) + for i in range(1, len(cu_seqlens)): + attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i], + cu_seqlens[i - 1]:cu_seqlens[i]] = True + output = F.scaled_dot_product_attention(q, + k, + v, + attention_mask, + dropout_p=0.0) + context_layer = rearrange(output, "b h s d -> b s h d ") + else: + from xformers import ops as xops + from xformers.ops.fmha.attn_bias import BlockDiagonalMask + + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, + kv_seqlen=None) + + context_layer = xops.memory_efficient_attention_forward( + q, k, v, attn_bias=attn_bias, p=0, scale=None) + context_layer = rearrange(context_layer, + "b s h d -> s b (h d)").contiguous() + + output, _ = self.proj(context_layer) + return output + + +class Qwen2VisionBlock(nn.Module): + + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float, + act_layer: Type[nn.Module] = QuickGELU, + norm_layer: Type[nn.Module] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = partial(nn.LayerNorm, eps=1e-6) + self.norm1 = norm_layer(dim) + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + self.attn = Qwen2VisionAttention(embed_dim=dim, + num_heads=num_heads, + projection_size=dim, + quant_config=quant_config) + self.mlp = Qwen2VisionMLP(dim, + mlp_hidden_dim, + act_layer=act_layer, + quant_config=quant_config) + + def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb) + x = x + self.mlp(self.norm2(x)) + return x + + +class Qwen2VisionPatchEmbed(nn.Module): + + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 2, + in_chans: int = 3, + embed_dim: int = 1152, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.embed_dim = embed_dim + + kernel_size = [temporal_patch_size, patch_size, patch_size] + self.proj = nn.Conv3d(in_chans, + embed_dim, + kernel_size=kernel_size, + stride=kernel_size, + bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + L, C = x.shape + x = x.view(L, -1, self.temporal_patch_size, self.patch_size, + self.patch_size) + x = self.proj(x).view(L, self.embed_dim) + return x + + +class Qwen2VisionPatchMerger(nn.Module): + + def __init__( + self, + d_model: int, + context_dim: int, + norm_layer: Type[nn.Module] = None, + spatial_merge_size: int = 2, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = context_dim * (spatial_merge_size**2) + if norm_layer is None: + norm_layer = partial(nn.LayerNorm, eps=1e-6) + self.ln_q = norm_layer(context_dim) + self.mlp = nn.ModuleList([ + ColumnParallelLinear(self.hidden_size, + self.hidden_size, + bias=True, + quant_config=quant_config), + nn.GELU(), + RowParallelLinear(self.hidden_size, + d_model, + bias=True, + quant_config=quant_config), + ]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.ln_q(x) + x = x.view(-1, self.hidden_size) + + mlp_fc1, mlp_act, mlp_fc2 = self.mlp + x_parallel, _ = mlp_fc1(x) + x_parallel = mlp_act(x_parallel) + out, _ = mlp_fc2(x_parallel) + return out + + +class Qwen2VisionRotaryEmbedding(nn.Module): + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + inv_freq = 1.0 / (theta + **(torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self._seq_len_cached = 0 + self._freqs_cached = None + + def update_freqs_cache(self, seqlen: int) -> None: + if seqlen > self._seq_len_cached: + seqlen *= 2 + self._seq_len_cached = seqlen + self.inv_freq = 1.0 / (self.theta**(torch.arange( + 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device) + / self.dim)) + seq = torch.arange(seqlen, + device=self.inv_freq.device, + dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + self._freqs_cached = freqs + + def forward(self, seqlen: int) -> torch.Tensor: + self.update_freqs_cache(seqlen) + return self._freqs_cached[:seqlen] + + +class Qwen2VisionTransformer(nn.Module): + + def __init__( + self, + vision_config: Qwen2VLVisionConfig, + norm_eps: float = 1e-6, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + + patch_size: int = vision_config.patch_size + temporal_patch_size: int = vision_config.temporal_patch_size + spatial_merge_size: int = vision_config.spatial_merge_size + in_chans: int = vision_config.in_chans + hidden_size: int = vision_config.hidden_size + embed_dim: int = vision_config.embed_dim + depth: int = vision_config.depth + num_heads: int = vision_config.num_heads + mlp_ratio: float = vision_config.mlp_ratio + + self.spatial_merge_size = spatial_merge_size + + self.patch_embed = Qwen2VisionPatchEmbed( + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + ) + + norm_layer = partial(nn.LayerNorm, eps=norm_eps) + head_dim = embed_dim // num_heads + self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList([ + Qwen2VisionBlock( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + norm_layer=norm_layer, + quant_config=quant_config, + ) for _ in range(depth) + ]) + self.merger = Qwen2VisionPatchMerger( + d_model=hidden_size, + context_dim=embed_dim, + norm_layer=norm_layer, + quant_config=quant_config, + ) + + @property + def dtype(self) -> torch.dtype: + return self.blocks[0].mlp.fc2.weight.dtype + + @property + def device(self) -> torch.device: + return self.blocks[0].mlp.fc2.weight.device + + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten() + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten() + pos_ids.append( + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + def forward( + self, + x: torch.Tensor, + grid_thw: torch.Tensor, + ) -> torch.Tensor: + # patchify + x = x.to(device=self.device, dtype=self.dtype) + x = self.patch_embed(x) + + # compute position embedding + rotary_pos_emb = self.rot_pos_emb(grid_thw) + + # compute cu_seqlens + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], + grid_thw[:, 0]).cumsum( + dim=0, dtype=torch.int32) + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) + + # transformers + x = x.unsqueeze(1) + for blk in self.blocks: + x = blk(x, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb) + + # adapter + x = self.merger(x) + return x + + +# === Vision input helpers === # + +cached_get_processor = lru_cache(get_processor) + + +def mm_input_mapper_for_qwen2_vl( + ctx: InputContext, + data: MultiModalData[object], + data_type_key: str, +) -> MultiModalInputs: + """Input mapper for Qwen2-VL.""" + if data_type_key == "image" and isinstance(data, dict): + return MultiModalInputs({ + "image_embeds": data.get("image_embeds"), + "image_grid_thw": data.get("image_grid_thw"), + }) + model_config = ctx.model_config + image_processor = cached_get_image_processor( + model_config.model, trust_remote_code=model_config.trust_remote_code) + if image_processor is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + + images = None + videos = None + if data_type_key == "image": + images = data + else: + assert data_type_key == "video" + videos = data + + try: + batch_data = image_processor \ + .preprocess(images=images, videos=videos, return_tensors="pt") \ + .data + except Exception: + logger.error("Failed to process image (%s)", data) + raise + + return MultiModalInputs(batch_data) + + +image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl, + data_type_key="image") +video_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl, + data_type_key="video") + + +def _get_vision_info( + image_processor, + height: int, + width: int, + min_pixels: int, + max_pixels: int, + do_resize: bool = True, + data_type_key: str = "image", + mm_count: int = 1, +): + """Get information (resized height / width and number of vision tokens) + of input image / video frame.""" + + if do_resize: + resized_height, resized_width = smart_resize( + height=height, + width=width, + factor=image_processor.patch_size * image_processor.merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + else: + resized_height, resized_width = height, width + + if data_type_key == "image": + grid_t = mm_count + else: + assert data_type_key == "video" + grid_t = max(mm_count // image_processor.temporal_patch_size, 1) + + grid_h = resized_height // image_processor.patch_size + grid_w = resized_width // image_processor.patch_size + vision_tokens = grid_t * grid_h * grid_w + llm_num_vision_tokens = (vision_tokens // image_processor.merge_size // + image_processor.merge_size) + + return resized_height, resized_width, llm_num_vision_tokens + + +def _get_max_image_info( + image_processor, + data_type_key: str = "image", + mm_count: int = 1, +): + return _get_vision_info( + image_processor, + height=9999999, + width=9999999, + + # Limit min / max pixels. + min_pixels=max(image_processor.min_pixels, 28 * 28), + max_pixels=min(image_processor.max_pixels, 1280 * 28 * 28), + data_type_key=data_type_key, + mm_count=mm_count, + ) + + +def get_max_qwen2_vl_mm_tokens(ctx: InputContext, data_type_key: str) -> int: + image_processor = cached_get_image_processor(ctx.model_config.model) + max_resized_height, max_resized_width, max_llm_image_tokens = \ + _get_max_image_info(image_processor, data_type_key=data_type_key, + mm_count=1) + return max_llm_image_tokens + + +get_max_qwen2_vl_image_tokens = partial(get_max_qwen2_vl_mm_tokens, + data_type_key="image") +get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens, + data_type_key="video") + + +def dummy_data_for_qwen2_vl( + ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int] +) -> Tuple[SequenceData, Optional[MultiModalDataDict]]: + image_processor = cached_get_image_processor(ctx.model_config.model) + + num_images = mm_counts["image"] + max_resized_height, max_resized_width, max_llm_image_tokens = \ + _get_max_image_info(image_processor, data_type_key="image", + mm_count=num_images) + if seq_len - max_llm_image_tokens - 2 < 0: + raise RuntimeError( + f"Qwen2-VL cannot process {num_images} images in a prompt, " + "please increase max_model_len or reduce image limit by " + "--limit-mm-per-prompt.") + + # Check video counts. + num_videos = mm_counts["video"] + max_resized_height, max_resized_width, max_llm_video_tokens = \ + _get_max_image_info(image_processor, data_type_key="video", + mm_count=num_videos) + if seq_len - max_llm_video_tokens - 2 < 0: + raise RuntimeError( + f"Qwen2-VL cannot process {num_images} videos in a prompt, " + "please increase max_model_len or reduce video limit by " + "--limit-mm-per-prompt.") + + hf_config = ctx.get_hf_config(Qwen2VLConfig) + + dummy_seqdata = SequenceData.from_token_counts( + (hf_config.vision_start_token_id, 1), + (hf_config.image_token_id, max_llm_image_tokens), + (hf_config.vision_end_token_id, 1), + (0, seq_len - max_llm_image_tokens - 2), + ) + + dummy_image = Image.new("RGB", (max_resized_width, max_resized_height), + color=0) + + return dummy_seqdata, { + "image": dummy_image if num_images == 1 else [dummy_image] * num_images + } + + +def _get_llm_num_vision_tokens( + mm_inputs: list, + data_type_key: str, + image_processor, +): + """Get number of vision tokens of multimodal inputs. + + This method is derived from `transformers.models.qwen2_vl. + image_processing_qwen2_vl.Qwen2VLImageProcessor._preprocess`. + """ + image = to_numpy_array(mm_inputs[0]) + input_data_format = infer_channel_dimension_format(image) + height, width = get_image_size(image, channel_dim=input_data_format) + _, _, llm_num_vision_tokens = _get_vision_info( + image_processor, + height=height, + width=width, + min_pixels=image_processor.min_pixels, + max_pixels=image_processor.max_pixels, + do_resize=image_processor.do_resize, + data_type_key=data_type_key, + mm_count=len(mm_inputs), + ) + return llm_num_vision_tokens + + +def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable, + data_type_key: str, image_processor: Any, + prompt_token_ids: List[int]) -> List[int]: + """ + Expand pad tokens for multi-modal inputs (e.g., images or videos). + + Args: + inputs (list): The multi-modal inputs (e.g., images or videos). + token_id (int): The token ID used to represent the multi-modal input. + make_batched_fn (Callable): A function to batch the inputs. + data_type_key (str): The type of the multi-modal input. + image_processor (Any): The image processor used to process the inputs. + prompt_token_ids (List[int]): The list of token IDs in the prompt. + + Returns: + List[int]: The list of token IDs for the multi-modal inputs. + """ + indices = [ + idx for idx, token in enumerate(prompt_token_ids) if token == token_id + ] + inputs = make_batched_fn(inputs) + assert len(indices) == len(inputs) + + prompt_token_ids_with_data = [] + for cnt, data in enumerate(inputs): + num_tokens = _get_llm_num_vision_tokens( + [data] if data_type_key == "image" else data, + data_type_key=data_type_key, + image_processor=image_processor, + ) + if cnt == 0: + end_idx = indices[cnt] + non_data_tokens = prompt_token_ids[:end_idx] + else: + non_data_tokens = prompt_token_ids[indices[cnt - 1] + + 1:indices[cnt]] + prompt_token_ids_with_data.extend(non_data_tokens) + prompt_token_ids_with_data.extend(token_id for _ in range(num_tokens)) + prompt_token_ids_with_data.extend(prompt_token_ids[indices[-1] + 1:]) + return prompt_token_ids_with_data + + +def input_processor_for_qwen2_vl(ctx: InputContext, + llm_inputs: LLMInputs) -> LLMInputs: + multi_modal_data = llm_inputs.get("multi_modal_data", None) + if multi_modal_data is None: + return llm_inputs + + image_inputs = multi_modal_data.get("image", None) + video_inputs = multi_modal_data.get("video", None) + + processor = cached_get_processor(ctx.model_config.model) + image_processor = processor.image_processor + hf_config = ctx.get_hf_config(Qwen2VLConfig) + + # To avoid redundant processing of vision objects (resize, rescale, etc.), + # we extract code of calculating number of vision tokens from + # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`. + # + # The following code is equivalent to: + # prompt = llm_inputs["prompt"] + # inputs = processor(text=[prompt], + # images=image_inputs, + # videos=video_inputs, + # padding=True, + # return_tensors="pt") + # prompt_token_ids = inputs["input_ids"][0].tolist() + + prompt_token_ids = llm_inputs.get("prompt_token_ids", None) + if prompt_token_ids is None: + prompt = llm_inputs["prompt"] + prompt_token_ids = processor.tokenizer( + prompt, + padding=True, + return_tensors=None, + )["input_ids"] + + # Expand image pad tokens. + + if image_inputs is not None: + if isinstance(image_inputs, dict): + prompt_token_ids_with_image = [] + image_indices = [ + idx for idx, token in enumerate(prompt_token_ids) + if token == hf_config.image_token_id + ] + image_cnt = len(image_indices) + embed_dim = image_inputs.get('image_embeds').size(0) + assert embed_dim % image_cnt == 0 + num_pad_tokens = embed_dim // image_cnt + for idx, token in enumerate(prompt_token_ids): + if idx in image_indices: + prompt_token_ids_with_image.extend([token] * + num_pad_tokens) + else: + prompt_token_ids_with_image.append(token) + prompt_token_ids = prompt_token_ids_with_image + else: + prompt_token_ids = _expand_pad_tokens(image_inputs, + hf_config.image_token_id, + make_batched_images, "image", + image_processor, + prompt_token_ids) + + if video_inputs is not None: + prompt_token_ids = _expand_pad_tokens(video_inputs, + hf_config.video_token_id, + make_batched_videos, "video", + image_processor, + prompt_token_ids) + + return LLMInputs( + prompt_token_ids=prompt_token_ids, + prompt=llm_inputs["prompt"], + multi_modal_data=multi_modal_data, + ) + + +@MULTIMODAL_REGISTRY.register_image_input_mapper( + image_input_mapper_for_qwen2_vl) +@MULTIMODAL_REGISTRY.register_input_mapper("video", + video_input_mapper_for_qwen2_vl) +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens) +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "video", get_max_qwen2_vl_video_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl) +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl) +class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP): + + def __init__(self, + config: Qwen2VLConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + + assert not cache_config.enable_prefix_caching, \ + "Qwen2-VL currently does not support prefix caching" + + self.config = config + self.multimodal_config = multimodal_config + + self.visual = Qwen2VisionTransformer( + config.vision_config, + norm_eps=getattr(config, "rms_norm_eps", 1e-6), + + # NOTE: Qwen2-VL vision encoder does not support any + # quantization method now. + quant_config=None, + ) + + self.model = Qwen2Model(config, cache_config, quant_config) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def _validate_and_reshape_mm_tensor(self, + mm_input: Union[torch.Tensor, + List[torch.Tensor]], + name: str) -> torch.Tensor: + if not isinstance(mm_input, (torch.Tensor, list)): + raise ValueError(f"Incorrect type of {name}. " + f"Got type: {type(mm_input)}") + if isinstance(mm_input, torch.Tensor): + if mm_input.ndim == 2: + return mm_input + if mm_input.ndim != 3: + raise ValueError(f"{name} should be 2D or batched 3D tensor. " + f"Got ndim: {mm_input.ndim}") + return torch.concat(list(mm_input)) + else: + return torch.concat(mm_input) + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[Qwen2VLImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + image_grid_thw = kwargs.pop("image_grid_thw", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + pixel_values = self._validate_and_reshape_mm_tensor( + pixel_values, "image pixel values") + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw") + + if not isinstance(pixel_values, (torch.Tensor, list)): + raise ValueError("Incorrect type of image pixel values. " + f"Got type: {type(pixel_values)}") + + return Qwen2VLImagePixelInputs(type="pixel_values", + data=pixel_values, + image_grid_thw=image_grid_thw) + + if image_embeds is not None: + image_embeds = self._validate_and_reshape_mm_tensor( + image_embeds, "image embeds") + + if not isinstance(image_embeds, torch.Tensor): + raise ValueError("Incorrect type of image embeddings. " + f"Got type: {type(image_embeds)}") + return Qwen2VLImageEmbeddingInputs(type="image_embeds", + data=image_embeds) + + def _parse_and_validate_video_input( + self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]: + pixel_values_videos = kwargs.pop("pixel_values_videos", None) + video_grid_thw = kwargs.pop("video_grid_thw", None) + + if pixel_values_videos is None: + return None + + pixel_values_videos = self._validate_and_reshape_mm_tensor( + pixel_values_videos, "video pixel values") + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw") + + return Qwen2VLVideoInputs( + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + ) + + def _process_image_input(self, + image_input: Qwen2VLImageInputs) -> torch.Tensor: + if image_input["type"] == "image_embeds": + return image_input["data"].type(self.visual.dtype) + + pixel_values = image_input["data"].type(self.visual.dtype) + image_embeds = self.visual(pixel_values, + grid_thw=image_input["image_grid_thw"]) + return image_embeds + + def _process_video_input(self, + video_input: Qwen2VLVideoInputs) -> torch.Tensor: + pixel_values_videos = video_input["pixel_values_videos"].type( + self.visual.dtype) + video_embeds = self.visual(pixel_values_videos, + grid_thw=video_input["video_grid_thw"]) + return video_embeds + + def _merge_multimodal_embeddings( + self, + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, + multimodal_embeddings: torch.Tensor, + placeholder_token_id: int, + ) -> torch.Tensor: + mask = (input_ids == placeholder_token_id) + inputs_embeds[mask, :] = multimodal_embeddings + return inputs_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for Qwen2-VL. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + positions: Flattened (concatenated) position ids corresponding to a + batch. + **NOTE**: If mrope is enabled (default setting for Qwen2-VL + opensource models), the shape will be `(3, seq_len)`, + otherwise it will be `(seq_len,). + pixel_values: Pixel values to be fed to a model. + `None` if no images are passed. + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM. + `None` if no images are passed. + pixel_values_videos: Pixel values of videos to be fed to a model. + `None` if no videos are passed. + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM. + `None` if no videos are passed. + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + image_input = self._parse_and_validate_image_input(**kwargs) + video_input = self._parse_and_validate_video_input(**kwargs) + + if image_input is None and video_input is None: + inputs_embeds = None + else: + rope_scaling = getattr(self.config, "rope_scaling", {}) + if rope_scaling.get("type", None) == "mrope": + assert positions.ndim == 2 and positions.size(0) == 3, ( + "multimodal section rotary embedding requires " + f"(3, seq_len) positions, but got {positions.size()}") + + inputs_embeds = self.model.embed_tokens(input_ids) + + if image_input is not None: + image_embeds = self._process_image_input(image_input) + inputs_embeds = self._merge_multimodal_embeddings( + input_ids, + inputs_embeds, + image_embeds, + placeholder_token_id=self.config.image_token_id, + ) + + if video_input is not None: + video_embeds = self._process_video_input(video_input) + inputs_embeds = self._merge_multimodal_embeddings( + input_ids, + inputs_embeds, + video_embeds, + placeholder_token_id=self.config.video_token_id, + ) + + input_ids = None + + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "up_proj", 1), + ("gate_up_proj", "gate_proj", 0), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if "visual" in name and "qkv.weight" in name: + visual_num_heads = self.config.vision_config.num_heads + visual_embed_dim = self.config.vision_config.embed_dim + head_size = visual_embed_dim // visual_num_heads + loaded_weight = loaded_weight.view(3, visual_num_heads, + head_size, + visual_embed_dim) + loaded_weight = loaded_weight.transpose(0, 1) + loaded_weight = loaded_weight.reshape(-1, visual_embed_dim) + elif "visual" in name and "qkv.bias" in name: + visual_num_heads = self.config.vision_config.num_heads + visual_embed_dim = self.config.vision_config.embed_dim + head_size = visual_embed_dim // visual_num_heads + loaded_weight = loaded_weight.view(3, visual_num_heads, + head_size) + loaded_weight = loaded_weight.transpose(0, 1) + loaded_weight = loaded_weight.reshape(-1) + try: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + except KeyError: + raise ValueError(f"Unexpected weight: {name}") from None + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/qwen3.py b/vllm/model_executor/models/qwen3.py new file mode 100644 index 00000000..3f63c5f6 --- /dev/null +++ b/vllm/model_executor/models/qwen3.py @@ -0,0 +1,357 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen3 model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Set, Tuple, Union + +import torch +from torch import nn +from transformers import Qwen3Config + +from vllm.attention import Attention, AttentionMetadata, AttentionType +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, LoRAConfig#VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import SamplerOutput, Sampler +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .qwen2 import Qwen2MLP as Qwen3MLP +from .qwen2 import Qwen2Model +from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix + +logger = init_logger(__name__) + + +class Qwen3Attention(nn.Module): + + def __init__(self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + head_dim: Optional[int] = None, + rms_norm_eps: float = 1e-06, + qkv_bias: bool = False, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + rope_scaling: Optional[Tuple] = None, + prefix: str = "", + attn_type: str = AttentionType.DECODER) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim or hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=self.rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + )#attn_type=attn_type) + self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + # Add qk-norm + q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, + self.head_dim) + q_by_head = self.q_norm.forward_cuda(q_by_head.contiguous()) + q = q_by_head.view(q.shape) + k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, + self.head_dim) + k_by_head = self.k_norm.forward_cuda(k_by_head.contiguous()) + k = k_by_head.view(k.shape) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Qwen3DecoderLayer(nn.Module): + + def __init__( + self, + config: Qwen3Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 1000000) + rope_scaling = getattr(config, "rope_scaling", None) + + # By default, Qwen3 uses causal attention as it is a decoder-only model. + # You can override the HF config with `is_causal=False` to enable + # bidirectional attention, which is used in some embedding models + # (e.g. Alibaba-NLP/gte-Qwen3-7B-instruct) + if getattr(config, "is_causal", True): + attn_type = AttentionType.DECODER + else: + attn_type = AttentionType.ENCODER_ONLY + + self.self_attn = Qwen3Attention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rms_norm_eps=config.rms_norm_eps, + qkv_bias=getattr(config, 'attention_bias', False), + head_dim=getattr(config, 'head_dim', None), + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=rope_scaling, + prefix=f"{prefix}.self_attn", + attn_type=attn_type, + ) + self.mlp = Qwen3MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + #prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +ALL_DECODER_LAYER_TYPES = { + "attention": Qwen3DecoderLayer, +} + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl, + # otherwise (seq_len, ). + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + }) +class Qwen3Model(Qwen2Model): + + def __init__(self, *, + config: Qwen3Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + decoder_layer_type=Qwen3DecoderLayer) + + +class Qwen3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + ] + embedding_modules = {} + embedding_padding_modules = [] + def __init__(self, *, + config: Qwen3Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = ""): + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = Qwen3Model(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "model")) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix( + prefix, "lm_head")) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, + torch.Tensor]]) -> Set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) \ No newline at end of file diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py new file mode 100644 index 00000000..26038f37 --- /dev/null +++ b/vllm/model_executor/models/qwen3_moe.py @@ -0,0 +1,538 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen3MoE model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig#, VllmConfig +from vllm.distributed import (get_pp_group, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import SamplerOutput, Sampler +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (extract_layer_index, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers, + maybe_prefix) + +logger = init_logger(__name__) + + +class Qwen3MoeMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Qwen3MoeSparseMoeBlock(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + + if self.tp_size > config.num_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.num_experts}.") + + self.experts = FusedMoE(num_experts=config.num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + reduce_results=False, + renormalize=config.norm_topk_prob, + quant_config=quant_config) + + self.gate = ReplicatedLinear(config.hidden_size, + config.num_experts, + bias=False, + quant_config=None) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_dim = hidden_states.shape[-1] + hidden_states = hidden_states.view(-1, hidden_dim) + + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states=hidden_states, + router_logits=router_logits) + final_hidden_states = final_hidden_states + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states.view(orig_shape) + + +class Qwen3MoeAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + head_dim: Optional[int] = None, + rms_norm_eps: float = 1e-06, + qkv_bias: bool = False, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim or (hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear(hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config) + + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn") + + self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + # Add qk-norm + q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, + self.head_dim) + q_by_head = self.q_norm.forward_native(q_by_head) + q = q_by_head.view(q.shape) + + k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, + self.head_dim) + k_by_head = self.k_norm.forward_native(k_by_head) + k = k_by_head.view(k.shape) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Qwen3MoeDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = Qwen3MoeAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + rms_norm_eps=config.rms_norm_eps, + qkv_bias=getattr(config, 'attention_bias', False), + head_dim=getattr(config, 'head_dim', None), + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + + # `mlp_only_layers` in the config. + layer_idx = extract_layer_index(prefix) + mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else + config.mlp_only_layers) + if (layer_idx not in mlp_only_layers) and ( + config.num_experts > 0 and + (layer_idx + 1) % config.decoder_sparse_step == 0): + self.mlp = Qwen3MoeSparseMoeBlock(config=config, + quant_config=quant_config) + else: + self.mlp = Qwen3MoeMLP(hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + +class Qwen3MoeModel(nn.Module): + + def __init__(self, *, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Qwen3MoeDecoderLayer(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix), + prefix=f"{prefix}.layers", + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class Qwen3MoeForCausalLM(nn.Module, SupportsPP): + + fall_back_to_pt_during_load = False + + def __init__(self, *, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + + self.config = config + self.quant_config = quant_config + + self.model = Qwen3MoeModel(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "model")) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, + torch.Tensor]]) -> Set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts) + + params_dict = dict(self.named_parameters()) + loaded_params: Set[str] = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if "mlp.experts" in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + # Skip loading extra bias for GPTQ models. + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in params_dict): + continue + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + # Remapping the name of FP8 kv-scale. + if name.endswith("kv_scale"): + remapped_kv_scale_name = name.replace( + ".kv_scale", ".attn.kv_scale") + if remapped_kv_scale_name not in params_dict: + logger.warning_once( + "Found kv scale in the checkpoint " + f"(e.g. {name}), but not found the expected " + f"name in the model " + f"(e.g. {remapped_kv_scale_name}). " + "kv-scale is not loaded.") + continue + else: + name = remapped_kv_scale_name + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params \ No newline at end of file diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py new file mode 100644 index 00000000..a2d063ca --- /dev/null +++ b/vllm/model_executor/models/registry.py @@ -0,0 +1,454 @@ +import importlib +import pickle +import subprocess +import sys +import tempfile +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from functools import lru_cache +from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union + +import cloudpickle +import torch.nn as nn + +from vllm.logger import init_logger +from vllm.utils import is_hip + +from .interfaces import (has_inner_state, is_attention_free, + supports_multimodal, supports_pp) +from .interfaces_base import is_embedding_model, is_text_generation_model + +logger = init_logger(__name__) + +# yapf: disable +_TEXT_GENERATION_MODELS = { + # [Decoder-only] + "AquilaModel": ("llama", "LlamaForCausalLM"), + "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2 + "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"), + "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b + "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b + "BloomForCausalLM": ("bloom", "BloomForCausalLM"), + # ChatGLMModel supports multimodal + "CohereForCausalLM": ("commandr", "CohereForCausalLM"), + "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"), + "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"), + "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"), + "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"), + "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"), + "FalconForCausalLM": ("falcon", "FalconForCausalLM"), + "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"), + "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"), + "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), + "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"), + "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"), + "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"), + "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"), + "GraniteForCausalLM": ("granite", "GraniteForCausalLM"), + "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"), + "InternLMForCausalLM": ("llama", "LlamaForCausalLM"), + "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"), + "JAISLMHeadModel": ("jais", "JAISLMHeadModel"), + "JambaForCausalLM": ("jamba", "JambaForCausalLM"), + "LlamaForCausalLM": ("llama", "LlamaForCausalLM"), + # For decapoda-research/llama-* + "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"), + "MambaForCausalLM": ("mamba", "MambaForCausalLM"), + "MistralForCausalLM": ("llama", "LlamaForCausalLM"), + "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"), + "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"), + # transformers's mpt class has lower case + "MptForCausalLM": ("mpt", "MPTForCausalLM"), + "MPTForCausalLM": ("mpt", "MPTForCausalLM"), + "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"), + "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"), + "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"), + "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"), + "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"), + "OPTForCausalLM": ("opt", "OPTForCausalLM"), + "OrionForCausalLM": ("orion", "OrionForCausalLM"), + "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"), + "PhiForCausalLM": ("phi", "PhiForCausalLM"), + "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"), + "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"), + "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"), + # QWenLMHeadModel supports multimodal + "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"), + "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"), + "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"), + "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"), + "RWForCausalLM": ("falcon", "FalconForCausalLM"), + "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"), + "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"), + "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"), + "SolarForCausalLM": ("solar", "SolarForCausalLM"), + "XverseForCausalLM": ("xverse", "XverseForCausalLM"), + # [Encoder-decoder] + "BartModel": ("bart", "BartForConditionalGeneration"), + "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"), +} + +_EMBEDDING_MODELS = { + "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"), + "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"), + "Gemma2Model": ("gemma2_embedding", "Gemma2EmbeddingModel"), +} + +_MULTIMODAL_MODELS = { + # [Decoder-only] + "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"), + "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501 + "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"), + "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"), + "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"), + "InternVLChatModel": ("internvl", "InternVLChatModel"), + "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"), + "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501 + "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501 + "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501 + "MiniCPMV": ("minicpmv", "MiniCPMV"), + "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"), + "NVLM_D": ("nvlm_d", "NVLM_D_Model"), + "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501 + "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"), + "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501 + "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"), + "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501 + "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501 + "UltravoxModel": ("ultravox", "UltravoxModel"), + # [Encoder-decoder] + "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501 +} + +_SPECULATIVE_DECODING_MODELS = { + "EAGLEModel": ("eagle", "EAGLE"), + "MedusaModel": ("medusa", "Medusa"), + "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"), +} +# yapf: enable + +_VLLM_MODELS = { + **_TEXT_GENERATION_MODELS, + **_EMBEDDING_MODELS, + **_MULTIMODAL_MODELS, + **_SPECULATIVE_DECODING_MODELS, +} + +# Models not supported by ROCm. +_ROCM_UNSUPPORTED_MODELS: List[str] = [] + +# Models partially supported by ROCm. +# Architecture -> Reason. +_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in " + "Triton flash attention. For half-precision SWA support, " + "please use CK flash attention by setting " + "`VLLM_USE_TRITON_FLASH_ATTN=0`") +_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = { + "Qwen2ForCausalLM": + _ROCM_SWA_REASON, + "MistralForCausalLM": + _ROCM_SWA_REASON, + "MixtralForCausalLM": + _ROCM_SWA_REASON, + "PaliGemmaForConditionalGeneration": + ("ROCm flash attention does not yet " + "fully support 32-bit precision on PaliGemma"), + "Phi3VForCausalLM": + ("ROCm Triton flash attention may run into compilation errors due to " + "excessive use of shared memory. If this happens, disable Triton FA " + "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`") +} + + +@dataclass(frozen=True) +class _ModelInfo: + is_text_generation_model: bool + is_embedding_model: bool + supports_multimodal: bool + supports_pp: bool + has_inner_state: bool + is_attention_free: bool + + @staticmethod + def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo": + return _ModelInfo( + is_text_generation_model=is_text_generation_model(model), + is_embedding_model=is_embedding_model(model), + supports_multimodal=supports_multimodal(model), + supports_pp=supports_pp(model), + has_inner_state=has_inner_state(model), + is_attention_free=is_attention_free(model), + ) + + +class _BaseRegisteredModel(ABC): + + @abstractmethod + def inspect_model_cls(self) -> _ModelInfo: + raise NotImplementedError + + @abstractmethod + def load_model_cls(self) -> Type[nn.Module]: + raise NotImplementedError + + +@dataclass(frozen=True) +class _RegisteredModel(_BaseRegisteredModel): + """ + Represents a model that has already been imported in the main process. + """ + + interfaces: _ModelInfo + model_cls: Type[nn.Module] + + @staticmethod + def from_model_cls(model_cls: Type[nn.Module]): + return _RegisteredModel( + interfaces=_ModelInfo.from_model_cls(model_cls), + model_cls=model_cls, + ) + + def inspect_model_cls(self) -> _ModelInfo: + return self.interfaces + + def load_model_cls(self) -> Type[nn.Module]: + return self.model_cls + + +@dataclass(frozen=True) +class _LazyRegisteredModel(_BaseRegisteredModel): + """ + Represents a model that has not been imported in the main process. + """ + module_name: str + class_name: str + + # Performed in another process to avoid initializing CUDA + def inspect_model_cls(self) -> _ModelInfo: + return _run_in_subprocess( + lambda: _ModelInfo.from_model_cls(self.load_model_cls())) + + def load_model_cls(self) -> Type[nn.Module]: + mod = importlib.import_module(self.module_name) + return getattr(mod, self.class_name) + + +@lru_cache(maxsize=128) +def _try_load_model_cls( + model_arch: str, + model: _BaseRegisteredModel, +) -> Optional[Type[nn.Module]]: + if is_hip(): + if model_arch in _ROCM_UNSUPPORTED_MODELS: + raise ValueError(f"Model architecture '{model_arch}' is not " + "supported by ROCm for now.") + + if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS: + msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch] + logger.warning( + "Model architecture '%s' is partially " + "supported by ROCm: %s", model_arch, msg) + + try: + return model.load_model_cls() + except Exception: + logger.exception("Error in loading model architecture '%s'", + model_arch) + return None + + +@lru_cache(maxsize=128) +def _try_inspect_model_cls( + model_arch: str, + model: _BaseRegisteredModel, +) -> Optional[_ModelInfo]: + try: + return model.inspect_model_cls() + except Exception: + logger.exception("Error in inspecting model architecture '%s'", + model_arch) + return None + + +@dataclass +class _ModelRegistry: + # Keyed by model_arch + models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict) + + def get_supported_archs(self) -> List[str]: + return list(self.models.keys()) + + def register_model( + self, + model_arch: str, + model_cls: Union[Type[nn.Module], str], + ) -> None: + """ + Register an external model to be used in vLLM. + + :code:`model_cls` can be either: + + - A :class:`torch.nn.Module` class directly referencing the model. + - A string in the format :code:`:` which can be used to + lazily import the model. This is useful to avoid initializing CUDA + when importing the model and thus the related error + :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`. + """ + if model_arch in self.models: + logger.warning( + "Model architecture %s is already registered, and will be " + "overwritten by the new model class %s.", model_arch, + model_cls) + + if isinstance(model_cls, str): + split_str = model_cls.split(":") + if len(split_str) != 2: + msg = "Expected a string in the format `:`" + raise ValueError(msg) + + model = _LazyRegisteredModel(*split_str) + else: + model = _RegisteredModel.from_model_cls(model_cls) + + self.models[model_arch] = model + + def _raise_for_unsupported(self, architectures: List[str]): + all_supported_archs = self.get_supported_archs() + + raise ValueError( + f"Model architectures {architectures} are not supported for now. " + f"Supported architectures: {all_supported_archs}") + + def _try_load_model_cls(self, + model_arch: str) -> Optional[Type[nn.Module]]: + if model_arch not in self.models: + return None + + return _try_load_model_cls(model_arch, self.models[model_arch]) + + def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]: + if model_arch not in self.models: + return None + + return _try_inspect_model_cls(model_arch, self.models[model_arch]) + + def _normalize_archs( + self, + architectures: Union[str, List[str]], + ) -> List[str]: + if isinstance(architectures, str): + architectures = [architectures] + if not architectures: + logger.warning("No model architectures are specified") + + return architectures + + def inspect_model_cls( + self, + architectures: Union[str, List[str]], + ) -> _ModelInfo: + architectures = self._normalize_archs(architectures) + + for arch in architectures: + model_info = self._try_inspect_model_cls(arch) + if model_info is not None: + return model_info + + return self._raise_for_unsupported(architectures) + + def resolve_model_cls( + self, + architectures: Union[str, List[str]], + ) -> Tuple[Type[nn.Module], str]: + architectures = self._normalize_archs(architectures) + + for arch in architectures: + model_cls = self._try_load_model_cls(arch) + if model_cls is not None: + return (model_cls, arch) + + return self._raise_for_unsupported(architectures) + + def is_text_generation_model( + self, + architectures: Union[str, List[str]], + ) -> bool: + return self.inspect_model_cls(architectures).is_text_generation_model + + def is_embedding_model( + self, + architectures: Union[str, List[str]], + ) -> bool: + return self.inspect_model_cls(architectures).is_embedding_model + + def is_multimodal_model( + self, + architectures: Union[str, List[str]], + ) -> bool: + return self.inspect_model_cls(architectures).supports_multimodal + + def is_pp_supported_model( + self, + architectures: Union[str, List[str]], + ) -> bool: + return self.inspect_model_cls(architectures).supports_pp + + def model_has_inner_state(self, architectures: Union[str, + List[str]]) -> bool: + return self.inspect_model_cls(architectures).has_inner_state + + def is_attention_free_model(self, architectures: Union[str, + List[str]]) -> bool: + return self.inspect_model_cls(architectures).is_attention_free + + +ModelRegistry = _ModelRegistry({ + model_arch: _LazyRegisteredModel( + module_name=f"vllm.model_executor.models.{mod_relname}", + class_name=cls_name, + ) + for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items() +}) + +_T = TypeVar("_T") + + +def _run_in_subprocess(fn: Callable[[], _T]) -> _T: + with tempfile.NamedTemporaryFile() as output_file: + # `cloudpickle` allows pickling lambda functions directly + input_bytes = cloudpickle.dumps((fn, output_file.name)) + + # cannot use `sys.executable __file__` here because the script + # contains relative imports + returned = subprocess.run( + [sys.executable, "-m", "vllm.model_executor.models.registry"], + input=input_bytes, + capture_output=True) + + # check if the subprocess is successful + try: + returned.check_returncode() + except Exception as e: + # wrap raised exception to provide more information + raise RuntimeError(f"Error raised in subprocess:\n" + f"{returned.stderr.decode()}") from e + + with open(output_file.name, "rb") as f: + return pickle.load(f) + + +def _run() -> None: + # Setup plugins + from vllm.plugins import load_general_plugins + load_general_plugins() + + fn, output_file = pickle.loads(sys.stdin.buffer.read()) + + result = fn() + + with open(output_file, "wb") as f: + f.write(pickle.dumps(result)) + + +if __name__ == "__main__": + _run() \ No newline at end of file diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py new file mode 100644 index 00000000..b2d081c9 --- /dev/null +++ b/vllm/model_executor/models/siglip.py @@ -0,0 +1,590 @@ +"""Implementation of SiglipVisionModel intended to be only used +within a vision language model.""" + +import math +from typing import Iterable, List, Optional, Tuple, Union + +import numpy as np +import torch +from PIL import Image +from torch import nn +from transformers import SiglipVisionConfig +from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention + +from vllm.config import ModelConfig +from vllm.distributed import divide, get_tensor_model_parallel_world_size +from vllm.inputs import LLMInputs +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import SequenceData + +try: + from xformers import ops as xops + USE_XFORMERS_OPS = True +except ImportError: + USE_XFORMERS_OPS = False + +USE_XFORMERS_OPS = True + + +def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int: + # Since interpolation is applied, the image size need not be divisible + # assert image_size % patch_size == 0 + return image_size // patch_size + + +def get_siglip_num_patches(*, image_size: int, patch_size: int) -> int: + grid_length = get_siglip_patch_grid_length(image_size=image_size, + patch_size=patch_size) + return grid_length * grid_length + + +def get_siglip_image_feature_size(hf_config: SiglipVisionConfig) -> int: + return get_siglip_num_patches(image_size=hf_config.image_size, + patch_size=hf_config.patch_size) + + +def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int: + return get_siglip_image_feature_size(hf_config) + + +def dummy_seq_data_for_siglip( + hf_config: SiglipVisionConfig, + seq_len: int, + num_images: int, + *, + image_token_id: int, + image_feature_size_override: Optional[int] = None, +): + if image_feature_size_override is None: + image_feature_size = get_siglip_image_feature_size(hf_config) + else: + image_feature_size = image_feature_size_override + + return SequenceData.from_token_counts( + (image_token_id, image_feature_size * num_images), + (0, seq_len - image_feature_size * num_images), + ) + + +def dummy_image_for_siglip( + hf_config: SiglipVisionConfig, + num_images: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + width = height = hf_config.image_size + if image_width_override is not None: + width = image_width_override + if image_height_override is not None: + height = image_height_override + + image = Image.new("RGB", (width, height), color=0) + return {"image": image if num_images == 1 else [image] * num_images} + + +def dummy_video_for_siglip( + hf_config: SiglipVisionConfig, + num_frames: int, + *, + image_width_override: Optional[int] = None, + image_height_override: Optional[int] = None, +): + pil_frame = dummy_image_for_siglip( + hf_config, + num_images=1, + image_width_override=image_width_override, + image_height_override=image_height_override) + np_frame = np.array(pil_frame["image"]) + mm_data_per_video = np.repeat([np_frame], num_frames, axis=0) + mm_data = {"video": mm_data_per_video} + return mm_data + + +def input_processor_for_siglip( + model_config: ModelConfig, + hf_config: SiglipVisionConfig, + llm_inputs: LLMInputs, + *, + image_token_id: int, + image_feature_size_override: Optional[Union[int, List[int]]] = None, +): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "image" not in multi_modal_data: + return llm_inputs + + tokenizer = cached_get_tokenizer(model_config.tokenizer) + + if image_feature_size_override is None: + image_data = multi_modal_data["image"] + if isinstance(image_data, Image.Image): + image_feature_size = get_siglip_image_feature_size(hf_config) + elif isinstance(image_data, torch.Tensor): + num_images, image_feature_size, hidden_size = image_data.shape + else: + raise TypeError(f"Invalid image type: {type(image_data)}") + else: + image_feature_size = image_feature_size_override + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=image_token_id, + repeat_count=image_feature_size, + ) + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs( + prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data, + ) + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa +class SiglipVisionEmbeddings(nn.Module): + + def __init__(self, config: SiglipVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + padding="valid", + ) + + self.num_patches = (self.image_size // self.patch_size)**2 + self.num_positions = self.num_patches + self.position_embedding = VocabParallelEmbedding( + self.num_positions, self.embed_dim) + self.register_buffer( + "position_ids", + torch.arange(self.num_positions, dtype=torch.int64).expand( + (1, -1)), + persistent=False, + ) + + def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, + width: int) -> torch.Tensor: + """ + This method is an adapted method for SigLIP (due to SigLIP not having + class embedding unlike other ViTs) that allows the model to interpolate + the pre-trained position encodings such that it can be usable on higher + resolution images. + + Source: + https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174 + """ + position_embeddings = self.position_embedding.weight.unsqueeze(0) + num_patches = embeddings.shape[1] + num_positions = position_embeddings.shape[1] + if num_patches == num_positions and height == width: + return position_embeddings + + dim = embeddings.shape[-1] + height = height // self.patch_size + width = width // self.patch_size + # we add a small number to avoid floating point error + # in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + height, width = height + 0.1, width + 0.1 + + patch_pos_embed = position_embeddings.reshape( + 1, int(math.sqrt(num_positions)), int(math.sqrt(num_positions)), + dim) + patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2) + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed, + scale_factor=( + height / math.sqrt(num_positions), + width / math.sqrt(num_positions), + ), + mode="bicubic", + align_corners=False, + ) + if (int(height) != patch_pos_embed.shape[-2] + or int(width) != patch_pos_embed.shape[-1]): + raise ValueError("Width or height does not match with " + "the interpolated position embeddings") + + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return patch_pos_embed + + def forward(self, + pixel_values: torch.Tensor, + interpolate_pos_encoding: bool = False) -> torch.Tensor: + _, _, height, width = pixel_values.shape + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to( + dtype=target_dtype)) # shape = [*, width, grid, grid] + embeddings = patch_embeds.flatten(2).transpose(1, 2) + + if interpolate_pos_encoding: + embeddings = embeddings + self.interpolate_pos_encoding( + embeddings, height, width) + else: + embeddings = embeddings + self.position_embedding( + self.position_ids) + return embeddings + + +class SiglipParallelAttention(nn.Module): + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError(f"embed_dim must be divisible by num_heads (got " + "`embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads}).") + + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + self.qkv_proj = QKVParallelLinear( + hidden_size=self.embed_dim, + head_size=self.head_dim, + total_num_heads=self.num_heads, + quant_config=quant_config, + ) + + self.out_proj = RowParallelLinear( + input_size=self.embed_dim, + output_size=self.embed_dim, + quant_config=quant_config, + ) + + self.tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = divide(self.num_heads, self.tp_size) + + def forward( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + """Input shape: Batch x Time x Channel""" + batch_size, q_len, _ = hidden_states.size() + + qkv_states, _ = self.qkv_proj(hidden_states) + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1) + + query_states = query_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + key_states = key_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + value_states = value_states.view(batch_size, q_len, + self.num_heads_per_partition, + self.head_dim) + query_states = query_states.transpose(1,2) + key_states = key_states.transpose(1,2) + value_states = value_states.transpose(1,2) + attn = (query_states @ (key_states * self.scale).permute(0,1,3,2)).float() + attn = torch.softmax(attn, dim=-1).to(value_states.dtype) + out = attn @ value_states + out = out.transpose(1,2).reshape(batch_size, q_len, -1) + + # out = xops.memory_efficient_attention_forward(query_states, + # key_states, + # value_states, + # p=self.dropout, + # scale=self.scale) + # out = out.contiguous().view(batch_size, q_len, -1) + attn_output, _ = self.out_proj(out) + + return attn_output, None + + +class SiglipMLP(nn.Module): + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + + # For quantization, we require the hidden size to be a multiple of 64 + quantizable = (config.hidden_size % 64 == 0 + and config.intermediate_size % 64 == 0) + self.fc1 = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + quant_config=quant_config if quantizable else None, + ) + self.fc2 = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + quant_config=quant_config if quantizable else None, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + return hidden_states + + +class SiglipEncoderLayer(nn.Module): + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.embed_dim = config.hidden_size + + num_heads = config.num_attention_heads + tp_size = get_tensor_model_parallel_world_size() + if USE_XFORMERS_OPS and num_heads % tp_size == 0: + self.self_attn = SiglipParallelAttention(config, + quant_config=quant_config) + else: + self.self_attn = SiglipSdpaAttention(config) + + self.layer_norm1 = nn.LayerNorm(self.embed_dim, + eps=config.layer_norm_eps) + self.mlp = SiglipMLP( + config, + quant_config=quant_config, + ) + self.layer_norm2 = nn.LayerNorm(self.embed_dim, + eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + ) -> Tuple[torch.Tensor, None]: + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn(hidden_states=hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states, None + + +class SiglipEncoder(nn.Module): + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None, + ): + super().__init__() + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + + self.layers = nn.ModuleList([ + SiglipEncoderLayer(config, quant_config=quant_config) + for _ in range(num_hidden_layers) + ]) + + def forward( + self, + inputs_embeds: torch.Tensor, + ) -> torch.Tensor: + hidden_states = inputs_embeds + for encoder_layer in self.layers: + hidden_states, _ = encoder_layer(hidden_states) + + return hidden_states + + +class SiglipMultiheadAttentionPoolingHead(nn.Module): + """Multihead Attention Pooling.""" + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + + self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size)) + # TODO(ChristopherCho): Implement vLLM version of MultiheadAttention + self.attention = torch.nn.MultiheadAttention( + config.hidden_size, config.num_attention_heads, batch_first=True) + self.layernorm = nn.LayerNorm(config.hidden_size, + eps=config.layer_norm_eps) + self.mlp = SiglipMLP(config=config, quant_config=quant_config) + + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: + batch_size = hidden_state.shape[0] + probe = self.probe.repeat(batch_size, 1, 1) + + hidden_state = self.attention(probe, hidden_state, hidden_state)[0] + + residual = hidden_state + hidden_state = self.layernorm(hidden_state) + hidden_state = residual + self.mlp(hidden_state) + + return hidden_state[:, 0] + + +class SiglipVisionTransformer(nn.Module): + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None, + ): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.embeddings = SiglipVisionEmbeddings(config) + self.encoder = SiglipEncoder( + config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + ) + + if len(self.encoder.layers) > config.num_hidden_layers: + raise ValueError( + f"The original encoder only has {config.num_hidden_layers} " + f"layers, but you requested {len(self.encoder.layers)} layers." + ) + elif len(self.encoder.layers) == config.num_hidden_layers: + self.post_layernorm = nn.LayerNorm(embed_dim, + eps=config.layer_norm_eps) + else: + # post_layernorm is unused when we extract intermediate features + # In this case, we can skip it to conserve memory + self.post_layernorm = None + + self.use_head = (True if not hasattr(config, "vision_use_head") else + config.vision_use_head) + if self.use_head: + self.head = SiglipMultiheadAttentionPoolingHead( + config=config, quant_config=quant_config) + + def forward( + self, + pixel_values: torch.Tensor, + interpolate_pos_encoding: bool = True, + ) -> torch.Tensor: + hidden_states = self.embeddings( + pixel_values, + interpolate_pos_encoding=interpolate_pos_encoding, + ) + + encoder_outputs = self.encoder(inputs_embeds=hidden_states) + + if self.post_layernorm is None: + return encoder_outputs + + last_hidden_state = self.post_layernorm(encoder_outputs) + # TODO: add this back when pooled_output is used in inference + # if self.use_head: + # pooled_output = self.head(last_hidden_state) + + return last_hidden_state + + +class SiglipVisionModel(nn.Module): + config_class = SiglipVisionConfig + main_input_name = "pixel_values" + + def __init__( + self, + config: SiglipVisionConfig, + quant_config: Optional[QuantizationConfig] = None, + num_hidden_layers_override: Optional[int] = None, + ): + super().__init__() + + num_heads = config.num_attention_heads + tp_size = get_tensor_model_parallel_world_size() + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0 + + self.vision_model = SiglipVisionTransformer( + config, + quant_config, + num_hidden_layers_override=num_hidden_layers_override, + ) + + def get_input_embeddings(self) -> nn.Module: + return self.vision_model.embeddings.patch_embedding + + def forward( + self, + pixel_values: torch.Tensor, + interpolate_pos_encoding: bool = False, + ) -> torch.Tensor: + return self.vision_model( + pixel_values=pixel_values, + interpolate_pos_encoding=interpolate_pos_encoding, + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] if self.shard_weight else [] + params_dict = dict(self.named_parameters()) + layer_count = len(self.vision_model.encoder.layers) + + for name, loaded_weight in weights: + # post_layernorm is optional in SiglipVisionModel + if (name.startswith("vision_model.post_layernorm") + and self.vision_model.post_layernorm is None): + continue + + # omit layers when num_hidden_layers_override is set + if name.startswith("vision_model.encoder.layers"): + layer_idx = int(name.split(".")[3]) + if layer_idx >= layer_count: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + + param = params_dict[name.replace(weight_name, param_name)] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/solar.py b/vllm/model_executor/models/solar.py new file mode 100644 index 00000000..b9298ed0 --- /dev/null +++ b/vllm/model_executor/models/solar.py @@ -0,0 +1,569 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Solar model compatible with HuggingFace weights.""" + +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + get_compressed_tensors_cache_scale) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import is_hip + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class SolarMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class SolarAttention(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr(config, "head_dim", + self.hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class SolarDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] \ + = config.original_max_position_embeddings + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + self.self_attn = SolarAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + cache_config=cache_config, + prefix=f"{prefix}.self_attn", + ) + self.mlp = SolarMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class SolarModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = ((lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0) + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: SolarDecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), + prefix=f"{prefix}.layers", + ) + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + bskcn_h_1 = None + bskcn_h_2 = None + bskcn_r_1 = None + bskcn_r_2 = None + bskcn_tv = (self.config.bskcn_tv[0] + if self.training else self.config.bskcn_tv[1]) + + for i in range(self.start_layer, self.end_layer): + if i in self.config.bskcn_1: + bskcn_h_1 = hidden_states.clone() + bskcn_r_1 = residual.clone() + if i in self.config.bskcn_2: + bskcn_h_2 = hidden_states.clone() + bskcn_r_2 = residual.clone() + if i in self.config.bskcn_3: + hidden_states = bskcn_h_1 * bskcn_tv + hidden_states * ( + 1 - bskcn_tv) + residual = bskcn_r_1 * bskcn_tv + residual * (1 - bskcn_tv) + if i in self.config.bskcn_4: + hidden_states = bskcn_h_2 * bskcn_tv + hidden_states * ( + 1 - bskcn_tv) + residual = bskcn_r_2 * bskcn_tv + residual * (1 - bskcn_tv) + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.model = SolarModel( + config, + cache_config, + quant_config, + lora_config=lora_config, + prefix="model", + ) + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else lora_config.lora_vocab_padding_size, + quant_config=quant_config, + ) + if config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + self.sampler = Sampler() + else: + self.lm_head = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return model_output + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + if scale_name := get_compressed_tensors_cache_scale(name): + # Loading kv cache scales for compressed-tensors quantization + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + + # If this function is called, it should always initialize KV cache scale + # factors (or else raise an exception). Thus, handled exceptions should + # make sure to leave KV cache scale factors in a known good (dummy) state + def load_kv_cache_scales(self, quantization_param_path: str) -> None: + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + for layer_idx, scaling_factor in kv_cache_scales_loader( + quantization_param_path, + tp_rank, + tp_size, + self.config.num_hidden_layers, + self.config.__class__.model_type, + ): + if not isinstance(self.model.layers[layer_idx], nn.Identity): + layer_self_attn = self.model.layers[layer_idx].self_attn + + if is_hip(): + # The scaling factor convention we are assuming is + # quantized_value * scaling_factor ~= true_value + # which is consistent with the practice of setting + # scaling_factor = tensor_amax / FPtype_max + scaling_factor *= 2 + if hasattr(layer_self_attn, "kv_scale"): + layer_self_attn.attn._kv_scale = scaling_factor + else: + raise RuntimeError("Self attention has no KV cache scaling " + "factor attribute!") diff --git a/vllm/model_executor/models/stablelm.py b/vllm/model_executor/models/stablelm.py new file mode 100644 index 00000000..083a4858 --- /dev/null +++ b/vllm/model_executor/models/stablelm.py @@ -0,0 +1,338 @@ +# coding=utf-8 +# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This code is based off the following work: +# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py +# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json +"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) +model compatible with HuggingFace weights.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class StablelmMLP(nn.Module): + + def __init__(self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_up_proj = MergedColumnParallelLinear( + config.hidden_size, [config.intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(config.intermediate_size, + config.hidden_size, + bias=False) + self.act_fn = SiluAndMul() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class StablelmAttention(nn.Module): + + def __init__(self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None) -> None: + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + self.num_heads = self.total_num_heads // tp_size + + self.total_num_key_value_heads = config.num_key_value_heads + if self.total_num_key_value_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_key_value_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_key_value_heads == 0 + self.num_key_value_heads = max( + 1, self.total_num_key_value_heads // tp_size) + self.head_dim = self.hidden_size // self.total_num_heads + self.max_position_embeddings = config.max_position_embeddings + rope_pct = getattr(config, "rope_pct", + getattr(config, "partial_rotary_factor", 1)) + self.rotary_ndims = int(self.head_dim * rope_pct) + self.scaling = self.head_dim**-0.5 + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_key_value_heads * self.head_dim + self.qkv_bias = getattr(config, "use_qkv_bias", False) + if (self.head_dim * self.num_heads * tp_size) != self.hidden_size: + raise ValueError(f"hidden_size must be divisible by num_heads " + f"(got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads}).") + + self.qkv_proj = QKVParallelLinear(self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_key_value_heads, + self.qkv_bias, + quant_config=quant_config) + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.rotary_ndims, + max_position=self.config.max_position_embeddings, + base=self.config.rope_theta, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_key_value_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class StablelmDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.self_attn = StablelmAttention(config, cache_config, quant_config) + self.mlp = StablelmMLP(config, quant_config) + norm_eps = getattr(config, "norm_eps", + getattr(config, "layer_norm_eps", 1e-05)) + self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states, residual + + +class StableLMEpochModel(nn.Module): + + def __init__(self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = '') -> None: + super().__init__() + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: StablelmDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers", + ) + norm_eps = getattr(config, "norm_eps", + getattr(config, "layer_norm_eps", 1e-05)) + self.norm = nn.LayerNorm(config.hidden_size, eps=norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class StablelmForCausalLM(nn.Module, SupportsPP): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = StableLMEpochModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/starcoder2.py b/vllm/model_executor/models/starcoder2.py new file mode 100644 index 00000000..81dd7c4d --- /dev/null +++ b/vllm/model_executor/models/starcoder2.py @@ -0,0 +1,336 @@ +# coding=utf-8 +# Copyright 2024 BigCode and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Starcoder2 model.""" +from typing import Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import Starcoder2Config + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class Starcoder2Attention(nn.Module): + + def __init__(self, + config: Starcoder2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + + self.hidden_size = config.hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = config.num_key_value_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = self.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = config.rope_theta + self.max_position_embeddings = config.max_position_embeddings + self.use_bias = config.use_bias + + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=self.use_bias, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=self.use_bias, + quant_config=quant_config, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=int(self.rope_theta), + is_neox_style=True, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class Starcoder2MLP(nn.Module): + + def __init__(self, + config: Starcoder2Config, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.c_fc = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + bias=config.use_bias, + quant_config=quant_config, + ) + self.c_proj = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + bias=config.use_bias, + quant_config=quant_config, + ) + self.act = get_act_fn(config.hidden_act, quant_config, + config.intermediate_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.c_fc(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.c_proj(hidden_states) + return hidden_states + + +class Starcoder2DecoderLayer(nn.Module): + + def __init__(self, + config: Starcoder2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Starcoder2Attention(config, + cache_config, + quant_config=quant_config) + self.mlp = Starcoder2MLP(config, quant_config=quant_config) + self.input_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.norm_epsilon) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, + eps=config.norm_epsilon) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class Starcoder2Model(nn.Module): + + def __init__(self, + config: Starcoder2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + # TODO: consider padding_idx (currently removed) + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, + config.hidden_size) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Starcoder2DecoderLayer( + config, cache_config, quant_config=quant_config), + prefix=f"{prefix}.layers", + ) + self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory(["hidden_states"], + config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states = layer(positions, hidden_states, + kv_caches[i - self.start_layer], + attn_metadata) + if not get_pp_group().is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class Starcoder2ForCausalLM(nn.Module, SupportsPP): + + def __init__(self, + config: Starcoder2Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None): + super().__init__() + self.config = config + self.model = Starcoder2Model(config, + cache_config, + quant_config=quant_config) + self.vocab_size = config.vocab_size + self.unpadded_vocab_size = config.vocab_size + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.unpadded_vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=DEFAULT_VOCAB_PADDING_SIZE, + quant_config=quant_config, + ) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: Optional[torch.Tensor], + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py new file mode 100644 index 00000000..e162e3af --- /dev/null +++ b/vllm/model_executor/models/ultravox.py @@ -0,0 +1,503 @@ +# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py +"""PyTorch Ultravox model.""" + +import math +from array import array +from functools import cached_property, lru_cache +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple, + TypedDict, Union, cast) + +import numpy as np +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import functional as F +from transformers.models.whisper import WhisperFeatureExtractor +from transformers.models.whisper.modeling_whisper import WhisperEncoder + +from vllm.attention import AttentionMetadata +from vllm.config import CacheConfig, MultiModalConfig +from vllm.inputs import INPUT_REGISTRY +from vllm.inputs.data import LLMInputs +from vllm.inputs.registry import InputContext +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.model_loader.loader import DefaultModelLoader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.base import MultiModalInputs, NestedTensors +from vllm.multimodal.utils import (cached_get_tokenizer, + repeat_and_pad_placeholder_tokens) +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors, + SequenceData) +from vllm.transformers_utils.configs.ultravox import UltravoxConfig +from vllm.utils import is_list_of + +from .interfaces import SupportsMultiModal, SupportsPP +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, + init_vllm_registered_model, merge_multimodal_embeddings) + +_AUDIO_PLACEHOLDER_TOKEN = 128002 +_AUDIO_TOKENS_PER_SECOND = 6.25 + + +class UltravoxAudioFeatureInputs(TypedDict): + type: Literal["audio_features"] + data: NestedTensors + """Shape: `(batch_size, num_audios, 80, M)""" + + +class UltravoxAudioEmbeddingInputs(TypedDict): + type: Literal["audio_embeds"] + data: NestedTensors + """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)""" + + +UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs, + UltravoxAudioEmbeddingInputs] + + +@lru_cache +def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor: + return WhisperFeatureExtractor.from_pretrained(model_id) + + +def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor: + return cached_feature_extractor( + ctx.get_hf_config(UltravoxConfig).audio_model_id) + + +def get_ultravox_max_audio_tokens(ctx: InputContext): + feature_extractor = whisper_feature_extractor(ctx) + return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND) + + +def dummy_seq_data_for_ultravox( + ctx: InputContext, + seq_len: int, + audio_count: int, +): + audio_placeholder = array( + VLLM_TOKEN_ID_ARRAY_TYPE, + [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx) + + # Add a separator between each chunk. + audio_token_ids = (audio_placeholder + + array(VLLM_TOKEN_ID_ARRAY_TYPE, [0])) * audio_count + other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + [0]) * (seq_len - len(audio_token_ids)) + + return SequenceData(audio_token_ids + other_token_ids) + + +def dummy_audio_for_ultravox( + ctx: InputContext, + audio_count: int, +): + feature_extractor = whisper_feature_extractor(ctx) + audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1) + return {"audio": [audio_and_sr] * audio_count} + + +def dummy_data_for_ultravox( + ctx: InputContext, + seq_len: int, + mm_counts: Mapping[str, int], +): + audio_count = mm_counts["audio"] + seq_data = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count) + mm_dict = dummy_audio_for_ultravox(ctx, audio_count) + + return (seq_data, mm_dict) + + +def input_mapper_for_ultravox(ctx: InputContext, data: object): + if not isinstance(data, list): + data = [data] + + # If the audio inputs are embeddings, no need for preprocessing + if is_list_of(data, torch.Tensor, check="all"): + return MultiModalInputs({"audio_embeds": data}) + + audio_features = [] + for audio_input in data: + if not isinstance(audio_input, tuple): + raise NotImplementedError( + f"Unsupported data type: {type(audio_input)}") + + (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], audio_input) + feature_extractor = whisper_feature_extractor(ctx) + + if sr != feature_extractor.sampling_rate: + try: + import librosa + except ImportError: + raise ImportError( + "Please install vllm[audio] for audio support.") from None + audio = librosa.resample(audio, + orig_sr=sr, + target_sr=feature_extractor.sampling_rate) + sr = feature_extractor.sampling_rate + + minimum_audio_length = feature_extractor.n_fft // 2 + 1 + if len(audio) < minimum_audio_length: + # Not enough audio; pad it. + audio = np.pad(audio, (0, minimum_audio_length - len(audio))) + + single_audio_features = feature_extractor( + audio, sampling_rate=sr, padding="longest", + return_tensors="pt")["input_features"] + + # Remove the batch dimension because we're wrapping it in a list. + audio_features.append(single_audio_features.squeeze(0)) + + return MultiModalInputs({"audio_features": audio_features}) + + +def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs): + multi_modal_data = llm_inputs.get("multi_modal_data") + if multi_modal_data is None or "audio" not in multi_modal_data: + return llm_inputs + + feature_extractor = whisper_feature_extractor(ctx) + audios = multi_modal_data["audio"] + if not isinstance(audios, list): + audios = [audios] + + audio_token_counts = [] + for audio in audios: + if isinstance(audio, torch.Tensor): + audio_num_tokens = audio.shape[1] + audio_token_counts.append(audio_num_tokens) + else: + audio_data, sample_rate = audio + audio_length = audio_data.shape[0] + if sample_rate != feature_extractor.sampling_rate: + # Account for resampling. + adjustment = feature_extractor.sampling_rate / sample_rate + audio_length = math.ceil(adjustment * audio_length) + + feature_extractor_output_length = math.ceil( + (audio_length - (feature_extractor.hop_length - 1)) / + feature_extractor.hop_length) + + uv_config = ctx.get_hf_config(UltravoxConfig) + audio_num_tokens = min( + max( + 1, + math.ceil(feature_extractor_output_length / + (uv_config.stack_factor * 2))), + get_ultravox_max_audio_tokens(ctx)) + audio_token_counts.append(audio_num_tokens) + + tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer) + + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens( + tokenizer, + llm_inputs.get("prompt"), + llm_inputs["prompt_token_ids"], + placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN, + repeat_count=audio_token_counts, + ) + + # NOTE: Create a defensive copy of the original inputs + return LLMInputs(prompt_token_ids=new_token_ids, + prompt=new_prompt, + multi_modal_data=multi_modal_data) + + +class StackAudioFrames(nn.Module): + """ + Stack the audio embedding frames to reduce the sequence length by a factor + of `stack_factor`. + """ + + def __init__(self, stack_factor: int = 8): + super().__init__() + self.stack_factor = stack_factor + + def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor: + B, T, C = audio_embeds.shape + T_pad = (T + self.stack_factor - + 1) // self.stack_factor * self.stack_factor + audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T)) + B, T, C = audio_embeds.shape + audio_embeds = audio_embeds.view(B, T // self.stack_factor, + C * self.stack_factor) + return audio_embeds + + +class FlippedSiluAndMul(SiluAndMul): + """Ultravox is trained with SwiGLU with flipped halves.""" + + def forward(self, x: torch.Tensor): + a, b = x.chunk(2, dim=-1) + flipped = torch.cat((b, a), dim=-1) + return super().forward(flipped) + + +class UltravoxProjector(nn.Module): + + def __init__(self, config: UltravoxConfig): + super().__init__() + self.hidden_dim = config.hidden_size + self._pad_and_stack = StackAudioFrames(config.stack_factor) + dim = config.audio_config.hidden_size * config.stack_factor + self.ln_pre = RMSNorm(dim) + self.linear_1 = nn.Linear(dim, self.hidden_dim, bias=False) + dim = self.hidden_dim + + if config.projector_act == "swiglu": + self.act = FlippedSiluAndMul() + dim = dim // 2 + else: + self.act = get_act_fn(config.projector_act) + + self.linear_2 = nn.Linear(dim, + config.text_config.hidden_size, + bias=False) + self.ln_post = RMSNorm(config.text_config.hidden_size) + + def forward(self, audio_features: torch.Tensor) -> torch.Tensor: + audio_features = self._pad_and_stack(audio_features) + audio_features = self.ln_pre(audio_features) + hidden_states = self.linear_1(audio_features) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + hidden_states = self.ln_post(hidden_states) + return hidden_states + + +class ModifiedWhisperEncoder(WhisperEncoder): + """ + Encoder portion of OpenAI's Whisper model. + + This implementation is a slightly modified version of HF Transformers' + Whisper Encoder, with only a few fixes: + 1. base_model_prefix updated to allow for doing `.from_pretrained` + directly on the encoder + 2. allow less than 30 second of audio padding to be passed in: + - relaxed ValueError check for `input_features` length to be less + than or equal to `expected_seq_length` instead of strictly equal + - embed_pos is now sliced to match the length of `inputs_embeds` + + Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py + See commentary: https://github.com/huggingface/transformers/issues/25744 + """ + + base_model_prefix = "model.encoder" + + def forward( + self, + input_features, + ): + expected_seq_length = (self.config.max_source_positions * + self.conv1.stride[0] * self.conv2.stride[0]) + if input_features.shape[-1] > expected_seq_length: + raise ValueError( + f"Whisper expects the mel input features to be of length " + f"{expected_seq_length} or less, but found " + f"{input_features.shape[-1]}. Make sure to pad the input mel " + f"features to {expected_seq_length}.") + + inputs_embeds = nn.functional.gelu(self.conv1(input_features)) + inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds)) + + inputs_embeds = inputs_embeds.permute(0, 2, 1) + embed_pos = self.embed_positions.weight[:inputs_embeds.size(-2)] + + hidden_states = inputs_embeds + embed_pos + hidden_states = nn.functional.dropout(hidden_states, + p=self.dropout, + training=self.training) + + for encoder_layer in self.layers: + layer_outputs = encoder_layer( + hidden_states, + None, + layer_head_mask=None, + ) + + hidden_states = layer_outputs[0] + + hidden_states = self.layer_norm(hidden_states) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_ultravox) +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens( + "audio", get_ultravox_max_audio_tokens) +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_ultravox) +@INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox) +class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP): + + def __init__(self, + config: UltravoxConfig, + multimodal_config: MultiModalConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional["QuantizationConfig"] = None): + super().__init__() + self.config = config + self.multi_modal_config = multimodal_config + assert self.multi_modal_config + + self.secondary_weights = [] + self.audio_tower = ModifiedWhisperEncoder(config.audio_config) + if config.audio_model_id is not None: + self.secondary_weights.append( + DefaultModelLoader.Source( + model_or_path=config.audio_model_id, + revision=None, + prefix="audio_tower.", + )) + self.multi_modal_projector = UltravoxProjector(config) + self.language_model = init_vllm_registered_model( + config.text_config, cache_config, quant_config) + if config.text_model_id is not None: + self.secondary_weights.append( + DefaultModelLoader.Source(model_or_path=config.text_model_id, + revision=None, + prefix="language_model.")) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + @cached_property + def sampler(self): + if hasattr(self.language_model, "sampler"): + return self.language_model.sampler + + return Sampler() + + def _audio_features_to_embeddings( + self, input_features: torch.Tensor) -> torch.Tensor: + audio_input = input_features.to(self.audio_tower.dtype) + audio_features = self.audio_tower(audio_input) + audio_features = audio_features.to(self.audio_tower.dtype) + audio_embeddings = self.multi_modal_projector(audio_features) + return audio_embeddings + + def _parse_and_validate_audio_input( + self, **kwargs: object) -> Optional[UltravoxAudioInputs]: + audio_features = kwargs.pop("audio_features", None) + audio_embeds = kwargs.pop("audio_embeds", None) + + if audio_features is None and audio_embeds is None: + return None + + if audio_features is not None: + if not isinstance(audio_features, (torch.Tensor, list)): + raise ValueError("Incorrect type of audio features. " + f"Got type: {type(audio_features)}") + + return UltravoxAudioFeatureInputs(type="audio_features", + data=audio_features) + + if audio_embeds is not None: + if not isinstance(audio_embeds, (torch.Tensor, list)): + raise ValueError("Incorrect type of audio embeds. " + f"Got type: {type(audio_embeds)}") + + return UltravoxAudioEmbeddingInputs(type="audio_embeds", + data=audio_embeds) + + raise AssertionError("This line should be unreachable.") + + def _process_audio_input( + self, audio_input: UltravoxAudioInputs) -> NestedTensors: + if audio_input["type"] == "audio_embeds": + return audio_input["data"] + + audio_features = audio_input["data"] + if isinstance(audio_features, torch.Tensor): + # Combine the B and N dimensions for the encoder/projector + flattened = flatten_bn(audio_features) + flattened_embeddings = self._audio_features_to_embeddings( + flattened) + + # Restore the original dimensions + embeddings = flattened_embeddings.unflatten( + 0, audio_features.shape[:2]) + return embeddings + + result = [] + # TODO: Batch heterogeneous tensors through the encoder/projector + for audio_features_item in audio_features: + if isinstance(audio_features_item, torch.Tensor): + result.append( + self._audio_features_to_embeddings(audio_features_item)) + else: + embeddings = [ + # Add a batch dimension to embed it, then remove it. + self._audio_features_to_embeddings(tensor.unsqueeze(0) + ).squeeze(0) + for tensor in audio_features_item + ] + result.append(embeddings) + + return result + + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[torch.Tensor], + **kwargs) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for Ultravox + + One key thing to understand is the `input_ids` already accounts for the + positions of the to-be-inserted audio embeddings. The to-be-inserted + audio has a size that is essentially 6.25 tokens per second of audio. + + This way, the `positions` and `attn_metadata` are consistent + with the `input_ids`. + + Args: + audio_features: A batch of audio inputs [B, N, 80, M]. + """ + if intermediate_tensors is not None: + input_ids = None + inputs_embeds = None + else: + audio_input = self._parse_and_validate_audio_input(**kwargs) + if audio_input is not None: + audio_embeddings = self._process_audio_input(audio_input) + inputs_embeds = self.language_model.model.get_input_embeddings( + input_ids) + + inputs_embeds = merge_multimodal_embeddings( + input_ids, inputs_embeds, audio_embeddings, + _AUDIO_PLACEHOLDER_TOKEN) + input_ids = None + else: + inputs_embeds = None + + hidden_states = self.language_model.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds) + return hidden_states + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + return self.language_model.sample(logits, sampling_metadata) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."}) + + loader = AutoWeightsLoader(self, + ignore_unexpected_prefixes=["audio_tower."]) + loader.load_weights(weights, mapper=hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py new file mode 100644 index 00000000..810da7a8 --- /dev/null +++ b/vllm/model_executor/models/utils.py @@ -0,0 +1,515 @@ +import itertools +from dataclasses import dataclass, field +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, + Protocol, Tuple, Union, overload) + +import torch +import torch.nn as nn +from torch.func import functional_call +from transformers import PretrainedConfig + +from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig, + SchedulerConfig) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.loader import build_model +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models import ModelRegistry +from vllm.multimodal.base import NestedTensors +from vllm.sequence import IntermediateTensors +from vllm.utils import is_pin_memory_available + +WeightsMapping = Mapping[str, Optional[str]] +"""If a key maps to a value of `None`, the corresponding weight is ignored.""" + + +@dataclass +class WeightsMapper: + """Maps the name of each weight if they match the following patterns.""" + + orig_to_new_substr: WeightsMapping = field(default_factory=dict) + orig_to_new_prefix: WeightsMapping = field(default_factory=dict) + orig_to_new_suffix: WeightsMapping = field(default_factory=dict) + + def _map_name(self, key: str) -> Optional[str]: + for substr, new_key in self.orig_to_new_substr.items(): + if substr in key: + if new_key is None: + return None + + key = key.replace(substr, new_key, 1) + + for prefix, new_key in self.orig_to_new_prefix.items(): + if key.startswith(prefix): + if new_key is None: + return None + + key = key.replace(prefix, new_key, 1) + + for suffix, new_key in self.orig_to_new_suffix.items(): + if key.endswith(suffix): + if new_key is None: + return None + + key = new_key.join(key.rsplit(suffix, 1)) + + return key + + def apply( + self, weights: Iterable[Tuple[str, torch.Tensor]] + ) -> Iterable[Tuple[str, torch.Tensor]]: + return ((out_name, data) for name, data in weights + if (out_name := self._map_name(name)) is not None) + + +class AutoWeightsLoader: + """ + Helper class to load weights into a :class:`torch.nn.Module`. It is able + to automatically detect child modules and parameters while iterating over + the weights only once. + + The weight loading logic for individual modules can be overridden + by defining a ``load_weights`` method. + + Similarly, the weight loading logic for individual parameters can be + overridden by defining a ``weight_loader`` method. + """ + + def __init__( + self, + module: nn.Module, + *, + skip_prefixes: Optional[List[str]] = None, + ignore_unexpected_prefixes: Optional[List[str]] = None, + ) -> None: + super().__init__() + + self.module = module + self.skip_prefixes = skip_prefixes or [] + self.ignore_unexpected_prefixes = ignore_unexpected_prefixes or [] + + def _groupby_prefix( + self, + weights: Iterable[Tuple[str, torch.Tensor]], + ) -> Iterable[Tuple[str, Iterable[Tuple[str, torch.Tensor]]]]: + weights_by_parts = ((weight_name.split(".", 1), weight_data) + for weight_name, weight_data in weights) + + for prefix, group in itertools.groupby(weights_by_parts, + key=lambda x: x[0][0]): + yield ( + prefix, + # Because maxsplit=1 in weight_name.split(...), + # the length of `parts` must either be 1 or 2 + (("" if len(parts) == 1 else parts[1], weights_data) + for parts, weights_data in group), + ) + + def _get_qualname(self, prefix: str, rest: str) -> str: + if prefix == "": + return rest + if rest == "": + return prefix + + return ".".join((prefix, rest)) + + def _can_skip(self, qualname: str) -> bool: + return any(qualname.startswith(p) for p in self.skip_prefixes) + + def _can_ignore_unexpected(self, qualname: str) -> bool: + return any( + qualname.startswith(p) for p in self.ignore_unexpected_prefixes) + + def _load_param( + self, + base_prefix: str, + param: nn.Parameter, + weights: Iterable[Tuple[str, torch.Tensor]], + ) -> None: + for weight_name, weight_data in weights: + weight_qualname = self._get_qualname(base_prefix, weight_name) + + if self._can_skip(weight_qualname): + continue + + if weight_name != "": + if not self._can_ignore_unexpected(weight_qualname): + raise ValueError( + f"Attempted to load nested weight '{weight_qualname}' " + f"into a single parameter '{base_prefix}'") + + continue + + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, weight_data) + + def _load_module( + self, + base_prefix: str, + module: nn.Module, + weights: Iterable[Tuple[str, torch.Tensor]], + ) -> None: + if isinstance(module, PPMissingLayer): + return + + # Avoid infinite recursion since this function is typically + # called inside load_weights of the module itself + if module != self.module: + module_load_weights = getattr(module, "load_weights", None) + if callable(module_load_weights): + module_load_weights(weights) + return + + child_modules = dict(module.named_children()) + child_params = dict(module.named_parameters(recurse=False)) + + for child_prefix, child_weights in self._groupby_prefix(weights): + prefix = self._get_qualname(base_prefix, child_prefix) + + if self._can_skip(prefix): + continue + + if child_prefix in child_modules: + self._load_module(prefix, child_modules[child_prefix], + child_weights) + elif child_prefix in child_params: + self._load_param(prefix, child_params[child_prefix], + child_weights) + else: + if not self._can_ignore_unexpected(prefix): + msg = f"There is no module or parameter named '{prefix}'" + raise ValueError(msg) + + def load_weights( + self, + weights: Iterable[Tuple[str, torch.Tensor]], + *, + mapper: Optional[WeightsMapper] = None, + ) -> None: + if mapper is not None: + weights = mapper.apply(weights) + + self._load_module("", self.module, weights) + + +def init_vllm_registered_model( + hf_config: PretrainedConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig], + *, + lora_config: Optional[LoRAConfig] = None, + multimodal_config: Optional[MultiModalConfig] = None, + scheduler_config: Optional[SchedulerConfig] = None, +) -> nn.Module: + """ + Helper function to initialize an inner model registered to vLLM, + based on the arguments passed to the outer vLLM model. + """ + model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures) + + return build_model( + model_class, + hf_config, + cache_config, + quant_config, + lora_config=lora_config, + multimodal_config=multimodal_config, + scheduler_config=scheduler_config, + ) + + +@overload +def flatten_bn(x: torch.Tensor) -> torch.Tensor: + ... + + +@overload +def flatten_bn(x: List[torch.Tensor]) -> List[torch.Tensor]: + ... + + +@overload +def flatten_bn( + x: Union[List[torch.Tensor], torch.Tensor], + *, + concat: Literal[True], +) -> torch.Tensor: + ... + + +def flatten_bn( + x: Union[List[torch.Tensor], torch.Tensor], + *, + concat: bool = False, +) -> Union[List[torch.Tensor], torch.Tensor]: + """ + Flatten the ``B`` and ``N`` dimensions of batched multimodal inputs. + + The input tensor should have shape ``(B, N, ...)```. + """ + if isinstance(x, torch.Tensor): + return x.flatten(0, 1) + + if concat: + return torch.cat(x) + + return [x_n for x_b in x for x_n in x_b] + + +def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor: + """ + Recursively flattens and concatenates NestedTensors on all but the last + dimension. + """ + + if isinstance(embeddings, torch.Tensor): + # Flatten all but the last dimension. + return embeddings.flatten(0, -2) + + return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings)) + + +def _embedding_count_expression(embeddings: NestedTensors) -> str: + """ + Constructs a debugging representation of the number of embeddings in the + NestedTensors. + """ + + if isinstance(embeddings, torch.Tensor): + return " x ".join([str(dim) for dim in embeddings.shape[:-1]]) + + return " + ".join( + _embedding_count_expression(inner) for inner in embeddings) + + +def merge_multimodal_embeddings(input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, + multimodal_embeddings: NestedTensors, + placeholder_token_id: int) -> torch.Tensor: + """ + Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the + positions in ``inputs_embeds`` corresponding to placeholder tokens in + ``input_ids``. + + Note: + This updates ``inputs_embeds`` in place. + """ + mask = (input_ids == placeholder_token_id) + num_expected_tokens = mask.sum().item() + assert isinstance(num_expected_tokens, int) + + flattened = _flatten_embeddings(multimodal_embeddings) + if flattened.shape[0] != num_expected_tokens: + expr = _embedding_count_expression(multimodal_embeddings) + raise ValueError( + f"Attempted to assign {expr} = {flattened.shape[0]} " + f"multimodal tokens to {num_expected_tokens} placeholders") + + inputs_embeds[mask] = flattened + return inputs_embeds + + +class LayerFn(Protocol): + + def __call__(self, prefix: str) -> torch.nn.Module: + ... + + +class PPMissingLayer(torch.nn.Identity): + """ + A placeholder layer for missing layers in a pipeline parallel model. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + + +_CPU_OFFLOAD_BYTES = 0 +_CPU_OFFLOAD_MAX_BYTES = 0 + + +def set_cpu_offload_max_bytes(max_bytes: int) -> None: + global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES + _CPU_OFFLOAD_BYTES = 0 + _CPU_OFFLOAD_MAX_BYTES = max_bytes + + +def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module: + device = next(module.parameters()).device + + if device == torch.device("cpu"): + return module + + global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES + if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES: + return module + + pin_memory = is_pin_memory_available() + + # offload parameters to CPU + # use pin_memory if possible, which helps cudagraph capture speed + offloaded_parameters = False + for p in module.parameters(): + if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES: + # we use per-parameter offloading + # one module might have some parameters offloaded and some not + break + + # `torch.empty_like` does not support `pin_memory` argument + cpu_data = torch.empty_strided(size=p.data.size(), + stride=p.data.stride(), + dtype=p.data.dtype, + layout=p.data.layout, + device='cpu', + pin_memory=pin_memory) + cpu_data.copy_(p.data) + p.data = cpu_data + _CPU_OFFLOAD_BYTES += p.data.numel() * p.data.element_size() + offloaded_parameters = True + + if offloaded_parameters: + original_forward = module.forward + + def forward(*args, **kwargs): + module.forward = original_forward + device_state = { + # here we blindly call `to(device)` + # if the parameter is already on the device, it will be a no-op + k: v.to(device, non_blocking=True) + for k, v in module.state_dict().items() + } + output = functional_call(module, + device_state, + args=args, + kwargs=kwargs) + module.forward = forward + return output + + module.forward = forward + + return module + + +def make_layers( + num_hidden_layers: int, + layer_fn: LayerFn, + prefix: str, +) -> Tuple[int, int, torch.nn.ModuleList]: + """Make a list of layers with the given layer function, taking + pipeline parallelism into account. + """ + from vllm.distributed.parallel_state import get_pp_group + from vllm.distributed.utils import get_pp_indices + start_layer, end_layer = get_pp_indices(num_hidden_layers, + get_pp_group().rank_in_group, + get_pp_group().world_size) + modules = torch.nn.ModuleList( + [PPMissingLayer() for _ in range(start_layer)] + [ + maybe_offload_to_cpu(layer_fn(prefix=f"{prefix}.{idx}")) + for idx in range(start_layer, end_layer) + ] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)]) + return start_layer, end_layer, modules + + +# NOTE: don't use lru_cache here because it can prevent garbage collection +_model_to_pp_missing_layer_names: Dict[int, List[str]] = {} + + +def get_pp_missing_layer_names(model: torch.nn.Module) -> List[str]: + """Get the names of the missing layers in a pipeline parallel model.""" + model_id = id(model) + if model_id in _model_to_pp_missing_layer_names: + return _model_to_pp_missing_layer_names[model_id] + + missing_layer_names = [] + for name, module in model.named_modules(): + if isinstance(module, PPMissingLayer): + # NOTE: the trailing dot is used to match the prefix of the layer. + # without the dot, we could match a layer that is not missing, + # e.g., 'encoder.layer.1' would match 'encoder.layer.11' + missing_layer_names.append(name + '.') + _model_to_pp_missing_layer_names[model_id] = missing_layer_names + + return missing_layer_names + + +def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool: + """Check if a parameter is missing in a pipeline parallel model.""" + if isinstance(model, PPMissingLayer): + return True + + return any( + name.startswith(missing_layer_name) + for missing_layer_name in get_pp_missing_layer_names(model)) + + +def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int): + + def make_empty_intermediate_tensors( + batch_size: int, + dtype: torch.dtype, + device: torch.device, + ) -> IntermediateTensors: + return IntermediateTensors({ + key: torch.zeros((batch_size, hidden_size), + dtype=dtype, + device=device) + for key in keys + }) + + return make_empty_intermediate_tensors + +def maybe_prefix(prefix: str, name: str) -> str: + """Add a prefix to a name if the prefix is non-empty. + + Args: + prefix: The prefix to add. If empty, no prefix will be added. + name: The name to potentially prefix. + + Returns: + The string "prefix.name" if prefix was non-empty, otherwise just "name". + """ + return name if not prefix else f"{prefix}.{name}" + +def extract_layer_index(layer_name: str) -> int: + """ + Extract the layer index from the module name. + Examples: + - "encoder.layers.0" -> 0 + - "encoder.layers.1.self_attn" -> 1 + - "2.self_attn" -> 2 + - "model.encoder.layers.0.sub.1" -> ValueError + """ + subnames = layer_name.split(".") + int_vals: List[int] = [] + for subname in subnames: + try: + int_vals.append(int(subname)) + except ValueError: + continue + assert len(int_vals) == 1, (f"layer name {layer_name} should" + " only contain one integer") + return int_vals[0] + +class LLMWrapper(nn.Module): + """ + To align with the key names of LoRA trained with PEFT, we need to add an + additional layer to the llm's implementation. + """ + + def __init__(self, llm: nn.Module, name: str) -> None: + super().__init__() + self.model_name = name + setattr(self, name, llm) + + def __getattr__(self, key: str): + llm = super().__getattr__(self.model_name) + if key == self.model_name: + return llm + + return getattr(llm, key) + + # We need to explicitly override this + def __call__(self, *args: Any, **kwargs: Any) -> Any: + llm = super().__getattr__(self.model_name) + return llm(*args, **kwargs) \ No newline at end of file diff --git a/vllm/model_executor/models/xverse.py b/vllm/model_executor/models/xverse.py new file mode 100644 index 00000000..3bded820 --- /dev/null +++ b/vllm/model_executor/models/xverse.py @@ -0,0 +1,404 @@ +# coding=utf-8 +# Adapted from +# https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Xverse model compatible with HuggingFace weights.""" +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.attention import Attention, AttentionMetadata +from vllm.config import CacheConfig, LoRAConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers) + + +class XverseMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config) + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate, _ = self.gate_up_proj(x) + x = self.act_fn(gate) + x, _ = self.down_proj(x) + return x + + +class XverseAttention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + cache_config: Optional[CacheConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + # partition the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=bias, + quant_config=quant_config, + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, kv_cache, attn_metadata) + output, _ = self.o_proj(attn_output) + return output + + +class XverseDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.self_attn = XverseAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=getattr(config, "bias", False), + cache_config=cache_config, + ) + self.mlp = XverseMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + kv_cache=kv_cache, + attn_metadata=attn_metadata, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class XverseModel(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: XverseDecoderLayer(config, cache_config, + quant_config), + prefix=f"{prefix}.layers", + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + hidden_states = self.embed_tokens(input_ids) + residual = None + else: + hidden_states = intermediate_tensors["hidden_states"] + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + kv_caches[i - self.start_layer], + attn_metadata, + residual, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + # LoRA specific attributes + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "gate_up_proj", + "down_proj", + "embed_tokens", + "lm_head", + ] + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings", + } + embedding_padding_modules = ["lm_head"] + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + lora_config: Optional[LoRAConfig] = None, + ) -> None: + super().__init__() + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = XverseModel(config, cache_config, quant_config) + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.sampler = Sampler() + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, kv_caches, + attn_metadata, intermediate_tensors) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + if ("rotary_emb.inv_freq" in name + or "rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/parameter.py b/vllm/model_executor/parameter.py new file mode 100644 index 00000000..7a6d7c90 --- /dev/null +++ b/vllm/model_executor/parameter.py @@ -0,0 +1,403 @@ +from fractions import Fraction +from typing import Callable, Optional, Union + +import torch +from torch.nn import Parameter + +from vllm.distributed import get_tensor_model_parallel_rank +from vllm.logger import init_logger + +__all__ = [ + "BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter", + "ModelWeightParameter", "ChannelQuantScaleParameter", + "GroupQuantScaleParameter", "PackedColumnParameter", "RowvLLMParameter" +] + +logger = init_logger(__name__) + + +class BasevLLMParameter(Parameter): + """ + Base parameter for vLLM linear layers. Extends the torch.nn.parameter + by taking in a linear weight loader. Will copy the loaded weight + into the parameter when the provided weight loader is called. + """ + + def __new__(cls, data: torch.Tensor, **kwargs): + + return super().__new__(cls, data=data, requires_grad=False) + + def __init__(self, data: torch.Tensor, weight_loader: Callable): + """ + Initialize the BasevLLMParameter + + :param data: torch tensor with the parameter data + :param weight_loader: weight loader callable + + :returns: a torch.nn.parameter + """ + + self._weight_loader = weight_loader + + @property + def weight_loader(self): + return self._weight_loader + + def _assert_and_load(self, loaded_weight: torch.Tensor): + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + def load_column_parallel_weight(self, loaded_weight: torch.Tensor): + self._assert_and_load(loaded_weight) + + def load_row_parallel_weight(self, loaded_weight: torch.Tensor): + self._assert_and_load(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs): + self._assert_and_load(loaded_weight) + + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs): + self._assert_and_load(loaded_weight) + + +class _ColumnvLLMParameter(BasevLLMParameter): + """ + Private class defining weight loading functionality + (load_merged_column_weight, load_qkv_weight) + for parameters being loaded into linear layers with column + parallelism. This includes QKV and MLP layers which are + not already fused on disk. Requires an output dimension + to be defined. Called within the weight loader of + each of the column parallel linear layers. + """ + + def __init__(self, output_dim: int, **kwargs): + self._output_dim = output_dim + super().__init__(**kwargs) + + @property + def output_dim(self): + return self._output_dim + + def load_column_parallel_weight(self, loaded_weight: torch.Tensor): + tp_rank = get_tensor_model_parallel_rank() + shard_size = self.data.shape[self.output_dim] + loaded_weight = loaded_weight.narrow(self.output_dim, + tp_rank * shard_size, shard_size) + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs): + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + if isinstance( + self, + (PackedColumnParameter, + PackedvLLMParameter)) and self.packed_dim == self.output_dim: + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size) + + param_data = self.data + + tp_rank = get_tensor_model_parallel_rank() + param_data = param_data.narrow(self.output_dim, shard_offset, + shard_size) + loaded_weight = loaded_weight.narrow(self.output_dim, + tp_rank * shard_size, shard_size) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs): + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + shard_id = kwargs.get("shard_id") + num_heads = kwargs.get("num_heads") + + if isinstance( + self, + (PackedColumnParameter, + PackedvLLMParameter)) and self.output_dim == self.packed_dim: + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size) + + param_data = self.data + tp_rank = get_tensor_model_parallel_rank() + shard_id = tp_rank if shard_id == "q" else tp_rank // num_heads + param_data = param_data.narrow(self.output_dim, shard_offset, + shard_size) + loaded_weight = loaded_weight.narrow(self.output_dim, + shard_id * shard_size, shard_size) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class RowvLLMParameter(BasevLLMParameter): + """ + Parameter class defining weight_loading functionality + (load_row_parallel_weight) for parameters being loaded + into linear layers with row parallel functionality. + Requires an input_dim to be defined. + """ + + def __init__(self, input_dim: int, **kwargs): + self._input_dim = input_dim + super().__init__(**kwargs) + + @property + def input_dim(self): + return self._input_dim + + def load_row_parallel_weight(self, loaded_weight: torch.Tensor): + tp_rank = get_tensor_model_parallel_rank() + shard_size = self.data.shape[self.input_dim] + loaded_weight = loaded_weight.narrow(self.input_dim, + tp_rank * shard_size, shard_size) + + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + +class ModelWeightParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for linear layer weights. Uses both column and + row parallelism. + """ + pass + + +class GroupQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + grouped quantization. Uses both column and row parallelism. + """ + pass + + +class ChannelQuantScaleParameter(_ColumnvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + channel-wise quantization. Equivalent to _ColumnvLLMParameter. + """ + pass + + +class PerTensorScaleParameter(BasevLLMParameter): + """ + Parameter class for scales where the number of scales is + equivalent to the number of logical matrices in fused linear + layers (e.g. for QKV, there are 3 scales loaded from disk). + This is relevant to weights with per-tensor quantization. + Adds functionality to map the scalers to a shard during + weight loading. + + Note: additional parameter manipulation may be handled + for each quantization config specifically, within + process_weights_after_loading + """ + + def __init__(self, **kwargs): + self.qkv_idxs = {"q": 0, "k": 1, "v": 2} + super().__init__(**kwargs) + + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int: + if isinstance(shard_id, int): + return shard_id + + # if not int, assume shard_id for qkv + # map to int and return + assert isinstance(shard_id, str) + assert shard_id in self.qkv_idxs + return self.qkv_idxs[shard_id] + + # For row parallel layers, no sharding needed + # load weight into parameter as is + def load_row_parallel_weight(self, *args, **kwargs): + super().load_row_parallel_weight(*args, **kwargs) + + def load_merged_column_weight(self, *args, **kwargs): + self._load_into_shard_id(*args, **kwargs) + + def load_qkv_weight(self, *args, **kwargs): + self._load_into_shard_id(*args, **kwargs) + + def load_column_parallel_weight(self, *args, **kwargs): + super().load_row_parallel_weight(*args, **kwargs) + + def _load_into_shard_id(self, loaded_weight: torch.Tensor, + shard_id: Union[str, int], **kwargs): + """ + Slice the parameter data based on the shard id for + loading. + """ + + param_data = self.data + shard_id = self._shard_id_as_int(shard_id) + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + param_data = param_data[shard_id] + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class PackedColumnParameter(_ColumnvLLMParameter): + """ + Parameter for model parameters which are packed on disk + and support column parallelism only. See PackedvLLMParameter + for more details on the packed properties. + """ + + def __init__(self, + packed_factor: Union[int, Fraction], + packed_dim: int, + marlin_tile_size: Optional[int] = None, + **kwargs): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + self._marlin_tile_size = marlin_tile_size + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + @property + def marlin_tile_size(self): + return self._marlin_tile_size + + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset): + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + marlin_tile_size=self.marlin_tile_size) + + +class PackedvLLMParameter(ModelWeightParameter): + """ + Parameter for model weights which are packed on disk. + Example: GPTQ Marlin weights are int4 or int8, packed into int32. + Extends the ModelWeightParameter to take in the + packed factor, the packed dimension, and optionally, marlin + tile size for marlin kernels. Adjusts the shard_size and + shard_offset for fused linear layers model weight loading + by accounting for packing and optionally, marlin tile size. + """ + + def __init__(self, + packed_factor: Union[int, Fraction], + packed_dim: int, + marlin_tile_size: Optional[int] = None, + **kwargs): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + self._marlin_tile_size = marlin_tile_size + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + @property + def marlin_tile_size(self): + return self._marlin_tile_size + + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset): + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + marlin_tile_size=self.marlin_tile_size) + + +def permute_param_layout_(param: BasevLLMParameter, input_dim: int, + output_dim: int, **kwargs) -> BasevLLMParameter: + """ + Permute a parameter's layout to the specified input and output dimensions, + useful for forcing the parameter into a known layout, for example, if I need + a packed (quantized) weight matrix to be in the layout + {input_dim = 0, output_dim = 1, packed_dim = 0} + then I can call: + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + to ensure x is in the correct layout (permuting it to the correct layout if + required, asserting if it cannot get it to the correct layout) + """ + + curr_input_dim = getattr(param, "input_dim", None) + curr_output_dim = getattr(param, "output_dim", None) + + if curr_input_dim is None or curr_output_dim is None: + assert param.data.dim() == 2,\ + "permute_param_layout_ only supports 2D parameters when either "\ + "input_dim or output_dim is not set" + + # if one of the dimensions is not set, set it to the opposite of the other + # we can only do this since we asserted the parameter is 2D above + if curr_input_dim is None: + assert curr_output_dim is not None,\ + "either input or output dim must be set" + curr_input_dim = (curr_output_dim + 1) % 2 + if curr_output_dim is None: + assert curr_input_dim is not None,\ + "either input or output dim must be set" + curr_output_dim = (curr_input_dim + 1) % 2 + + # create permutation from the current layout to the layout with + # self.input_dim at input_dim and self.output_dim at output_dim preserving + # other dimensions + perm = [ + i for i in range(param.data.dim()) + if i not in [curr_input_dim, curr_output_dim] + ] + perm.insert(input_dim, curr_input_dim) + perm.insert(output_dim, curr_output_dim) + + if "packed_dim" in kwargs: + assert hasattr(param, "packed_dim") and\ + param.packed_dim == perm[kwargs["packed_dim"]],\ + "permute_param_layout_ currently doesn't support repacking" + + param.data = param.data.permute(*perm) + if hasattr(param, "_input_dim"): + param._input_dim = input_dim + if hasattr(param, "_output_dim"): + param._output_dim = output_dim + if "packed_dim" in kwargs and hasattr(param, "_packed_dim"): + param._packed_dim = kwargs["packed_dim"] + + return param + + +def _adjust_shard_indexes_for_marlin(shard_size, shard_offset, + marlin_tile_size): + return shard_size * marlin_tile_size, shard_offset * marlin_tile_size + + +def _adjust_shard_indexes_for_packing(shard_size, shard_offset, packed_factor, + marlin_tile_size): + shard_size = shard_size // packed_factor + shard_offset = shard_offset // packed_factor + if marlin_tile_size is not None: + return _adjust_shard_indexes_for_marlin( + shard_size=shard_size, + shard_offset=shard_offset, + marlin_tile_size=marlin_tile_size) + return shard_size, shard_offset diff --git a/vllm/model_executor/pooling_metadata.py b/vllm/model_executor/pooling_metadata.py new file mode 100644 index 00000000..b86cafce --- /dev/null +++ b/vllm/model_executor/pooling_metadata.py @@ -0,0 +1,69 @@ +from dataclasses import dataclass +from typing import Any, Dict, List, Tuple + +import torch + +from vllm.pooling_params import PoolingParams +from vllm.utils import is_pin_memory_available + + +class PoolingMetadata: + """Metadata for pooling operations in the Pooler layer. + + This class holds the necessary information for pooling operations, + providing context for how to perform pooling and other related operations. + + Attributes: + seq_groups: List of (seq_ids, pooling_params). + seq_data: A mapping of sequence ID to additional sequence data. + prompt_lens: List of the lengths of each prompt. + """ + + def __init__( + self, + seq_groups: List[Tuple[List[int], PoolingParams]], + seq_data: Dict[int, Any], # Specific data related to sequences + prompt_lens: List[int], + ) -> None: + self.seq_groups = seq_groups + self.seq_data = seq_data + self.prompt_lens = prompt_lens + + def __repr__(self) -> str: + return ("PoolingMetadata(" + f"seq_groups={self.seq_groups}, " + f"seq_data={self.seq_data}, " + f"prompt_lens={self.prompt_lens})") + + +@dataclass +class PoolingTensors: + """Tensors for pooling.""" + + prompt_lens: torch.Tensor + + @classmethod + def from_pooling_metadata( + cls, + pooling_metadata: "PoolingMetadata", + device: torch.device, + ) -> "PoolingTensors": + """ + Create PoolingTensors from PoolingMetadata. + + Args: + pooling_metadata: PoolingMetadata instance to convert. + device: Device to store the tensors. + """ + # Convert prompt lengths to tensor + pin_memory = is_pin_memory_available() + + prompt_lens_t = torch.tensor( + pooling_metadata.prompt_lens, + device="cpu", + dtype=torch.long, + pin_memory=pin_memory, + ) + + return cls(prompt_lens=prompt_lens_t.to(device=device, + non_blocking=True), ) diff --git a/vllm/model_executor/sampling_metadata.py b/vllm/model_executor/sampling_metadata.py new file mode 100644 index 00000000..ee02368b --- /dev/null +++ b/vllm/model_executor/sampling_metadata.py @@ -0,0 +1,587 @@ +from array import array +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch + +from vllm.sampling_params import SamplingParams, SamplingType +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData, + SequenceGroupMetadata) +from vllm.utils import (PyObjectCache, async_tensor_h2d, + is_pin_memory_available, make_tensor_with_pad) + +_SAMPLING_EPS = 1e-5 + + +@dataclass +class SequenceGroupToSample: + # |---------- N-1 iteration --------| + # |---------------- N iteration ---------------------| + # |- tokenA -|......................|-- newTokens ---| + # |---------- context_len ----------| + # |-------------------- seq_len ----------------------| + # |-- query_len ---| + + # Sequence ids for the sequence group in a previous step. + seq_ids: List[int] + sampling_params: SamplingParams + # seq_id -> sequence data. + seq_data: Dict[int, SequenceData] + # The length of the sequence (all tokens seen in the past + new token to + # compute attention) of the sequence group. None if it is in a decode + # stage. + seq_len: Optional[int] + # The length of new query tokens to compute in the current step. None if it + # is in a decode stage. The length of query_len <= seq_len if chunked + # prefill is enabled. + query_len: Optional[int] + # A random number generator for sampling. + generator: Optional[torch.Generator] + # True if the sequence group is in prefill stage. False if it is in a + # decode stage. + is_prompt: bool + # Query token indices from logits. to compute prompt logprob. Empty if + # prompt logprob is not required. + prompt_logprob_indices: List[int] + # Sample token indices from logits. Empty if sampling is not required. + sample_indices: List[int] + + @property + def do_sample(self): + return len(self.sample_indices) > 0 + + def __post_init__(self): + if len(self.prompt_logprob_indices) > 0: + assert self.sampling_params.prompt_logprobs is not None + if self.is_prompt: + assert self.seq_len is not None + assert self.query_len is not None + + +def gen_seq_group_to_sample_builder(num_seqs: int): + return lambda: SequenceGroupToSample( + seq_ids=[0] * num_seqs, + sampling_params=None, + seq_data=None, # type: ignore + seq_len=0, + query_len=0, + generator=None, + is_prompt=True, + prompt_logprob_indices=[], + sample_indices=[], + ) + + +class SamplingMetadataCache: + """Used to cache SamplingMetadata objects between scheduler iterations""" + + def __init__(self): + self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {} + + def get_cached_seq_group_to_sample(self, num_seqs): + if num_seqs not in self._seq_group_to_sample_cache: + self._seq_group_to_sample_cache[num_seqs] = PyObjectCache( + gen_seq_group_to_sample_builder(num_seqs)) + + obj = self._seq_group_to_sample_cache[num_seqs].get_object() + return obj + + def reset(self): + for cache in self._seq_group_to_sample_cache.values(): + cache.reset() + + +class SamplingMetadata: + """Metadata for input sequences. Used in sampler. + + The usage is as follow; + ``` + hidden_states = execute_model(...) + logits = hidden_states[sampling_metadata.selected_token_indices] + sample(logits) + + def sample(logits): + # Use categorized_sample_indices for sampling.... + ``` + + Args: + seq_groups: List of batched sequence groups. + selected_token_indices: (num_query_tokens_to_logprob). Indices to find + logits from the initial model output hidden states. + categorized_sample_indices: SamplingType -> token indices to sample. + Each token indices is 2D tensor of (num_indices, num_indices) where + the first item means the sample index within the returned logit + (before pruning padding), and the second item means the sample + index after pruning using selected_token_indices. + For example, if the returned logit is [1, 2, 3], and we select + [1, 2] for sampling, the pruned logit will be [2, 3]. In this case, + The first tuple is [1, 2] (sampled index within original logit), + and the second tuple is [0, 1] (sampled index within pruned logit). + num_prompts: Number of prompt sequence groups in seq_groups. + skip_sampler_cpu_output: Indicates if we want to skip the GPU=>CPU + serialization of token outputs. + reuse_sampling_tensors: Indicates if we want to reuse sampling + tensors that are part of the sampler forward pass. Currently, + it is mainly used for multi-step decode. + + """ + + def __init__( + self, + seq_groups: List[SequenceGroupToSample], + selected_token_indices: torch.Tensor, + categorized_sample_indices: Dict[SamplingType, torch.Tensor], + num_prompts: int, + skip_sampler_cpu_output: bool = False, + reuse_sampling_tensors: bool = False, + ) -> None: + self.seq_groups = seq_groups + self.selected_token_indices = selected_token_indices + self.categorized_sample_indices = categorized_sample_indices + self.num_prompts = num_prompts + self.skip_sampler_cpu_output = skip_sampler_cpu_output + self.reuse_sampling_tensors = reuse_sampling_tensors + + @staticmethod + def prepare( + seq_group_metadata_list: List[SequenceGroupMetadata], + seq_lens: List[int], + query_lens: List[int], + device: str, + pin_memory: bool, + generators: Optional[Dict[str, torch.Generator]] = None, + cache: Optional[SamplingMetadataCache] = None, + ) -> "SamplingMetadata": + ( + seq_groups, + selected_token_indices, + categorized_sample_indices, + num_prompts, + ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens, + device, generators, cache) + selected_token_indices = async_tensor_h2d( + selected_token_indices, + dtype=torch.long, + target_device=device, + pin_memory=pin_memory, + ) + categorized_sample_indices = { + t: async_tensor_h2d( + seq_ids, + dtype=torch.int, + target_device=device, + pin_memory=pin_memory, + ) + for t, seq_ids in categorized_sample_indices.items() + } + + sampling_metadata = SamplingMetadata( + seq_groups=seq_groups, + selected_token_indices=selected_token_indices, + categorized_sample_indices=categorized_sample_indices, + num_prompts=num_prompts, + ) + return sampling_metadata + + def __repr__(self) -> str: + return ( + "SamplingMetadata(" + f"seq_groups={self.seq_groups}, " + f"selected_token_indices={self.selected_token_indices}, " + f"categorized_sample_indices={self.categorized_sample_indices}), ") + + +def _prepare_seq_groups( + seq_group_metadata_list: List[SequenceGroupMetadata], + seq_lens: List[int], + query_lens: List[int], + device: str, + generators: Optional[Dict[str, torch.Generator]] = None, + cache: Optional[SamplingMetadataCache] = None, +) -> Tuple[List[SequenceGroupToSample], List[int], Dict[SamplingType, + List[int]], int, ]: + """Prepare sequence groups and indices for sampling. + + Args: + seq_group_metadata_list: A list of sequence group to batch. + seq_lens: A list of sequence lens per sequence group. + Index of prompt len should match with seq_group_metadata_list. + query_lens: A list of query lengths. Prompt lens include the length + of entire prompt tokens, and it could be shorter. + device: A device to use for random number generators, + `SequenceGroupToSample.generator`. + generators: A store of per-request random number generators used + for seeded requests. + + Returns: + seq_groups: A list of sequence group to sample. + selected_token_indices: See the definition from `SamplingMetadata`. + categorized_sample_indices: See the definition from `SamplingMetadata`. + num_prompts: Total number of prompts from `seq_group_metadata_list`. + """ + # Batched sequence groups for the current model forward stsep. + seq_groups: List[SequenceGroupToSample] = [] + # A list of token indices to sample/compute logprob. It is used to + # prune the outcome logits from the model for the performance. + selected_token_indices: List[int] = [] + # Used for selected_token_indices. + model_output_idx = 0 + + # Sampling type -> ( + # indices to sample/prompt logprob within pruned output logits, + # indices to sample within pruned logits) + categorized_sample_indices: Dict[SamplingType, List[int]] = { + t: [] + for t in SamplingType + } + # Index of logits to compute logprob. Logits include both prompt logprob + # and sample logprob indices. + logit_idx = 0 + # Total number of prompts from given sequence groups. + num_prompts = 0 + + for i, seq_group_metadata in enumerate(seq_group_metadata_list): + seq_ids = seq_group_metadata.seq_data.keys() + + if cache is not None: + sample_obj = cache.get_cached_seq_group_to_sample(len(seq_ids)) + + for j, seq_id in enumerate(seq_ids): + sample_obj.seq_ids[j] = seq_id + + sample_obj.prompt_logprob_indices.clear() + sample_obj.sample_indices.clear() + + sampling_params = seq_group_metadata.sampling_params + is_prompt = seq_group_metadata.is_prompt + generator: Optional[torch.Generator] = None + # If the current seq group is in decode stage, it is None. + seq_len: Optional[int] = None + query_len: Optional[int] = None + prompt_logprob_indices: List[int] = (sample_obj.prompt_logprob_indices + if cache is not None else []) + sample_indices: List[int] = (sample_obj.sample_indices + if cache is not None else []) + do_sample = seq_group_metadata.do_sample + + if seq_group_metadata.is_prompt: + if sampling_params.seed is not None: + generator = torch.Generator(device=device).manual_seed( + sampling_params.seed) + if generators is not None: + generators[seq_group_metadata.request_id] = generator + + num_prompts += 1 + num_prefill_sample = len(seq_ids) + assert num_prefill_sample == 1 + assert query_lens is not None and seq_lens is not None + query_len, seq_len = query_lens[i], seq_lens[i] + # If we need sampling, exclude num_prefill_sample tokens from + # prompt logprob. + prompt_logprob_len = (query_len - num_prefill_sample + if do_sample else query_len) + sample_len = num_prefill_sample if do_sample else 0 + else: + # Decode + prompt_logprob_len = 0 + query_len = query_lens[i] if query_lens is not None else 1 + sample_len = len(seq_ids) * query_len if do_sample else 0 + + if sampling_params.seed is not None and generators is not None: + generator = generators.get(seq_group_metadata.request_id) + + # Update indices to select from the model output. + """ + This blocks computes selected_token_indices which is used in the + following way. + + hidden_states = model(...) + logits = hidden_states[selected_token_indices] + """ + + if sampling_params.prompt_logprobs is not None: + selected_token_indices.extend( + range(model_output_idx, model_output_idx + prompt_logprob_len)) + model_output_idx += prompt_logprob_len + if do_sample: + selected_token_indices.extend( + range(model_output_idx, model_output_idx + sample_len)) + model_output_idx += sample_len + + # We now find indices for logprob computation and sampling. + """ + This block computes categorized_sample_indices which is used in the + following way. + + hidden_states = model(...) + logits = hidden_states[selected_token_indices] + def sample(logits): + # Use categorized_sample_indices for sampling. + # prompt_logprob_indices to find prompt logprob indices. + # sample_indices to find sample indices. + """ + + if sampling_params.prompt_logprobs is not None: + prompt_logprob_indices.extend( + range(logit_idx, logit_idx + prompt_logprob_len)) + logit_idx += prompt_logprob_len + if do_sample: + sample_indices.extend(range(logit_idx, logit_idx + sample_len)) + categorized_sample_indices[sampling_params.sampling_type].extend( + list(range(logit_idx, logit_idx + sample_len))) + logit_idx += sample_len + + if cache is not None: + sample_obj.sampling_params = sampling_params + sample_obj.seq_data = seq_group_metadata.seq_data + sample_obj.seq_len = seq_len + sample_obj.query_len = query_len + sample_obj.generator = generator + sample_obj.is_prompt = is_prompt + else: + sample_obj = SequenceGroupToSample( + seq_ids=list(seq_ids), + sampling_params=sampling_params, + seq_data=seq_group_metadata.seq_data, + seq_len=seq_len, + query_len=query_len, + generator=generator, + is_prompt=is_prompt, + prompt_logprob_indices=list(prompt_logprob_indices), + sample_indices=list(sample_indices), + ) + + seq_groups.append(sample_obj) + + if cache is not None: + cache.reset() + + return (seq_groups, selected_token_indices, categorized_sample_indices, + num_prompts) + + +@dataclass +class SamplingTensors: + """Tensors for sampling.""" + + temperatures: torch.Tensor + top_ps: torch.Tensor + top_ks: torch.Tensor + min_ps: torch.Tensor + presence_penalties: torch.Tensor + frequency_penalties: torch.Tensor + repetition_penalties: torch.Tensor + prompt_tokens: torch.Tensor + output_tokens: torch.Tensor + + @classmethod + def from_sampling_metadata( + cls, + sampling_metadata: "SamplingMetadata", + vocab_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> Tuple["SamplingTensors", bool, bool, bool]: + prompt_tokens: List[array] = [] + output_tokens: List[array] = [] + top_ks: List[int] = [] + temperatures: List[float] = [] + top_ps: List[float] = [] + min_ps: List[float] = [] + presence_penalties: List[float] = [] + frequency_penalties: List[float] = [] + repetition_penalties: List[float] = [] + do_penalties = False + do_top_p_top_k = False + do_min_p = False + + assert sampling_metadata.seq_groups is not None + for seq_group in sampling_metadata.seq_groups: + seq_ids = seq_group.seq_ids + sampling_params = seq_group.sampling_params + temperature = sampling_params.temperature + p = sampling_params.presence_penalty + f = sampling_params.frequency_penalty + r = sampling_params.repetition_penalty + top_p = sampling_params.top_p + min_p = sampling_params.min_p + + # k should not be greater than the vocab size. + top_k = min(sampling_params.top_k, vocab_size) + top_k = vocab_size if top_k == -1 else top_k + if temperature < _SAMPLING_EPS: + # NOTE: Zero temperature means deterministic sampling + # (i.e., greedy sampling or beam search). + # Set the temperature to 1 to avoid division by zero. + temperature = 1.0 + if not do_top_p_top_k and (top_p < 1.0 - _SAMPLING_EPS + or top_k != vocab_size): + do_top_p_top_k = True + if not do_min_p and min_p > _SAMPLING_EPS: + do_min_p = True + if not do_penalties and (abs(p) >= _SAMPLING_EPS + or abs(f) >= _SAMPLING_EPS + or abs(r - 1.0) >= _SAMPLING_EPS): + do_penalties = True + + is_prompt = seq_group.is_prompt + if is_prompt and sampling_params.prompt_logprobs is not None: + # For tokens in the prompt that we only need to get + # their logprobs + query_len = seq_group.query_len + assert query_len is not None + prefill_len = len(seq_group.prompt_logprob_indices) + temperatures += [temperature] * prefill_len + top_ps += [top_p] * prefill_len + top_ks += [top_k] * prefill_len + min_ps += [min_p] * prefill_len + presence_penalties += [0] * prefill_len + frequency_penalties += [0] * prefill_len + repetition_penalties += [1] * prefill_len + + if seq_group.do_sample: + sample_lens = len(seq_group.sample_indices) + assert sample_lens >= len(seq_ids) + temperatures += [temperature] * sample_lens + top_ps += [top_p] * sample_lens + top_ks += [top_k] * sample_lens + min_ps += [min_p] * sample_lens + presence_penalties += [p] * sample_lens + frequency_penalties += [f] * sample_lens + repetition_penalties += [r] * sample_lens + + if do_penalties: + for seq_group in sampling_metadata.seq_groups: + seq_ids = seq_group.seq_ids + if (seq_group.is_prompt + and sampling_params.prompt_logprobs is not None): + prefill_len = len(seq_group.prompt_logprob_indices) + prompt_tokens.extend( + array(VLLM_TOKEN_ID_ARRAY_TYPE) + for _ in range(prefill_len)) + output_tokens.extend( + array(VLLM_TOKEN_ID_ARRAY_TYPE) + for _ in range(prefill_len)) + if seq_group.do_sample: + for seq_id in seq_ids: + seq_data = seq_group.seq_data[seq_id] + prompt_tokens.append(seq_data.prompt_token_ids_array) + output_tokens.append(seq_data.output_token_ids_array) + + sampling_tensors = SamplingTensors.from_lists( + temperatures, + top_ps, + top_ks, + min_ps, + presence_penalties, + frequency_penalties, + repetition_penalties, + prompt_tokens, + output_tokens, + vocab_size, + device, + dtype, + ) + return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p) + + @classmethod + def from_lists( + cls, + temperatures: List[float], + top_ps: List[float], + top_ks: List[int], + min_ps: List[float], + presence_penalties: List[float], + frequency_penalties: List[float], + repetition_penalties: List[float], + prompt_tokens: List[array], + output_tokens: List[array], + vocab_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> "SamplingTensors": + # Note that the performance will be very bad without + # pinned memory. + pin_memory = is_pin_memory_available() + + do_penalties = prompt_tokens or output_tokens + + if do_penalties: + prompt_t = make_tensor_with_pad( + prompt_tokens, + vocab_size, + device="cpu", + dtype=torch.int64, + pin_memory=pin_memory, + ) + output_t = make_tensor_with_pad( + output_tokens, + vocab_size, + device="cpu", + dtype=torch.int64, + pin_memory=pin_memory, + ) + else: + empty_tensor = torch.empty(0, device=device, dtype=torch.long) + prompt_t = empty_tensor + output_t = empty_tensor + + temperatures_t = torch.tensor( + temperatures, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + top_ps_t = torch.tensor( + top_ps, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + min_ps_t = torch.tensor( + min_ps, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + presence_penalties_t = torch.tensor( + presence_penalties, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + frequency_penalties_t = torch.tensor( + frequency_penalties, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + repetition_penalties_t = torch.tensor( + repetition_penalties, + device="cpu", + dtype=dtype, + pin_memory=pin_memory, + ) + top_ks_t = torch.tensor( + top_ks, + device="cpu", + dtype=torch.int, + pin_memory=pin_memory, + ) + # Because the memory is pinned, we can do non-blocking + # transfer to device. + + return cls( + temperatures=temperatures_t.to(device=device, non_blocking=True), + top_ps=top_ps_t.to(device=device, non_blocking=True), + top_ks=top_ks_t.to(device=device, non_blocking=True), + min_ps=min_ps_t.to(device=device, non_blocking=True), + presence_penalties=presence_penalties_t.to(device=device, + non_blocking=True), + frequency_penalties=frequency_penalties_t.to(device=device, + non_blocking=True), + repetition_penalties=repetition_penalties_t.to(device=device, + non_blocking=True), + prompt_tokens=prompt_t.to(device=device, non_blocking=True), + output_tokens=output_t.to(device=device, non_blocking=True), + ) diff --git a/vllm/model_executor/utils.py b/vllm/model_executor/utils.py new file mode 100644 index 00000000..d7eec818 --- /dev/null +++ b/vllm/model_executor/utils.py @@ -0,0 +1,31 @@ +"""Utils for model executor.""" +from typing import Any, Dict, Optional + +import torch + +from vllm.utils import seed_everything + + +def set_random_seed(seed: int) -> None: + seed_everything(seed) + + +def set_weight_attrs( + weight: torch.Tensor, + weight_attrs: Optional[Dict[str, Any]], +): + """Set attributes on a weight tensor. + + This method is used to set attributes on a weight tensor. This method + will not overwrite existing attributes. + + Args: + weight: The weight tensor. + weight_attrs: A dictionary of attributes to set on the weight tensor. + """ + if weight_attrs is None: + return + for key, value in weight_attrs.items(): + assert not hasattr( + weight, key), (f"Overwriting existing tensor attribute: {key}") + setattr(weight, key, value) diff --git a/vllm/multimodal/__init__.py b/vllm/multimodal/__init__.py new file mode 100644 index 00000000..489e1e51 --- /dev/null +++ b/vllm/multimodal/__init__.py @@ -0,0 +1,24 @@ +from .base import (BatchedTensorInputs, MultiModalDataBuiltins, + MultiModalDataDict, MultiModalInputs, MultiModalPlugin, + NestedTensors) +from .registry import MultiModalRegistry + +MULTIMODAL_REGISTRY = MultiModalRegistry() +""" +The global :class:`~MultiModalRegistry` is used by model runners to +dispatch data processing according to its modality and the target model. + +See also: + :ref:`input_processing_pipeline` +""" + +__all__ = [ + "BatchedTensorInputs", + "MultiModalDataBuiltins", + "MultiModalDataDict", + "MultiModalInputs", + "MultiModalPlugin", + "NestedTensors", + "MULTIMODAL_REGISTRY", + "MultiModalRegistry", +] diff --git a/vllm/multimodal/__pycache__/__init__.cpython-310.pyc b/vllm/multimodal/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab4b7072d4c5709fe27f4a23cda488f300713be9 GIT binary patch literal 489 zcmaixy-ve05XbF&H7%`#sKm^gp+ucn5Fb?$s?-!z%77OuZLGwSokX?+>dc$K#K0@? zAYPex1tx4%M433rKk5Fb`=9Py^?Ds(V)*)2JUalq+hjY82F}cr78%GO0~{jA9prEq zx!gk@_mR&76hK>dSc``!gn)#P0JYm72buc>Q77|mK;(ZwV-p3sKT6fAAlZVHvJ#WB zZj{vfNy8LPsw`#WRHdVanzWR9@2@_lE2VdTwN3Yya?P5Ymby11QvJ-35!^PKlblK= z)=?ES+t|QWh%)(2BFxD`tA~K zYuds%WenpNAbNI^#ph|PcpcZY9;j60L=BF3Q}7{wh#6hR^;#8G8Q(L;V{Y%n?VVzb eX-O5vhxJ;A$3|-yIk88U_FoS`*9{y8cEc|L!h#49J1Lwx1LnH!ffvrd~FveJxqXLsh!n;Dmzn>`2C!OvgQx$8K;@vyom9F9QrD^T3w zBzGi<3BkRayV8q2Nn=W!4-WS@y>U1d^xBI%yz|)^`hUR78IopjQdG4$E(;^B%+PJN zPwU)drzKBvd!tt^pYr-VD^he_T|OL+K=gZ1!ifoYVwaQHGM7$IB5EBYo3)Y%Ph5oS6pfWMd|kjkPO8Fn$0NmU7TOPZZ=X-;I73n`$n z{GC#z8d8M;RUg#=RJ$XDYp^yTGufQX24jHNWoasV+|34Wi<>TEyiCFDX`>YW$qGp~ zetGs)q())S)izqZtT5^^9+xES4$H4PpineY4q&9V+@9x>-rXLxcjKWS@8VpJY*dgio^+q&dL?`+dfI)O>m%Va(X;Ne(R1!|(ev)}(K+{ARC8;(CY}~A z?C9=`VnU4g)fr8Uh~sPL@r$x)&S>t1vL-5GR$OQrc)BP~i%b6KyoOe7?InL~U6X(F zw&ox4$M0zVxG3zH?#po%El2w;{ju~0?GyeadQOU$@%)N7>Q7nIa_83-8D1nOhAH1PaXn zgKBzSxE^^c0u>{UQS!jo!@x`NFmfaDMR+xdo52bmtJmXJJHulpZfEp5h=Z&iCMzpK z)(Uy)4{p7?uqcJdofVPQvFv11`1SQoPp;IHINYiy8$!y!7inGz(t3!E)RSg@WGzkN zdgQh0q3|~F@{|;55^jik6xCZYX^1pUq@t7NqYEOwy05tg9n=FOKUTj1?g!Yl7b;S3c&S);VCJQIJ@z6| zujiF|Jxcs`NcC#H{&Cw2)td*JtbrsK|Ml9%ENWeB1+5Dib}zCE-;3I-(M!?h#V}aD z*xJI;#}_xkFuEAgjG*X+7nd>FY->wC12Rw35-k)Z&Cm$6~<%w$w%Qr;9H$|ao`IcX}LkeW0)fPoxgV>v*gf~UM#Ba)Y zQx+w^A}VVmYgK-0`!EjtcGMq*I*jGx`C)WN&$2`|R%hdQ7AJ@knr$W$!sjo|a<6kX$ zS+t`swfnkn+yRAa+K#@hXXUO=w0zUIJ~z`#T^d=*MyPE@>*_nkFSV@tnby?(f^Xj` z-Q{sRx_t94&8*)Wy~|W<#mR&S?$-x4V zW0pv`cLA@2XQ?o?j8MxZRSM-$bXBCNh40M(fOevkmkILm6NQgBZ+;8zL{T z{N}c@Z7|5KycqkQl-`zGY04z3$L*-Km776K20To>%(d0(bC#6A()@WQrm=nNgR@<}@b!+Ug@4&-PBn_X7^ut)Z_0jY$xz8R$AapOs= zuo&l|W``Fz(tI(eFCp&86%NFDbR07h&3eHx?St~nY+}HnV z*N(`G=v6c2%lOIlb+;U3B4V&`(29`5U@Ou<=d^^v)@-Bl$bQs$;efa@XNJtkU@)+Y z8H!kvoaPqsAurIRX>J2$iP*0d|BxAL&=u&H zv2A1)`w{X#r7pbEwuv5vuDJtyT{9O3#4r>#P!XHNZ#sse3*!hQiNT+%?u#|ut;T?9 z%do=h5D%D1BTG=~w;$;82F47T$I^W0Ma#Z-`C}qT#(hldoIkKzFYST4p+o5ntl7$# z%@E<&sFP?eaGofv6%e}FzQLiBC5oc7Rz~_Z~H~Y zzeTi^fR4)uCM&?sh8Stue#IZ*x7D)gpZ2R_RE$xG>5uwjcM9$i{|G|Y{)lmLRM=vI z&Vxsn5GqgHvE0cXWkXB>jZgYhO!1h149^CKz?M7B^pAo5ai))O81zpteKqc+e+sl_ zd=sa8W*DWv7q-a@{Bz&MQc&bHLND!|tvb}y@gW5`WDeNKgCH^t8V*3X<2h+dG=e4` zIn;r0^nC91*lBqZ+S3j_dEVI+j^`&W$P2vi3M?@~ftHtLLIS@g@W&~h;6L|G#h%G) zz~{2JMNjfxm`a4svY@rJQ%K!8xVIGXr&Rt8V(ACXrjUpwROi%*ye%fy+XmBNm(Mc} zx`$h}S!Zz-j(4z3)B`LM0fm}uSrM9JMj<%a)y|u}oh<>E2iOdf0RRHD=ha|^07@u; z=6M|-EJa(UPjO(#GT2QUW#ajY=3)5t7;G*ZGXwEHpKJlX-4PwlCi_db2`^i}d z??F#M_WCNxHYpRi>$)m+A71==QwH_FC-UUJY`vm)W_DSlL~XX;?|_#gPep6{k_Knmr+Q8Aygu0U+aluwX_gk(SiM&Rp^{P zu#4fIx!K5W=N1iYm>AXn&^~BW>>&gyHQgKUD;llsCda6djp5)pfmA~;QW9#w(8mVG zMBrS&Z6|< z6j~pl>kzW;;bJ};Oa>$Fr0rHKkzhATxJ)*kIdQu&w?qjmK7d}}PP5A<@y3!3{K$Zc zH5B>eAt|E$Cw8pfORMb>+JC}ub#!B$=MEg*p+j<8$$Nt|r2nDA>;thc$yxgVwZ4&Q zO#raZbl6`6`**4o*pS!s{%!B!l8uALdvPx$P&1IN^LqA+?Z*%wKkUWwz1LDS1cHA$ z1quk5z3g!#4g^%tGO!)UG;Xl_9q3F(Mod`+=UfUq(vz2A;a#dJ6NEG0fcf;DW;^Cg z4I-EUp&;IX@*{|W@$E~%t~MTsxyXzu1FeAD&<-IfOAARXmMB2#&&ey`B!<&F&b=H% zSGMsKO$*N-B6rX@l0aQ>l!@fo`)NipmPp!(5Fka_Na{+Tes>Wg7rW16lgHkCY0CJ?jj|zp~)%|JV-^*^LX?C_}UY>L>JL$Ou1585v`-c`t5C_6Prgcj*L*iVob0?1A+=3wedoBg^c3 z?|`cH*SJSJcO!xxP0aigP()nF7J#SEXbn7lN;O`?u|vuIE8ereIWb{X4^j6+2j$h? zF>?M}u7gH{G`;^Rqe7VaD!%Ck3W}FPEz-9wUF#xov7;l*Eo|F}rjXR=7VhiY#ct6z z+nRhTLq4^u{Yvu-T_A7#+P^OB=%4D{B6{2F#&#(yB2`uDnrJEc#m}v6)F^?tqj!y_ z0l#eio&Fb;{3vOTYPL8)To5o= zi8uq^!vM(F($X*|mtfWh)7j1K44j zY^BAvVs7(atrdP6xqb;^3spQmclg8(F@XLd7286YIz(*t*kU_ zlN!jks#i>hbUxSXtn}_A%xpUFk4ixHyWBFGl|KI`hNct}SC3YWDl#Bs6Dp4?Bd`k; z$Kuca?X*1{diOBkfW#hmORksj5-FWjc&YQZ18Krt83huV%9#O^22ne5kTqR~kEMJ# z!_)HzaiItBL+P^trG8SD-P8a+vwI~^)Q@Nt1nU9I(2G~vh-Q^v8_J3?b}@#q3mdIn z!+B3VSbDQs9l6?-^E-p9&)bi)vQ|JZzV4>CW%OS8vI&i-~OFRRHB0x(_u zSK6Gg|Equv)}$irAsaGe6u&8!g1KSML0X0E)3<_B z3P^nuWW?yyS zvd%XHOGW8nxw1tCB|GID6`X+j1=T*GLZSdaFm@e0L;Zmi5I#N;$ znXVj==)U(jTq_P>C)oz^K62t!>>{T>!7GKF&U`i1i?m8ojT1+#BZruwuO7We$>_Ry z0G$354+G$2064+&FmR%~ACsWdhAt=Rf*y4f_dq6K6hIThs9IJ4j561322up4dN2y$ z32;hY&Pv?^U{o0Z$^Z~6WEG;`)B&Gtf={5N-5VijhM5Ztre48D2dedU&%_;+6gEZ| zpmew$QEp{0i1ZO*OEaKAK^Y1WCCFs8Q}Pb*L%rB;uby%wrr3zSJ)i?X9R>~h{|^qV z`WWlHvvi5WBXxx>hU1<89S}PlJ$(z+2HP}w8^uEBb>Nzp9wy?g+Q2Gy)%BuSJF6@&t{laG!P-AgM5e1|{F8s76^Qh2~lM*%{4K8{uyG zODfJ%;iAZ^2>od>fakDN&rmyMA>;>Cd`JadgGe&V?!@j{Uqj;Z8g5(VpV3>Ei9bgz zx8Y^+uQnW<^T`}00C{O1%8hR1>a_Gd6cue;AJ^ygl3CJAdR6@y^!o(bPC{j-EGSW7 k!Z=}#TcsN(&|;MEuLzYfD3f7SE2LD0FL8xgeufRA4IgFqLnJF5`X??=6k=q#oS!Wf_47KUxuB) zvVOzO>~dgp5uWlV0Nmn4S{Y%4AWkLCY-VQ;b268?=FFBJ^We^rjm&4hIdf$*Yq6HW zJ=x9z78txC=dup#7~Gfh*#cWIcvCKBOKi#DEqNqcX3K!Lc_5EwD{Lh@#*SsHY?WAI zL9Fc(c3cp#1n-xwS$vLn9$UO4R(EaoGPiG8-TB|aa;rIC- zX`(CtlaWq~JeHe4y3Og*09y$fyV1J zPjw`V!9Xb39vKK7r6?*IsUi_&S>SOiEhCwhIx70V6ST^%Iyc5rryx9*mt!4Q3(xmP zMLBzNQ;r8|o}fds%ZJTHcuEOCSd4IsQBIi6sc`y~+uV8VFt_RQ1?~xtH$+4D!V%3r zU9;A#zQcXqeC)CoZ;3WRmDo+UrW1gVT`wMY=K7{8FEHh3(AlI7*QrR2-#nVqc7-G`8t3p<<{6z zUTQt0+M1GW(kGn2o2Fz+zb9YrIlZXW80tcatk43~aV*l~rdoM)NHJBeF-*M-I=Z%6 zftNat;3WhH5mU!tTOBhJhW!^S5^2jN0KV%GN*v<+<52pH+TPR2YqK%=AO6Zt^n=R5 zkp3qM4G`Ic=VO%rD**7G+yR+D zze~0|-`d1_WZkcI&TlV(&Ra&O!PRHl=I#{oV(ePv9`Mm!@|E>9z(=Ip*qmJ2EW(e* z1L(oQ)i@F1L2rj?8OHbGRK{CUgu2+j3H4CybwaVVEnwQv5sVAgsT(5FB8SOPPQJSe zyn@einz9Ygyis<;`>7rpNnG?}DC{tcMi9HJ<51c+58E^kwTkl+S~e4^4A1DDk$5izwQqqa9_ED(?_IxsqsIcn8i{O6 z@N%-ek>87D%J&4;T-_y8ggEEYywq`?2!+L<<`JAk;30r7jip`%Pz9fkb7)j?MX5qn zt&tLYF|C?c9wfp{=E|$*YUK(;<)E_4kD@%zL=;u7E>tpP-fa`is(Bl(jl6E7Zj2Ds z8SSP!uus^0?ae_3S!)XizL^g%_RrC1FYr-{Q$MQ1e;UM;?*j0=mfa$&Huey7(KFxm z38mEYpr0s(eGvR>a8j!3nj$5Z|Ae>IiILTBbj@ z!8h#pE}e1qg#qx&f$%JT5IvubkuL5+(Cz@cS_3dSd#FW+GIhx&r28yzRjnw3X*8CY zj3A0WAIEYpHH|3ZMFPimsg=T+tMFr`Tm(2M6;4c33JNEN>Hw&`x>xF|Ifo~9nrqcV zg6EzY4Ck9#1c1Bp6aXJ4dIzHKJcS%Q>)5TQG(gO==~L>qo~*oW=({JXW`CR~x+r9+ zFtn8ibp@SWn`DeLX1)n`%)YsC-PA5SHQO$||5On$M_bT#rsi22KSaIy%VE4`%jU$? zP}x&r0KWv(j`4esxUO@BKgrlhLp-Z28tYV+y@Qffka~-?44t_$eN{Qe1Po*yzy-uG qPYNqmoj`yQJ1mryf1%Fg`&eiteq91xrtrF?SbkblCsxpIkf^=)EF`F#)QjnkYcha}Cy zBk=H+?|sYfTM*97R22L!{==`f_g`0(f1`)-pN@yCxT1eX<0y{mDHi=_p6WA;sdTS- zny*{BU$6>((JJ~StK=J&A>Zj<*{@g?d0+6Ve$A@kzUY*^8NY7T<-Os}`g7KtKX1+Z z3)X^v#5&?1wT}A7tYh+h**or^uujPPinr)5Sxc(IYy9M%YMpvdajH)3vEtM?+taMm zc%E_Uc&_g;>x`qWD~;LTf_7!Cp{B;2h>II;D{d50?OM=Fb$Uok@7tY@8*HVkk?%c+6 zC?X=_rf+ZYoCrMLbse75)FKXy|BCrT^mW{D#Zn!`VvcHQ4&%C`@d7XMQkyL)ZT(ZF ztvb3>cwDoLPZeFU%HU}+oo|k)y5;(A93`JrO#D6)b|*47nx5=zZuXi!@o&Ryhr+Z? zKNQ?_13dV4>^|fs7XmMQ9z~#DuCTMs1JjOU?n4~GmKlep-D+WgoQ}23c-xLmC*+YC zgt5sVxlz05F{-b;&u$6XP$*w~RtlS$V#T+vlDNhwzLlqYIm-B&+S)^W0!^vljCwB5ilH+(-obaqyG z?90hFmdzWkWA?%>abSzbbe@*&dFF=Hq~WY4^r6`iAx^#VZbMHCQI|BN+H7Ky3vM)< zsqtOj`!+EwEy(3DOpmzN7RPX+RO63YspdypPt~;CY_>c*iki)*$`|jh#C~U`<905@ zHj!}Qn|^oOzv%C*c<$y(rx$OB!OBC=^H+S*Tr6#SD}rx4`-we*#xjN5j;* z{HrkI$Lv{h=1ZEeJVtV(k{ZosVEY{8SDK^@U61Z-&E|uy?d5Mu&88E!noVI~13|({ zHEf=0ZV=1e)0zB0E$(?3XVN&?oTa;0sX2;f9#=%_Z790>RzdcCN&QI?uF}A-!+^)t zpX+-HFW@TfDQ(Rv@sh)Sp~4^xUS3kj4B9%u*W;p9anP6dl%2|B-KsiOUTZU_=FH$} zrd)8|q#mcv>wK0>$(eQL9v7@R`DW0|oOx#fz4Nm72w!lH@*~bM%sC2>J4QxGJM-gh z!#VDpz{nF0Mq!^tX9@c(4)-~UIZJY%Q_g9OJ1O4`dYN;^Ig8#?G#l%k20gDhCRR9; z^~kfm>b!=Yv-y};K*8w@2AtQOH}J-E3NZh-#v$&*Tu+i+X@b|xf#T$5>P8(qZf(O@ z9e|+;4JTX94Ljb>Y<^WL>*~gjNDG%oTDUwynGD%7+v0X>8&Co@b>m)U`ar-^MYbOrASlPaYeM7i=iKvb^#qvlzlAwH|zmh&$JZO)nIr{VnGua)nF-G{#vSb#uS-! zIIUcA;FDTFfh*@}U8486lB|sQFyJ?=Oae%A`H~j`P@*f#qq)O#WBMea3+Rx97Kl(s z$%t&If;c3pX)r-sr|SI!#0lcr3NA_20vH0@&Uv1mYC3LYZ+hGrK)=K@DG_Of>D=JO zz$aj%Q?#iRmq~ZBk$eDXWT!iaereEi`hr$t$*Hg4L|V+KnLvI*`~t3_)HCtyvOeo8 z_tb}qn2XsH4T#U+3p5piqy9d-1oVg0wdAj*Vv}>rc+aIHBc!|_bmuQhb>HaI4}h9)g$u})gru|)vN8d@f`Kcbg2QlV)Z(&_{HlFnvOKz>c?!8@1| z&7m<$6{f39{^`%^h2*sF9P&CQjOBi>6v*HT+ zZsC$}G*?N^jVQ{FjNoPJ$`e{d2uqqoTJ*ab8sa5s%syE^OMdopd~_-L&aDms>;DD& z(t_L+*iD=mpy?D&rKd)0ATj~1(1Kh*e}TD@gSB!p=5D8O@JmzEnpBczOD0FDPR5fS zK}4H7`A{_P12wjy7BP76o54*P7co5gB^p=R)%%Qm<_+cJ@BM)77Iupeg4!?atGgux z-^EzlV`4GZ`^7yqP#tyGfTXOngEGfI?zR@q=*Tp+=(P8}p!gj4+MyvM?w1v-0 zd+H}@zx1R8XfO7)eQmeWujKnJV!!HctzYfya0MS?g<@><*&YDHDfO#>4PDNz_4R)B zlJelngJN9nqYuhq*#8KxInMxB2aPzlky=GVgmHX(TE;I*YH`?0KIoY z;h2zCmmn@0F7?p0<%9|380b^(Y-B=q@Hph;4uc9{L#Bk&;9XuDIr5skZNsr=GP>9x)@@w4 zAamIZzJCEB&4oNgJ8y0RNZVl-o;5iM$ItypavDO6knT{Ri&SnoStPNJGcPjFf!opL zjk8vK|>0j|F9PjhW1m{Tc2BnWZ2{WmSESY|%kr*NQDedU{ZW z%!h*5hqR>hdTBZ6`b|lSh1DPBx0MWJTy`(MeHJYqccCvBVj23 z6CD|2Wc+ZXC5f28?4M$Y#O(S^^464-PnnX=kZ6F-v8*8-8cvAISQ||E^^ggfCxC$w zjA0Si@A@W0zKO1|4ZWnA)OjfH7!YM41Z8<%mf(@QALu8|@`r_n3>haN&o;jllbpxy zDEio9wA>^PdUmkYB^R8=Oexz@GCFRPjQt9RV6o)pe;Pd8UvtdMF%HI6!{e^`zgX=^Vw*p4p|sCaHX` zj$;o0t2`tn4O8{<|Hora6cmWVlv2M#oPG<&t{te20kqfEtsUkEoG|a2I#~ge#&d zW~EYR1{~B>jXsos35_NmQ{*f)`VnqqX+lAqT8q^u&}J%9@1tOi|GG?hSyvG!C^p{J zg0p>gulN(?Gv!GM)hOkjLH`#i)<$|S=WBhPW)DmJWEloo#je)ZaF2EKB1kd$+o7i4 z#0L;KE|fN2!b-BhXMp^XbdeO%4p#$Q%PQUw-%-%;!W~3l5P2R%<&HH&_BD}NlZ&5hNAH~9^piBmNhs1xD&bV6vzK}S}6ZS^XSi*G#S9B7MEZ~)?fCuS9 zN{+J6hJ`$eZ2`xBO8>G_p0uc^sTpqEtoEY#BDp%n zhzh$?j;&mzS|^jmY%?8iQdq&o3l+FzqE7Onn( zmZB8asz?!087j)`@-OL?govd>N&Q!hkqA~Xs2-)O`tzc0AmA*r1yrbHjf&~d@D^}_ zI@L7qbm-y}woEdl>pJa7BFgM*i3-bV@*hKcc<-g0&tYaodM?drxZWUadP%by&ih}> zy`Y+er(*zW(xVjAqQu3)ceF9_|2u}r2%%C<-kze1hs_grFtPEfc#XDrotihOd6Sw2 zYO2(bU0Jg?zWtqRcW&KoesJr~dpFnC1zDV!rRE)K{)8HGb~4`*^o>bSDMJ=t#5KAj z*car0WTYf+)7>9XLvBO-kQxGf(WJ(lY=}Fefrk2{duZtU z)?9%ViqDD~d&acNvxRDXo>lc{jIrlMJhM8KnN^;%S^f9a$1tw0v){8qba?B!7s7 zR#kq}$)HEFDDTPY%7RKkR>7t(=u#S~zG>q-y=0^0FJ-8i93=DT3!e-*r2wtML;B7w vMVLuhBFs@5MEsh@$kez5Gr?!bD=J1ri)ai~WKXL#R>Nh={|1}W7H0noV7?6M literal 0 HcmV?d00001 diff --git a/vllm/multimodal/__pycache__/utils.cpython-310.pyc b/vllm/multimodal/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d04d3df17638203eeb5c1b1c4f4accbdb976ffb7 GIT binary patch literal 8511 zcmcgxO>i4WcAn|^!2kpylA=gamgJFTtqj|uSsZ7u<7 z8vXP7&HL-GUpKnr;}s2`7yj;_SO4Z&P5V!341cE4cneSbqM>P0lX|Fm^eaNWBRru~ z-3X12>6u(NL#tzZcBkZ(I*#Xb%3irs@hY7$Z;Z$FuT+NQc1z4@;5a9bfa7WT1oJ5e<7>|ZV?jN5HXjF#DUZrypn2{O(j1ph zg68>x<_KoF@&tO$u$0pQ^*kj{qGx`H=4p8fJ!gk#ekM<&=bU_oB$4%?4tY;Pw`b*Z z=zjsaJulCEBjh~k%-Wx2$xpT%c}~6n4PV3aSuPcGUkRR- zuQGqKo+Ho8*S@m6moWpjk*~`OpnpZa5gb^1RbGV6AWOj!J&@>4`EyX5FDQuHG|^(@ z7xFEPyjF}{;&f>=&Q`S9bo_yt%Z)iLL6pR!tSykHdd3Q*JyU5C}`5$y2uWH zc>Vf~#--a=7H&4)xw`cBwZ_uIjjK0rFBKyTH!i(PA0I z+Bdq=53l%%f2G|_z`6=qm%4X?sJ$7eYuf+NAdbP0gb9@Z#B| z(>vR1_s%4~f)r=YclxWH7dq=_!}jvo-bS+8jn3W;!_L_bsnH=d&i0db7|-`MvKgr9 z%La37v{bj#SSD{A0_WI9QPUmW)YTllx13Y;Rk%+%5^$j0VKh>08{jC#eeto_(zisS zr{W$3fU)>cfBsuC4D4scy|A5RGmpIAkAqiVt{aN%l<5txn!k3L`&s9_pjTyckH|0K zq6TDq0>d%MH6a{bxkPNd|L$e@vywA}B?9#n_1h{)`YIY|MH14?fmYYh$hEq!tI0%P zgXApGUesy>6F?x;G|**6D#lCSYo9#YQ{2)Fc+6)V>t!1T%_O@aCCYVF1v;dO!$RnZJfdwd<3`VK-T_X> zzXWwm;VOWoh(NRYgV6Ea;b%vYo>@7cGm5Pi#s(-P(MYs=T8^p+P;BxDys1R28Nefx z@yLi~d4zE26l#_QxksNw%^S;wVrcTJMdPu%P60GbavM`Q#4Bb+ zP>4GS@d>>OsF^a$X~eU} zz_aHb2SdDKi}g_KLXYwEqIwnb?)g}qr{-%Y9`(jq8X9;*U7+S0RP1+xx=4K^OKBgB zu#Hzil`o;sD^_mh+OU;>j`4j~Hqb#20|_Mx31H)%p06B#X-K-Q_yj)QH9^tWwiE5L$F2kqk z@P-B%Xm5HT@;&cyd)&zDdAt4sbNKZR5Jp`^Yv1d4jqy7ob5T&pkvR}p+O+~shK-_r z0V2pkW>uG{7_q_J-Zn8}iN6FjCywItDYC^oXzvqd*A^@cd1k*bdp1#IEeJuIgt|h- zA0q_YVwk04J;Aw8oUX2KEiuL5jMJ6}z86REx5c#K_ z=VcGAe$qPgTD_DbnSobk*1BpZ0WTLuu7EjZ4q^_eSU!TvE5!VGOI1H&aa3tA{ zwxz+=bPf%&*A_OSZ3H#87)qE$YNR*@6qYgumclqJB{qM1#v*=v@zXcuJ*&eH4NlfeNee?`$i8>OPAzRkaNVG(U^$=HlaMCxA^J#(E z=6mse+u*oFN3=bh#lm?EdZ^&8Ko zOtbr;B|;A`5n9;Wd;sCSUqi-tY-FV z&|X1k+{|U2W z+Uja%O((pWKoX!nMn@CR;6pwh8Kc|9>lvOZ}s}NF-q1AJmt3K|O!dU&9h&&WtEsrf(5N3Ki zvv`g-zHemI&Ws8bc-GWsM3sCR zqBj$bC6@Amks%ygvJd>K^>7u(mxzrpXeHT!y|!XRvv^Q-!A13`_!PxpCo#|< z!xFZ?jVGpaYO1X;K*Ll*YJFYRQh^?GmNOdenMJG^D?0bUoE#ynC#kY^0^z^*g6Kec->WG~4k^54QJZ5e0N z{OZ+ihy&fAd$3E<@|ea`#=|DK3)scJWFmzV%1O-?9frU~L0?k`q_$OxR`WCkdz6z} z+>+xYgH6sgqM7s>N1SS^z^xcuxoD%>wksw`$Vbrn9+|*3xoXk@nYgc0T|R ztfu30T6$nsI}aT`{T21d(j2!|w5Bp`nA-c9ccK1*-{a60~H z`SrJ^BG@QLF3J!X80zD=qmKF7Dk=brbI{Dnal+33n#fA)zKWoJW`y{o;22GBp~%d& zZad=jmDzY<5XsE;d-yX!)~8jU$nH|HPQ_<1K&^Pk)#-k~9*73(cdlsB~pO=vP) zc?b9RF=TnOkIrW$ylSzuag0u4wF)u3$ze2X;2_lT4h&&UgI+vNxmk{pfP=@0*Os3# zZ0(;5;!&(RF@jZ(NE&|+sbT`CO2x)~j;&V!;|{LwssMGdQ!yOe_~05(m_Jlas+l`g z(*~WrV`3ZUF&d|l@6DYl2b%z9Ob`!z?Y#WHVmo@(m_f@i#lx!U2=x_SimZ)xwECOH zd9c-wkezpthY-SNqTS8^9I}!1APN+LpP~&wQTC-MT~L&9Dni4|LM-<-ltkUDa(zCZ z=NVBi@eKSAktfL$bWb|J_cvP1SHoVJyMyR%oZsuv-Mu%)oq0GaI=Fb#+_U#r(>xti z)Bzeg!9#qspd($N;zs^I)ByJt0fw4HkvWY?+R{FjZSHNK;m4U zb4stYc;yoFqNFL$ZD>nZ9V%!;W5*2z8lMm8nj>>A0)+Zu@N-(`@xP&PG|G`&1Qz{P z`O>gT!66(`Givs6Tg+5`6hi!Hipq~OGf!8xLve*hsR5*H5q1%3l(UpeK>4M>(M&%P1`JnHJQT~)67Tva_qyWR9)T=@03 z%^NMx`vWHjM*x#c@KnD8;T|W_Q-l$MJe5@W%vS*mRLDXVu}IZe&F%TJ&Ke*S$fjzs zmRk#DtlF&Y_(;yF4(m9+Cg;@xTX1|`9#V^J(eVwrqz<#gz&CkI9#P9|Ssi6Z-8`1Z z)CyZ6p6H0?tkg^ z=AS`SUXR#@)T5-I_BTZ@wEpFMY=f&=-`L>OtTML#S!uE&Po)iS=fJFkM(b9=g}hSa zgKP-X_-0;?3`~u9U~oIjGn2?-I20PDi$h_Odl?r+QtF~Fs;a={HZGguuE?{AlWS&G zBH_rS7(65B8ntt`M$%+(S}L!mCbjeXb9ZKir`=TPP<$>&!z`aP^Kv!ksn+SZkKr90 zb(ma&r;Aa7@?+)bgxdP9hn2?-+LAuUDr}+zBU{DD;(1&5uk-sjk8C9;8S$W>18c;s}ff{Y9 z3+n!w%r@4_vDqy0wR=*kHHGjhgm=xETP?>n#(P1BlDlI3CLT+H1Y~mbKn$w~h1P8h zHiS(2E+g!sQu{ZZ+`SJ@RDB<;8H77VJwBc_ylBTq+}_mCQIO74&W;RN0^BUAb_ z`C%5cT`+nC38IB6jDX{~==pZVR~`A2}rOVQ2K)U(eue02Qm2%ee3tl0~;WuFQHIFlkj|k_P+xH z@p=yf0}&f~$KRs*y{Y#kmQ$ffbUXeN`GfHZM%K8kJ?C4Hy^;Gg-Qs-%Dl2 zcNEOH6d!hb9e2wXI!cyTCe8c8(!6A~oz|dF1KKwJVw6Mni)&gJTA#tC#kV2Iv8TQyjrDZAf`(UOz2PBGOA9E72@1Nq&qqnBU;9Vhbu1Lt3l9tiH0B09C_uu2~!`~79s>8smd zOCJL=dFQnry;iA5)_aF+JxTIZi6pVjB!RLXN#x@s`F51bot;{e@S+dPer2>iiicr^ z=?DpS9F48VUB7N1*8yUqS-NU>$M52bpXElMM}htD9EXiUV=urvt4hWSX9 zV~q`xwXk2W&Kj(Xi7mMmmloBk7DM=Q>amMzM|hUN8vo<4Gfvp(G_Rnhl+aZ&GMTJa z_o`aMKcN4S+xt-K80DJIo-;A4ieMwHgoD$;XhT=hdI5y>DiZ9X`XuTu9q8A4W?9qa vgZrvGz+Q*EtzVp7{70Cat^#831gQh!G1>ji17D}`gKD58B6K_0UW)$(wZo@R literal 0 HcmV?d00001 diff --git a/vllm/multimodal/audio.py b/vllm/multimodal/audio.py new file mode 100644 index 00000000..04d71826 --- /dev/null +++ b/vllm/multimodal/audio.py @@ -0,0 +1,17 @@ +from vllm.inputs.registry import InputContext +from vllm.multimodal.base import MultiModalInputs, MultiModalPlugin + + +class AudioPlugin(MultiModalPlugin): + """Plugin for audio data.""" + + def get_data_key(self) -> str: + return "audio" + + def _default_input_mapper(self, ctx: InputContext, data: object, + **mm_processor_kwargs) -> MultiModalInputs: + raise NotImplementedError("There is no default audio input mapper") + + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int: + raise NotImplementedError( + "There is no default maximum multimodal tokens") diff --git a/vllm/multimodal/base.py b/vllm/multimodal/base.py new file mode 100644 index 00000000..84e71cbf --- /dev/null +++ b/vllm/multimodal/base.py @@ -0,0 +1,368 @@ +import sys +from abc import ABC, abstractmethod +from collections import UserDict, defaultdict +from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, + TypedDict, TypeVar, Union, cast, final) + +import numpy as np +import torch +import torch.types +from PIL import Image +from torch import nn +from typing_extensions import TypeAlias + +from vllm.config import ModelConfig +from vllm.inputs import InputContext +from vllm.logger import init_logger +from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of, + json_map_leaves, resolve_mm_processor_kwargs) + +logger = init_logger(__name__) + +NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor] +""" +Uses a list instead of a tensor if the dimensions of each element do not match. +""" + +BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors] +""" +A dictionary containing nested tensors which have been batched via +:meth:`MultiModalInputs.batch`. +""" + +if sys.version_info < (3, 9): + # UserDict cannot be subscripted + class _MultiModalInputsBase(UserDict): + pass +else: + + class _MultiModalInputsBase(UserDict[str, NestedTensors]): + pass + + +class MultiModalInputs(_MultiModalInputsBase): + """ + A dictionary that represents the keyword arguments to + :meth:`~torch.nn.Module.forward`. + """ + + @staticmethod + def _try_stack(nested_tensors: NestedTensors) -> NestedTensors: + """ + Recursively stacks lists of tensors when they all have the same shape. + """ + if isinstance(nested_tensors, torch.Tensor): + return nested_tensors + + if isinstance(nested_tensors, np.ndarray): + return torch.from_numpy(nested_tensors) + + if isinstance(nested_tensors, (int, float)): + return torch.tensor(nested_tensors) + + stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors] + if not is_list_of(stacked, torch.Tensor, check="all"): + # Only tensors (not lists) can be stacked. + return stacked + + tensors_ = cast(List[torch.Tensor], stacked) + if any(t.shape != tensors_[0].shape for t in tensors_): + # The tensors have incompatible shapes and can't be stacked. + return tensors_ + + return torch.stack(tensors_) + + @staticmethod + def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs: + """ + Batch multiple inputs together into a dictionary. + + The resulting dictionary has the same keys as the inputs. + If the corresponding value from each input is a tensor and they all + share the same shape, the output value is a single batched tensor; + otherwise, the output value is a list containing the original value + from each input. + """ + if len(inputs_list) == 0: + return {} + + item_lists: Dict[str, List[NestedTensors]] = defaultdict(list) + + for inputs in inputs_list: + # For models that supports multiple modalities (e.g. Qwen2-VL), + # different modalities will return different data keys, + # so batch() should skip the same key check. + + for k, v in inputs.items(): + item_lists[k].append(v) + + return { + k: MultiModalInputs._try_stack(item_list) + for k, item_list in item_lists.items() + } + + @staticmethod + def as_kwargs( + batched_inputs: BatchedTensorInputs, + *, + device: torch.types.Device, + ) -> BatchedTensorInputs: + json_inputs = cast(JSONTree[torch.Tensor], batched_inputs) + + json_mapped = json_map_leaves( + lambda x: x.to(device, non_blocking=True), + json_inputs, + ) + + return cast(BatchedTensorInputs, json_mapped) + + +_T = TypeVar("_T") + +MultiModalData: TypeAlias = Union[_T, List[_T]] +""" +Either a single data instance, or a list of data instances. + +The number of data instances allowed per modality is restricted by +`--limit-mm-per-prompt`. +""" + + +@final +class MultiModalDataBuiltins(TypedDict, total=False): + """Modality types that are predefined by vLLM.""" + + image: MultiModalData[Image.Image] + """The input image(s).""" + + audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]] + """The input audio item(s) and corresponding sampling rate(s).""" + + +MultiModalDataDict = Union[MultiModalDataBuiltins, + Mapping[str, MultiModalData[object]]] +""" +A dictionary containing an item for each modality type to input. + +Note: + This dictionary also accepts modality keys defined outside + :class:`MultiModalDataBuiltins` as long as a customized plugin is registered + through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`. + Read more on that :ref:`here `. +""" + +MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]], + MultiModalInputs] +""" +Return a dictionary to be passed as keyword arguments to +:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers +and processors in HuggingFace Transformers. + +If the data is not supported, throw :exc:`TypeError`. +""" + +MultiModalTokensCalc = Union[int, Callable[[InputContext], int]] +""" +Calculate the maximum number of multimodal tokens input to the language +model. This does not include tokens that correspond to the input text. +""" + +N = TypeVar("N", bound=Type[nn.Module]) + + +class MultiModalPlugin(ABC): + """ + Base class that defines data processing logic for a specific modality. + + In particular, we adopt a registry pattern to dispatch data processing + according to the model being used (considering that different models may + process the same data differently). This registry is in turn used by + :class:`~MultiModalRegistry` which acts at a higher level + (i.e., the modality of the data). + + See also: + :ref:`adding_multimodal_plugin` + """ + + def __init__(self) -> None: + self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {} + self._max_mm_tokens: Dict[Type[nn.Module], MultiModalTokensCalc] = {} + + @abstractmethod + def get_data_key(self) -> str: + """ + Get the data key corresponding to the modality. + """ + raise NotImplementedError + + @abstractmethod + def _default_input_mapper( + self, + ctx: InputContext, + data: MultiModalData[object], + **mm_processor_kwargs, + ) -> MultiModalInputs: + """ + Return a dictionary to be passed as keyword arguments to + :meth:`~torch.nn.Module.forward`. This is similar in concept to + tokenizers and processors in HuggingFace Transformers. + + If the data is not supported, throw :exc:`TypeError`. + """ + raise NotImplementedError + + def register_input_mapper( + self, + mapper: Optional[MultiModalInputMapper] = None, + ): + """ + Register an input mapper to a model class. + + When the model receives input data that matches the modality served by + this plugin (see :meth:`get_data_key`), the provided function is + invoked to transform the data into a dictionary of model inputs. + + If `None` is provided, then the default input mapper is used instead. + + See also: + - :ref:`input_processing_pipeline` + - :ref:`enabling_multimodal_inputs` + """ + + def wrapper(model_cls: N) -> N: + if model_cls in self._input_mappers: + logger.warning( + "Model class %s already has an input mapper " + "registered to %s. It is overwritten by the new one.", + model_cls, self) + + self._input_mappers[model_cls] = mapper \ + or self._default_input_mapper + + return model_cls + + return wrapper + + def map_input(self, model_config: ModelConfig, + data: MultiModalData[object], + mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs: + """ + Transform the data into a dictionary of model inputs using the + input mapper registered for that model. + + The model is identified by ``model_config``. + + Raises: + TypeError: If the data type is not supported. + + See also: + - :ref:`input_processing_pipeline` + - :ref:`enabling_multimodal_inputs` + """ + # Avoid circular import + from vllm.model_executor.model_loader import get_model_architecture + + model_cls, _ = get_model_architecture(model_config) + + mapper = self._input_mappers.get(model_cls) + + if mapper is None: + raise KeyError(f"No input mapper in {self} is registered for " + f"model class {model_cls.__name__}.") + + # In the case of the default mapper, we have to get resource + # processor through its HuggingFace autoclass; since this goes + # through **kwargs, we can't inspect it the same way, so we allow + # drop mm_processor_kwargs based on signature inspection + # if we're using the default mapper. + # + # This should be safe in general due to the sanitation, since the + # transformers resource should filter unused kwargs anyway. + uses_default_mapper = mapper == self._default_input_mapper + mm_processor_kwargs = resolve_mm_processor_kwargs( + model_config.mm_processor_kwargs, + mm_processor_kwargs, + callable=mapper, + allow_var_kwargs=uses_default_mapper, + ) + return mapper(InputContext(model_config), data, **mm_processor_kwargs) + + @abstractmethod + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int: + """ + Calculate the maximum number of tokens, corresponding to a single + instance of multimodal data, that are passed to the language model. + """ + raise NotImplementedError + + def _validate_max_multimodal_tokens(self, max_mm_tokens: int): + if max_mm_tokens < 1: + raise ValueError("You should set the number of tokens to a " + f"positive integer. Found: {max_mm_tokens}") + + def register_max_multimodal_tokens( + self, + max_mm_tokens: Optional[MultiModalTokensCalc] = None, + ): + """ + Register the maximum number of tokens, corresponding to a single + instance of multimodal data, that are passed to the language model + for a model class. + + If `None` is provided, then the default calculation is used instead. + + See also: + :ref:`enabling_multimodal_inputs` + """ + + def wrapper(model_cls: N) -> N: + if model_cls in self._max_mm_tokens: + logger.warning( + "Model class %s already calculates maximum number of " + "tokens in %s. It is overwritten by the new one.", + model_cls, self) + + if isinstance(max_mm_tokens, int): + self._validate_max_multimodal_tokens(max_mm_tokens) + + self._max_mm_tokens[model_cls] = max_mm_tokens \ + or self._default_max_multimodal_tokens + + return model_cls + + return wrapper + + def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int: + """ + Get the maximum number of multi-modal tokens + for profiling the memory usage of a model. + + If this registry is not applicable to the model, `0` is returned. + + The model is identified by ``model_config``. + + See also: + :ref:`enabling_multimodal_inputs` + """ + # Avoid circular import + from vllm.model_executor.model_loader import get_model_architecture + + model_cls, _ = get_model_architecture(model_config) + + if model_cls not in self._input_mappers: + return 0 + + max_mm_tokens = self._max_mm_tokens.get(model_cls) + if max_mm_tokens is None: + raise KeyError(f"No maximum number of multi-modal tokens is given " + f"for model class {model_cls.__name__} in {self}.") + + if callable(max_mm_tokens): + mm_processor_kwargs = get_allowed_kwarg_only_overrides( + max_mm_tokens, overrides=model_config.mm_processor_kwargs) + max_mm_tokens = max_mm_tokens(InputContext(model_config), + **mm_processor_kwargs) + + self._validate_max_multimodal_tokens(max_mm_tokens) + + return max_mm_tokens diff --git a/vllm/multimodal/image.py b/vllm/multimodal/image.py new file mode 100644 index 00000000..5f74bcea --- /dev/null +++ b/vllm/multimodal/image.py @@ -0,0 +1,88 @@ +from functools import lru_cache +from typing import Any, Dict, Optional + +import torch +from PIL import Image +from transformers.image_processing_base import BatchFeature + +from vllm.config import ModelConfig +from vllm.inputs.registry import InputContext +from vllm.logger import init_logger +from vllm.transformers_utils.processor import get_image_processor +from vllm.utils import is_list_of + +from .base import MultiModalData, MultiModalInputs, MultiModalPlugin + +logger = init_logger(__name__) + +cached_get_image_processor = lru_cache(get_image_processor) + + +class ImagePlugin(MultiModalPlugin): + """Plugin for image data.""" + + def get_data_key(self) -> str: + return "image" + + def _get_hf_image_processor( + self, + model_config: ModelConfig, + mm_processor_kwargs: Optional[Dict[str, Any]] = None, + ): + if mm_processor_kwargs is None: + mm_processor_kwargs = {} + return cached_get_image_processor( + model_config.model, + trust_remote_code=model_config.trust_remote_code, + **mm_processor_kwargs) + + def _default_input_mapper( + self, + ctx: InputContext, + data: MultiModalData[object], + **mm_processor_kwargs, + ) -> MultiModalInputs: + model_config = ctx.model_config + + # Processed by input processor + if isinstance(data, BatchFeature): + return MultiModalInputs(data.data) + + # PIL image + if isinstance(data, Image.Image) or is_list_of(data, Image.Image): + image_processor = self._get_hf_image_processor( + model_config, + mm_processor_kwargs, + ) + + if image_processor is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + try: + # NOTE: It may make sense to forward the mm_processor_kwargs + # here too. For now, to keep it simple, we only allow it be + # used for the initialization call though, just in case the + # signatures of the preprocessor initializer don't match + # preprocess() + batch_data = image_processor \ + .preprocess(data, return_tensors="pt") \ + .data + except Exception: + logger.error( + "Failed to process image (%s) with the default mapper. " + "This is most likely an edge-case with this model's image " + "processor in transformers (type: %s), and not vLLM.", + data, + type(image_processor).__name__) + raise + + return MultiModalInputs(batch_data) + + # Image embedding + elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor): + return MultiModalInputs({"image_embeds": data}) + + raise TypeError(f"Invalid image type: {type(data)}") + + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int: + return 3000 diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py new file mode 100644 index 00000000..5e9b8bd5 --- /dev/null +++ b/vllm/multimodal/registry.py @@ -0,0 +1,243 @@ +import functools +from collections import UserDict +from typing import Any, Dict, Mapping, Optional, Sequence + +from vllm.config import ModelConfig +from vllm.logger import init_logger + +from .audio import AudioPlugin +from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs, + MultiModalPlugin, MultiModalTokensCalc, NestedTensors) +from .image import ImagePlugin +from .video import VideoPlugin + +logger = init_logger(__name__) + + +class _MultiModalLimits(UserDict): + """ + Wraps `_limits_by_model` for a more informative error message + when attempting to access a model that does not exist. + """ + + def __getitem__(self, key: ModelConfig) -> Dict[str, int]: + try: + return super().__getitem__(key) + except KeyError as exc: + msg = (f"Cannot find `mm_limits` for model={key.model}. Did you " + "forget to call `init_mm_limits_per_prompt`?") + raise KeyError(msg) from exc + + +class MultiModalRegistry: + """ + A registry that dispatches data processing to the + :class:`~vllm.multimodal.MultiModalPlugin` for each modality. + """ + + DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin()) + + def __init__( + self, + *, + plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None: + self._plugins = {p.get_data_key(): p for p in plugins} + + # This is used for non-multimodal models + self._disabled_limits_per_plugin = {k: 0 for k in self._plugins} + + self._limits_by_model = _MultiModalLimits() + + def register_plugin(self, plugin: MultiModalPlugin) -> None: + """ + Register a multi-modal plugin so it can be recognized by vLLM. + + See also: + :ref:`adding_multimodal_plugin` + """ + data_type_key = plugin.get_data_key() + + if data_type_key in self._plugins: + logger.warning( + "A plugin is already registered for data type %s, " + "and will be overwritten by the new plugin %s.", data_type_key, + plugin) + + self._plugins[data_type_key] = plugin + + def _get_plugin(self, data_type_key: str): + plugin = self._plugins.get(data_type_key) + if plugin is not None: + return plugin + + msg = f"Unknown multi-modal data type: {data_type_key}" + raise NotImplementedError(msg) + + def register_input_mapper( + self, + data_type_key: str, + mapper: Optional[MultiModalInputMapper] = None, + ): + """ + Register an input mapper for a specific modality to a model class. + + See :meth:`MultiModalPlugin.register_input_mapper` for more details. + """ + return self._get_plugin(data_type_key).register_input_mapper(mapper) + + def register_image_input_mapper( + self, + mapper: Optional[MultiModalInputMapper] = None, + ): + """ + Register an input mapper for image data to a model class. + + See :meth:`MultiModalPlugin.register_input_mapper` for more details. + """ + return self.register_input_mapper("image", mapper) + + def map_input( + self, + model_config: ModelConfig, + data: MultiModalDataDict, + mm_processor_kwargs: Optional[Dict[str, Any]] = None, + ) -> MultiModalInputs: + """ + Apply an input mapper to the data passed to the model. + + The data belonging to each modality is passed to the corresponding + plugin which in turn converts the data into into keyword arguments + via the input mapper registered for that model. + + See :meth:`MultiModalPlugin.map_input` for more details. + + Note: + This should be called after :meth:`init_mm_limits_per_prompt`. + """ + merged_dict: Dict[str, NestedTensors] = {} + + for data_key, data_value in data.items(): + plugin = self._get_plugin(data_key) + + num_items = len(data_value) if isinstance(data_value, list) else 1 + max_items = self._limits_by_model[model_config][data_key] + if num_items > max_items: + raise ValueError( + f"You set {data_key}={max_items} (or defaulted to 1) in " + f"`--limit-mm-per-prompt`, but found {num_items} items " + "in the same prompt.") + + input_dict = plugin.map_input(model_config, data_value, + mm_processor_kwargs) + for input_key, input_tensor in input_dict.items(): + if input_key in merged_dict: + raise ValueError(f"The input mappers (keys={set(data)}) " + f"resulted in a conflicting keyword " + f"argument to `forward()`: {input_key}") + + merged_dict[input_key] = input_tensor + + return MultiModalInputs(merged_dict) + + def create_input_mapper(self, model_config: ModelConfig): + """ + Create an input mapper (see :meth:`map_input`) for a specific model. + """ + # NOTE - we currently make the assumption that if a model has multiple + # supported modalities, they take the same kwargs. For the default, + # this could be an issue in the future if it falls back to two HF + # resources and we can't inspect the signature easily since it's + # getting initialized through the autoclass. + # + # If this is a problem in the future, we should revisit it, but since + # it potentially introduces a lot of complexity for a currently + # uncommon case, we do not for simplicity of both use & implementation + return functools.partial(self.map_input, model_config) + + def register_max_multimodal_tokens( + self, + data_type_key: str, + max_mm_tokens: Optional[MultiModalTokensCalc] = None, + ): + """ + Register the maximum number of tokens, corresponding to a single + instance of multimodal data belonging to a specific modality, that are + passed to the language model for a model class. + """ + return self._get_plugin(data_type_key) \ + .register_max_multimodal_tokens(max_mm_tokens) + + def register_max_image_tokens( + self, + max_mm_tokens: Optional[MultiModalTokensCalc] = None, + ): + """ + Register the maximum number of image tokens, corresponding to a single + image, that are passed to the language model for a model class. + """ + return self.register_max_multimodal_tokens("image", max_mm_tokens) + + def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int: + """ + Get the maximum number of multi-modal tokens + for profiling the memory usage of a model. + + See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details. + + Note: + This should be called after :meth:`init_mm_limits_per_prompt`. + """ + limits_per_plugin = self._limits_by_model[model_config] + + return sum((limits_per_plugin[key] * + plugin.get_max_multimodal_tokens(model_config)) + for key, plugin in self._plugins.items()) + + def init_mm_limits_per_prompt( + self, + model_config: ModelConfig, + ) -> None: + """ + Initialize the maximum number of multi-modal input instances for each + modality that are allowed per prompt for a model class. + """ + if model_config in self._limits_by_model: + logger.warning( + "`mm_limits` has already been set for model=%s, and will " + "be overwritten by the new values.", model_config.model) + + multimodal_config = model_config.multimodal_config + if multimodal_config is None: + limits_per_plugin = self._disabled_limits_per_plugin + else: + config_limits_per_plugin = multimodal_config.limit_per_prompt + + extra_keys = config_limits_per_plugin.keys() - self._plugins.keys() + if extra_keys: + logger.warning( + "Detected extra keys in `--limit-mm-per-prompt` which " + "are not registered as multi-modal plugins: %s. " + "They will be ignored.", extra_keys) + + # NOTE: Currently the default is set to 1 for each plugin + # TODO: Automatically determine the limits based on budget + # once more models support multi-image inputs + limits_per_plugin = { + key: config_limits_per_plugin.get(key, 1) + for key in self._plugins + } + + self._limits_by_model[model_config] = limits_per_plugin + + def get_mm_limits_per_prompt( + self, + model_config: ModelConfig, + ) -> Mapping[str, int]: + """ + Get the maximum number of multi-modal input instances for each modality + that are allowed per prompt for a model class. + + Note: + This should be called after :meth:`init_mm_limits_per_prompt`. + """ + return self._limits_by_model[model_config] diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py new file mode 100644 index 00000000..3c801464 --- /dev/null +++ b/vllm/multimodal/utils.py @@ -0,0 +1,323 @@ +import base64 +from functools import lru_cache +from io import BytesIO +from typing import Any, List, Optional, Tuple, TypeVar, Union + +import numpy as np +import numpy.typing as npt +from PIL import Image + +from vllm.connections import global_http_connection +from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT +from vllm.logger import init_logger +from vllm.multimodal.base import MultiModalDataDict +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer + +logger = init_logger(__name__) + +cached_get_tokenizer = lru_cache(get_tokenizer) + + +def _load_image_from_bytes(b: bytes): + image = Image.open(BytesIO(b)) + image.load() + return image + + +def _load_image_from_data_url(image_url: str): + # Only split once and assume the second part is the base64 encoded image + _, image_base64 = image_url.split(",", 1) + return load_image_from_base64(image_base64) + + +def fetch_image(image_url: str, *, image_mode: str = "RGB") -> Image.Image: + """ + Load a PIL image from a HTTP or base64 data URL. + + By default, the image is converted into RGB format. + """ + if image_url.startswith('http'): + image_raw = global_http_connection.get_bytes( + image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT) + image = _load_image_from_bytes(image_raw) + + elif image_url.startswith('data:image'): + image = _load_image_from_data_url(image_url) + else: + raise ValueError("Invalid 'image_url': A valid 'image_url' must start " + "with either 'data:image' or 'http'.") + + return image.convert(image_mode) + + +async def async_fetch_image(image_url: str, + *, + image_mode: str = "RGB") -> Image.Image: + """ + Asynchronously load a PIL image from a HTTP or base64 data URL. + + By default, the image is converted into RGB format. + """ + if image_url.startswith('http'): + image_raw = await global_http_connection.async_get_bytes( + image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT) + image = _load_image_from_bytes(image_raw) + + elif image_url.startswith('data:image'): + image = _load_image_from_data_url(image_url) + else: + raise ValueError("Invalid 'image_url': A valid 'image_url' must start " + "with either 'data:image' or 'http'.") + + return image.convert(image_mode) + + +def try_import_audio_packages() -> Tuple[Any, Any]: + try: + import librosa + import soundfile + except ImportError: + raise ImportError( + "Please install vllm[audio] for audio support.") from None + return librosa, soundfile + + +def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]: + """ + Load audio from a URL. + """ + librosa, _ = try_import_audio_packages() + + if audio_url.startswith("http"): + audio_bytes = global_http_connection.get_bytes( + audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT) + elif audio_url.startswith("data:audio"): + _, audio_base64 = audio_url.split(",", 1) + audio_bytes = base64.b64decode(audio_base64) + else: + raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start " + "with either 'data:audio' or 'http'.") + + return librosa.load(BytesIO(audio_bytes), sr=None) + + +async def async_fetch_audio( + audio_url: str) -> Tuple[np.ndarray, Union[int, float]]: + """ + Asynchronously fetch audio from a URL. + """ + librosa, _ = try_import_audio_packages() + + if audio_url.startswith("http"): + audio_bytes = await global_http_connection.async_get_bytes( + audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT) + elif audio_url.startswith("data:audio"): + _, audio_base64 = audio_url.split(",", 1) + audio_bytes = base64.b64decode(audio_base64) + else: + raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start " + "with either 'data:audio' or 'http'.") + + return librosa.load(BytesIO(audio_bytes), sr=None) + + +def get_and_parse_audio(audio_url: str) -> MultiModalDataDict: + audio, sr = fetch_audio(audio_url) + return {"audio": (audio, sr)} + + +def get_and_parse_image(image_url: str) -> MultiModalDataDict: + image = fetch_image(image_url) + return {"image": image} + + +async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict: + audio, sr = await async_fetch_audio(audio_url) + return {"audio": (audio, sr)} + + +async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict: + image = await async_fetch_image(image_url) + return {"image": image} + + +def encode_audio_base64( + audio: np.ndarray, + sampling_rate: int, +) -> str: + """Encode audio as base64.""" + _, soundfile = try_import_audio_packages() + + buffered = BytesIO() + soundfile.write(buffered, audio, sampling_rate, format="WAV") + + return base64.b64encode(buffered.getvalue()).decode('utf-8') + + +def encode_image_base64( + image: Image.Image, + *, + image_mode: str = "RGB", + format: str = "JPEG", +) -> str: + """ + Encode a pillow image to base64 format. + + By default, the image is converted into RGB format before being encoded. + """ + buffered = BytesIO() + image = image.convert(image_mode) + image.save(buffered, format) + return base64.b64encode(buffered.getvalue()).decode('utf-8') + + +def load_image_from_base64(image: Union[bytes, str]) -> Image.Image: + """Load image from base64 format.""" + return _load_image_from_bytes(base64.b64decode(image)) + + +def rescale_image_size(image: Image.Image, + size_factor: float, + transpose: int = -1) -> Image.Image: + """Rescale the dimensions of an image by a constant factor.""" + new_width = int(image.width * size_factor) + new_height = int(image.height * size_factor) + image = image.resize((new_width, new_height)) + if transpose >= 0: + image = image.transpose(Image.Transpose(transpose)) + return image + + +def try_import_video_packages() -> Any: + try: + import cv2 + except ImportError: + raise ImportError( + "Please install vllm[video] for video support.") from None + return cv2 + + +def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray: + cv2 = try_import_video_packages() + + num_frames, _, _, channels = frames.shape + new_height, new_width = size + resized_frames = np.empty((num_frames, new_height, new_width, channels), + dtype=frames.dtype) + for i, frame in enumerate(frames): + resized_frame = cv2.resize(frame, (new_width, new_height)) + resized_frames[i] = resized_frame + return resized_frames + + +def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray: + _, height, width, _ = frames.shape + new_height = int(height * size_factor) + new_width = int(width * size_factor) + + return resize_video(frames, (new_height, new_width)) + + +def sample_frames_from_video(frames: npt.NDArray, + num_frames: int) -> npt.NDArray: + total_frames = frames.shape[0] + if num_frames == -1: + return frames + else: + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) + sampled_frames = frames[frame_indices, ...] + return sampled_frames + + +# Utilities for input processors +_T = TypeVar("_T", str, int) + + +def repeat_and_pad_token( + token: _T, + *, + repeat_count: int = 1, + pad_token_left: Optional[_T] = None, + pad_token_right: Optional[_T] = None, +) -> List[_T]: + replacement = [token] * repeat_count + if pad_token_left is not None: + replacement = [pad_token_left] + replacement + if pad_token_right is not None: + replacement = replacement + [pad_token_right] + + return replacement + + +def repeat_and_pad_placeholder_tokens( + tokenizer: AnyTokenizer, + prompt: Optional[str], + prompt_token_ids: List[int], + *, + placeholder_token_id: int, + repeat_count: Union[int, List[int]], + pad_token_left: Optional[int] = None, + pad_token_right: Optional[int] = None, +) -> Tuple[Optional[str], List[int]]: + if isinstance(repeat_count, int): + repeat_count = [repeat_count] + + if prompt is None: + new_prompt = None + else: + placeholder_token_str = tokenizer.decode(placeholder_token_id) + pad_token_str_left = (None if pad_token_left is None else + tokenizer.decode(pad_token_left)) + pad_token_str_right = (None if pad_token_right is None else + tokenizer.decode(pad_token_right)) + + placeholder_token_count = prompt.count(placeholder_token_str) + # This is an arbitrary number to distinguish between the two cases + if placeholder_token_count > 16: + logger.warning( + "Please follow the prompt format that is " + "documented on HuggingFace which does not involve " + "repeating %s tokens.", placeholder_token_str) + if placeholder_token_count < len(repeat_count): + logger.warning( + "The number of multi-modal placeholder tokens in the prompt " + "is less than the number of multi-modal inputs. Extra " + "placeholder tokens will be treated as plain text") + repeat_count = repeat_count[:placeholder_token_count] + + prompt_parts = prompt.split(placeholder_token_str, + maxsplit=len(repeat_count)) + new_prompt = "" + for i, repeat_count_item in enumerate(repeat_count): + replacement_str = "".join( + repeat_and_pad_token( + placeholder_token_str, + repeat_count=repeat_count_item, + pad_token_left=pad_token_str_left, + pad_token_right=pad_token_str_right, + )) + # The image tokens are removed to be consistent with HuggingFace + new_prompt += prompt_parts[i] + replacement_str + new_prompt += prompt_parts[-1] + + new_token_ids: List[int] = [] + placeholder_token_idx = 0 + for i, token in enumerate(prompt_token_ids): + if token == placeholder_token_id: + replacement_ids = repeat_and_pad_token( + placeholder_token_id, + repeat_count=repeat_count[placeholder_token_idx], + pad_token_left=pad_token_left, + pad_token_right=pad_token_right, + ) + new_token_ids.extend(replacement_ids) + placeholder_token_idx += 1 + + # No need to further scan the list since we replaced all tokens + if placeholder_token_idx >= len(repeat_count): + new_token_ids.extend(prompt_token_ids[i + 1:]) + break + else: + new_token_ids.append(token) + + return new_prompt, new_token_ids diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py new file mode 100644 index 00000000..4a9dbf20 --- /dev/null +++ b/vllm/multimodal/video.py @@ -0,0 +1,86 @@ +from functools import lru_cache +from typing import Any, Dict, List, Optional, Union + +import numpy as np + +from vllm.config import ModelConfig +from vllm.inputs.registry import InputContext +from vllm.logger import init_logger +from vllm.transformers_utils.processor import get_video_processor +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.utils import is_list_of + +from .base import MultiModalData, MultiModalInputs +from .image import ImagePlugin + +logger = init_logger(__name__) + +cached_get_video_processor = lru_cache(get_video_processor) +cached_get_tokenizer = lru_cache(get_tokenizer) + +VideoInput = Union[ + "np.ndarray", # single video input + List["np.ndarray"], + # TODO: support more types + # List[Image.Image], List[List[Image.Image]], + # "torch.Tensor", + # List["torch.Tensor"], + # List[List["np.ndarrray"]], + # List[List["torch.Tensor"]], +] + + +class VideoPlugin(ImagePlugin): + """Plugin for video data.""" + + def get_data_key(self) -> str: + return "video" + + def _get_hf_video_processor( + self, + model_config: ModelConfig, + mm_processor_kwargs: Optional[Dict[str, Any]] = None, + ): + if mm_processor_kwargs is None: + mm_processor_kwargs = {} + return cached_get_video_processor( + model_config.model, + trust_remote_code=model_config.trust_remote_code, + **mm_processor_kwargs) + + def _default_input_mapper( + self, + ctx: InputContext, + data: MultiModalData[object], + **mm_processor_kwargs, + ) -> MultiModalInputs: + model_config = ctx.model_config + + # single video input as np.ndarray + if isinstance(data, np.ndarray): + video_processor = self._get_hf_video_processor( + model_config, + mm_processor_kwargs, + ) + if video_processor is None: + raise RuntimeError("No HuggingFace processor is available " + "to process the image object") + try: + # NOTE: Similar to image; it may be a good idea to filter and + # pass mm_processor_kwargs here too, but for now we don't to + # avoid extra complexity if the initializer and preprocess + # signatures of the processor don't align + batch_data = video_processor(data, return_tensors="pt").data + except Exception: + logger.error("Failed to process image (%s)", data) + raise + + return MultiModalInputs(batch_data) + elif is_list_of(data, np.ndarray): + raise NotImplementedError( + "Multi video for a prompt is not supported yet") + + raise TypeError(f"Invalid video type: {type(data)}") + + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int: + return 4096 diff --git a/vllm/outputs.py b/vllm/outputs.py new file mode 100644 index 00000000..07650241 --- /dev/null +++ b/vllm/outputs.py @@ -0,0 +1,318 @@ +import time +from dataclasses import dataclass +from typing import List, Optional +from typing import Sequence as GenericSequence +from typing import Union + +from vllm.lora.request import LoRARequest +from vllm.sampling_params import RequestOutputKind +from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs, + SequenceGroup, SequenceStatus) + + +@dataclass +class CompletionOutput: + """The output data of one completion output of a request. + + Args: + index: The index of the output in the request. + text: The generated output text. + token_ids: The token IDs of the generated output text. + cumulative_logprob: The cumulative log probability of the generated + output text. + logprobs: The log probabilities of the top probability words at each + position if the logprobs are requested. + finish_reason: The reason why the sequence is finished. + stop_reason: The stop string or token id that caused the completion + to stop, None if the completion finished for some other reason + including encountering the EOS token. + lora_request: The LoRA request that was used to generate the output. + """ + + index: int + text: str + token_ids: GenericSequence[int] + cumulative_logprob: Optional[float] + logprobs: Optional[SampleLogprobs] + finish_reason: Optional[str] = None + stop_reason: Union[int, str, None] = None + lora_request: Optional[LoRARequest] = None + + def finished(self) -> bool: + return self.finish_reason is not None + + def __repr__(self) -> str: + return (f"CompletionOutput(index={self.index}, " + f"text={self.text!r}, " + f"token_ids={self.token_ids}, " + f"cumulative_logprob={self.cumulative_logprob}, " + f"logprobs={self.logprobs}, " + f"finish_reason={self.finish_reason}, " + f"stop_reason={self.stop_reason})") + + +@dataclass +class EmbeddingOutput: + """The output data of one completion output of a request. + + Args: + embedding: The embedding vector, which is a list of floats. The + length of vector depends on the model as listed in the embedding guide. + """ + + embedding: List[float] + + def __repr__(self) -> str: + return (f"EmbeddingOutput(" + f"embedding={len(self.embedding)})") + + +class RequestOutput: + """The output data of a completion request to the LLM. + + Args: + request_id: The unique ID of the request. + prompt: The prompt string of the request. + For encoder/decoder models, this is the + decoder input prompt. + prompt_token_ids: The token IDs of the prompt. + For encoder/decoder models, this is the + decoder input prompt token ids. + prompt_logprobs: The log probabilities to return per prompt token. + outputs: The output sequences of the request. + finished: Whether the whole request is finished. + metrics: Metrics associated with the request. + lora_request: The LoRA request that was used to generate the output. + encoder_prompt: The encoder prompt string of the request; + None if decoder-only + encoder_prompt_token_ids: The token IDs of the encoder prompt; + None if decoder-only + """ + + def __init__( + self, + request_id: str, + prompt: Optional[str], + prompt_token_ids: Optional[List[int]], + prompt_logprobs: Optional[PromptLogprobs], + outputs: List[CompletionOutput], + finished: bool, + metrics: Optional[RequestMetrics] = None, + lora_request: Optional[LoRARequest] = None, + encoder_prompt: Optional[str] = None, + encoder_prompt_token_ids: Optional[List[int]] = None, + ) -> None: + self.request_id = request_id + self.prompt = prompt + self.prompt_token_ids = prompt_token_ids + self.prompt_logprobs = prompt_logprobs + self.outputs = outputs + self.finished = finished + self.metrics = metrics + self.lora_request = lora_request + self.encoder_prompt = encoder_prompt + self.encoder_prompt_token_ids = encoder_prompt_token_ids + + @classmethod + def from_seq_group(cls, seq_group: SequenceGroup, + use_cache: bool) -> Optional["RequestOutput"]: + sampling_params = seq_group.sampling_params + if sampling_params is None: + raise ValueError( + "Sampling parameters are missing for a CompletionRequest.") + + finished = seq_group.is_finished() + if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and ( + not finished): + return None + + # Init cache (if needed) + if use_cache and seq_group.cached_request_output is None: + seq_group.cached_request_output = RequestOutput( # type: ignore + request_id="", + prompt=None, + prompt_token_ids=[], + prompt_logprobs=None, + outputs=[], + finished=False) + + seqs = seq_group.get_seqs() + if len(seqs) == 1: + top_n_seqs = seqs + else: + # Get the top-n sequences. + n = sampling_params._real_n or sampling_params.n + sorting_key = lambda seq: seq.get_cumulative_logprob() + sorted_seqs = sorted(seqs, key=sorting_key, reverse=True) + top_n_seqs = sorted_seqs[:n] + + # Create the outputs. + # NOTE: We need omit logprobs here explicitly because the sequence + # always has the logprobs of the sampled tokens even if the + # logprobs are not requested. + include_logprobs = sampling_params.logprobs is not None + text_buffer_length = sampling_params.output_text_buffer_length + delta = sampling_params.output_kind == RequestOutputKind.DELTA + + outputs = [] + include_prompt = True + for i, seq in enumerate(top_n_seqs): + output_text = seq.get_output_text_to_return( + text_buffer_length, delta) + + output_token_ids = seq.get_output_token_ids_to_return(delta) + num_output_tokens = 1 if isinstance(output_token_ids, + int) else len(output_token_ids) + + output_logprobs = seq.output_logprobs if include_logprobs else None + + if delta: + # Slice logprobs delta if applicable + if output_logprobs: + output_logprobs = output_logprobs[-num_output_tokens:] + # Don't include prompt if this is after the first output + # containing decode token ids + if include_prompt and seq.get_output_len() > num_output_tokens: + include_prompt = False + + if use_cache: + # Get cached output object + cached_outputs = seq_group.cached_request_output.outputs # type: ignore + if i >= len(cached_outputs): + cached_outputs.append( + CompletionOutput(index=i, + text="", + token_ids=[], + cumulative_logprob=None, + logprobs=None, + finish_reason=None, + stop_reason=None)) + output = cached_outputs[i] + + # Init cached output object + assert output.index == i + output.text = output_text + + if isinstance(output_token_ids, int): + output.token_ids.clear() + output.token_ids.append(output_token_ids) + else: + output.token_ids = output_token_ids + + output.cumulative_logprob = seq.get_cumulative_logprob() \ + if include_logprobs else None + output.logprobs = output_logprobs + output.finish_reason = SequenceStatus.get_finished_reason( + seq.status) + output.stop_reason = seq.stop_reason + + else: + output = CompletionOutput( + seqs.index(seq), output_text, [output_token_ids] + if isinstance(output_token_ids, int) else output_token_ids, + seq.get_cumulative_logprob() if include_logprobs else None, + output_logprobs, + SequenceStatus.get_finished_reason(seq.status), + seq.stop_reason) + + outputs.append(output) + + # Every sequence in the sequence group should have the same prompt. + if include_prompt: + prompt = seq_group.prompt + prompt_token_ids = seq_group.prompt_token_ids + encoder_prompt = seq_group.encoder_prompt + encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids + prompt_logprobs = seq_group.prompt_logprobs + else: + prompt = None + prompt_token_ids = None + encoder_prompt = None + encoder_prompt_token_ids = None + prompt_logprobs = None + finished_time = time.time() if finished else None + seq_group.set_finished_time(finished_time) + + init_args = (seq_group.request_id, prompt, prompt_token_ids, + prompt_logprobs, outputs, finished, seq_group.metrics, + seq_group.lora_request, encoder_prompt, + encoder_prompt_token_ids) + + if use_cache: + request_output = seq_group.cached_request_output + request_output.__init__(*init_args) # type: ignore + + else: + request_output = cls(*init_args) + + return request_output + + def __repr__(self) -> str: + return (f"RequestOutput(request_id={self.request_id}, " + f"prompt={self.prompt!r}, " + f"prompt_token_ids={self.prompt_token_ids}, " + f"encoder_prompt={self.encoder_prompt!r}, " + f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, " + f"prompt_logprobs={self.prompt_logprobs}, " + f"outputs={self.outputs}, " + f"finished={self.finished}, " + f"metrics={self.metrics}, " + f"lora_request={self.lora_request})") + + +class EmbeddingRequestOutput: + """ + The output data of an embedding request to the LLM. + + Args: + request_id (str): A unique identifier for the embedding request. + outputs (EmbeddingOutput): The embedding results for the given input. + prompt_token_ids (List[int]): A list of token IDs used in the prompt. + finished (bool): A flag indicating whether the embedding is completed. + """ + + def __init__(self, request_id: str, outputs: "EmbeddingOutput", + prompt_token_ids: List[int], finished: bool): + self.request_id = request_id + self.prompt_token_ids = prompt_token_ids + self.finished = finished + self.outputs = outputs + + @classmethod + def from_seq_group(cls, + seq_group: 'SequenceGroup') -> "EmbeddingRequestOutput": + if seq_group.embeddings is None: + raise ValueError( + "Embeddings are missing in seq_group for EmbeddingRequest.") + output = EmbeddingOutput(seq_group.embeddings) + prompt_token_ids = seq_group.prompt_token_ids + finished = seq_group.is_finished() + + return cls(seq_group.request_id, output, prompt_token_ids, finished) + + def __repr__(self): + """ + Returns a string representation of an EmbeddingRequestOutput instance. + + The representation includes the request_id and the number of outputs, + providing a quick overview of the embedding request's results. + + Returns: + str: A string representation of the EmbeddingRequestOutput instance. + """ + return (f"EmbeddingRequestOutput(request_id='{self.request_id}', " + f"outputs={repr(self.outputs)}, " + f"prompt_token_ids={self.prompt_token_ids}, " + f"finished={self.finished})") + + +class RequestOutputFactory: + + @staticmethod + def create(seq_group: SequenceGroup, use_cache: bool = False): + # Determine the type based on a condition, for example: + if hasattr(seq_group, + 'embeddings') and seq_group.embeddings is not None: + return EmbeddingRequestOutput.from_seq_group(seq_group) + else: + return RequestOutput.from_seq_group(seq_group, use_cache) diff --git a/vllm/platforms/__init__.py b/vllm/platforms/__init__.py new file mode 100644 index 00000000..65f02ba1 --- /dev/null +++ b/vllm/platforms/__init__.py @@ -0,0 +1,81 @@ +from .interface import Platform, PlatformEnum, UnspecifiedPlatform + +current_platform: Platform + +# NOTE: we don't use `torch.version.cuda` / `torch.version.hip` because +# they only indicate the build configuration, not the runtime environment. +# For example, people can install a cuda build of pytorch but run on tpu. + +is_tpu = False +try: + # While it's technically possible to install libtpu on a non-TPU machine, + # this is a very uncommon scenario. Therefore, we assume that libtpu is + # installed if and only if the machine has TPUs. + import libtpu # noqa: F401 + is_tpu = True +except Exception: + pass + +is_cuda = False + +try: + import pynvml + pynvml.nvmlInit() + try: + if pynvml.nvmlDeviceGetCount() > 0: + is_cuda = True + finally: + pynvml.nvmlShutdown() +except Exception: + is_cuda = True + +is_rocm = False + +try: + import amdsmi + amdsmi.amdsmi_init() + try: + if len(amdsmi.amdsmi_get_processor_handles()) > 0: + is_rocm = True + finally: + amdsmi.amdsmi_shut_down() +except Exception: + pass + +is_xpu = False + +try: + import torch + if hasattr(torch, 'xpu') and torch.xpu.is_available(): + is_xpu = True +except Exception: + pass + +is_cpu = False +try: + from importlib.metadata import version + is_cpu = "cpu" in version("vllm") +except Exception: + pass + +if is_tpu: + # people might install pytorch built with cuda but run on tpu + # so we need to check tpu first + from .tpu import TpuPlatform + current_platform = TpuPlatform() +elif is_cuda: + from .cuda import CudaPlatform + current_platform = CudaPlatform() +elif is_rocm: + from .rocm import RocmPlatform + current_platform = RocmPlatform() +elif is_xpu: + from .xpu import XPUPlatform + current_platform = XPUPlatform() +elif is_cpu: + from .cpu import CpuPlatform + current_platform = CpuPlatform() +else: + current_platform = UnspecifiedPlatform() + +__all__ = ['Platform', 'PlatformEnum', 'current_platform'] diff --git a/vllm/platforms/__pycache__/__init__.cpython-310.pyc b/vllm/platforms/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b50dc419131ff82cbc52614604ab580326d0e5b GIT binary patch literal 1323 zcmZWpzjM?^6yB9|I-O5m&Z%B zZlCqx8|d@qv7=u?|F@NsZ%>JSiHgg!USEAtyE^V-Z6LmrYix}Y-hws0a7;&ptb*xj|sb=euhB*gLo(|$_=)G_(qNY1Reb^;vKOmx7ZfqTQz1iDBw zRk=xCGH)b=M6~@NFV8eE4EIVN&&T(&M9#(SM|bzdYB7E zk?Cj{D=wg5y(wFuVpL1ot>R2s!(=)cR?T5t#KvfiuQFjFlJO``Mf^lS-4%|zT1sV} zX@k98mSAEYn^@booVnPmwqJvKWR++8)NRXncGo(zRL7c@HC;4S6K5wx6#Yr`iUqCt z@A|;Ve2}O4C6vZaE?t%7P+pOT1H9=VKQhBi4eX&0^63Lo45E4nQFJ*!s)B1W z_TejDpjjb-o@@os56P5uR5WP=J1})tMSkIeZ;)yGX<>e&GbSy NK-Q{!ZG9(nJ^^MQRwe)d literal 0 HcmV?d00001 diff --git a/vllm/platforms/__pycache__/cpu.cpython-310.pyc b/vllm/platforms/__pycache__/cpu.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49e4ed2e5127723bdc89aee0623beb4eee83351e GIT binary patch literal 1014 zcmZ`%OK;RL5cVU)nsl>KW0 z{!k8p(#Q=@t7dn-L1wNm)iO&}oJkiLWgAnZ4DvfcA91IkH4dfijm;cStLBE>FmUqP zAZyd#nV+vBtE;HWsso#t3uO;p>Sm#j^rtA#K19{ZF3KWW=DCil?NsUr>K#@qw+HZe z%c)4T!e^?Hg~!&zxWTqEQbtQ0BI?OZ2hcD=}*`Usv!aR7hxRZu;bVb zVl2_*=#S#~BMfJ=WAOC_KRtUd{8+))b;C4I>RKy!P3d@6SjTH?%oqtiK=TOA-LKeq zI7{P$>4Db!U_ug5#-32dQ75$Bod#wPk=TmpLv3Z|NvaITBr<7H|0?dn*P903U~e@J LH6$V3XOHizN9n#2Ue#q=}e#d)*ph`KKUm!ZR&n!NnI#Ag|v&kVfXC0edjP9AFpV5 z-uv4>+J8T(Y5%0o(Z@pP21?r3HH~YI#&ymD-O=$j0u~yM5t@#v)3_NF!lF|QOHL`Y z94jn4Wi`%%ic`USAs7p*PF0N+gYj^}nLxY5t>8pB=}d-G&Qv(o-S{TnP+Etbx+3*DogrDw~>kv-?*aJze`ki}WpRNE7Y%8ovm7 z-}+eNm$<#Faa&C77|z?At!lNo-d77T7q%azK3-YdPwkc;2)pS;cIvf6(6zlZ?SvTV zaL-NxFKfjzyljj5THW@;B$k;UtyPk4v>66<`;&M}YzleV_A@(e$DM%NQJi6aKl8o7 z?+y29#!)7oX7$Q*(u+oo<>f$j+@{xTi@dNUy(F!fxwYU0f%iBNxp~J=v%GM>lb}<& z-%Z2=PuBFjyy8hOTunq1oiRGa4dS&mA)iCDw3?AmE{aXRDHgoMd+Y~(*3GSz;T`ho zerqY}ggJ9pQ111{TvzSm)|0MhV4E-Ej_pZdcT!x`_G~T^DVkm;xcz8w?MHUvH8;F9 zQMXqD;iZb78ECg3MVmhNy(?jGCFwpQHfO@)mu)ZNcIa&gJMCcSEVerl&8bg@d)DoT zZ4udz>hij3hpfn5%c-|}%*M*9(If{Umr)EP~$18Rhnu}tgsst|TN%aZi! z{Cv_){CX_c<`WsOi)MxalAsOg+F=Z${D=hYWqushbJmLTq7+$2Mor@E{=?8Hp%^IX zPf=ysAG96Vai*^`&QKfN$jnyp_gV|K%nQ5K9erEx>(ISoBR4&{mgdEcEqco-eBVSl zcxZR>6_-%b%cwH#nZ5@d?^XYx^)+AX>wnSzs6V7}M&qjotu5_4y#e_b2iI~ey4*7K z!>LkhwcW{hycN}$oW{NqUJ4(`NqTADU!BjwWIpkeD;WqA*_98%PCI-r+?o&k$6!>} zj-&a_APDF8?IE2f^MJ&UBy>I|xU>ZP6Ez;IebRb;kqmgG4!pA$7OlTzqC zO<{?7nJe3HeLf;nk}q+70Y%*a-iNJ`w=f#d%NUfjLeWsu)%jwLNGJNx84@L3bog)C_qFulyXbbYSKm zi|iAaRv>P5mm@BoI#VNa4KEbmF-3A-Y^#L>sue#K>1SR=dW7$Oj-yf&RfSb_N&J0@ zY?@VM6St3$&CoW8@O5+NfcSzUUfNii8-A2Ir$a!??JG~+P2{t<0Pcbw6BSKwnJ*KfJp zZ?%LJQ8rY}w0>ZPhhI8V3!g9EbQhN%EH5m%pWa(sy603@7M9&xpRc;hi@k3aMFyt{ z2sqddxIjpg!f}$bw$mjls4(L3Q<2$^UOMH&!$TesE_j=<&w&o&DKL)C zdo0@CrXS0Cu2*VfN{O855!G3UI}xx_deNH5jX*@XfwOWJr+G;Lk7SGyC{A7+KqXuj zSOaX!&Gp!aOUx?=jv;lb!#3GKdYsp1f0U_8qb`8;mTu|O`mD}$_JZl=3lr|pV%87l ze=H_{fdvHI?%lY#aX03jK#XXGfq5!ww;N~xShRJ0QIu$ z%?u7Z`vxO@?X0%Vps$SVt`9w|6&lWrV)}#S)#Y1vmWCo;{a(MEYrW52HDx0vB4I%q zB22Qbon{i;x@>Q?{bt(^g%_pv$19%$%M%KRXg&8+id%Rc9(^8kOY(VnAx#j1+>(LPf8$C(I@o^Bmj=zP)_941;FQiI5xl+#Sj$|kOl8Q1vC!)tc}jV zKS0VlN%^;I+t@bYuZ+IQ^=}EZ7)6(b&|8lsUVaoyaGMyRut$!eRV(C1Ge`$AxJFaI zqUr`!3K|aa;S_n?A@>gvt&NomZY@^Tt9ozl=ymF^{G@sGR;1(9S9(q`KYFWuz#9lq zfe-F*6YGwUe2mZ&M@}W%jj&oF0jWao!jBP1OGii}vIy@LX~cO>u^VdyL^D11&@Hng z1RbO0^buN)qEm{`m47x_@68{h=;#Oe$(L;-9U+(ElPZcL_s?ih@UjaGQRoFECM{Xi zzFZh;%6|VU`i1Qx^ro<($-BLoOh>R`_KOt4?it&qehDoT2v>gyegM^sRis`|PMR9V z3*9@qwrwFWoy>~sK*PSp3*Q3!9wHs%#ck^=%niO7^tOsWRPO3uxRwqC{hTt(8VnNrAHUmG}uz&KN-Hwym!8Rjaltf5&qp+2t zMun{<6;Q!T?v0=;JB&-@J6LBmvAt}7rprr+4-qUgd)-<(q4tGAN4+M zJh(%2a|NSFG3=Iyw3FBE`;>y>A9fmg0db-mcaV}GGxQ^*FCI^8U(d=4GAKIoEh__l}FcyKr*_(B4>F`i+a- zmm9a8>4Uk|`#0|c_&>evlxb|>xZejC@`B2(YUP3QFJZ_j4I;}_*3gr4v`Sxh#u6c< zJD74R$@__mbPp8BPiWOKgP9oRVHZnMI<8n`;P)hAISYVl4*vdIF)9GoNoEqrvf>ZZ zru8-ZH~SZRVNUeU{~y^jkn)gi*Cunzb;(V5=1%$ zm-w5HH3R%Q5DFwjTmhMKr}7ppGys#5%up^;MGBakkK;Iyzo!1-ja7?+Z+RV+Kv4mp zr4?}-1P%ohlinAYS|M*^N$s7yA)yxi(U%?#V8lJs5WQF$>#mK&`}@>_}m zNnj@_V;!i))L?;&-y#Hl0*RwkQkLY5QFd1!#7B~Ric?06E@aDVioqSp$taGbHOrjT y&mcqqbg>x*d;mgJl{T3#=Ekk@i@F8=TZLI>6Bbo{K850nl=;4aaQUku1YJjphI(AqlJ+DsTVq@0sFV zDc-ZiyQ<`1S?!7Mq}@mcyJ0VAHRiI4ZHExawzH~j2NCc2^j)*R>du=$jsg6C~-6slAh_Z#@U&cEPSH4FIawhv3y>LDoGsY46NXXhQ0yj%6$4h0ZxGR&1V%j77uir5Z4n;{iCI_?vuD%wxPR7S^?{$#3zcXQiild$E{Z7PrKbm>2V1wyx!{SW)Vh`3k?SsFy`; zxYG)DS>BifhN zUgI`Tqtm#_$PIo*1V071&WomAmlvhOiM+}XK8<_t4p}XX9}6?D%W2{JIDH=XHYRhz z%_wlC}($?EtRyMv&?Y)MBc}~V_8fUq1zKz zegH~!9JAcNJ3P)r(yTgS=#pIq7oB2gr(Xl1NWNlR|3d$ZabmHF;o3YPuWvc z%q!zcKP&BPSH@Sy*INGFF#9iuCE-9MsdaH8P);Oj3aU(8^c7T3+$e~VykSY}!b?vq z=Vlojt5OFjNuAC}_)f<{4}+2wrl@6;E?F(UT1vti5XG)RkG=|$XbY;2LNAS)~=rOB$D40p=gs#X+N*q<7Fiq88 zsyHHBx0l?WlsJz)I^_-fpP;k2;W-~-bwb&-RPBh;-aYz{T;bNiq->h zx&E=py8-GzErjI2NKLd!%dIHg3!q*yg5bMSMpCK-G^)MIYZt~`kv94R7^pmN8vQ$y z{FR8N$|rw?<&f>&fov(70N)Bk5XsAN8PWb&N0+Hcn(}=tnJC)Ym}qJrJKyI|wBR=Y zeQT0{!j)d#<5@96~taai@e&I-A9Xak9<`XOFxd^kw!tcV+ zU3p}oiNaJIM>3tp-r5U`)d;pk+@}ul5@GB6R@d?U;cmprN~6IxVPqdmhj(BGD?(zI z9!Kw(rvi$nh>ybXpn=Al4%=V|6H576{kj$qSt??#dzYlYu?`-U0c(~1+R$Lvm$$H< zl6K?HEB$2NoxFvq%sh%BU;YH2Qnd&7=&M0MmGGYv&7A9_S#S}JZe}vWe90JlSvG1f*bFnx7fZmH z!Co-4_FuNTsxyE+2-4M+!dBVXIM|sI6Pr zNxnm1WR1~LC?TB@FVaFqtLlW4M82KyuwFWhu4&jtETpDbB4zzgg-$ fYKKY8vdVV;yZnGoSp1pZ(JScd7ia6YRzCS3lm*Im literal 0 HcmV?d00001 diff --git a/vllm/platforms/__pycache__/rocm.cpython-310.pyc b/vllm/platforms/__pycache__/rocm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1aab345aa6379e0a4b28ab9a23713a41281ac31 GIT binary patch literal 1574 zcmaJ>PjA~c6z3mFmTWo6x^(S^9K4|yA8MnAVZcxf1=3&}mOn+(Z2&PqQA9eHCrTPc zJGGJQl3x2Al4B413i}|u?v$^v(;j8Vae;IM_((mz_xSt$NUCVJn+V#ozy2P+aS-~) zeyld|p%>7rKf!RsafX;3Nrok11kZhxQSi~skuGz==WsXkWP>%N&wLrM03+dw<|Sq= zfkxp~n|mBz6W*90^|feSt#SVvLtbE4x%V7>k9d=}-Xh)-{v~DW93P=>cmo*Feixe{ zQ{yO!lc9i*R+^_e%8Judp>8llbg9{RDJE$mw&OBBNwZW>jsGf(^`KDFgtxPud@R}4 zFT1;Y(XR)GKkXbwdq3~K{_)k}!FIH_^ZJK_mtA5U$oYMLHaNE9aYqV0EO;laI(eZx z)wnDRr3LStOgoS!d!7F)XVZ#_P%7nI9)Vu`M{27J7L zUgcl}VwfXF1RXid6`p8t%*i{<+uRX8cSSI8K;Xt(_Y#BT&3htR5aIJ+5fKV+nd`%f zl^$|gn};C#nh6h!MBb`);{ybcy&>Z_RXP*ON^SyP3m&E1cuMH8$`bN9aA? z?^3k^UiAqWv%XR#sfto4or;Quy3lbJNg)e01ujdtEL@zG4X)<6YX%m)d9*@w zwZfOMSZf^;56_-_jM8cf{2c202$$B6B1<-kOf!mP!N-}Mhf(y$IL_uP%04URz{A~R zZY1D$U|LBQSJnKLD=VgM^XYP+9U(0W4VRm38^Z!-V+wF(yuJ z=@+<;=x!KNR3;dV^F$W~+*rF^Ei!A_8`zuw(v&?N761yN25}lrmO-Stwp)1+~mpsj(P_K;oVR0If$6q{}9V!bvRCqQ{26p^;&kdjJL z+uFyh&wYmW=tEzj529;NIrr8}JEM((ZO{l?QmM@x=@)bs&X!uQi1PdD;;;0$P%U{V-yrFPyZqplW zRyMl*=*4_~z>g2VpMJ}yhX+Ud^BJGdUd-l~pB^7Q-xHOzdu8SJY*W{jaeMovXma?| zn1xj>%b!)2Be2cY2jdBBy8s~xQpY?fxW@w77X#rzp}vexdPvt*u%ay9@Q2{i`T2W(y&0V&Yc~sZgpDlP2!tcQ(%;Cl zH53@Ipc_9HmK8rWnHW(sgs69sZ5QQx{XL06#>5UD<%z z(TPkNtP}~eSWHCv)bs9A&&Rz@yVKFF)~l?!3+F7x85qlQX<4t6yPaeGB)bPYR`_DA z!J1jxoz|*3ElO+rE(3s*<|NCd3Bc}3X+y|Wn5s0NV94O<4~wr34*B6?Ix``9qW#6w s=TmsY9iacKqm3S~x%v|OXzziDfb-N#X-bD);tlDx=h5Cjk$=bk7djL~n*aa+ literal 0 HcmV?d00001 diff --git a/vllm/platforms/__pycache__/xpu.cpython-310.pyc b/vllm/platforms/__pycache__/xpu.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5ebb41d76601fa1fff811c0b52d6224ab01afdb GIT binary patch literal 1241 zcmZ`&&2H2%5cW@!&F*Gtfo=IaAx?W}m!1$3LX`^QwyIPip&W?P*kx12n_znjTeYX{ zm3L^5J@5)V0FU4+r@jK0GWPy7NbN{7vB%@_H{VRkPN#)HwtoH^{w4_hv75Shu-OJx zFTpUxaDigP2%`lSBqoB!RCuw6(Fepl=AR+vb2_E5&+s1V1y^tj4X!ZQ9yP1)_)(tm zowQ2#^CH(16YdtNJ}9Ly(PH&}G!~h?R+j)a+n_22!%>VGiV3HOUhMOLwHe_JMtOMP zIS~F7$IY6*1^$3F&aA)9qh)*p{2^;Pe}}aIaAew_c0VqlNQ>JvnM0lA%mk9_u^eUA zzYxoi)g4gBl17L4oSdKw%J3ceh)b21 zBZD`4#P}jTEG2yMQ7L)WmSdRKfMIrjtygRB7Pj6Ne zxdQi_plTINKnNz+Rs$SfcggYd8h4ww;{1;}%c2GBn6nsfnA<~_c+jJ=173L#j9IDW z8>NEx@T@>vkt<7f?P?v21*u?|QACcP-2us_$ZX6)ERZFb2e6rg!57fD6LLuRkX!>d z5W55w*PBtvvf@(bT*Z+tby_5Xi&9Qt?rByihc=GpowLDe2HO>2v71M=66+akgt8+R zkpLgRxTB)_jRCOZ{<(W5OcGmbl9*PKh?0#9yN;6N>o_fDC(?)9F@?ObpK$p7Or&(G z^Gt9U1t#yqv70Q{enXX(j%U-(id3n2)MFcU(~V_W(*C str: + return "cpu" + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + return psutil.virtual_memory().total + + @classmethod + def inference_mode(cls): + return torch.no_grad() diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py new file mode 100644 index 00000000..991ac38b --- /dev/null +++ b/vllm/platforms/cuda.py @@ -0,0 +1,145 @@ +"""Code inside this file can safely assume cuda platform, e.g. importing +pynvml. However, it should not initialize cuda context. +""" + +import os +from functools import lru_cache, wraps +from typing import Callable, List, Tuple, TypeVar + +import pynvml +from typing_extensions import ParamSpec + +from vllm.logger import init_logger + +from .interface import DeviceCapability, Platform, PlatformEnum + +logger = init_logger(__name__) + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +if pynvml.__file__.endswith("__init__.py"): + logger.warning( + "You are using a deprecated `pynvml` package. Please install" + " `nvidia-ml-py` instead, and make sure to uninstall `pynvml`." + " When both of them are installed, `pynvml` will take precedence" + " and cause errors. See https://pypi.org/project/pynvml " + "for more information.") + +# NVML utils +# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`, +# all the related functions work on real physical device ids. +# the major benefit of using NVML is that it will not initialize CUDA + + +def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]: + + @wraps(fn) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + pynvml.nvmlInit() + try: + return fn(*args, **kwargs) + finally: + pynvml.nvmlShutdown() + + return wrapper + + +@lru_cache(maxsize=8) +def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]: + return [9, 0] + + +@lru_cache(maxsize=8) +def get_physical_device_name(device_id: int = 0) -> str: + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id) + return pynvml.nvmlDeviceGetName(handle) + + +@lru_cache(maxsize=8) +# @with_nvml_context +def get_physical_device_total_memory(device_id: int = 0) -> int: + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id) + return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total) + + +# @with_nvml_context +def warn_if_different_devices(): + device_ids: int = pynvml.nvmlDeviceGetCount() + if device_ids > 1: + device_names = [get_physical_device_name(i) for i in range(device_ids)] + if len(set(device_names)) > 1 and os.environ.get( + "CUDA_DEVICE_ORDER") != "PCI_BUS_ID": + logger.warning( + "Detected different devices in the system: \n%s\nPlease" + " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to " + "avoid unexpected behavior.", "\n".join(device_names)) + + +try: + from sphinx.ext.autodoc.mock import _MockModule + + if not isinstance(pynvml, _MockModule): + warn_if_different_devices() +except ModuleNotFoundError: + pass + # warn_if_different_devices() + + +def device_id_to_physical_device_id(device_id: int) -> int: + if "CUDA_VISIBLE_DEVICES" in os.environ: + device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",") + if device_ids == [""]: + raise RuntimeError("CUDA_VISIBLE_DEVICES is set to empty string," + " which means GPU support is disabled.") + physical_device_id = device_ids[device_id] + return int(physical_device_id) + else: + return device_id + + +class CudaPlatform(Platform): + _enum = PlatformEnum.CUDA + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability: + physical_device_id = device_id_to_physical_device_id(device_id) + major, minor = get_physical_device_capability(physical_device_id) + return DeviceCapability(major=major, minor=minor) + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + physical_device_id = device_id_to_physical_device_id(device_id) + return get_physical_device_name(physical_device_id) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + return 32 * 1024 * 1024 * 1024 + # physical_device_id = device_id_to_physical_device_id(device_id) + # return get_physical_device_total_memory(physical_device_id) + + @classmethod + # @with_nvml_context + def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool: + """ + query if the set of gpus are fully connected by nvlink (1 hop) + """ + handles = [ + pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids + ] + for i, handle in enumerate(handles): + for j, peer_handle in enumerate(handles): + if i < j: + try: + p2p_status = pynvml.nvmlDeviceGetP2PStatus( + handle, peer_handle, + pynvml.NVML_P2P_CAPS_INDEX_NVLINK) + if p2p_status != pynvml.NVML_P2P_STATUS_OK: + return False + except pynvml.NVMLError as error: + logger.error( + "NVLink detection failed. This is normal if your" + " machine has no NVLink equipped.", + exc_info=error) + return False + return True diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py new file mode 100644 index 00000000..00742a29 --- /dev/null +++ b/vllm/platforms/interface.py @@ -0,0 +1,108 @@ +import enum +from typing import NamedTuple, Optional, Tuple, Union + +import torch + + +class PlatformEnum(enum.Enum): + CUDA = enum.auto() + ROCM = enum.auto() + TPU = enum.auto() + XPU = enum.auto() + CPU = enum.auto() + UNSPECIFIED = enum.auto() + + +class DeviceCapability(NamedTuple): + major: int + minor: int + + def as_version_str(self) -> str: + return f"{self.major}.{self.minor}" + + def to_int(self) -> int: + """ + Express device capability as an integer ````. + + It is assumed that the minor version is always a single digit. + """ + assert 0 <= self.minor < 10 + return self.major * 10 + self.minor + + +class Platform: + _enum: PlatformEnum + + def is_cuda(self) -> bool: + return self._enum == PlatformEnum.CUDA + + def is_rocm(self) -> bool: + return self._enum == PlatformEnum.ROCM + + def is_tpu(self) -> bool: + return self._enum == PlatformEnum.TPU + + def is_xpu(self) -> bool: + return self._enum == PlatformEnum.XPU + + def is_cpu(self) -> bool: + return self._enum == PlatformEnum.CPU + + def is_cuda_alike(self) -> bool: + """Stateless version of :func:`torch.cuda.is_available`.""" + return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM) + + @classmethod + def get_device_capability( + cls, + device_id: int = 0, + ) -> Optional[DeviceCapability]: + """Stateless version of :func:`torch.cuda.get_device_capability`.""" + return None + + @classmethod + def has_device_capability( + cls, + capability: Union[Tuple[int, int], int], + device_id: int = 0, + ) -> bool: + """ + Test whether this platform is compatible with a device capability. + + The ``capability`` argument can either be: + + - A tuple ``(major, minor)``. + - An integer ````. (See :meth:`DeviceCapability.to_int`) + """ + current_capability = cls.get_device_capability(device_id=device_id) + if current_capability is None: + return False + + if isinstance(capability, tuple): + return current_capability >= capability + + return current_capability.to_int() >= capability + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + """Get the name of a device.""" + raise NotImplementedError + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + """Get the total memory of a device in bytes.""" + raise NotImplementedError + + @classmethod + def inference_mode(cls): + """A device-specific wrapper of `torch.inference_mode`. + + This wrapper is recommended because some hardware backends such as TPU + do not support `torch.inference_mode`. In such a case, they will fall + back to `torch.no_grad` by overriding this method. + """ + return torch.inference_mode(mode=True) + + +class UnspecifiedPlatform(Platform): + _enum = PlatformEnum.UNSPECIFIED diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py new file mode 100644 index 00000000..fd8afc92 --- /dev/null +++ b/vllm/platforms/rocm.py @@ -0,0 +1,36 @@ +import os +from functools import lru_cache + +import torch + +from vllm.logger import init_logger + +from .interface import DeviceCapability, Platform, PlatformEnum + +logger = init_logger(__name__) + +if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]: + logger.warning("`fork` method is not supported by ROCm. " + "VLLM_WORKER_MULTIPROC_METHOD is overridden to" + " `spawn` instead.") + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + +class RocmPlatform(Platform): + _enum = PlatformEnum.ROCM + + @classmethod + @lru_cache(maxsize=8) + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability: + major, minor = torch.cuda.get_device_capability(device_id) + return DeviceCapability(major=major, minor=minor) + + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + device_props = torch.cuda.get_device_properties(device_id) + return device_props.total_memory diff --git a/vllm/platforms/tpu.py b/vllm/platforms/tpu.py new file mode 100644 index 00000000..8ba973b2 --- /dev/null +++ b/vllm/platforms/tpu.py @@ -0,0 +1,33 @@ +import os + +import torch + +import vllm.envs as envs +from vllm.compilation.levels import CompilationLevel +from vllm.plugins import set_torch_compile_backend + +from .interface import Platform, PlatformEnum + +if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ: + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE) + +assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR,\ + "TPU does not support Inductor." + +set_torch_compile_backend("openxla") + + +class TpuPlatform(Platform): + _enum = PlatformEnum.TPU + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + raise NotImplementedError + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + raise NotImplementedError + + @classmethod + def inference_mode(cls): + return torch.no_grad() diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py new file mode 100644 index 00000000..d00e0dca --- /dev/null +++ b/vllm/platforms/xpu.py @@ -0,0 +1,22 @@ +import torch + +from .interface import DeviceCapability, Platform, PlatformEnum + + +class XPUPlatform(Platform): + _enum = PlatformEnum.XPU + + @staticmethod + def get_device_capability(device_id: int = 0) -> DeviceCapability: + major, minor, *_ = torch.xpu.get_device_capability( + device_id)['version'].split('.') + return DeviceCapability(major=int(major), minor=int(minor)) + + @staticmethod + def get_device_name(device_id: int = 0) -> str: + return torch.xpu.get_device_name(device_id) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + device_props = torch.xpu.get_device_properties(device_id) + return device_props.total_memory diff --git a/vllm/plugins/__init__.py b/vllm/plugins/__init__.py new file mode 100644 index 00000000..211fedbc --- /dev/null +++ b/vllm/plugins/__init__.py @@ -0,0 +1,56 @@ +import logging +from typing import Callable, Dict, Optional, Union + +import vllm.envs as envs + +logger = logging.getLogger(__name__) + + +def load_general_plugins(): + """WARNING: plugins can be loaded for multiple times in different + processes. They should be designed in a way that they can be loaded + multiple times without causing issues. + """ + import sys + if sys.version_info < (3, 10): + from importlib_metadata import entry_points + else: + from importlib.metadata import entry_points + + allowed_plugins = envs.VLLM_PLUGINS + + discovered_plugins = entry_points(group='vllm.general_plugins') + for plugin in discovered_plugins: + logger.info("Found general plugin: %s", plugin.name) + if allowed_plugins is None or plugin.name in allowed_plugins: + try: + func = plugin.load() + func() + logger.info("Loaded general plugin: %s", plugin.name) + except Exception: + logger.exception("Failed to load general plugin: %s", + plugin.name) + + +_torch_compile_backend: Optional[Union[Callable, str]] = None + + +def set_torch_compile_backend(backend: Union[Callable, str]): + global _torch_compile_backend + _torch_compile_backend = backend + + +def get_torch_compile_backend() -> Optional[Union[Callable, str]]: + return _torch_compile_backend + + +_inductor_additional_configs: Dict = {} + + +def set_inductor_additional_configs(configs: Dict): + global _inductor_additional_configs + _inductor_additional_configs = configs + + +def get_inductor_additional_configs() -> Dict: + return _inductor_additional_configs diff --git a/vllm/plugins/__pycache__/__init__.cpython-310.pyc b/vllm/plugins/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01e729b56d0559f0a6aca199393714c11262fc5f GIT binary patch literal 1869 zcmaJ?&2Jnv6u0MVXE(b^8lsSdLZ_4hE7Yu74wb41fm&Ks3JYpTD@H=2$=JI)>3m^( z5;p9f=#~E=IYxpT;>aK3E2o@$1Bvq9?5>hjp_csY=l49n_kMo%4+#13JE(S-Oo8VOAe-$+g0OfBC^ZQrI7yOY-Z8YP?@xx9W%{RT6c z^@8|wjD}5aahn^w)ic(}8sYQI-XzR9Tk9FD#@rW{zrgAjeZkUTb6{DFNqB{uu(t_& zTigW8JX-+E66`^3kzE1pEk?IVd+8J+C*8KC-TPsZgu4mX)`K`w+Wopvah`>Vws$g^ zMCdqcKnAHpzYksh3W8BcnT*MZ>``VuCt?v+tON)cQ%W+Y(yWQ5>^)~}j3~32^9voD zBl7|I{?oCgYDX@sjjX-;u|dhmU`9#g9n?q0J`vl8izDl(aZJYc$o`p7Qu(_x|7YX} z&mwAW_=JM3vNldqVFZ z9C~UHDiGjx0jK)TYd8n7f)6QhEM=Vay;#Z;Fe+C;f$DDCI5lBB#oBCB-GqR|Fevgk zQ}S?mKS|P!KF_!alVF-1c-eiCm&M`g!@SIxH?>c*zvtbQhihAt)Gskze;CFIq@(hR zpG$VPTiTREshj&;NGO&d&U(4O6sJWlR1)t7DOVv2RVZ8tTAvvbD#JbK*S)=rZ5`&?)XKc}rMOrbjQh;;qSA0u8BsIK$bwCxDwq{8qQpq%pq&So4y z`NuNKA;7aW?Mw*LRe_Vjzf_2K5FVoe5WE^fSm0!ya7tZR4= z;4#pS8Wu1LJQ9JAfIz+l{)hZ}b@TBSi5BR(4k%lb`{M=y-kl&2^XQJXRn-GrS|#*C z5QJHls}QM`;Dje#abNLO "PoolingParams": + """Returns a deep copy of the PoolingParams instance.""" + return PoolingParams(additional_data=self.additional_data, ) + + def __repr__(self) -> str: + return (f"PoolingParams(" + f"additional_metadata={self.additional_data})") diff --git a/vllm/prompt_adapter/__init__.py b/vllm/prompt_adapter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/prompt_adapter/__pycache__/__init__.cpython-310.pyc b/vllm/prompt_adapter/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0d7d61e7b7d6cefdb8d03bc4a0b63efa94651bf GIT binary patch literal 160 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hseo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_enC-wZb3C| literal 0 HcmV?d00001 diff --git a/vllm/prompt_adapter/__pycache__/layers.cpython-310.pyc b/vllm/prompt_adapter/__pycache__/layers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8291d1887d5065bbbe6553344f3b13bc7b3bcaaa GIT binary patch literal 2922 zcmb7GTW=gS6t+E{+1c4kN^Ue1N&$(oK(kbpKtd5}RTW-nTG6UV^D>&Oz1i8!W%10W z&1x$i($c5?hxV}#`~`l9UwP_J04aoXJhMr&sZf}a&)A(B%wZ~SZ6UyEZ;J9E9O22jK@5&ecNDm>?Ae6X6jDtCUw7V z>b2NQ8h(QkNq->174;`X)Maho+NXXK5KlA!X-Iq6+PD0+u{!|jf;3EI5NIz5l1z+a ztT%(;mvI=EJ+Fwu8IAb+Mz=^t-BC2UQiN(-7FWJW#)IT)@~|66TiwxKG04*HZX74w zkx?cHOD(%`xF=P75w9q)*2cyZFXv(_uhK)7S=b|OrfH%bgBH|oMK;IFIfVRYvL0N?^ zixoiI&j=^}0uZ^XFTdRwKS#yj1jX9h3Ue74t+k_MF;?kjj~R_hfpshvl>BQD1=%Mr z=u1M$luk*&rep{uumqjbAs^biM6DrKq2wO8HX3)+=RamY-RL;l&c`FE(56uu6+xgD zU|L<5$yT+IdJq_ogswVtfn)Go#(gvH6;;cCm9|Fvs6`M68He(yLtWaWZORWPU!9SK z_1RAJ%1L(!dd9m?5rRg)iA&xGQIH|3&Y)PJI8e|pzs|5`+Ro4w^f_QbeKr=p-dtZ} zij+dB=mM>6QS6PRc7)uGda~2hc9E&xz_%w-Wx2nUgpY!g8_xanr@^4ec4VrSV6T1) zSS|xI5K*F?hcen86zV+msWlLK@n}FE6f(^-m1`bn>9$^|VkC%Vx?K$N4pS@m+q)=^ zLQkPh^anj9!$QvOE+|*aFTjxeA_&gl1&gr**LJD<7q4^5!G7)mhxX*wS?%|qt7$t* zg?x0%i0IMAh;68V_mUxa^oF7T728ypr#e)N10#@d$<_ymM@NmYUhq@{MGXZWKgvPzAqZ_nX<;Hrp{J?$ zK={smlStL4h@jsoj0oMTj=7$UV>s%LePI=cn6nL26E2WrOuA&1u^OY^VS9;LjIn=Q z%R68V4mA$7i=M*qodw-Ey41;BVII;BoFs4?JKmJS^xP>r_Z>QqF<+B3smq$QiG?!0NLTs4#)#sD literal 0 HcmV?d00001 diff --git a/vllm/prompt_adapter/__pycache__/models.cpython-310.pyc b/vllm/prompt_adapter/__pycache__/models.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfb83a68f17fe83d9f297a6857244ac4d9f4fc50 GIT binary patch literal 13057 zcmbVS&2t>bb)V^(on0&zi^T^)f}bN&61g%3O8$@>7OjvbB}Fj_(G(@clc*+(?FO*m zez9jjikqdsK$q<#PT7v*IAyyEtV#|#B&S@H$|?UrE>o4NROOJ;A%~PJsj?Ib`MozY zi=D*+a%EA|=3;9^d%yR(J5y6d1HTvl_8-^v7YyUysWAMTLg6x=z%va)8q#bU zUi{zmOq4}av@FkR*`D3XdAXM3Ijy{xZxy_P8MmA8CeY4mPPB?%k?ZW{WUJ(rT2tOs ztL&AzF4vrHRlG`T#+zx)db6!LZ>}}(&9@GDhxonITxcEk4s$-=Jknb97CB!){-}49 z^ApWut>fNt(@@6t33c+Z>79~B&6Pzt`N;5|k|jBX|EFbH9hK9{QF(P{(O5KOMNMox zz3RvrIs3@=o_XDnb8`NXA?MZ9mgPN*@*%l^@`5U(d=}-y@(9XDR7pL%XrS&K>K5fu z)E!l4QLd`9tEN1riW|?#fGlwMIRR?CXsnj0$gd!$zlFYex3%2VGS)BhhYGt!Vm+taU@B zuitw6YOTJmqEb?Eqa#%_nj0?NsI_ZrO5;P5H7WgC+%uY$s#Xsh_iCXU)J?9ba8Q_P zVg_j`h{{^EI`?+l&IP+|{Z_5pZM4@gzqy;b)9QxF1g>`4s|~6?`%b4`Te(@&Sfy&d z+FDUkqQkq5a6QujP3JQeHwGimG&?ovchzd>-&c*b^)RRt1&4nEg{ydi5|R+I`?68P ze^6U2e+Y3lF#Mwb`t>)zeC4`-^VXX;Zr=8q3_ooD%9ni#QN0bst2Bh??2A zenZkH0vky|*rq;;Or2(y{^(5(4=>{~GHb>|Q<@J&-?(PHH~-M;TT87 z=v#fUCLWun(HGKs*`WH++!lTFF$tQztmy+E?%<~vkc0dM5geLZqQ)ebtk zYDf7-TQ=${@McuXBR{E%a`nz;JB)JoYRyfxODesHF{9)4PWztHp&xeqJzYdH1=BIhqG%R{K23e|iTFfEN2%48>U|oqAtOhta8tMI#N+fw#IlknxQvW6rFaC6 z_suQynz3bm-}tV9eQ8PiQK4^aaNR=@ioVz~A6lWcZTCSq5LfiARRLnEieylDDqz+z&9ATXP2;sWbgnYGHky zzLlAS+mQHP+`@JDC)Y)n^hY|2kue7S({V89m@mbWu=_Rcc8j?%w9sxX6_AVWZ5AW9 zv+4%xwXSl1pBLc=jh=dMFqe2?yz;P-|rdLgnJ@_kDDQ4ua)g6M~JsIONp(a2;c9m9dR+MhV0 zcP5qagn3Jsn$T&@;FYD_ewk^Kq&j%!Xt1Y|tkE>f56wR8=p9>^zh_|cVsk$hZKOZq z$yNpLR(%ZZp{*?F5LBH-Nmr44V0x37I&BFO!FUP00gIPNU-Rs*pkYuz;uu0K>^M6I z&3(|Tnu{h>d$Xl9OoBJXUB2JTYO?pPa%|<_(|o zEM-sRKoNv=e*84+Y-x~!kUSL!5h>5uZn z+^cv3(x`p2Z}_B#4c~;^89qjT%h1o_6$u@*S0&l11L8!pAhmz5p~KBu6a2h`tpojQ z2dcRm&5UkrsXvFY^k*p<*r=GA1s|f&2z~zsUIrv@j_sI5vv+LoT9(qbkzL=EwP-d0 z13nuqzgr6#6B_)r$N`^Vy(34ed$0+08aw^P(v^_lcTkJP+Z!a+Ga+QTZ~Yudk!tYY z`XxyGZ4+A8?B}+`L#OXZd(qgY9yYXZz7NBN{`oD7H1Kk7elKas_XqC=pQ@Sh4 zz4RsMllMohE0giuM`qPaDL9~aP|bNq_UjTA-yxg!s@5H?4`DTt2s;qwjAi#_ND!=`NS12mGV#jhwWw zn{7Nfm0vALN9FuBX;muL`8C6N8#7bD2Yf#1;<~(ezPaweXto}y59Fr^VEE;}u|56Rfaghd6*d{U+Rq3t zY6Y9=*0MP326+N*tu5U|CAg~{jYeyYd#WANC4r8F!?N&^YM5XbL1h(Z38F#u#e>OR zT*hsOsG4-Cz29hUwp`G9MQO%l>h7j$1KcOW)oX26Z?+kKW^fn{7{EINGnp^rKb2+K zejkk)n*HLQk1+z?|BGXVA^u=SPTg2c&JV_4{*aW2HL#rA6I$lMiIzDK2!hms& zCaV_NE!ad@OVw!VH&CGoazqOwGfFmuS4tS|lcm^&OHpa17N`^^M+;mUSf)XJ9t?*@ z4x*EOods&tyru7_d}!w|#bzJm^yg`2v>u-H?nH;wG;3(m?QJ{(xt-2T(Xw|OF%NGn z_Q*;mfn7j{(;0%X94Fg9g-N^gHxD#)t`R4ijza-OUlc2V-YesMor+dLw-BGaBaikQ~Iev@t`L zj1QehPtv8VM}LY8>Cp}2o}u%k;eh54RDe$nO}b&lUbVD=;-Fily=my!B^7SvN4sB@%#K-&`~v=0ayXFP(mFqtLe=TL@H84&(JLZZ)x8d`#gDa^Y;khuf3D zgSm~;V__P1za;Z8N`>B6Zy^4ZRk6@rP>MD8^_y?IE1RpUP;`UMPg*S2n45u;q~%DH zLIvNiY3X)>ZAX3@aT+i`&IdS5y~o<`zyL!M_k=5p`&a>rfON{s{s zTo&Bk7p}x(K&y{5SRxuCtZ>nNMxN)^sDNup<=fS%sN!>&a1i(lA{)yH^%52kD@t;f zqj_JeH8`M3!-^*xAc&l1XAQB(C@0m*=9+$y`rM-Cxkjj3fj3J*#O&r}xLnQ08i4?| zH@(`>Pis zswf$e+&O=5sQB;Ej(v7vmO!gnqz-81m|~~*{Qi_0Gg4H}@;^g7(zUcDnRpaHjFnU0 z+<D5YB%<1J>mgv-j))Gal-~L67Ck1j#2JXixeu z#U8<5bBnYg^da&oSBtFOa{U}}6?%|#+tltNavbc4*+_eYIT%e6|B^sT)rr@3hTf7W z>8~P*%sXS*HV=qBh~K9}{1ZCFx`(X(&a5bt?m57+jvFMSHEH!k1!j>(hNOD~F#suk zU_z%~i1jBBw%8CG7VOGPA=Xdv*S-Z@ZfrOlx;D0f#>sAwg?I=XvR#OK^lhRP_QC|- z7x7F|na+RSGVbOnMTq;gs_9jd6AHQ6Z8qv0#LApHR7;GJyeaZ@W#az^(S+}_x9j`* zGN!L7T<4t_kHC_HhZqIUXw_}j>QG9XD={>Src))DjIK9m!~v_4O&3kZ&Dj}`oTP8D zt?nn?JdDFbr?VW9Wc?O~{bxL^!m#PgGBzKF3BXR9ayvyKkkTOofjsd)_9wkB9w6r9 z*BehQ&|O*vF}F@GSNbEi6MSISlTxA62x&sKiEV+fCPRns!0UWWst!y~YAuEY2>Cv>fAk+nb`}mO;PB-atep z$_0>xFv?>q)WQ&lVDS!OtUOp$zrKI>t=jFOyi^S|KX!0Jh>42=h?@R|Kd zx<@+nU39NoNYWqW>A8w0pr|JKt1))k)e9!;bD#^>YdmQhXmoxZ)cq@sLprl;_NoWX zEVk5%v}F_;^`?iXM0Sgg(I|XZ6(23#WM=FgI0aW}V4zNCl1sfcv~LsQXasJ%6N0JT zZbyeP7~4Ba>SDYdC^tRW+_x3`(l~RZ@ewDVTkeE!v=Dx1fsaaZB=3X76tIaKF$I(8 zn!=QS!z$S)Kzs_DrT5IfA6**il~ID^UJR)AEDFTNZ8HuQa6lpsJCHfB(|X9f=v}zQ z$Eq&QMYGWM21gGlx)(sdf(ss0ICkzeBRif$`@( z%mq60DdFrEr#?8}C?@9{eDdQ>%1Kqkn-b18=o%RCA+CYZDGzG6_q42_cbRL1jIV-C z@3!G9VYXMHnxV6wxJ1{%Y;TtNKL`FVMo*LB_wbe z^KEk|XsRGbuFkvMi5vp5Qg@`+M{CvU%o%N#G_L$O&RXhXNcHtXxDR-rk16ZdNY zl$KO-)PfV8pCTHN^(x_37vN5={0UN^%1@`q5ct^}n@*`g|AWg&;_we>F&lD}dW%wV zC#PreA2$jJaECSo9dT#A*G09sB*zg3&?`nJc}AX*)l_XZe$iBdE;>!AaM;aqzKYN3 zA0kQVKyC!xLJM%vIV#xH_&gDT>}$!6jSu_{0Q zr#(t6y^1Fw?Et|BYne$&V3b50f-Lj{yuMu(k?WK1mDPq*bX~_w4C%u%OVKMOM3;G9!0GNiyB06#e!aw*x=g9G;7T zk-j)aj_?psFx7kmqh!VYBk$9{L{B}Ay+6f^8w2*@#wnt`|Nk`%M5#JIijN-A3}~z9 z?@*Ej+nUiYAX`+QT+ffCTT_Ikr{`L%O5XH z&)Q-##>C|(<%?TRP}o#NN9QFhs9o(2+Cj`0gf?0;fV_8tA`5r^va3W4*ONzeywP z(v3i9v`;WMS>|mbCh2W^A{Kan5?oJmKDHhGb5vK)M|nENLZC_Cpy3>x?o)xi8QSnM zdvlZ{Eux7=@##Magm;Xti7%xQ6Wq{jb=tuaL%~2#P+QU%dXm~69ou#@Y~acn3(OQX zB~u$qQiXElWqFg_Grj@AZeB80R%I?VYY&tTILOWZoVUpRMq0%mJcMzh3Amcq`4le} z`9GvD9;SpY5$MyDkhFVFeET;(#b8-~joy8a5*9t)7(~$_{xGMQNrg+G2>eu1#zycC zk^;g#F0PB0%z5F8l0~)@hfQz=3vls78Qh{M_a`pru`Aw-;v8^ LRP(XOmw)|#|8u97 literal 0 HcmV?d00001 diff --git a/vllm/prompt_adapter/__pycache__/request.cpython-310.pyc b/vllm/prompt_adapter/__pycache__/request.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29f2651f5018a7cbbb201eb2c962c55adb27d499 GIT binary patch literal 1327 zcma)4y>AmS6u0j#m&-Lt=?4;(iW(-Ap^3`EQXxJDwyL7Ac(IPN(_Fc)`g}!+*jlL* z{{xI2_!sy?yfQI&Lkzrkxi)PH61M!_v;FMf`}k#@j!$6heEUA$vI+Tto14Rf&2w1A z6A*$38WV=vifLjoi;~ww*ups@!ja_27L97UpIzn%dO-TkpYV?C^{MH;5J9e`dLxfZ zS?CbYZw^4OMsWm65=I4K);mCCwscrSy0R&(T_QcuHfY>C^}ex7&dG=hS2WK&){?&P zMC**4Qnm!PFP1914R%{}D!X$|9KyN~Vb`qht1QX&?f9mT{T_@u+TJKry`VR9>;lSt_j9CY?UrbS5W@}NT7 zf`uh3uz#AIk}vctIUpBwuWy+~QRY$^kMnU*j5)uc#wlbx0^Q;~jDwwe$wV2W z-RArlXjI=cITu;Txx$jM4){qO)O;{@p_N(YJV?__2Rh19V2QD#R98(l9*_WNr6+0y z<61+p4#L1c)gPgZWB7aCtv0$~%M?--GcK@zK5-h3?O7hh*|sd(r?a(6E6=xmoB4rK z!Gy=r2gy2F5@{~vC@5oH7_a9d6z?aKyadEdyrPh8AnE=IfGsjiD{0 z)BXpvF*ZA>u%DzdNMb+i;G#ARLhkXGK_;`JIU0JSeAi>iK^!JN1KG8HEX8;;ksC=K zv%V0C0JNfJJ+%}DtwAmXkENG}z8oeZnzVi9r;-aVN*E8jyFom;brH(^A(uSP5|I^_ z;4&BSK&iL-U>H1tFKYpj#vwW~gr!Vj7|by@3o^)=Uzme5?_i;v?$(3MQ=sRE!9M@h z5H&zG`S)Q@Mrkh%(tFYudtBao5apxjess_agWX;_mZK!@y$Hjomx?4xrMFPHm&+i` zy6L!Zf_MlvG5#VL@HM5F1;j=KUQq|Ou(iEjzzSu{Nx~Q;2}lN7kCM`3W=$=9+P21k*Vsip`C<>PrP3fq(wdc}4Vstry0TiQ zIa_lUtGq4E?5RB~O>OamtV|7Q4RHXL{(^o)GZNpO)HGUNMV}e*9b(Ap_%~#zSJqf{ zYW!rdnx>8{17DXG(1u2uKi6KN@6mxVEr~y*GbP}=)>}NQOe*0C@6trTih&=7XL+D34nG`#n^BFa}4X5bEH1*|39j?w@ zWa8P;aDswkKjJhQUJ#MV*-2_Yt@>4(cF#oT$$GU)i}_VwEZvaAGCAMqQvLAu58$q4 zurUh;^0Kk(j0N*LL_sQ#poC<~uB5{tW@q7ol1L|n1~Hv{rLyB)4@@d^7r8{2$<(D!L+)ps4n};i z4-)wo$`d@thczes+T>rXXIIAOJDz6Yl zN|3!+-UZ7uFt<}KWD-Z_M$v~O!=Q$CbSiJJgtFM+h$Yo{3F z6PsA*I&L7KNKpsd$U#+t5jk~E6vZY!#xTaH3NJ?1f22%csY1vJHtl00MjLLr=;v4QnHS#$T}sapm2BxZFb%h*MTc6$WpifLiHT+ArQAYn%AZw)F#!~ z&qLzAe`%_#+nkP++g|ZxS4T#uPA5K4dp1;$>@HhEV&TAc4 te5w1fcBuUcl)^mZpF-s|G77}T4zh6zd;n zZPyM;jYtXB$)y%}W(%ww$ z54&-*)AobJxGCbq{CKk~FgELR?)lvB#zJ}zI`XcN$%5bQ2Adf+%Fy#cGl~yjS0e#r zBW~Uo`LUXwEL>TSVu|5T)Yc#PZU45A7?}G=cEWCaB|Z6Sr@hg<4RTIjzj5oT-?$@2 zhCk?V5e(KQvj>Ngm4mo!`iX`Vek{DMIxi#93->6QA6a`t-0O)bZV>N=f0aQeI-_fl zgsXGSWw)RWt|3gfAS_|7Xrjp3MNQZnIyXd#oA?T%+?wE)sPH1@*elvb`AZGsBtK|@ zsPeLy+F*QwS00($x;xD$v0|n*3;f)M$*X+ok?GF!X|Vu|4a^Lm1!i%8nF3~x&jYi> z7jPc!X7WY81k5sD7AsReb4e2fH}#}@fROy?kTKZ5hdJ-@t62{j5uFvrskf8_SKTrROBPv9SEiW@1w_@ z_>Lbtsdyr%?BF*lIwvG;h@4IvQ-zc2tP?f4sF&6BYxjlRbYv>XSJk9JNATQUk@KJ# z1kSoh4+=Uy$DH%Mb|9jNR&F}J6pkN2hxn$Gj@MpIXZvmNOP$rhOqttyZ)a?#wyh_X zOzXTx5G9q+|J-Z$LN5~cqGU1at@|<5pL=oVu4qSz#l`(*LnI5MOz;}YVv^+?TJeUq zZ&*XxOKoHDMX#c|O1T=SX1f`CUd>9(NCX@ETgV04j3&z^fE_(4d06j9k>~Ab zPv5@~husU^X7{z&m$yay+M8kTPWXEG;6l(`ztG)`?{wN1?gv45Vf6ABQdP*9T<>nm zW7z%_zKG1%v@JdTo7VehkJi9?MtNdM5%*-f0h8bRQJcQ2_`adoK+blDt$i%%pdDQv z$3N9hfrD;RZbqJH`|E+=b@abs04D378$-nYilm^V+Mq?*kEB_|r>J-ljabRE1(~)W z(UyVFT^(g(#y^s1tth83CAUJ(0+&qZeW`p2=p7?v z0JYu4!*|YcM;-A7=0~(IGnsA<{;Y4UqNVrW8pEcs-aW|^Fz^E-tRxvy20*NgehbiF z0P{4kQU@z_uu^ZqFj|G!0xWLpDO*2*>D+|3i|x%21xH6Ug(sg){X8U&D#q^8LhTnuz z)yi@Ol)DQXh;m-gX<#FGk;dI_r`e9(mE4)>gsvaok+0FR)4B13^rWV3AeJkNlG3%$ z8$u-%N##bb9XCU9P0Eghb}9WQCzDi1w@q}BlYBA>TK&EJ<1!XSF9R^cVx$8Mf2MBh z=2LUmtX98G`_?YoWd?g@|AFoH&yUfKk>wYw`nRlBFvvCl^MF`521%1U^SD0|9D@VaY_d+0N{7 zXK;g`U?6t``4(`bx)Tnp3U=S2D1XI_XU`hOJ4yLx`Ue$nlv5?h^iOfPQVv;iI=ZkU^04(AiNH zAhqfz@JN(z4O5cLW%~p{rU#LH0c-MwG7f`$6dG`<^2O~w#z;<?PL{;dG!L4lmNM8MQ;WQ*GzpL{$x8&t6eTHZszoFh#UaGHG{nBPonCZAZK_U1eVSE~ zPIE(5v<;%zY2pU!@S;w`@A{2qyjf3<T;tWZ7vtmM?M@QBGj>!FQ z(l|NvbhB5emGhERGW-!=MD^lmKF^BguEqM7zgzzG@tOV7Xm06cfopB#W~A49k&E&k zu50=BEhr#0aKDw2!R#e<~&i7fZjyC}#5bZQEn9qq) z&6BU=hKY6G5AYn9CI^Gkf^uzlZgfoqE$>RIA#;pVp7_teM-BkoC9|Ge1xq*KEE$pr zXrp00Gj}V63flcQ{&y}aaUD=PGO4R{Yzsii8+rQ^?Y=J0#;QK2GIEK+s?jo`dP~%L zT%a5Ps%24zkIJ2J#Vv-m(JrZ;O?y4T<6{pvg(;;NtTUj7Z0&2fO%3%R6?SQ8{)Ad& zv5B3z33p+T19Sq8*Ul`pNJHBdnmU1pmp$|^ff=ss6sKJ)b$WS@)AGmoB1(#@Y|$v| zWxc9fCh9o^BeH+gcDs1~sDBS`4`r#fS@|xulnnUlX`a;6nl* z5!eslDp*ewPKD?-Y7xOhmr%~kA>0js9zIo*QaENM=nX1~;lP`PST>Ue-s-MeS38~* zgU;>SsFV^b-lU2tC4)30%^4KAM5(( None: + super().__init__() + self.base_layer = base_layer + self.emb_layer = self.base_layer + if 'LoRA' in base_layer.__class__.__name__: + self.emb_layer = self.base_layer.base_layer + + def create_prompt_adapter_weights( + self, prompt_adapter_config: PromptAdapterConfig): + self.embeddings_tensors = torch.zeros( + ( + prompt_adapter_config.max_prompt_adapters, + prompt_adapter_config.max_prompt_adapter_token, + self.emb_layer.embedding_dim, + ), + dtype=self.emb_layer.weight.dtype, + device=self.emb_layer.weight.device, + ) + self.adapter_lengths = torch.zeros( + prompt_adapter_config.max_prompt_adapters, + dtype=torch.long, + device=self.emb_layer.weight.device) + + self.indices_gpu: torch.Tensor + self.embedding_indices_gpu: torch.Tensor + + def reset_prompt_adapter(self, index: int): + self.embeddings_tensors[index] = 0 + + def set_prompt_adapter( + self, + index: int, + adapter_model: Optional[torch.Tensor], + ): + self.reset_prompt_adapter(index) + if adapter_model is not None: + length = adapter_model.shape[0] + self.embeddings_tensors[index, :length] = adapter_model + self.adapter_lengths[index] = length + + def set_mapping( + self, + prompt_indices: torch.Tensor, + prompt_embedding_indices: torch.Tensor, + ): + self.indices_gpu = prompt_indices.to( + device=self.emb_layer.weight.device) + self.embedding_indices_gpu = prompt_embedding_indices.to( + device=self.emb_layer.weight.device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + hidden_states = self.base_layer(x) + if self.embedding_indices_gpu.ndim > 1: + valid_mask = self.indices_gpu != -1 + gathered_embeddings = self.embeddings_tensors[ + self.embedding_indices_gpu[:, 0], + self.embedding_indices_gpu[:, 1]] + + # Update hidden states + hidden_states[valid_mask] = gathered_embeddings + return hidden_states \ No newline at end of file diff --git a/vllm/prompt_adapter/models.py b/vllm/prompt_adapter/models.py new file mode 100644 index 00000000..18a5f86c --- /dev/null +++ b/vllm/prompt_adapter/models.py @@ -0,0 +1,355 @@ +import logging +import math +from typing import Any, Callable, Dict, List, Optional, Type + +import torch +from torch import nn + +from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel, + AdapterModelManager) +from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter, + get_adapter, list_adapters, + remove_adapter, set_adapter_mapping) +from vllm.config import PromptAdapterConfig +from vllm.prompt_adapter.layers import ( + VocabParallelEmbeddingWithPromptAdapter) # yapf: disable +from vllm.prompt_adapter.layers import PromptAdapterMapping +from vllm.prompt_adapter.utils import load_peft_weights + +logger = logging.getLogger(__name__) + +_GLOBAL_PROMPT_ADAPTER_ID = 0 + + +def get_prompt_adapter_id(): + global _GLOBAL_PROMPT_ADAPTER_ID + _GLOBAL_PROMPT_ADAPTER_ID += 1 + return _GLOBAL_PROMPT_ADAPTER_ID + + +def convert_to_embedding_indices(indices): + embedding_indices = [] + count = 0 + + for value in indices: + if value == -1: + count = 0 + else: + embedding_indices.append([value, count]) + count += 1 + + return torch.tensor(embedding_indices) + + +def convert_mapping( + mapping: PromptAdapterMapping, + prompt_adapter_index_to_id: List[Optional[int]], +) -> torch.Tensor: + """Converts PromptAdapterMapping to index tensors. + + Args: + mapping: PromptAdapterMapping mapping rows in a + batch to PromptAdapter ids. + prompt_adapter_index_to_id: List mapping PromptAdapter + ids to PromptAdapter indices. + + Returns: + pa_indices: Tensor of shape [batch_size] mapping batch rows to + PromptAdapter indices. + """ + id_to_index = { + id_: idx + for idx, id_ in enumerate(prompt_adapter_index_to_id) + if id_ is not None + } + pa_indices = ([ + id_to_index.get(id_, -1) if id_ > 0 else -1 + for id_ in mapping.index_mapping + ]) + + pa_embedding_mapping = convert_to_embedding_indices(pa_indices) + pa_indices = torch.tensor(pa_indices) + return pa_indices, pa_embedding_mapping + + +class PromptAdapterModel(AdapterModel): + + def __init__(self, + prompt_adapter_id=None, + num_virtual_tokens=None, + prompt_embedding=None) -> None: + self.id = prompt_adapter_id + self.prompt_embedding = prompt_embedding + self.num_virtual_tokens = num_virtual_tokens + + @classmethod + def from_local_checkpoint( + cls, + adapter_model_path: str, + prompt_adapter_id: int, + num_virtual_tokens: int, + config: PromptAdapterConfig, + device: str = "cuda", + ) -> "PromptAdapterModel": + + if num_virtual_tokens > config.max_prompt_adapter_token: + raise ValueError( + f'num_virtual_tokens ({num_virtual_tokens}) should be <= ' + f'max_prompt_adapter_token({config.max_prompt_adapter_token})') + + adapters_weights = load_peft_weights(adapter_model_path, device) + prompt_embedding = adapters_weights["prompt_embeddings"].to( + config.prompt_adapter_dtype) + + return cls(prompt_adapter_id, num_virtual_tokens, prompt_embedding) + + +class PromptAdapterModelManager(AdapterModelManager): + """A manager that manages multiple Prompt Adapter models.""" + + def __init__( + self, + model: nn.Module, + max_num_seqs: int, + max_num_batched_tokens: int, + prompt_adapter_config: PromptAdapterConfig, + ): + """Create a PromptAdapterModel and adapter for a given model. + + Args: + model: the model to be adapted. + max_num_seqs: the maximum number of sequences model can run in a + single batch. + max_num_batched_tokens: the maximum number of tokens model can run + in a single batch. + prompt_adapter_config: the PromptAdapter config, + """ + self.model: nn.Module = model + # Dict instead of a Set for compatibility with LRUCache. + self.prompt_adapter_index_to_id: List[ + Optional[int]] = [None] * self.prompt_adapter_slots + self.max_num_seqs = max_num_seqs + self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8 + self.prompt_adapter_config = prompt_adapter_config + self.model.prompt_adapter_manager = self + self.adapter_type = 'PromptAdapter' + + self.base_indices = torch.tensor([-1]) + self.base_embedding_indices = torch.tensor([]) + + self.modules: Dict[str, nn.Module] = {} + self._create_prompt_adapter_modules() + self._last_mapping: Optional[PromptAdapterMapping] = None + + @property + def prompt_adapter_slots(self) -> int: + return self.prompt_adapter_config.max_prompt_adapters + + @property + def adapter_slots(self) -> int: + return self.prompt_adapter_slots + + @property + def capacity(self) -> int: + return self.prompt_adapter_config.max_cpu_prompt_adapters + + def activate_adapter( + self, + prompt_adapter_id: int, + ) -> bool: + """Move PromptAdapter into a GPU buffer + to be used in the forward pass.""" + if prompt_adapter_id in self._active_adapters: + return False + first_free_slot = next( + ((i, prompt_adapter_id) for i, prompt_adapter_id in enumerate( + self.prompt_adapter_index_to_id) if prompt_adapter_id is None), + None) + if first_free_slot is None: + raise ValueError("No free prompt_adapter slots") + index, _ = first_free_slot + self._active_adapters[prompt_adapter_id] = None + prompt_adapter_model = (self._registered_adapters[prompt_adapter_id]) + logger.debug("Activating prompt_adapter. int id: %d, slot index: %d", + prompt_adapter_model.id, index) + self.prompt_adapter_index_to_id[index] = prompt_adapter_model.id + for _, v in self.modules.items(): + v.set_prompt_adapter(index, prompt_adapter_model.prompt_embedding) + return True + + def _deactivate_adapter(self, prompt_adapter_id: int): + try: + index = self.prompt_adapter_index_to_id.index(prompt_adapter_id) + self.prompt_adapter_index_to_id[index] = None + for _, v in self.modules.items(): + v.reset_prompt_adapter(index) + except ValueError: + pass + + def _add_adapter(self, prompt_adapter: PromptAdapterModel): + self._registered_adapters[prompt_adapter.id] = prompt_adapter + + def _set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None: + base_indices, base_embedding_indices = convert_mapping( + mapping, self.prompt_adapter_index_to_id) + for k, v in self.modules.items(): + v.set_mapping(base_indices, base_embedding_indices) + + def _create_prompt_adapter_modules(self): + for module_name, module in self.model.named_modules( + remove_duplicate=False): + if "VocabParallel" in module.__class__.__name__: + new_module = VocabParallelEmbeddingWithPromptAdapter(module) + new_module.create_prompt_adapter_weights( + self.prompt_adapter_config) + replaced_module = self.replace_submodule( + self.model, module_name, new_module) + self.register_module(module.__class__.__name__, + replaced_module) + replaced_module.set_mapping(self.base_indices, + self.base_embedding_indices) + break + + def replace_submodule(self, model: nn.Module, module_name: str, + new_module: nn.Module) -> nn.Module: + """Replace a submodule in a model with a new module.""" + parent = model.get_submodule(".".join(module_name.split(".")[:-1])) + target_name = module_name.split(".")[-1] + setattr(parent, target_name, new_module) + return new_module + + def register_module(self, module_name: str, module: nn.Module): + self.modules[module_name] = module + + def pin_adapter(self, prompt_adapter_id: int) -> bool: + """Pin a PromptAdapterModel in the manager cache.""" + raise NotImplementedError( + "Pinning is not supported in PromptAdapterModelManager." + "Use LRUCachePromptAdapterModelManager for pinning" + ) # type: ignore + + def remove_all_adapters(self): + """Remove all PromptAdapterModel from the manager.""" + self._registered_adapters.clear() + self.prompt_adapter_index_to_id = [None] * self.prompt_adapter_slots + self._active_adapters.clear() + + def deactivate_adapter(self, adapter_id: int) -> bool: + return deactivate_adapter(adapter_id, self._active_adapters, + self._deactivate_adapter) + + def add_adapter(self, adapter: PromptAdapterModel) -> bool: + return add_adapter(adapter, self._registered_adapters, self.capacity, + self._add_adapter) + + def set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None: + self._last_mapping = set_adapter_mapping(mapping, self._last_mapping, + self._set_adapter_mapping) + + def remove_adapter(self, adapter_id: int) -> bool: + return remove_adapter(adapter_id, self._registered_adapters, + self.deactivate_adapter) + + def list_adapters(self) -> Dict[int, Any]: + return list_adapters(self._registered_adapters) + + def get_adapter(self, adapter_id: int) -> Optional[Any]: + return get_adapter(adapter_id, self._registered_adapters) + + +class PromptAdapterLRUCache(AdapterLRUCache[PromptAdapterModel]): + + def __init__(self, capacity: int, + deactivate_prompt_adapter_fn: Callable[[int], bool]): + super().__init__(capacity, deactivate_prompt_adapter_fn) + + +class LRUCachePromptAdapterModelManager(PromptAdapterModelManager): + """A model manager that manages multiple prompt_adapters with LRU cache.""" + + def __init__( + self, + model: nn.Module, + max_num_seqs: int, + max_num_batched_tokens: int, + prompt_adapter_config: PromptAdapterConfig, + ): + self.prompt_adapter_config = prompt_adapter_config + super().__init__(model, max_num_seqs, max_num_batched_tokens, + prompt_adapter_config) + self._registered_adapters = PromptAdapterLRUCache( + self.capacity, self.deactivate_adapter) + self._active_adapters = PromptAdapterLRUCache( + self.prompt_adapter_slots, self._deactivate_adapter) + + def list_adapters(self) -> Dict[int, PromptAdapterModel]: + """List all registered PromptAdapterModel.""" + return dict(self._registered_adapters.cache) + + def add_adapter(self, prompt_adapter: PromptAdapterModel) -> bool: + """Add a PromptAdapterModel to the manager.""" + if prompt_adapter.id not in self._registered_adapters: + self._add_adapter(prompt_adapter) + was_added = True + else: + # We always touch to update the LRU cache order + self._registered_adapters.touch(prompt_adapter.id) + was_added = False + return was_added + + def activate_adapter( + self, + prompt_adapter_id: int, + ) -> bool: + if prompt_adapter_id not in self._active_adapters and len( + self._active_adapters) >= self.prompt_adapter_slots: + self._active_adapters.remove_oldest() + result = super().activate_adapter(prompt_adapter_id) + # We always touch to update the LRU cache order + self._active_adapters.touch(prompt_adapter_id) + return result + + def remove_oldest_adapter(self) -> bool: + if len(self._registered_adapters) > 0: + self._registered_adapters.remove_oldest() + return True + return False + + def pin_adapter(self, prompt_adapter_id: int) -> bool: + """Pin a PromptAdapterModel in the manager cache.""" + self._pin_prompt_adapter_in_cpu_cache(prompt_adapter_id) + self._pin_prompt_adapter_in_gpu_cache(prompt_adapter_id) + return True + + def _pin_prompt_adapter_in_cpu_cache(self, prompt_adapter_id: int): + try: + self._registered_adapters.pin(prompt_adapter_id) + except ValueError as err: + raise ValueError( + "Pinning failed. " + f"Prompt Adapter {prompt_adapter_id} is not registered." + ) from err + + def _pin_prompt_adapter_in_gpu_cache(self, prompt_adapter_id: int): + if prompt_adapter_id not in self._active_adapters: + # move adapter to gpu if not already active + self.activate_adapter(prompt_adapter_id) + self._active_adapters.pin(prompt_adapter_id) + + +def create_prompt_adapter_manager( + model: nn.Module, + max_num_seqs: int, + max_num_batched_tokens: int, + prompt_adapter_config: PromptAdapterConfig, + prompt_adapter_manager_cls: Type[ + PromptAdapterModelManager] = PromptAdapterModelManager, + **kwargs) -> PromptAdapterModelManager: + """Create a PromptAdapterModel for a given model.""" + prompt_adapter_manager = prompt_adapter_manager_cls( + model=model, + max_num_seqs=max_num_seqs, + max_num_batched_tokens=max_num_batched_tokens, + prompt_adapter_config=prompt_adapter_config, + **kwargs) + return prompt_adapter_manager diff --git a/vllm/prompt_adapter/request.py b/vllm/prompt_adapter/request.py new file mode 100644 index 00000000..775dd11d --- /dev/null +++ b/vllm/prompt_adapter/request.py @@ -0,0 +1,34 @@ +import msgspec + +from vllm.adapter_commons.request import AdapterRequest + + +class PromptAdapterRequest( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True, # type: ignore[call-arg] + frozen=True): # type: ignore[call-arg] + """ + Request for a Prompt adapter. + """ + __metaclass__ = AdapterRequest + + prompt_adapter_name: str + prompt_adapter_id: int + prompt_adapter_local_path: str + prompt_adapter_num_virtual_tokens: int + + def __hash__(self): + return super().__hash__() + + @property + def adapter_id(self): + return self.prompt_adapter_id + + @property + def name(self): + return self.prompt_adapter_name + + @property + def local_path(self): + return self.prompt_adapter_local_path diff --git a/vllm/prompt_adapter/utils.py b/vllm/prompt_adapter/utils.py new file mode 100644 index 00000000..aae81de8 --- /dev/null +++ b/vllm/prompt_adapter/utils.py @@ -0,0 +1,95 @@ +# code borrowed from: https://github.com/huggingface/peft/blob/v0.12.0/src/peft/utils/save_and_load.py#L420 + +import os +from typing import Optional + +import torch +from huggingface_hub import file_exists, hf_hub_download +from huggingface_hub.utils import EntryNotFoundError +from safetensors.torch import load_file as safe_load_file + +from vllm.platforms import current_platform + +WEIGHTS_NAME = "adapter_model.bin" +SAFETENSORS_WEIGHTS_NAME = "adapter_model.safetensors" + + +# Get current device name based on available devices +def infer_device() -> str: + if current_platform.is_cuda_alike(): + return "cuda" + return "cpu" + + +def load_peft_weights(model_id: str, + device: Optional[str] = None, + **hf_hub_download_kwargs) -> dict: + r""" + A helper method to load the PEFT weights from the HuggingFace Hub or locally + + Args: + model_id (`str`): + The local path to the adapter weights or the name of the adapter to + load from the HuggingFace Hub. + device (`str`): + The device to load the weights onto. + hf_hub_download_kwargs (`dict`): + Additional arguments to pass to the `hf_hub_download` method when + loading from the HuggingFace Hub. + """ + path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"]) + if hf_hub_download_kwargs.get("subfolder", None) is not None else + model_id) + + if device is None: + device = infer_device() + + if os.path.exists(os.path.join(path, SAFETENSORS_WEIGHTS_NAME)): + filename = os.path.join(path, SAFETENSORS_WEIGHTS_NAME) + use_safetensors = True + elif os.path.exists(os.path.join(path, WEIGHTS_NAME)): + filename = os.path.join(path, WEIGHTS_NAME) + use_safetensors = False + else: + token = hf_hub_download_kwargs.get("token", None) + if token is None: + token = hf_hub_download_kwargs.get("use_auth_token", None) + + hub_filename = (os.path.join(hf_hub_download_kwargs["subfolder"], + SAFETENSORS_WEIGHTS_NAME) + if hf_hub_download_kwargs.get("subfolder", None) + is not None else SAFETENSORS_WEIGHTS_NAME) + has_remote_safetensors_file = file_exists( + repo_id=model_id, + filename=hub_filename, + revision=hf_hub_download_kwargs.get("revision", None), + repo_type=hf_hub_download_kwargs.get("repo_type", None), + token=token, + ) + use_safetensors = has_remote_safetensors_file + + if has_remote_safetensors_file: + # Priority 1: load safetensors weights + filename = hf_hub_download( + model_id, + SAFETENSORS_WEIGHTS_NAME, + **hf_hub_download_kwargs, + ) + else: + try: + filename = hf_hub_download(model_id, WEIGHTS_NAME, + **hf_hub_download_kwargs) + except EntryNotFoundError: + raise ValueError( # noqa: B904 + f"Can't find weights for {model_id} in {model_id} or \ + in the Hugging Face Hub. " + f"Please check that the file {WEIGHTS_NAME} or \ + {SAFETENSORS_WEIGHTS_NAME} is present at {model_id}.") + + if use_safetensors: + adapters_weights = safe_load_file(filename, device=device) + else: + adapters_weights = torch.load(filename, + map_location=torch.device(device), weights_only=True) + + return adapters_weights diff --git a/vllm/prompt_adapter/worker_manager.py b/vllm/prompt_adapter/worker_manager.py new file mode 100644 index 00000000..ddc1ef89 --- /dev/null +++ b/vllm/prompt_adapter/worker_manager.py @@ -0,0 +1,176 @@ +import logging +from typing import Any, Optional, Set, Type + +import torch + +from vllm.adapter_commons.utils import (add_adapter_worker, + apply_adapters_worker, + list_adapters_worker, + set_active_adapters_worker) +from vllm.adapter_commons.worker_manager import AbstractWorkerManager +from vllm.config import PromptAdapterConfig +from vllm.prompt_adapter.models import (LRUCachePromptAdapterModelManager, + PromptAdapterModel, + PromptAdapterModelManager, + create_prompt_adapter_manager) +from vllm.prompt_adapter.request import PromptAdapterRequest + +logger = logging.getLogger(__name__) + + +class WorkerPromptAdapterManager(AbstractWorkerManager): + """WorkerPromptAdapterManager that manages + prompt_adapter models on the worker side. + + Every request, the requested prompt_adapters will be + loaded (unless they are already loaded), + and every other prompt_adapter will be unloaded.""" + + _manager_cls: Type[PromptAdapterModelManager] = PromptAdapterModelManager + + def __init__( + self, + max_num_seqs: int, + max_num_batched_tokens: int, + device: torch.device, + prompt_adapter_config: PromptAdapterConfig, + prompt_adapter_model_cls: Type[PromptAdapterModel] = PromptAdapterModel + ): + self._adapter_manager: PromptAdapterModelManager + self.max_num_seqs = max_num_seqs + self.max_num_batched_tokens = max_num_batched_tokens + self._prompt_adapter_model_cls = prompt_adapter_model_cls + self.prompt_adapter_config = prompt_adapter_config + super().__init__(device) + + @property + def is_enabled(self) -> bool: + return True + + def create_prompt_adapter_manager( + self, + model: torch.nn.Module, + ) -> Any: + prompt_adapter_manager = create_prompt_adapter_manager( + model, + max_num_seqs=self.max_num_seqs, + max_num_batched_tokens=self.max_num_batched_tokens, + prompt_adapter_config=self.prompt_adapter_config, + prompt_adapter_manager_cls=self._manager_cls, + ) + self._adapter_manager = prompt_adapter_manager + return prompt_adapter_manager.model + + def _load_adapter( + self, prompt_adapter_request: PromptAdapterRequest + ) -> PromptAdapterModel: + try: + prompt_adapter = ( + self._prompt_adapter_model_cls.from_local_checkpoint( + prompt_adapter_request.prompt_adapter_local_path, + prompt_adapter_id=prompt_adapter_request.prompt_adapter_id, + num_virtual_tokens=prompt_adapter_request. + prompt_adapter_num_virtual_tokens, + config=self.prompt_adapter_config, + device=str(self.device), + )) + except Exception as e: + raise RuntimeError( + f"Loading prompt_adapter " + f"{prompt_adapter_request.prompt_adapter_local_path}" + f" failed") from e + return prompt_adapter + + def add_dummy_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + return True + + def pin_adapter(self, adapter_id: int) -> bool: + return self._adapter_manager.pin_adapter(adapter_id) + + def set_active_adapters(self, requests: Set[Any], + mapping: Optional[Any]) -> None: + set_active_adapters_worker(requests, mapping, self._apply_adapters, + self._adapter_manager.set_adapter_mapping) + + def add_adapter(self, adapter_request: Any) -> bool: + return add_adapter_worker(adapter_request, self.list_adapters, + self._load_adapter, + self._adapter_manager.add_adapter, + self._adapter_manager.activate_adapter) + + def _apply_adapters(self, adapter_requests: Set[Any]) -> None: + apply_adapters_worker(adapter_requests, self.list_adapters, + self._adapter_manager.adapter_slots, + self.remove_adapter, self.add_adapter) + + def remove_adapter(self, adapter_id: int) -> bool: + return self._adapter_manager.remove_adapter(adapter_id) + + def remove_all_adapters(self): + self._adapter_manager.remove_all_adapters() + + def list_adapters(self) -> Set[int]: + return list_adapters_worker(self._adapter_manager.list_adapters) + + +class LRUCacheWorkerPromptAdapterManager(WorkerPromptAdapterManager): + """WorkerPromptAdapterManager that manages + prompt_adapter models on the worker side. + + Uses an LRU Cache. Every request, the requested + prompt_adapters will be loaded (unless they are already loaded) + and least recently used prompt_adapters will + be unloaded if the cache is above capacity.""" + + _prompt_adapter_manager_cls: Type[ + LRUCachePromptAdapterModelManager] = LRUCachePromptAdapterModelManager + + def create_prompt_adapter_manager( + self, + model: torch.nn.Module, + ) -> Any: + prompt_adapter_manager = create_prompt_adapter_manager( + model, + max_num_seqs=self.max_num_seqs, + max_num_batched_tokens=self.max_num_batched_tokens, + prompt_adapter_config=self.prompt_adapter_config, + prompt_adapter_manager_cls=self._prompt_adapter_manager_cls) + self._adapter_manager: LRUCachePromptAdapterModelManager = ( + prompt_adapter_manager) + return prompt_adapter_manager.model + + def _apply_adapters( + self, prompt_adapter_requests: Set[PromptAdapterRequest]) -> None: + prompt_adapters_map = { + prompt_adapter_request.prompt_adapter_id: prompt_adapter_request + for prompt_adapter_request in prompt_adapter_requests + if prompt_adapter_request + } + if len(prompt_adapters_map + ) > self._adapter_manager.prompt_adapter_slots: + raise RuntimeError( + f"Number of requested prompt_adapters " + f"({len(prompt_adapters_map)}) is greater " + "than the number of GPU prompt_adapter slots " + f"({self._adapter_manager.prompt_adapter_slots}).") + for prompt_adapter in prompt_adapters_map.values(): + self.add_adapter(prompt_adapter) + + def add_adapter(self, + prompt_adapter_request: PromptAdapterRequest) -> bool: + if prompt_adapter_request.prompt_adapter_id not in self.list_adapters( + ): + # Remove before we load the new prompt_adapter to save memory + if len(self._adapter_manager) + 1 > self._adapter_manager.capacity: + self._adapter_manager.remove_oldest_adapter() + prompt_adapter = self._load_adapter(prompt_adapter_request) + loaded = self._adapter_manager.add_adapter(prompt_adapter) + else: + # If the prompt_adapter is already loaded, just touch it to + # update its position in the caches + loaded = self._adapter_manager.get_adapter( + prompt_adapter_request.prompt_adapter_id) is not None + self._adapter_manager.activate_adapter( + prompt_adapter_request.prompt_adapter_id) + return loaded diff --git a/vllm/py.typed b/vllm/py.typed new file mode 100644 index 00000000..33b3ad73 --- /dev/null +++ b/vllm/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The vllm package uses inline types. diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py new file mode 100644 index 00000000..4f2ae75e --- /dev/null +++ b/vllm/sampling_params.py @@ -0,0 +1,491 @@ +"""Sampling parameters for text generation.""" +import copy +from dataclasses import dataclass +from enum import Enum, IntEnum +from functools import cached_property +from typing import Any, Callable, Dict, List, Optional, Set, Union + +import msgspec +import torch +from pydantic import BaseModel +from typing_extensions import Annotated + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_SAMPLING_EPS = 1e-5 +_MAX_TEMP = 1e-2 + + +class SamplingType(IntEnum): + GREEDY = 0 + RANDOM = 1 + RANDOM_SEED = 2 + + +LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor], + Callable[[List[int], List[int], torch.Tensor], + torch.Tensor]] +"""LogitsProcessor is a function that takes a list +of previously generated tokens, the logits tensor +for the next token and, optionally, prompt tokens as a +first argument, and returns a modified tensor of logits +to sample from.""" + + +# maybe make msgspec? +@dataclass +class GuidedDecodingParams: + """One of these fields will be used to build a logit processor.""" + json: Optional[Union[str, Dict]] = None + regex: Optional[str] = None + choice: Optional[List[str]] = None + grammar: Optional[str] = None + json_object: Optional[bool] = None + """These are other options that can be set""" + backend: Optional[str] = None + whitespace_pattern: Optional[str] = None + + @staticmethod + def from_optional( + json: Optional[Union[Dict, BaseModel, str]], + regex: Optional[str] = None, + choice: Optional[List[str]] = None, + grammar: Optional[str] = None, + json_object: Optional[bool] = None, + backend: Optional[str] = None, + whitespace_pattern: Optional[str] = None, + ) -> "GuidedDecodingParams": + # Extract json schemas from pydantic models + if isinstance(json, (BaseModel, type(BaseModel))): + json = json.model_json_schema() + return GuidedDecodingParams( + json=json, + regex=regex, + choice=choice, + grammar=grammar, + json_object=json_object, + backend=backend, + whitespace_pattern=whitespace_pattern, + ) + + def __post_init__(self): + """Validate that some fields are mutually exclusive.""" + guide_count = sum([ + self.json is not None, self.regex is not None, self.choice + is not None, self.grammar is not None, self.json_object is not None + ]) + if guide_count > 1: + raise ValueError( + "You can only use one kind of guided decoding but multiple are " + f"specified: {self.__dict__}") + + +class RequestOutputKind(Enum): + # Return entire output so far in every RequestOutput + CUMULATIVE = 0 + # Return only deltas in each RequestOutput + DELTA = 1 + # Do not return intermediate RequestOuputs + FINAL_ONLY = 2 + + +class SamplingParams( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True): # type: ignore[call-arg] + """Sampling parameters for text generation. + + Overall, we follow the sampling parameters from the OpenAI text completion + API (https://platform.openai.com/docs/api-reference/completions/create). + In addition, we support beam search, which is not supported by OpenAI. + + Args: + n: Number of output sequences to return for the given prompt. + best_of: Number of output sequences that are generated from the prompt. + From these `best_of` sequences, the top `n` sequences are returned. + `best_of` must be greater than or equal to `n`. By default, + `best_of` is set to `n`. + presence_penalty: Float that penalizes new tokens based on whether they + appear in the generated text so far. Values > 0 encourage the model + to use new tokens, while values < 0 encourage the model to repeat + tokens. + frequency_penalty: Float that penalizes new tokens based on their + frequency in the generated text so far. Values > 0 encourage the + model to use new tokens, while values < 0 encourage the model to + repeat tokens. + repetition_penalty: Float that penalizes new tokens based on whether + they appear in the prompt and the generated text so far. Values > 1 + encourage the model to use new tokens, while values < 1 encourage + the model to repeat tokens. + temperature: Float that controls the randomness of the sampling. Lower + values make the model more deterministic, while higher values make + the model more random. Zero means greedy sampling. + top_p: Float that controls the cumulative probability of the top tokens + to consider. Must be in (0, 1]. Set to 1 to consider all tokens. + top_k: Integer that controls the number of top tokens to consider. Set + to -1 to consider all tokens. + min_p: Float that represents the minimum probability for a token to be + considered, relative to the probability of the most likely token. + Must be in [0, 1]. Set to 0 to disable this. + seed: Random seed to use for the generation. + stop: List of strings that stop the generation when they are generated. + The returned output will not contain the stop strings. + stop_token_ids: List of tokens that stop the generation when they are + generated. The returned output will contain the stop tokens unless + the stop tokens are special tokens. + include_stop_str_in_output: Whether to include the stop strings in + output text. Defaults to False. + ignore_eos: Whether to ignore the EOS token and continue generating + tokens after the EOS token is generated. + max_tokens: Maximum number of tokens to generate per output sequence. + min_tokens: Minimum number of tokens to generate per output sequence + before EOS or stop_token_ids can be generated + logprobs: Number of log probabilities to return per output token. + When set to None, no probability is returned. If set to a non-None + value, the result includes the log probabilities of the specified + number of most likely tokens, as well as the chosen tokens. + Note that the implementation follows the OpenAI API: The API will + always return the log probability of the sampled token, so there + may be up to `logprobs+1` elements in the response. + prompt_logprobs: Number of log probabilities to return per prompt token. + detokenize: Whether to detokenize the output. Defaults to True. + skip_special_tokens: Whether to skip special tokens in the output. + spaces_between_special_tokens: Whether to add spaces between special + tokens in the output. Defaults to True. + logits_processors: List of functions that modify logits based on + previously generated tokens, and optionally prompt tokens as + a first argument. + truncate_prompt_tokens: If set to an integer k, will use only the last k + tokens from the prompt (i.e., left truncation). Defaults to None + (i.e., no truncation). + guided_decoding: If provided, the engine will construct a guided + decoding logits processor from these parameters. Defaults to None. + logit_bias: If provided, the engine will construct a logits processor + that applies these logit biases. Defaults to None. + allowed_token_ids: If provided, the engine will construct a logits + processor which only retains scores for the given token ids. + Defaults to None. + """ + + n: int = 1 + best_of: Optional[int] = None + _real_n: Optional[int] = None + presence_penalty: float = 0.0 + frequency_penalty: float = 0.0 + repetition_penalty: float = 1.0 + temperature: float = 1.0 + top_p: float = 1.0 + top_k: int = -1 + min_p: float = 0.0 + seed: Optional[int] = None + stop: Optional[Union[str, List[str]]] = None + stop_token_ids: Optional[List[int]] = None + ignore_eos: bool = False + max_tokens: Optional[int] = 16 + min_tokens: int = 0 + logprobs: Optional[int] = None + prompt_logprobs: Optional[int] = None + # NOTE: This parameter is only exposed at the engine level for now. + # It is not exposed in the OpenAI API server, as the OpenAI API does + # not support returning only a list of token IDs. + detokenize: bool = True + skip_special_tokens: bool = True + spaces_between_special_tokens: bool = True + # Optional[List[LogitsProcessor]] type. We use Any here because + # Optional[List[LogitsProcessor]] type is not supported by msgspec. + logits_processors: Optional[Any] = None + include_stop_str_in_output: bool = False + truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None + output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE + + # The below fields are not supposed to be used as an input. + # They are set in post_init. + output_text_buffer_length: int = 0 + _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set) + + # Fields used to construct logits processors + guided_decoding: Optional[GuidedDecodingParams] = None + logit_bias: Optional[Dict[int, float]] = None + allowed_token_ids: Optional[List[int]] = None + + @staticmethod + def from_optional( + n: Optional[int] = 1, + best_of: Optional[int] = None, + presence_penalty: Optional[float] = 0.0, + frequency_penalty: Optional[float] = 0.0, + repetition_penalty: Optional[float] = 1.0, + temperature: Optional[float] = 1.0, + top_p: Optional[float] = 1.0, + top_k: int = -1, + min_p: float = 0.0, + seed: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + include_stop_str_in_output: bool = False, + ignore_eos: bool = False, + max_tokens: Optional[int] = 16, + min_tokens: int = 0, + logprobs: Optional[int] = None, + prompt_logprobs: Optional[int] = None, + detokenize: bool = True, + skip_special_tokens: bool = True, + spaces_between_special_tokens: bool = True, + logits_processors: Optional[List[LogitsProcessor]] = None, + truncate_prompt_tokens: Optional[Annotated[int, + msgspec.Meta(ge=1)]] = None, + output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE, + guided_decoding: Optional[GuidedDecodingParams] = None, + logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None, + allowed_token_ids: Optional[List[int]] = None, + ) -> "SamplingParams": + if logit_bias is not None: + logit_bias = { + int(token): bias + for token, bias in logit_bias.items() + } + + return SamplingParams( + n=1 if n is None else n, + best_of=best_of, + presence_penalty=0.0 + if presence_penalty is None else presence_penalty, + frequency_penalty=0.0 + if frequency_penalty is None else frequency_penalty, + repetition_penalty=1.0 + if repetition_penalty is None else repetition_penalty, + temperature=1.0 if temperature is None else temperature, + top_p=1.0 if top_p is None else top_p, + top_k=top_k, + min_p=min_p, + seed=seed, + stop=stop, + stop_token_ids=stop_token_ids, + include_stop_str_in_output=include_stop_str_in_output, + ignore_eos=ignore_eos, + max_tokens=max_tokens, + min_tokens=min_tokens, + logprobs=logprobs, + prompt_logprobs=prompt_logprobs, + detokenize=detokenize, + skip_special_tokens=skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + logits_processors=logits_processors, + truncate_prompt_tokens=truncate_prompt_tokens, + output_kind=output_kind, + guided_decoding=guided_decoding, + logit_bias=logit_bias, + allowed_token_ids=allowed_token_ids, + ) + + def __post_init__(self) -> None: + # how we deal with `best_of``: + # if `best_of`` is not set, we default to `n`; + # if `best_of`` is set, we set `n`` to `best_of`, + # and set `_real_n`` to the original `n`. + # when we return the result, we will check + # if we need to return `n` or `_real_n` results + if self.best_of: + if self.best_of < self.n: + raise ValueError( + f"best_of must be greater than or equal to n, " + f"got n={self.n} and best_of={self.best_of}.") + self._real_n = self.n + self.n = self.best_of + if 0 < self.temperature < _MAX_TEMP: + logger.warning( + "temperature %s is less than %s, which may cause numerical " + "errors nan or inf in tensors. We have maxed it out to %s.", + self.temperature, _MAX_TEMP, _MAX_TEMP) + self.temperature = max(self.temperature, _MAX_TEMP) + if self.seed == -1: + self.seed = None + else: + self.seed = self.seed + if self.stop is None: + self.stop = [] + elif isinstance(self.stop, str): + self.stop = [self.stop] + else: + self.stop = list(self.stop) + if self.stop_token_ids is None: + self.stop_token_ids = [] + else: + self.stop_token_ids = list(self.stop_token_ids) + self.logprobs = 1 if self.logprobs is True else self.logprobs + self.prompt_logprobs = (1 if self.prompt_logprobs is True else + self.prompt_logprobs) + + # Number of characters to hold back for stop string evaluation + # until sequence is finished. + if self.stop and not self.include_stop_str_in_output: + self.output_text_buffer_length = max(len(s) for s in self.stop) - 1 + + self._verify_args() + + if self.temperature < _SAMPLING_EPS: + # Zero temperature means greedy sampling. + self.top_p = 1.0 + self.top_k = -1 + self.min_p = 0.0 + self._verify_greedy_sampling() + # eos_token_id is added to this by the engine + self._all_stop_token_ids = set(self.stop_token_ids) + + def _verify_args(self) -> None: + if not isinstance(self.n, int): + raise ValueError(f"n must be an int, but is of " + f"type {type(self.n)}") + if self.n < 1: + raise ValueError(f"n must be at least 1, got {self.n}.") + if not -2.0 <= self.presence_penalty <= 2.0: + raise ValueError("presence_penalty must be in [-2, 2], got " + f"{self.presence_penalty}.") + if not -2.0 <= self.frequency_penalty <= 2.0: + raise ValueError("frequency_penalty must be in [-2, 2], got " + f"{self.frequency_penalty}.") + if not 0.0 < self.repetition_penalty <= 2.0: + raise ValueError("repetition_penalty must be in (0, 2], got " + f"{self.repetition_penalty}.") + if self.temperature < 0.0: + raise ValueError( + f"temperature must be non-negative, got {self.temperature}.") + if not 0.0 < self.top_p <= 1.0: + raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.") + if self.top_k < -1 or self.top_k == 0: + raise ValueError(f"top_k must be -1 (disable), or at least 1, " + f"got {self.top_k}.") + if not isinstance(self.top_k, int): + raise TypeError( + f"top_k must be an integer, got {type(self.top_k).__name__}") + if not 0.0 <= self.min_p <= 1.0: + raise ValueError("min_p must be in [0, 1], got " + f"{self.min_p}.") + if self.max_tokens is not None and self.max_tokens < 1: + raise ValueError( + f"max_tokens must be at least 1, got {self.max_tokens}.") + if self.min_tokens < 0: + raise ValueError(f"min_tokens must be greater than or equal to 0, " + f"got {self.min_tokens}.") + if self.max_tokens is not None and self.min_tokens > self.max_tokens: + raise ValueError( + f"min_tokens must be less than or equal to " + f"max_tokens={self.max_tokens}, got {self.min_tokens}.") + if self.logprobs is not None and self.logprobs < 0: + raise ValueError( + f"logprobs must be non-negative, got {self.logprobs}.") + if self.prompt_logprobs is not None and self.prompt_logprobs < 0: + raise ValueError(f"prompt_logprobs must be non-negative, got " + f"{self.prompt_logprobs}.") + if (self.truncate_prompt_tokens is not None + and self.truncate_prompt_tokens < 1): + raise ValueError(f"truncate_prompt_tokens must be >= 1, " + f"got {self.truncate_prompt_tokens}") + assert isinstance(self.stop, list) + if any(not stop_str for stop_str in self.stop): + raise ValueError("stop cannot contain an empty string.") + if self.stop and not self.detokenize: + raise ValueError( + "stop strings are only supported when detokenize is True. " + "Set detokenize=True to use stop.") + if self.best_of != self._real_n and self.output_kind == ( + RequestOutputKind.DELTA): + raise ValueError("best_of must equal n to use output_kind=DELTA") + + def _verify_greedy_sampling(self) -> None: + if self.n > 1: + raise ValueError("n must be 1 when using greedy sampling, " + f"got {self.n}.") + + def update_from_generation_config( + self, + generation_config: Dict[str, Any], + model_eos_token_id: Optional[int] = None) -> None: + """Update if there are non-default values from generation_config""" + + if model_eos_token_id is not None: + # Add the eos token id into the sampling_params to support + # min_tokens processing. + self._all_stop_token_ids.add(model_eos_token_id) + + # Update eos_token_id for generation + if (eos_ids := generation_config.get("eos_token_id")) is not None: + # it can be either int or list of int + eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) + if model_eos_token_id is not None: + # We don't need to include the primary eos_token_id in + # stop_token_ids since it's handled separately for stopping + # purposes. + eos_ids.discard(model_eos_token_id) + if eos_ids: + self._all_stop_token_ids.update(eos_ids) + if not self.ignore_eos: + eos_ids.update(self.stop_token_ids) + self.stop_token_ids = list(eos_ids) + + @cached_property + def sampling_type(self) -> SamplingType: + if self.temperature < _SAMPLING_EPS: + return SamplingType.GREEDY + if self.seed is not None: + return SamplingType.RANDOM_SEED + return SamplingType.RANDOM + + @property + def all_stop_token_ids(self) -> Set[int]: + return self._all_stop_token_ids + + def clone(self) -> "SamplingParams": + """Deep copy excluding LogitsProcessor objects. + + LogitsProcessor objects are excluded because they may contain an + arbitrary, nontrivial amount of data. + See https://github.com/vllm-project/vllm/issues/3087 + """ + + logit_processor_refs = None if self.logits_processors is None else { + id(lp): lp + for lp in self.logits_processors + } + return copy.deepcopy(self, memo=logit_processor_refs) + + def __repr__(self) -> str: + return ( + f"SamplingParams(n={self.n}, " + f"presence_penalty={self.presence_penalty}, " + f"frequency_penalty={self.frequency_penalty}, " + f"repetition_penalty={self.repetition_penalty}, " + f"temperature={self.temperature}, " + f"top_p={self.top_p}, " + f"top_k={self.top_k}, " + f"min_p={self.min_p}, " + f"seed={self.seed}, " + f"stop={self.stop}, " + f"stop_token_ids={self.stop_token_ids}, " + f"include_stop_str_in_output={self.include_stop_str_in_output}, " + f"ignore_eos={self.ignore_eos}, " + f"max_tokens={self.max_tokens}, " + f"min_tokens={self.min_tokens}, " + f"logprobs={self.logprobs}, " + f"prompt_logprobs={self.prompt_logprobs}, " + f"skip_special_tokens={self.skip_special_tokens}, " + "spaces_between_special_tokens=" + f"{self.spaces_between_special_tokens}, " + f"truncate_prompt_tokens={self.truncate_prompt_tokens}), " + f"guided_decoding={self.guided_decoding}") + + +class BeamSearchParams( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True): # type: ignore[call-arg] + """Beam search parameters for text generation.""" + beam_width: int + max_tokens: int + ignore_eos: bool = False + temperature: float = 0.0 + length_penalty: float = 1.0 diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py new file mode 100644 index 00000000..eb491dd1 --- /dev/null +++ b/vllm/scalar_type.py @@ -0,0 +1,35 @@ +from ._core_ext import NanRepr, ScalarType + +# naming generally follows: https://github.com/jax-ml/ml_dtypes +# for floating point types (leading f) the scheme is: +# `float_em[flags]` +# flags: +# - no-flags: means it follows IEEE 754 conventions +# - f: means finite values only (no infinities) +# - n: means nans are supported (non-standard encoding) +# for integer types the scheme is: +# `[u]int[b]` +# - if bias is not present it means its zero + + +class scalar_types: + int4 = ScalarType.int_(4, None) + uint4 = ScalarType.uint(4, None) + int8 = ScalarType.int_(8, None) + uint8 = ScalarType.uint(8, None) + float8_e4m3fn = ScalarType.float_(4, 3, True, + NanRepr.EXTD_RANGE_MAX_MIN.value) + float8_e5m2 = ScalarType.float_IEEE754(5, 2) + float16_e8m7 = ScalarType.float_IEEE754(8, 7) + float16_e5m10 = ScalarType.float_IEEE754(5, 10) + + # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main + float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value) + + # "gptq" types + uint4b8 = ScalarType.uint(4, 8) + uint8b128 = ScalarType.uint(8, 128) + + # colloquial names + bfloat16 = float16_e8m7 + float16 = float16_e5m10 diff --git a/vllm/scripts.py b/vllm/scripts.py new file mode 100644 index 00000000..4e4c0717 --- /dev/null +++ b/vllm/scripts.py @@ -0,0 +1,201 @@ +# The CLI entrypoint to vLLM. +import argparse +import os +import signal +import sys +from typing import List, Optional + +import uvloop +from openai import OpenAI +from openai.types.chat import ChatCompletionMessageParam + +from vllm.engine.arg_utils import EngineArgs +from vllm.entrypoints.openai.api_server import run_server +from vllm.entrypoints.openai.cli_args import (make_arg_parser, + validate_parsed_serve_args) +from vllm.logger import init_logger +from vllm.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + + +def register_signal_handlers(): + + def signal_handler(sig, frame): + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTSTP, signal_handler) + + +def serve(args: argparse.Namespace) -> None: + # The default value of `--model` + if args.model != EngineArgs.model: + raise ValueError( + "With `vllm serve`, you should provide the model as a " + "positional argument instead of via the `--model` option.") + + # EngineArgs expects the model name to be passed as --model. + args.model = args.model_tag + + uvloop.run(run_server(args)) + + +def interactive_cli(args: argparse.Namespace) -> None: + register_signal_handlers() + + base_url = args.url + api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY") + openai_client = OpenAI(api_key=api_key, base_url=base_url) + + if args.model_name: + model_name = args.model_name + else: + available_models = openai_client.models.list() + model_name = available_models.data[0].id + + print(f"Using model: {model_name}") + + if args.command == "complete": + complete(model_name, openai_client) + elif args.command == "chat": + chat(args.system_prompt, model_name, openai_client) + + +def complete(model_name: str, client: OpenAI) -> None: + print("Please enter prompt to complete:") + while True: + input_prompt = input("> ") + + completion = client.completions.create(model=model_name, + prompt=input_prompt) + output = completion.choices[0].text + print(output) + + +def chat(system_prompt: Optional[str], model_name: str, + client: OpenAI) -> None: + conversation: List[ChatCompletionMessageParam] = [] + if system_prompt is not None: + conversation.append({"role": "system", "content": system_prompt}) + + print("Please enter a message for the chat model:") + while True: + input_message = input("> ") + conversation.append({"role": "user", "content": input_message}) + + chat_completion = client.chat.completions.create(model=model_name, + messages=conversation) + + response_message = chat_completion.choices[0].message + output = response_message.content + + conversation.append(response_message) # type: ignore + print(output) + + +def _add_query_options( + parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + parser.add_argument( + "--url", + type=str, + default="http://localhost:8000/v1", + help="url of the running OpenAI-Compatible RESTful API server") + parser.add_argument( + "--model-name", + type=str, + default=None, + help=("The model name used in prompt completion, default to " + "the first model in list models API call.")) + parser.add_argument( + "--api-key", + type=str, + default=None, + help=( + "API key for OpenAI services. If provided, this api key " + "will overwrite the api key obtained through environment variables." + )) + return parser + + +def env_setup(): + # The safest multiprocessing method is `spawn`, as the default `fork` method + # is not compatible with some accelerators. The default method will be + # changing in future versions of Python, so we should use it explicitly when + # possible. + # + # We only set it here in the CLI entrypoint, because changing to `spawn` + # could break some existing code using vLLM as a library. `spawn` will cause + # unexpected behavior if the code is not protected by + # `if __name__ == "__main__":`. + # + # References: + # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing + # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors + # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders + if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ: + logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'") + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + +def main(): + env_setup() + + parser = FlexibleArgumentParser(description="vLLM CLI") + subparsers = parser.add_subparsers(required=True, dest="subparser") + + serve_parser = subparsers.add_parser( + "serve", + help="Start the vLLM OpenAI Compatible API server", + usage="vllm serve [options]") + serve_parser.add_argument("model_tag", + type=str, + help="The model tag to serve") + serve_parser.add_argument( + "--config", + type=str, + default='', + required=False, + help="Read CLI options from a config file." + "Must be a YAML with the following options:" + "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server" + ) + serve_parser = make_arg_parser(serve_parser) + serve_parser.set_defaults(dispatch_function=serve) + + complete_parser = subparsers.add_parser( + "complete", + help=("Generate text completions based on the given prompt " + "via the running API server"), + usage="vllm complete [options]") + _add_query_options(complete_parser) + complete_parser.set_defaults(dispatch_function=interactive_cli, + command="complete") + + chat_parser = subparsers.add_parser( + "chat", + help="Generate chat completions via the running API server", + usage="vllm chat [options]") + _add_query_options(chat_parser) + chat_parser.add_argument( + "--system-prompt", + type=str, + default=None, + help=("The system prompt to be added to the chat template, " + "used for models that support system prompts.")) + chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat") + + args = parser.parse_args() + if args.subparser == "serve": + validate_parsed_serve_args(args) + + # One of the sub commands should be executed. + if hasattr(args, "dispatch_function"): + args.dispatch_function(args) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/vllm/sequence.py b/vllm/sequence.py new file mode 100644 index 00000000..3bb35ea9 --- /dev/null +++ b/vllm/sequence.py @@ -0,0 +1,1380 @@ +"""Sequence and its related classes.""" +import copy +import enum +from abc import ABC, abstractmethod +from array import array +from collections import defaultdict +from dataclasses import dataclass +from functools import cached_property, reduce +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional +from typing import Sequence as GenericSequence +from typing import Set, Tuple, Union, cast + +import msgspec +import torch + +from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs +from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs +from vllm.lora.request import LoRARequest +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import SamplingParams +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics + +if TYPE_CHECKING: + from vllm.multimodal.base import MultiModalDataDict + +VLLM_TOKEN_ID_ARRAY_TYPE = "l" + +VLLM_INVALID_TOKEN_ID = -1 + + +# We use dataclass for now because it is used for +# openai server output, and msgspec is not serializable. +# TODO(sang): Fix it. +@dataclass +class Logprob: + """Infos for supporting OpenAI compatible logprobs and token ranks. + + Attributes: + logprob: The logprob of chosen token + rank: The vocab rank of chosen token (>=1) + decoded_token: The decoded chosen token index + """ + logprob: float + rank: Optional[int] = None + decoded_token: Optional[str] = None + + +# {token_id -> logprob} per each sequence group. None if the corresponding +# sequence group doesn't require prompt logprob. +PromptLogprobs = List[Optional[Dict[int, Logprob]]] +# {token_id -> logprob} for each sequence group. +SampleLogprobs = List[Dict[int, Logprob]] + + +class SequenceStatus(enum.IntEnum): + """Status of a sequence.""" + WAITING = 0 + RUNNING = 1 + SWAPPED = 2 + # Note: anything after SWAPPED (2) will be considered + # as a finished status. + FINISHED_STOPPED = 3 + FINISHED_LENGTH_CAPPED = 4 + FINISHED_ABORTED = 5 + FINISHED_IGNORED = 6 + + @staticmethod + def is_finished(status: "SequenceStatus") -> bool: + return status > SequenceStatus.SWAPPED + + @staticmethod + def get_finished_reason(status: "SequenceStatus") -> Union[str, None]: + if status == SequenceStatus.FINISHED_STOPPED: + finish_reason = "stop" + elif status == SequenceStatus.FINISHED_LENGTH_CAPPED: + finish_reason = "length" + elif status == SequenceStatus.FINISHED_ABORTED: + finish_reason = "abort" + elif status == SequenceStatus.FINISHED_IGNORED: + # The ignored sequences are the sequences whose prompt lengths + # are longer than the model's length cap. Therefore, the stop + # reason should also be "length" as in OpenAI API. + finish_reason = "length" + else: + finish_reason = None + return finish_reason + + +class SequenceStage(enum.Enum): + PREFILL = enum.auto() + DECODE = enum.auto() + + +@dataclass +class RequestMetrics: + """Metrics associated with a request. + + Attributes: + arrival_time: The time when the request arrived. + first_scheduled_time: The time when the request was first scheduled. + first_token_time: The time when the first token was generated. + time_in_queue: The time the request spent in the queue. + finished_time: The time when the request was finished. + scheduler_time: The time spent in the scheduler when this request was + being considered by the scheduler. + model_forward_time: The time spent in the model forward pass when this + request was in the batch. + model_execute_time: The time spent in the model execute function. This + will include model forward, block/sync across + workers, cpu-gpu sync time and sampling time. + """ + arrival_time: float + last_token_time: float + first_scheduled_time: Optional[float] + first_token_time: Optional[float] + time_in_queue: Optional[float] + finished_time: Optional[float] = None + scheduler_time: Optional[float] = None + model_forward_time: Optional[float] = None + model_execute_time: Optional[float] = None + + +class SequenceDataDelta( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Delta SequenceData to send to workers per step.""" + # A new token to be appended to existing SequenceData. + new_output_token_ids: List[int] + # Overwriting existing `cumulative_logprob` + new_cumulative_logprob: float + # Overwriting existing `num_computed_tokens`. + new_num_computed_tokens: int + # Overwriting existing `stage`. + new_stage: SequenceStage + + +class SequenceData(msgspec.Struct, + omit_defaults=True): # type: ignore[call-arg] + """Data associated with a sequence. + + Args: + prompt_token_ids: The token IDs of the prompt. + output_token_ids: The token IDs of the output. Set to an empty list if + None. + + Attributes: + prompt_token_ids: The token IDs of the prompt. + output_token_ids: The token IDs of the output. + cumulative_logprob: The cumulative log probability of the output. + """ + # NOTE: we cannot use Union[List, array] because msgspec cannot support + # union of 2 list types. + _prompt_token_ids: array + _output_token_ids: array = msgspec.field( + default_factory=lambda: array(VLLM_TOKEN_ID_ARRAY_TYPE, [])) + + ### The below fields should not be passed as an argument ### + _cumulative_logprob: float = 0.0 + _prompt_token_ids_tuple: Tuple[int, + ...] = msgspec.field(default_factory=tuple) + # The number of tokens that are computed (that run against the model). + _num_computed_tokens: int = 0 + _stage: SequenceStage = SequenceStage.PREFILL + _cached_all_token_ids: List[int] = msgspec.field(default_factory=list) + + # It is used to get delta input. It is reset when `get_delta_and_reset` + # is called. + _new_appended_tokens: List[int] = msgspec.field(default_factory=list) + + # It is used to compute mrope_position_ids. + _mrope_position_delta: Optional[int] = None + + @staticmethod + def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData": + if len(token_counts) == 0: + return SequenceData.from_seqs([]) + + arrs = [ + array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count + for token_id, count in token_counts + ] + + return SequenceData(reduce(array.__add__, arrs)) + + @staticmethod + def from_seqs( + prompt_token_ids: GenericSequence[int], + output_token_ids: Optional[GenericSequence[int]] = None, + ) -> "SequenceData": + prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE, + prompt_token_ids) + + if output_token_ids is None: + return SequenceData(prompt_token_ids_arr) + + output_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE, + output_token_ids) + + return SequenceData(prompt_token_ids_arr, + _output_token_ids=output_token_ids_arr) + + def __post_init__(self) -> None: + assert self._prompt_token_ids.typecode == "l" + assert self._output_token_ids.typecode == "l" + self._prompt_token_ids_tuple: Tuple[int, ...] = tuple( + self._prompt_token_ids) + self._update_cached_all_tokens() + + def _update_cached_all_tokens(self): + assert isinstance(self._prompt_token_ids, array) + assert isinstance(self._output_token_ids, array) + self._cached_all_token_ids: List[int] = list(self._prompt_token_ids + + self._output_token_ids) + + @property + def cumulative_logprob(self) -> float: + return self._cumulative_logprob + + @property + def prompt_token_ids(self) -> Tuple[int, ...]: + return self._prompt_token_ids_tuple + + @prompt_token_ids.setter + def prompt_token_ids(self, new_prompt_token_ids) -> None: + raise NotImplementedError + + @property + def prompt_token_ids_array(self) -> array: + """Return the prompt token ids in array type. + + Note that the array is in "I" type, and it is not compatible + with torch.long (2 bytes vs 4 bytes). So beware of the usage. + """ + return self._prompt_token_ids + + @property + def output_token_ids(self) -> Tuple[int, ...]: + return tuple(self._output_token_ids) + + @output_token_ids.setter + def output_token_ids(self, new_output_token_ids: List[int]) -> None: + self._output_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, + new_output_token_ids) + self._update_cached_all_tokens() + + @property + def output_token_ids_array(self) -> array: + """Return the prompt token ids in array type. + + Note that the array is in "I" type, and it is not compatible + with torch.long (2 bytes vs 4 bytes). So beware of the usage. + """ + assert isinstance(self._output_token_ids, array) + return self._output_token_ids + + @property + def mrope_position_delta(self) -> Optional[int]: + return self._mrope_position_delta + + @mrope_position_delta.setter + def mrope_position_delta(self, new_mrope_position_delta): + self._mrope_position_delta = new_mrope_position_delta + + def append_token_id(self, token_id: int, logprob: float) -> None: + self._output_token_ids.append(token_id) + self._new_appended_tokens.append(token_id) + self._cached_all_token_ids.append(token_id) + self._cumulative_logprob += logprob + + def get_len(self) -> int: + return len(self._output_token_ids) + len(self._prompt_token_ids) + + def get_prompt_len(self) -> int: + return len(self._prompt_token_ids) + + def get_output_len(self) -> int: + return len(self._output_token_ids) + + def get_token_ids(self) -> List[int]: + return self._cached_all_token_ids + + def get_prefix_token_ids( + self, num_tokens: int + ) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]: + """Get prefix tokens, and make the return value hashable""" + prompt_length = self.get_prompt_len() + if num_tokens > prompt_length: + return (self._prompt_token_ids_tuple, + tuple(self._output_token_ids[:num_tokens - prompt_length])) + else: + return (self._prompt_token_ids_tuple[:num_tokens], None) + + def get_num_computed_tokens(self) -> int: + """Return the number of prefill tokens that are already computed.""" + return self._num_computed_tokens + + def update_num_computed_tokens(self, num_new_computed_tokens: int): + """Update number of tokens computed so far.""" + self._num_computed_tokens += num_new_computed_tokens + assert self._num_computed_tokens <= self.get_len(), ( + self._num_computed_tokens, self.get_len()) + # If all tokens are computed, it means it is in decoding phase. + if self.get_num_uncomputed_tokens() == 0: + self._stage = SequenceStage.DECODE + + def reset_state_for_recompute(self) -> None: + """Reset the number of computed tokens from this sequence. It is + supposed to be called when a sequence needs to be started from + the beginning again (e.g., sequence is preempted). + """ + self._num_computed_tokens = 0 + self._stage = SequenceStage.PREFILL + self._new_appended_tokens = [] + + def get_num_uncomputed_tokens(self) -> int: + """Return the number of prefill tokens that are not computed.""" + # we use `get_len()` which includes prompt_len + output_len instead + # of prompt_len here. This is because during recompute we need to + # prefill for both prompt and output. + return self.get_len() - self.get_num_computed_tokens() + + def get_last_token_id(self) -> int: + if not self._output_token_ids: + return self._prompt_token_ids[-1] + return self._output_token_ids[-1] + + def get_prompt_token_ids(self) -> Tuple[int, ...]: + return self.prompt_token_ids + + def get_output_token_ids(self) -> Tuple[int, ...]: + return self.output_token_ids + + def get_delta_and_reset(self) -> SequenceDataDelta: + delta = SequenceDataDelta(self._new_appended_tokens, + self._cumulative_logprob, + self.get_num_computed_tokens(), self.stage) + # Reset delta state. + self._new_appended_tokens = [] + return delta + + def apply_delta(self, delta: SequenceDataDelta): + self._num_computed_tokens = delta.new_num_computed_tokens + self._cumulative_logprob = delta.new_cumulative_logprob + self._stage = delta.new_stage + self._output_token_ids.extend(delta.new_output_token_ids) + self._cached_all_token_ids.extend(delta.new_output_token_ids) + + @property + def stage(self) -> SequenceStage: + return self._stage + + def __repr__(self) -> str: + return (f"SequenceData(" + f"prompt_token_ids={self._prompt_token_ids}, " + f"output_token_ids={self.output_token_ids}, " + f"cumulative_logprob={self.cumulative_logprob}, " + f"get_num_computed_tokens={self.get_num_computed_tokens()}") + + +class Sequence: + """Stores the data, status, and block information of a sequence. + + The sequence is constructed from the LLMInputs instance passed + in through the `inputs` constructor argument. + + For encoder/decoder models, LLMInputs encapsulates both a + decoder and encoder prompt, creating an ambiguity about which + prompt to construct the sequence from. The `from_decoder_prompt` + constructor argument signals whether to construct the Sequence + from the LLMInputs decoder prompt, or encoder prompt. + + Args: + seq_id: The ID of the sequence. + inputs: The inputs of the sequence. + block_size: The block size of the sequence. Should be the same as the + block size used by the block manager and cache engine. + eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM. + lora_request: LoRA request. + prompt_adapter_request: Prompt Adapter request. + from_decoder_prompt: Construct Sequence from LLMInputs decoder prompt + (True) or encoder prompt (False.) Must be True + for decoder-only model. + + """ + + def __init__( + self, + seq_id: int, + inputs: "LLMInputs", + block_size: int, + eos_token_id: Optional[int] = None, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + from_decoder_prompt: bool = True, + ) -> None: + self.seq_id = seq_id + self.inputs = inputs + self.block_size = block_size + self.eos_token_id = eos_token_id + self.lora_request = lora_request + self.prompt_adapter_request = prompt_adapter_request + self.from_decoder_prompt = from_decoder_prompt + + # For decoder-only models, a Sequence is constructed + # from an LLMInputs instance (the `inputs` arg.) + # + # For encoder/decoder models the same `inputs` + # instance could be utilized to construct either an + # encoder sequence or a decoder sequence, because + # `LLMInputs` has both decoder- and encoder-oriented + # member variables (i.e. it encapsulates both an encoder + # and a decoder prompt.) The decision of which type of sequence + # to generate is determined by the `from_decoder_prompt` argument. + # + # When constructing a encoder sequence + # (`from_decoder_prompt` False) it matters that + # the `LLMInputs` instance stored in `inputs` is valid + # in the sense that its encoder-related member variables are + # populated; below, an exception is raised if this is + # not the case. + # + # When constructing a decoder sequence (`from_decoder_prompt` True) + # it does not matter whether `inputs` has its encoder-related + # member variables populated. + if not (from_decoder_prompt + or is_valid_encoder_decoder_llm_inputs(inputs)): + raise ValueError("Cannot extract encoder input prompt from " + f"invalid input {inputs}; did you forget the " + "encoder input prompt fields?") + + self.data = SequenceData.from_seqs(self.prompt_token_ids) + self.output_logprobs: SampleLogprobs = [] + self.output_text = "" + + self.status = SequenceStatus.WAITING + self.stop_reason: Union[int, str, None] = None + + # These are used to keep track of delta outputs + self._last_output_token_ids_offset: int = 0 + self._last_output_text_offset: int = 0 + + # Used for incremental detokenization + self.prefix_offset = 0 + self.read_offset = 0 + # Input + output tokens + self.tokens: Optional[List[str]] = None + + @property + def n_blocks(self) -> int: + return (self.get_len() + self.block_size - 1) // self.block_size + + @cached_property + def prompt(self) -> Optional[str]: + # Select decoder or encoder input prompt str, as appropriate + prompt_key: str = ("prompt" + if self.from_decoder_prompt else "encoder_prompt") + + return cast(Optional[str], self.inputs.get(prompt_key)) + + @cached_property + def prompt_token_ids(self) -> List[int]: + # Select decoder or encoder input prompt token ids, as appropriate + prompt_token_ids_key: str = ("prompt_token_ids" + if self.from_decoder_prompt else + "encoder_prompt_token_ids") + + # Cache computed prompt token ids + return cast(List[int], self.inputs.get(prompt_token_ids_key)) + + @property + def multi_modal_data(self) -> "MultiModalDataDict": + if self.inputs.get("multi_modal_data") and self.inputs.get( + "encoder_multi_modal_data"): + raise ValueError( + "Multi-modal data in both encoder and decoder is not supported." + ) + inputs = self.inputs + return self.inputs.get("multi_modal_data") or (cast( + EncoderDecoderLLMInputs, + inputs).get("encoder_multi_modal_data")) or {} + + @property + def mm_processor_kwargs(self) -> Dict[str, Any]: + return self.inputs.get("mm_processor_kwargs") or {} + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + def get_output_text_to_return(self, buffer_length: int, + delta: bool) -> str: + """If delta is True, only new text since the last call to + this method is returned""" + + # We return the full output text if the sequence is finished. + truncate = buffer_length and not self.is_finished() + if not delta: + return self.output_text[:-buffer_length] if truncate else ( + self.output_text) + length = len(self.output_text) + if truncate: + length -= buffer_length + last_offset = self._last_output_text_offset + if last_offset < length: + self._last_output_text_offset = length + return self.output_text[last_offset:length] + return "" + + def get_output_token_ids_to_return( + self, delta: bool) -> Union[GenericSequence[int], int]: + """If delta is True, only new tokens since the last call to + this method are returned""" + if not delta: + return self.get_output_token_ids() + + output_len = self.get_output_len() + + # Get the number of new tokens + num_new_tokens = output_len - self._last_output_token_ids_offset + self._last_output_token_ids_offset = output_len + + # Return new tokens + if num_new_tokens == 1: + # Optimization for single decode token case + # (which is what we have most of the time) + return self.data._cached_all_token_ids[-1] + + return self.data._cached_all_token_ids[-num_new_tokens:] + + def hash_of_block(self, logical_idx: int) -> int: + # TODO This can produce incorrect hash when block size > prompt size + + # Compute the number of tokens in the sequence + # TODO: The current hashing function is O(L^2). We should optimize + # this in the future. + num_tokens = self.num_hashed_tokens_of_block(logical_idx) + hashed_tokens = self.data.get_prefix_token_ids(num_tokens) + return hash((hashed_tokens, self.lora_int_id)) + + def num_hashed_tokens_of_block(self, logical_idx: int): + return logical_idx * self.block_size + self.block_size + + def reset_state_for_recompute(self): + """Reset the sequence states for recomputation.""" + self.data.reset_state_for_recompute() + + def append_token_id(self, token_id: int, logprobs: Dict[int, + Logprob]) -> None: + assert token_id in logprobs + self.output_logprobs.append(logprobs) + self.data.append_token_id(token_id, logprobs[token_id].logprob) + + def get_len(self) -> int: + return self.data.get_len() + + def get_prompt_len(self) -> int: + return self.data.get_prompt_len() + + def get_output_len(self) -> int: + return self.data.get_output_len() + + def get_token_ids(self) -> List[int]: + return self.data.get_token_ids() + + def get_prompt_token_ids(self) -> Tuple[int, ...]: + return self.data.get_prompt_token_ids() + + def get_last_token_id(self) -> int: + return self.data.get_last_token_id() + + def get_output_token_ids(self) -> Tuple[int, ...]: + return self.data.get_output_token_ids() + + def get_cumulative_logprob(self) -> float: + return self.data.cumulative_logprob + + def is_finished(self) -> bool: + return SequenceStatus.is_finished(self.status) + + def fork(self, new_seq_id: int) -> "Sequence": + new_seq = copy.deepcopy(self) + new_seq.seq_id = new_seq_id + return new_seq + + def get_num_new_tokens(self) -> int: + """Get the number of new tokens to be computed. + + Returns: + The new number of tokens to be computed. I.e., 1 for decode, or + the remaining prompt size for prefill. + """ + if self.data.stage == SequenceStage.DECODE: + return 1 + return self.data.get_num_uncomputed_tokens() + + def is_prefill(self) -> bool: + return self.data.stage == SequenceStage.PREFILL + + def __repr__(self) -> str: + return (f"Sequence(seq_id={self.seq_id}, " + f"status={self.status.name}, " + f"num_blocks={self.n_blocks}, ") + + +class SequenceGroupState(msgspec.Struct, + omit_defaults=True): # type: ignore[call-arg] + """Mutable state tied to a specific sequence group""" + + # for multi-step decoding + num_steps: int = 1 + current_step: int = 0 + + @property + def remaining_steps(self) -> int: + return self.num_steps - self.current_step + + +class SequenceGroup: + """A group of sequences that are generated from the same prompt. + + Args: + request_id: The ID of the request. + seqs: The list of sequences. + sampling_params: The sampling parameters used to generate the outputs. + arrival_time: The arrival time of the request. + lora_request: LoRA request. + embeddings: The embeddings vectors of the prompt of the sequence group + for an embedding model. + pooling_params: The pooling parameters used to generate the pooling + for an embedding model. + encoder_seq: Optional, the single encoder sequence. Should be None + unless you are working with an encoder/decoder model. + trace_headers: OpenTelemetry trace headers. + prompt_adapter_request: Prompt Adapter request. + priority: User-defined priority of the request. + """ + + def __init__( + self, + request_id: str, + seqs: List[Sequence], + arrival_time: float, + sampling_params: Optional[SamplingParams] = None, + lora_request: Optional[LoRARequest] = None, + embeddings: Optional[List[float]] = None, + pooling_params: Optional[PoolingParams] = None, + encoder_seq: Optional[Sequence] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + self.request_id = request_id + self.seqs = seqs + self.arrival_time = arrival_time + self.is_single_seq = len(seqs) == 1 + self.seqs_dict = {seq.seq_id: seq for seq in seqs} + + self.sampling_params = sampling_params + self.metrics = RequestMetrics(arrival_time=arrival_time, + last_token_time=arrival_time, + first_scheduled_time=None, + first_token_time=None, + time_in_queue=None) + self.lora_request = lora_request + self.prompt_logprobs: Optional[PromptLogprobs] = None + self.state = SequenceGroupState() + self.embeddings = embeddings + self.pooling_params = pooling_params + self.prompt_adapter_request = prompt_adapter_request + self.encoder_seq = encoder_seq + self.trace_headers = trace_headers + self.priority = priority + + self.cached_request_output = None + + @property + def prompt(self) -> Optional[str]: + # All sequences in the group should have the same prompt. + # We use the prompt of an arbitrary sequence. + return self.seqs[0].prompt + + @property + def prompt_token_ids(self) -> List[int]: + # All sequences in the group should have the same prompt. + # We use the prompt of an arbitrary sequence. + return self.seqs[0].prompt_token_ids + + @property + def encoder_prompt(self) -> Optional[str]: + # There are either 0 or 1 encoder sequences + # If one is present, its prompt is distinct + # from the decoder's. + return (self.encoder_seq.prompt + if self.encoder_seq is not None else None) + + @property + def encoder_prompt_token_ids(self) -> Optional[List[int]]: + # There are either 0 or 1 encoder sequences + # If one is present, its prompt token ids are + # distinct from the decoder's. + return (self.encoder_seq.prompt_token_ids + if self.encoder_seq is not None else None) + + @property + def multi_modal_data(self) -> "MultiModalDataDict": + # All sequences in the group should have the same multi-modal data. + # We use the multi-modal data of an arbitrary sequence. + return self.seqs[0].multi_modal_data + + @property + def mm_processor_kwargs(self) -> Dict[str, Any]: + # As with multi-modal data, all sequences in the group should have the + # same processor kwargs (i.e., mm_processor_kwargs are optionally + # provided per request; note that are independent of whether the model + # decoder-only or an encoder-decoder). + return self.seqs[0].mm_processor_kwargs + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + @property + def prompt_adapter_num_virtual_tokens(self) -> int: + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\ + if self.prompt_adapter_request else 0 + + def init_multi_step(self, num_steps: int) -> None: + self.state.num_steps = num_steps + self.state.current_step = 0 + + def init_multi_step_from_lookahead_slots(self, num_lookahead_slots: int, + num_scheduler_steps: int, + is_multi_step: bool, + enable_chunking: bool) -> None: + + if not is_multi_step: + self.init_multi_step(num_steps=num_scheduler_steps) + return + + # Multi-Step case + is_prefill = self.is_prefill() + + # The asserts below reflect the expectations of the current system. + if is_prefill and enable_chunking: + assert num_lookahead_slots == num_scheduler_steps + self.init_multi_step(num_steps=num_lookahead_slots) + else: + is_decode: bool = not is_prefill + # If it is a prefill, num_lookahead_slots must be 0 + assert num_lookahead_slots == 0 or is_decode + # If it is a decode, num_lookahead_slots + 1 must match + # the scheduler steps. + assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill + self.init_multi_step(num_steps=num_lookahead_slots + 1) + + def get_last_latency(self, now: float) -> Optional[float]: + """Sets the last token time for Request level timings.""" + # If still in prefill phase, raise Error. + if self.is_prefill(): + raise ValueError( + "seq_group.get_last_latency() should not be called " + "if the seq_group is in prefill phase.") + + # Otherwise return token latency. + latency = now - self.metrics.last_token_time + self.metrics.last_token_time = now + return latency + + def maybe_set_first_token_time(self, time: float) -> None: + """Sets the first token time for Request level timings.""" + # Note: in a case where a sequence_group is swapped and + # recomputed, the time between iterations is counted + # in TPOT, rather than recalculating TTFT (since from the ) + # POV of the user, there is simply a long generation delay. + if (self.metrics.first_token_time is None + and self.seqs[0].get_output_len() == 1): + self.metrics.first_token_time = time + + def maybe_set_first_scheduled_time(self, time: float) -> None: + """Sets the first scheduled time and time in queue for Request + level timings.""" + if self.metrics.first_scheduled_time is None: + self.metrics.first_scheduled_time = time + self.metrics.time_in_queue = time - self.metrics.arrival_time + + def set_finished_time(self, time: Optional[float]) -> None: + """Sets the finished time for Request level timings.""" + self.metrics.finished_time = time + + def get_max_num_running_seqs(self) -> int: + """The maximum number of sequences running in parallel in the remaining + lifetime of the request.""" + if self.sampling_params: + n = self.sampling_params.n + assert isinstance(n, int) + if n > self.num_seqs(): + # At prompt stage, the sequence group is not yet filled up + # and only have one sequence running. However, in the + # generation stage, we will have `n` sequences + # running. + return n + # At sampling stages, return the number of actual sequences + # that are not finished yet. + return self.num_unfinished_seqs() + + def get_seqs( + self, + status: Optional[SequenceStatus] = None, + ) -> List[Sequence]: + if status is None: + return self.seqs + + if self.is_single_seq: + return self.seqs if self.seqs[0].status == status else [] + + return [seq for seq in self.seqs if seq.status == status] + + def is_encoder_decoder(self) -> bool: + return self.encoder_seq is not None + + def get_encoder_seq(self) -> Optional[Sequence]: + return self.encoder_seq + + def get_unfinished_seqs(self) -> List[Sequence]: + if self.is_single_seq: + return self.seqs if not self.seqs[0].is_finished() else [] + + return [seq for seq in self.seqs if not seq.is_finished()] + + def get_finished_seqs(self) -> List[Sequence]: + if self.is_single_seq: + return self.seqs if self.seqs[0].is_finished() else [] + + return [seq for seq in self.seqs if seq.is_finished()] + + def update_num_computed_tokens(self, num_new_computed_tokens: int): + """Update number of tokens computed so far.""" + for seq in self.seqs: + if not seq.is_finished(): + seq.data.update_num_computed_tokens(num_new_computed_tokens) + + def get_num_uncomputed_tokens(self) -> int: + num_uncomputed_tokens = 0 + for seq in self.seqs: + if not seq.is_finished(): + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens() + return num_uncomputed_tokens + + def num_seqs(self, status: Optional[SequenceStatus] = None) -> int: + # Optimization. We don't need to call get_seqs if we don't need to + # filter by states. + if status is None: + return len(self.seqs) + + if self.is_single_seq: + return 1 if self.seqs[0].status == status else 0 + + return len(self.get_seqs(status)) + + def num_unfinished_seqs(self) -> int: + if self.is_single_seq: + return 1 if not self.seqs[0].is_finished() else 0 + + return len(self.get_unfinished_seqs()) + + def num_finished_seqs(self) -> int: + if self.is_single_seq: + return 1 if self.seqs[0].is_finished() else 0 + + return len(self.get_finished_seqs()) + + def find(self, seq_id: int) -> Sequence: + if seq_id not in self.seqs_dict: + raise ValueError(f"Sequence {seq_id} not found.") + return self.seqs_dict[seq_id] + + def add(self, seq: Sequence) -> None: + if seq.seq_id in self.seqs_dict: + raise ValueError(f"Sequence {seq.seq_id} already exists.") + self.seqs_dict[seq.seq_id] = seq + self.seqs.append(seq) + self.is_single_seq = len(self.seqs) == 1 + + def remove(self, seq_id: int) -> None: + seq = self.seqs_dict.pop(seq_id, None) + if seq is None: + raise ValueError(f"Sequence {seq_id} not found.") + self.seqs.remove(seq) + self.is_single_seq = len(self.seqs) == 1 + + def is_finished(self) -> bool: + if self.is_single_seq: + return self.seqs[0].is_finished() + + return all(seq.is_finished() for seq in self.seqs) + + def is_prefill(self) -> bool: + # Every sequence should be in the same stage. + return self.seqs[0].is_prefill() + + def __repr__(self) -> str: + return (f"SequenceGroup(request_id={self.request_id}, " + f"sampling_params={self.sampling_params}, " + f"num_seqs={len(self.seqs)})") + + +class SequenceGroupMetadataDelta( + msgspec.Struct, + tag=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Delta of SequenceGroupMetadata. + + After sending the first SequenceGroupMetadata, vLLM scheduler + only sends delta to reduce the data payload size. + """ + seq_data_delta: Dict[int, SequenceDataDelta] + request_id: str + block_tables: Dict[int, List[int]] + is_prompt: bool + do_sample: bool = True + token_chunk_size: Optional[int] = None + computed_block_nums: Optional[List[int]] = None + state: Optional[SequenceGroupState] = msgspec.field( + default_factory=lambda: SequenceGroupState()) + + +class SequenceGroupMetadata( + msgspec.Struct, + tag=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """Metadata for a sequence group. Used to create `AttentionMetadata`. + + Args: + request_id: The ID of the request. + is_prompt: Whether the request is at prompt stage. + seq_data: The sequence data. (Seq id -> sequence data) + sampling_params: The sampling parameters used to generate the outputs. + block_tables: The block tables. (Seq id -> list of physical block + numbers) + do_sample: True if sampling is required. Sampling is not required when + e.g., prefill is chunked, and the current iteration only computes + query tokens for prefill, we don't need sampling. + token_chunk_size: The number of tokens to be processed (per sequence). + None if chunking is not required. + lora_request: LoRA request. + computed_block_nums: The block numbers that are already computed, + used in prefix caching. + state: Internal state tied to this sequence group. + multi_modal_data: Multi modal data. + mm_processor_kwargs: Multimodal input processor / mapper overrides. + encoder_seq_data: Optional sequence data for encoder prompt + (SequenceGroup.encoder_seq). Should be None + unless you are working with an encoder/decoder + model. + cross_block_table: Optional cross-attention block table associated + with the encoder prompt + (SequenceGroup.encoder_seq). Should be None + unless you are working with an encoder/decoder + model. + prompt_adapter_request: Prompt Adapter request. + """ + + request_id: str + is_prompt: bool + seq_data: Dict[int, SequenceData] + sampling_params: Optional[SamplingParams] + block_tables: Dict[int, List[int]] + do_sample: bool = True + pooling_params: Optional[PoolingParams] = None + lora_request: Optional[LoRARequest] = None + computed_block_nums: Optional[List[int]] = None + state: Optional[SequenceGroupState] = msgspec.field( + default_factory=lambda: SequenceGroupState()) + # "MultiModalDataDict" types. We have to use Any due to msgspec + # doesn't allow to have union of 2 different dicts. + multi_modal_data: Optional[Any] = None + mm_processor_kwargs: Optional[Dict[str, Any]] = None + encoder_seq_data: Optional[SequenceData] = None + cross_block_table: Optional[List[int]] = None + prompt_adapter_request: Optional[PromptAdapterRequest] = None + token_chunk_size: Optional[int] = None + + ### Stateful fields that are lazily defined. ### + # The number of speculative tokens adopted in this request. + # None means specuative decoding is not used. + # Zero means speculative decoding is disabled for some reasons. + # TODO: We should maintain this states out of the sequence group. + num_speculative_tokens: Optional[int] = None + + def __post_init__(self): + if self.seq_data is not None and self.token_chunk_size is None: + if self.is_prompt: + self.token_chunk_size = next(iter( + self.seq_data.values())).get_len() + else: + self.token_chunk_size = 1 + + @property + def lora_int_id(self) -> int: + return self.lora_request.lora_int_id if self.lora_request else 0 + + @property + def prompt_adapter_id(self) -> int: + return self.prompt_adapter_request.prompt_adapter_id \ + if self.prompt_adapter_request else 0 + + @property + def prompt_adapter_num_virtual_tokens(self) -> int: + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \ + if self.prompt_adapter_request else 0 + + # Multi-Step Chunked-Prefill property + @property + def is_single_step_prompt(self) -> bool: + # do_sample is true, only when the token_chunk_size matches the + # num_uncomputed_tokens of the sequence. This indicates that + # the prompt will finish processing in a single `execute_model` + # step. + return self.is_prompt and self.do_sample + + def get_first_seq_id(self) -> int: + # This is an efficient way of fetching the seq_id when + # we know this SequenceGroup has only one sequence. + return next(iter(self.seq_data)) + + def apply_delta(self, + sequence_group_metadata_delta: SequenceGroupMetadataDelta): + for id, delta in sequence_group_metadata_delta.seq_data_delta.items(): + self.seq_data[id].apply_delta(delta) + assert self.request_id == sequence_group_metadata_delta.request_id + self.block_tables = sequence_group_metadata_delta.block_tables + self.token_chunk_size = sequence_group_metadata_delta.token_chunk_size + self.do_sample = sequence_group_metadata_delta.do_sample + self.is_prompt = sequence_group_metadata_delta.is_prompt + + def finish_step(self) -> None: + assert self.state is not None + assert self.state.current_step < self.state.num_steps, \ + f"current step {self.state.current_step}, num_steps {self.state.num_steps}" # noqa + self.state.current_step += 1 + + +class SequenceOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """The model output associated with a sequence. + + Args: + parent_seq_id: The ID of the parent sequence (for forking in beam + search). + output_token: The output token ID. + logprobs: The logprobs of the output token. + (Token id -> logP(x_i+1 | x_0, ..., x_i)) + """ + parent_seq_id: int + output_token: int + logprobs: Dict[int, Logprob] + + def __repr__(self) -> str: + return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, " + f"output_token={self.output_token}, " + f"logprobs={self.logprobs})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SequenceOutput): + raise NotImplementedError() + equal = (self.parent_seq_id == other.parent_seq_id + and self.output_token == other.output_token) + log_probs_equal = other.logprobs == self.logprobs + return equal and log_probs_equal + + +class SequenceGroupOutput(ABC): + """The base class for model outputs associated with a sequence group.""" + + @abstractmethod + def __repr__(self) -> str: + pass + + @abstractmethod + def __eq__(self, other: object) -> bool: + pass + + +class CompletionSequenceGroupOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + __metaclass__ = SequenceGroupOutput + """The model output associated with a completion sequence group.""" + samples: List[SequenceOutput] + # Prompt logprob for each prompt query token. + prompt_logprobs: Optional[PromptLogprobs] + + def __repr__(self) -> str: + return (f"CompletionSequenceGroupOutput(samples={self.samples}, " + f"prompt_logprobs={self.prompt_logprobs})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CompletionSequenceGroupOutput): + raise NotImplementedError() + return (self.samples == other.samples + and self.prompt_logprobs == other.prompt_logprobs) + + +class EmbeddingSequenceGroupOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True, # type: ignore[call-arg] +): + """The model output associated with an embedding sequence group.""" + __metaclass__ = SequenceGroupOutput + embeddings: List[int] + + def __repr__(self) -> str: + return (f"EmbeddingSequenceGroupOutput(" + f"embeddings_shape={len(self.embeddings)})") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, EmbeddingSequenceGroupOutput): + raise NotImplementedError() + return self.embeddings == other.embeddings + + +# cannot use msgspec.Struct here because Dynamo does not support it +@dataclass +class IntermediateTensors: + """For all pipeline stages except the last, we need to return the hidden + states and residuals to be sent to the next stage. This data structure + contains the hidden states and residuals for a request. + """ + + tensors: Dict[str, torch.Tensor] + + def __getitem__(self, key: Union[str, slice]): + if isinstance(key, str): + return self.tensors[key] + elif isinstance(key, slice): + return self.__class__({k: v[key] for k, v in self.tensors.items()}) + + def __setitem__(self, key: str, value): + self.tensors[key] = value + + def __len__(self): + return len(self.tensors) + + def __eq__(self, other: object): + return isinstance(other, self.__class__) and self + + def __repr__(self) -> str: + return f"IntermediateTensors(tensors={self.tensors})" + + +class PoolerOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """The output from a pooling operation in the embedding model.""" + outputs: List[EmbeddingSequenceGroupOutput] + + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None + + def __getitem__(self, idx: int): + return self.outputs[idx] + + def __setitem__(self, idx: int, value): + self.outputs[idx] = value + + def __len__(self): + return len(self.outputs) + + def __eq__(self, other: object): + return isinstance(other, + self.__class__) and self.outputs == other.outputs + + +def get_all_seq_ids( + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]: + """Given a list of SequenceGroupMetadata, create a list of all + sequence ids. + """ + return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data] + + +def get_all_seq_ids_and_request_ids( + seq_group_metadata_list: List[SequenceGroupMetadata] +) -> Tuple[List[int], Dict[str, Set[int]]]: + """Given a list of SequenceGroupMetadata, create a list of all + sequence ids. + """ + seq_ids: List[int] = [] + request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set) + for sg in seq_group_metadata_list: + for seq_id in sg.seq_data: + seq_ids.append(seq_id) + request_id_seq_ids_mapping[sg.request_id].add(seq_id) + return seq_ids, request_id_seq_ids_mapping + + +class HiddenStates(msgspec.Struct, array_like=True, + omit_defaults=True): # type: ignore[call-arg] + """Hidden states corresponding to in-progress sequences. + Used in speculative decoding to pass hidden states from + the target model to the proposer model. + + seq_ids are the sequence ids of each entry of the batch + dimension of the hidden_states tensor""" + # Scorer hidden states. For prefill step, it is used for hidden states of + # all tokens, whereas for decode step, it use used for last accepted tokens. + hidden_states: torch.Tensor + # The sequence group metadata list. Only needed for decode step. + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None + # Scorer hidden states of the 2nd last token proposed by the proposer ( + # irrespective of whether it was accepted or not). Only used for cases when + # last proposed token is accepted (i.e., in case of bonus tokens). For the + # case of no bonus tokens, these are ignored. + second_last_token_hidden_states: Optional[torch.Tensor] = None + + _seq_ids: List[int] = msgspec.field(default_factory=list) + + def __post_init__(self): + if self.seq_group_metadata_list is not None: + assert len(self.seq_group_metadata_list) == len(self.hidden_states) + self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list) + + @property + def seq_ids(self) -> List[int]: + return self._seq_ids + + def update(self, + hidden_states: torch.Tensor, + seq_group_metadata_list: List[SequenceGroupMetadata], + second_last_token_hidden_states: Optional[torch.Tensor] = None): + """Update hidden states from target model invocation. Only used for + decode steps""" + assert len(seq_group_metadata_list) == len(hidden_states) + self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list)) + self.hidden_states = torch.cat([self.hidden_states, hidden_states]) + + if self.second_last_token_hidden_states is not None: + # Adding dummy hidden_states to this to maintain same shape + self.second_last_token_hidden_states = torch.cat([ + self.second_last_token_hidden_states, + torch.zeros_like(hidden_states) + if second_last_token_hidden_states is None else + second_last_token_hidden_states + ]) + + def prune(self, + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None: + """Prune to provided list of sequence ids. Only used for decode steps. + """ + # Currently this prunes all seq_ids not present in + # seq_group_metadata_list which might cause problems where a sequence + # may be "paused" then "resumed" later. This should only prune sequences + # which are confirmed to be aborted. + seq_ids = get_all_seq_ids(seq_group_metadata_list) + if seq_ids != self._seq_ids: + # Batch contents changed - prune removed sequences. + index = [self._seq_ids.index(seq_id) for seq_id in seq_ids] + self.hidden_states = self.hidden_states[index] + if self.second_last_token_hidden_states is not None: + self.second_last_token_hidden_states = self\ + .second_last_token_hidden_states[index] + self._seq_ids = seq_ids + + def expand_with_bonus_tokens( + self, seq_with_bonus_token_in_last_step: set) -> None: + """Expand hidden states for sequences with bonus tokens. This is in + alignment with `MultiStepWorker._expand_execute_model_request`.""" + if self.second_last_token_hidden_states is None \ + or not seq_with_bonus_token_in_last_step: + return + + index = [] + for seq_id in self._seq_ids: + i = self._seq_ids.index(seq_id) + if seq_id in seq_with_bonus_token_in_last_step: + index.append(i + len(self._seq_ids)) + index.append(i) + + self.hidden_states = torch.cat( + [self.hidden_states, self.second_last_token_hidden_states])[index] + + +class ExecuteModelRequest( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True): # type: ignore[call-arg] + """The model execution request, containing CPU metadata only. The LLM + engine should create an instance of this class for each request batch.""" + # The sequence group metadata list. + seq_group_metadata_list: List[Union[SequenceGroupMetadata, + SequenceGroupMetadataDelta]] + # Blocks to swap in. List of CPU -> GPU block number. + blocks_to_swap_in: List[Tuple[int, + int]] = msgspec.field(default_factory=list) + # Blocks to swap out. List of GPU -> CPU block number. + blocks_to_swap_out: List[Tuple[int, + int]] = msgspec.field(default_factory=list) + # Blocks to copy. Source to dest block. + blocks_to_copy: List[Tuple[int, int]] = msgspec.field(default_factory=list) + # Virtual engine ID for pipeline parallel. + virtual_engine: int = 0 + # The number of slots for lookahead decoding. + num_lookahead_slots: int = 0 + # The number of requests in the running queue. + running_queue_size: int = 0 + # Optional hidden states from prior step. + previous_hidden_states: Optional[HiddenStates] = None + # The number of forward steps to run. + num_steps: int = 1 + # Finished request ids since last step. + finished_requests_ids: List[str] = msgspec.field(default_factory=list) + # The last sampled token ids for multi step decoding. + last_sampled_token_ids: Optional[torch.Tensor] = None + # Async callback + async_callback: Optional[Callable] = None + + @property + def is_first_multi_step(self) -> bool: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + first_seq_group = self.seq_group_metadata_list[0] + assert first_seq_group.state is not None + return first_seq_group.state.current_step == 0 + + @property + def is_last_step(self) -> bool: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + first_seq_group = self.seq_group_metadata_list[0] + assert first_seq_group.state is not None + return first_seq_group.state.remaining_steps == 1 + + @property + def current_step(self) -> int: + # TODO(will) make this be able to handle batches with variable number of + # steps + assert len(self.seq_group_metadata_list) > 0 + state = self.seq_group_metadata_list[0].state + assert state is not None + return state.current_step + + def clone( + self, seq_group_metadata_list: List[Union[SequenceGroupMetadata, + SequenceGroupMetadataDelta]] + ) -> "ExecuteModelRequest": + """Clone the request with a new sequence group metadata list.""" + return ExecuteModelRequest( + seq_group_metadata_list=seq_group_metadata_list, + blocks_to_swap_in=self.blocks_to_swap_in.copy(), + blocks_to_swap_out=self.blocks_to_swap_out.copy(), + blocks_to_copy=self.blocks_to_copy.copy(), + virtual_engine=self.virtual_engine, + num_lookahead_slots=self.num_lookahead_slots, + running_queue_size=self.running_queue_size, + previous_hidden_states=self.previous_hidden_states, + num_steps=self.num_steps, + finished_requests_ids=self.finished_requests_ids, + last_sampled_token_ids=self.last_sampled_token_ids.clone() + if self.last_sampled_token_ids is not None else None, + async_callback=self.async_callback) diff --git a/vllm/spec_decode/__init__.py b/vllm/spec_decode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/spec_decode/__pycache__/__init__.cpython-310.pyc b/vllm/spec_decode/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bccb28a17a771b00f447f404e333e1b30ca0a290 GIT binary patch literal 157 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_esMu+a(qf^a(+sxetdjpUS>&r ayk0@&Ee@O9{FKt1R6CH##Y{kgg#iFcC?yI2 literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/batch_expansion.cpython-310.pyc b/vllm/spec_decode/__pycache__/batch_expansion.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c3fbc242f674e42b75bf380beb6dfee94156edf GIT binary patch literal 12452 zcmd5?&6C{5bq6pQ3ivNRM=cGzfIpmgut6Ztvb8{80Ii!3~l`Hz-=J#F$n3*NlPIAg@ z0Sk0DUcc^sAHVmy!>K7t!SCun{Bq~f9Yy(fdKv%ec=-;l=$E+pim!H*uHq;vJ!>7c zt2tV?Fou%%wv)o;AR=TUsDwkC|=ep;e^W6*11yu=Na9+f9 z5!XvW?Xlw6w-n#gA_8oWQ?t9m7Z`^S2zw_OjcioK}=_(iJukQ7C-@X0=_x=y=-Aos5J_?$H zIJgt~LFe7z!5~2A^-X%{HG>;o?4?Wj+bt0e`gelZ^Qm}dJBVGc({ZEVf!p?@Mk!s~ z?1Naxi`)CbJrVZ9$m>Mu+*s*mGZcY%wkX1sSiSxDkt@93c933(FwofD^5W)>yLI69 zGiq)JY|&hf|CaFb9bD0Ws)`aQj_NCp=BrLA(0%Q(>X(ADuLp)-4k|6fH+=1<`eW5G z{Yp>`taFNQ;(s-$om22kPj&n+2UGH?xT9rIZ|UchmdPuuJJWtWsQXjF3{#)R^Gx>j zZ1#T+WkhLqcdoeR=?PSb(yTx4FZ@(@778lAtF_@T`b%iNSa80Kesa!Nf~8=YIp!FVZf*H5_%EXMYH-fKi1sf9=Y2a^_FtCt{a1nu7>U)Y`=9ea z&-3>JT7Cgzr*WUbbunfde?D##<2L*+`d?xyFM`S?^l_d!=Ml7Yzu|wGYcHbqD;Wo6 z{LjXU7Hj@xZt;@84*j~4uD(H{a`RE&>qXF;`(gj8lr`_1rYIZGo1h!?;>h;wXe6X| z#PVpzJ6>!Lq9C$a;_M)&W5=NlIk#;5AcI@pRvU8JM#Hc*K`0LQf*zw;u@RtpnX>(wPE+V$Zj`gHGJW2yAGjIIy8t;34cqObs1* zb}x9Cu}*yDAFtVg*W9rmw&NWdH~4sl#=3ofr;T1Zo$#Tg*9wJw=X=*LciMX+_S?OQ z(fUC%ZyV#m^~6J@uQsJnVnPD+01N$E%W&=^*R=CF#I@~(lp zN@Du@6V)$2QHNG+CKfE5v8P2}dt313iRDAbhE`IEuH&m}VkMeieWDF(y;;1| zH|XIp+9g_2O)3ddt@+lI%CMHy&|`f#HJl#K#Pwt4 zlAb`KCYXln%cPZ<+Gx=b#w|>yjF9-=HuUgeIOxE#K57PFgP84lkK|NkE!L>CSMb&g zN94jDu>_&>2TgoUn#z3UjF0hgU2I3M6-0=$XTTB(}so1Cm^yP zVzKP@b`K;kEy%#ah4C37Y_5 zuj3&n2#~gd$#S;|%DLSP%ehpauH*~q^0X7ClNIwt1!P#3VJ0quE&N--zu(Y&9$vsG zf@X|)m!9S_3~%3tcYwMGYJquMl5^umgk%HZX0L28Y?`a;ci9Z6_oNUr}Mo* z*Ugquau=nuy|CxNY;Ktmj6ULEPp!u3j^(g4Hcm<>|+B9J=;pTjy7)rr2ucezg zY&Y6#_apH|`rr$=r5E#sdA@Ln&r?fU5XZ_{5B$_3zPSg?F+FE9%L`FV^c>YpO!i1^ z#0%8>44aRUvn9Twd$?JvmTHtV)i}`&UDfcXe`cQGo!|5m<7Ba9s=B6YRA)Zb4QNU6 zr>n+))veLH@hPtqPm!=ezDbIo|D>K}?wUF$1CW#+Yd^x@P#>0Yhcku?RTCYqQ`yyG zIBU3H74BGjQW~1Eo|tgs)x%haOQm1?4}4H$M8NQEckGTtf923xf|N?&{d|P1u=;$ zc_Bwu@*_2^kged@PEOn!W+h!16BV)%ZA7jOLr@f!UT`HFg|bSp#A8h_=9Lo!_q%eg~%AeBTR(fu*6rXj7-gF(VQxa7g&=5>`WDwfXhgp#n1A!B+n zpHzHxMLE`nkg;7IuFKrjexX7(%Rg7&RsLFmELvz+-PMi(7n7Q=4OH<~QX6Yq+tvKi zIVEdf0|fv=NwfK6?<016dQUpPg5F|p8k5t?B7<;QAG;nqulw^i=xFVIsZr-o{q%3}w)Q>D*Jyw&2r0gSf|C!d(t|`e>R=Sv%qE16y33bz1-O_XF zmbvd=0k~qc(LcSd02I$;?a`m?VT_g`ehnR6NDR=eCX4*vN|yM)r1k5hmMru80_vBN zW!C44_X-y9&CF@BRnp(IFW1;KY&>XHK}q`hF8E~lodVX zsypGo0nLDmfv!h=!7Pm|iFg^pe)l&g_yGgx)SF|q@Qu6gbq4T>?OqE|;yY;7sHIEr z>XiJ+{e6ZJ*!Gm89j_lqBT<1>q_?!@?gQv-xlGrD`Ey%?PRFUz(_tXOC@r@-p%;rU zqXVZp=tY>N;4o;cNb@pMUWRnf(cpA88+YMN(|T4SJ>E?AR`|TdtYE_JIp+#Xb()^D zbh?TOJFBOQ^B$ZRPuEO#wUA3s=d{(xqyPQE+uQ+$)k zPZ`v!DEk{+Q42Sn4XX8-^wssUSvJ*Kt*+{-c~aL*4F(nl*Tma2d-YWf&+zV6Nmu7l zs+V-Mnm#d0i}J#^rnZQ3YA65d+V6IH|8#~kA-P90R{|$ z18h9>!~ghyZaI*B@uFzZMQ{cw7z?tH4I0@Aj=1_F*l&k}2)HGeqtogy{undfFa_zc zpe0D>WE9|!;tg?w$`>ZfSrNod`m8}W(&9!{s%{Y?2v%b9w&E7u-lE$E-QK3#DVcp0 zWxvK15i|7}gD~WYsgZ1wq#Aev5P)os0kqA#@3~0$JxCmoL*m;zexU__@ki1Lz7z!-2yhjEgKRg5c2j_I*MJl_d_;OtB0eTP3Z2)&r zGl&+zz7<>ad_#Hv0;CM#0pO^$r-?U`3fh~)sib;T1L&^r0g3t%w zN~(~+vbUh+1_PQqX(nu*13OO%$qcET>pxqGJk%^ys0capW|^7T%wXuP@eRW@r1Aq+ za|%cc8f=e9ZP}vqkQJRw)s7jHyB>n)Q8pCM#Xdx44kG)zxv65=&A{{-UCTkqhBEfG z-wGcF*o$FS$+~&IZ+rGu*c%X6*kQm{1oDl{YrDmv_34qoSvH$NA2}?U$;J0A2L1B; zu6MwBXa>+Qo0L^1JIrj13=YQxtwFCz$?Gh($LK~$@#oGlc*OYkMsVHp_q|^1Z3FSk zjghgTj5<5Kvd@+5DZxG&n<{;M9T=y^aTSj?l_1W0rYfYEEoa z@gKP}V4PB2omK0UCp$szjPqp%-}O(BX{#Q-_InL#&eJma9D z9~s9AY-9~dkLpncmwAk{BCD{nuz@edw1;NlLW(1i2|U=#b4nW+9mxRwNri3X7UuBH zJeQX3naWYv&(*mx1NH2t3r)kMbCG)831%uy)` z1DQw{dmgZm^my75{21ilCQGo62W!rNu4}Laun1D)pXyKR^+muaL^-VWmZqIh3*)zk z*MG0(|2{jJPET&`G-)RDA8nr%+0Vbg3jhn0owmXgbyaC;L(Z!sxJJG*3(TorKURve zG9VN3o5ec?-^#AxtH^9#rKh3>xzrwx0D%skXiFWzcNHNxvh<17GC3pv!%a#yu?x<$ zTmk|)`;cGA`M5`;BpI?Om&eKqp_ku`g#2zpyL-1$5#Pl?oiYL9j<}8Iw9FZ*^ zDI~hb;@XaNZX#-qoRy-{WQF(!aWhxc z=E)sro~cY`k(P^Q6XmBR?hTMU%q32n<2{%Y{lwA@M_TCyZ>%hL*X*}2eEx!8Tr zu%|S+oFMkc+zw;Kc7DW(k2^qaF{=E;6CW}Js>M#H0IxWt6Wdqqukg_W;6fyK3br6n z6r58O{p=|Rw&=LAod%>UGr@)?bj*5M!bt}4hj=PxrTyj~IOVQNTlfyU97T@QGD9_8Clmx-_w9Lm{Ir>53 zhq`xW5Xr;`V{pVnsbs20Pt4i-c=|`;j1Y>xU})2-Md@?x@YVkjtiy zS}vV$8+7Qo?z)sQ%W5jF>xWI(6_i*KQ*@i7+XCIbMK{ts20&@$KH)bH_36vt|j(qzT$Y5R?TKw3ca28r#j_!G_QIA3@O#4tU_GM!N~{yc7Cq zSyofzAbw0Y@Vn1*B0w5NbSpJ{siQ|%UZexF)+ zGj#3K^JVK3?G*}>_@Z{WkP7~cF8XF#r_`~9?{X?di9JrqAWAP%8=|d&wo%qTF%0Wt z?HlA8(F0K@%Ba&&ry2E+wJTX2>Zdgd*#O<1YL@`jP-3X{6RjqnEp7IxpwTs6Vd#q3 z8ZIsl!%oEcIccJe_@EDa!ILbPi)Wt~XdfRtIvr3FG^Nff^tjGo2z%XhNIVql9q%9z z(RzmPcs-pdZkYQoJ|V@xS;`25X-37fMD>DfOjt&qfLe!y1fu2P0FRU>^585~&eAh& z2XVV2<&RDqhymRW==P9qG2Q;0ZZr|kTmRI&CZ+pz=#CPRD%F))brpL|b?h|R>SAdY MYeTr$&=yPo4F>h(rvLx| literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/draft_model_runner.cpython-310.pyc b/vllm/spec_decode/__pycache__/draft_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50e0db840ea07ac1ce3a332b0c386f0906890b74 GIT binary patch literal 7875 zcmbtZ%X1vZd7qh`9n3zlcn}XhN2DlnOCo5>POLDJB8C)HR3wn*!-^&qPX^l!U>5UO zcFzLF)}l&fD&>=soN~&6blC?Va!6H9IV8E{>OWwv$wif;9Y>a8A-}I@W&u!eRjRUE z)7$Uv`JTV;+oaX1Yxvyw`+wg0#qVj_zfm` z3Vti08CBh?>N7(tnsTRfZFF{IW2_q5QO&LC+DBSEV`{w48ym*n>$*l?j9XzdYSH+| zTCDHVthvdo_jJu|W2Y*gW>Zgf-u{un47dGf?On{oT087SGww_@>&`}V?p!qQ&PS)* zQ_+IE5G}fk(UQ9qExXI=oLYD~I^&*E^?GGUK7vo%x(-)-^KogFvR4eQ%Hk zN$iJfT|JwVJoPpbvFi)w^^!Q{KTOfxdgw=kkc)f6bTFijh1JlPTes4b$F%rEp8CvB zeT^62OXcZ9!@I#I%S=*R)@4MRT-cDTcMFJVEv>GXy|V9RjNNn#XoI2a@% zmA4)Eghtb&AH^y>-dOj=vs+LAqCTw(tAG55vPr_eBy7(p$_Qq+Aq}rXS*9HUyFK z4v?Y|#6jwX$>t^(Xm&mvhG~G){P1oJGD@&u?%f}P3Mp52c)*_wIXE(R7jzd9XMvyc zM?98^pt&!rNowh8BHp|I_@f}*Qj7uRN)BZC>pU37F&C?kdc^9ZpVLJ{{s$^tb9JV< z2GiY2&8SssCOzzBR^=wMc$H0Yi`jgN)ws>-yv7=&4$B&A;TT4gQSYEW zy{v64{aD-3*$kWgwB>f#9Gm}Cb7$BoKFb#P99!hG8x~t)%b%L=JUh)#Z5ZqfJB!u= zUwpRoT*E!g+BtSvtvJt@`C|VxPPl;G^O}R&8T7ow&tkVaX3Q;XIJe0zvP+*qb8gi@$M2O4m)e6m9UUa33yJPhI< zchW7$@Zpf-hr9lsba>8OXPY)8m2w8Ylw3N!B$mW?$UdYzP1IJK94rIJG5x4+qzQyu zIzjCCjs&Meer$N6Ix^*h*VIP490v?T=4|rR5dm|O4ZNOAo!j>xUxVB@axd;}i6jn2 zP`?BlL$gSIomBX-+~7iOlk9M@#eF87^*v{s^8qPnq;xkR`BlpjWf2_M0g_T^twRSu zTNF*wAmS@^wcDe6ckaoXs(NF^xjoVIQpIzHSE_vtsvW=P_>7eXKrti%l?ADN>=ha> za~{0D;yla^h4bjX^R{!txteYT(h<<7K!DsbiO9ytU=xH~=RJQYxdSS(#EFv>Y=Ru5 z>T2MqAMZKq0=A;(%amR@h-JWd$xKgCpqrnZnf@?MR@bto>LrUwhG%+E7>u%`BTM5@ z_OzI?O=Yv$BfC|$8X=B9(hz>xXm9U$`Bawf4S3cGq{jrrNO-%6*ybWTHBiRMgFPne zR4$wsJROSI+X@)tu_sdqn#|^tqvo!d+iM*$7*ucLso;^{K{0L!D}`%)s{cq&^`~0j z80%wWTp63=>ew1jjqP!5Tpu^aO=f)7Bo_r8tgMaRKIsAuxI`Q%gaB$tnsw~>? zD~Ye~gkf}@l-*Nm%&#jWl8cy^+oF}hp16WnAlgfM7jxRs4c$DjO!__PTmRMQ>`62% zU$TsRmI~OE?XD%R;sV)>>W1=rQGskSn=XVa$V^=go>~MrQ4g}(gHFYYbWyWckPxna`U6ia@K$h zre1ECGb>lS9SQ7r!!C{8RFRmIAJ-TM{Ac(4^LHTov8lW9X;E-Ciu{p!Az zT4O7ndTPjvs?U06!xB`-TE9lt;5pzm_O0(vjkRYD*ahmVpik>J(ci+;hP|-Je)KDN zrtx&}%;2dr9oD11Z;$PMy+6w;5TnO*3ae0$Hgaqtf}aoi=fP=^VT4d z0XbEfwUnzm6d)242(rxTBs_b0y%{F3?vXzr@0zt#i~JUFpv-0?LX2b%{LtG5fY_Ac z1{f#4h2qhuvG(pq&fT^9k00H1Msp4v6})nB9p^3QvK%dxos(PT{nL(9-iA8ZXvT34 zFO_$7;0nptmM?H~v`B!bT<7I3|0W$XuLi;-EnnoYcRfsc+g|Fghg?!`_q4jaxJIw> z78P%!C=~8cpP3YIQhz-F~N8jv4Ycu0* z;{N5$h(23bE5DDTUaeQ^x>af7Y3sIOEB|5Z`#`5LrUG?OanFv_6cJL zWSg+Zq;49c%O_f?7jKcx6gar_Yy3$M;Cu2R(?o4$j_#n6YSLu-Loy%aRqTBOCdd%e z02Ana0)x`8JTspop5LyBWi?alSH=0FW;GS$82#rRPGM9K0|5pbIX*A1ki&d}s!GMK z5w)D29|Ija;5*CV9f|cJQZ3*lBlOO49d~@eb4*K9=>u$gk0RIHpG&7im1+<`RSh1Z z++)MvUUAkE=Yx9>ZXN6Xj_NKP*l}1lnYTKsA8y*6$`*jYlihC!gu6(hshsFUue}gi z0Hu^rt|Q<&vw>Q?*pGOQtq4(dE4j43kGenxrf_!3?J1A9Si*UAbZ)+gwm-vMbZa(gr5o} znc@|+ENaDX&>Ha$B)UrTM%nQRJf7(M+qv$svLtKQ8NJGfs>-LE!=)?mi8M=rS4pTYbo z@sFs__UD)lJZIw#=FvX23>SztoILF>{vlm>wm7b&3;iWP|4Kf4N!tWsL>&(MFO6mG zC)8TTbDGsZt85z2_5FH`_d-d4;SBAzU*FXB8|m3G=m)L$_X#GLgMSOO{Z?_-FIZz- z5A;|^UpqZVH+`!AHS%EN8l!v9;yI6C1pQ-^HUF-fUg$gH7EUVJi^ zKSB=BM1tTc07-1$(Bv1ppc#EX#zQxH_7Z{^(BrwWKRs?dyGVWgOJf2l)2#g&-T?EO zM8_pZ~${&1^{X>BtWoAfATNm24&m)285|d^B~$G6iLGOVoG2a$sB~dJARKx z$beDIuAIvxw4UfXVrdH!2g;daxYlmj52+7GNQSvV; z@&HnqzHivGyDHc(BYapqC&!`4D@<TjI?lN`oNjk_YhXe%jlj5FVKe%4HyQ9`Hj(=CvAJe zDDGtSKPy3j-S3q({%BOehdZIf;5NHpC90(kW}|_ zo;kQot5i0DJf4DyIbbB{=yoVU?)NBjhs>_^Daa-$rGlsUI=ohqydvN%cZmVD7rY**3-rWjBF* znuhVU!rfolUt4v{ze}iW(sFCc8g(yFE{vd_=Vf(|Kr;TwK)vaCParJC z$du=?r002pk_Upaf`U+!pnQU`sUQF)2+xZKimU=)nVIWJ5{hZ+q-3H>W(kVZ#3?GO zRFH^>IVwJ+;!jby?I1=bnp3Ptd>8Pdnrl5$iDog6Mpgm76MsRqR*`w@@i4^w4%ytr zs32r27Ew?hO{OSp4YWN~=FZGt8sLXfF@B4H=YNc<@s(kL)>FoBEZg|Psu}-n)Qw+T z4dcIzrtvGQ)&8Y@$(SKNUHNS7r0XVPe7Si`s+<;oqJIY9TomWtc3m^7jmW0F-68=B1}mBuiVNcO|D`C zk8F#IqO-XV6^GZ7bTt{qEYFHI?nd-yn`*dM)R`x2C6Zp`9ocURHML;to zA<4yusrVBrNJNgvj*=o3)!1)h7ih%W6nRLB p(mR?BS%4}--8=Lph5lsLR3fFs!v4fKc&*xm`qMa+AEscw^1n2((3t=L literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/interfaces.cpython-310.pyc b/vllm/spec_decode/__pycache__/interfaces.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a5a06525556a5686f05554ff96066a10e6ff2dc GIT binary patch literal 2896 zcmbVOOOG2x5T5QCkH_Qju6H+(2MSDpgcuT=oDkA3WZt*Jjo><&_Odpe|VI(!~gTyzIC0J4C0gMAd9^26}j2nCj#!F`0LIYh_3OtjFU803f6dE56eN3$EmhAMcIYvwL9O7 zWK@c;3NF%b#G{c=C5Zj+3i(jTo3RoJp1;^^7+i<0ZUYfkM7R~vZ^1W_Eu8I&a5-HE z`wVB#IA%8Wsv;8yGeWk*yQLpOgY$|IUGq?d{bD!F;(RYu z^INA!h&bto3LTLrBJ9ViZMN&x>0Qj+#RgpIrI`$vp3;rkI0bD!(?MRasYoRR+%;{; zSE1j6u0kN3>|5K`&wwcSl5Gl{4V(>M=V=vuROyZ{pX8*=(+lQ$ZH!m0%v^&C^wh?< zxgbLAHlCAK%aI6wSqGvksvi%941j+|w4+42^~{p1FnoTqQ)a`?a4@`D#`1wEuYQz` z`q{N?x04Qfo#7rpE${54Y1To9GAeToG zcKwgkfTTHV9Z8T11iz+N7QC~W6oCKn;J>n??2AyDwPCS!0@f#ak-!Q*$RA*4O?TiB z94yiU@K+3oA}=i-7*5w_Nzm1e{(y6lGX>5S$6;OpR~VR|17-sAZE^r~YB>8T+URu0 zXO0<(=tlGgS0?$1X}2K}JuO?l2_LzPNG6dKQX;sBXF3;gu&2JQ^N%!E;r;wqVXZQxk>MD`HaGF;zqg+4o3agu;d{ zWV?g1&w52ZQq!`LGk7CP2%;EjR|+V&d18Ye(}p}&Eez$K_$-#z59Ar?LakWq7UkV+ zn2HQ)vfy{5EF^?y1s2J-k-UTCaAfo<)DfnzTtJ8?_5KDdQxu5fjW3;GQTzCoZmV`Q zTBxW*S4tO%wxJA}WR)$H&9yL`0m%iJ_!PQAk!Mj#3G|n4h5kj|=5vYj3G_p47OZs; zPlE-avLRFsgsCzOQ{;*&cZJ8RqQ*VpgB)LZ3}2nsA$-B{6tQ|IxXIRO(lT+Vx@nshFoGh@0|jg)h=LYaG+3-SLusXb zX=Z7e1O>E_ft;FSf!-vb7a#gB^weYjg1z>XbJ0s0$Nhb?q)1tAy3!11=ke|L{C(eK z?RL|^bLX#r+yBRkVf>R`=ARl~eu<|3gpM1W#fDG6X3P@PHxtXZ65F?vnqOnpy5l=o zx8i!@`mUa{<3`f-n|fS}7m}9WO4@!~zdP|_a?QV{$MtwAS@xIp*o{|`Rex2F8}apI z&0k~22L^BQg+~To5X)li)b`go+c7$=FTtC!-LXsiUZjfB{V*@0EDht*+7U%r`(%*E zqQlCioiNGcDBZg+ijapzh`F|&6Y|4Bkq?TFS+2hKQ1k|cxSw$me=Lp$0uQR@U`Oo3Hh4zRou=v+=oM8~!E?wOO|A3qDX`CGlvZ`t|ym z^s0q!)5Ftab9%5aw$ZMksqdgGj8o%)jag)j&A+fO7&FEO`N@v)m~F#?H8seEEZrc8 z(x?c6j#b)9#Qo9^<(?{?!(;k=%*sX(^x{ydAb4zi_1;#Id z$?;Yk?QZ46Vn0i_9>j67rC{=aiyj#$pLk#p zi?q{_OQ2LPXeDs*AnFO(qM7=!2oI&`mo@BvI4qrN(QiJ;df{%MqLJ|1L@uEaAX?B= zMOw(mOadkhxklXvb(_@PKu1k2qH8kKbeXOHT3?SgFDYP-ohv#ZJ^TvoU!**eO_I^8 z&%Agsy7+2&Q;7mb?!^Nxg1vk|T<)r%5UI-K%S6aTXR}e~5)0pU>VNViUm^y+ou5~U zvf9rEF(>6k{b8^th2X;pV^NyNVNbkDl)i^<^yU@5{{QK6SyTu$v5yo-Wm6Hd32m!a zlsF0^u7cyJ*bjEIbO4QHha!ch0=PRkEJR*9QWOK3_DI#!hf;xyMmA6!-PkxY#_XIz zj2*BO_}rN}w$It`%#+#)c_R5HJE)zrlX~Ht)yMX@Hl}%()(#ru`nhvbjhhDx*x5R0 zk6n1LdD0j+xY;-1zgBo3hW~hwdd0r*w9^qHAWFHHBRst#^Y%n4WLOAqCaBjRKmdVN z$fhdbAn^ED%Y{=Q79$;aS$}d$dYMM~+g_Fm4;bOa;ZVriUKnTTo=5ufqLfFyu*jqa zBBF=m$NK`*Bt9j3`(EW<*!c`uFNLXz*eN{kz zs$AAzM-mK@Nfvy+vskVLGK`c6qJA()fuDIM3&G0`@})h>c^A6_(rLL4j~00-aUw5dp(|Yy2Y9B+CO$8Rlktp0`9o|2 z_-P?3^Qa1bW}vfp*?b+GQ1qSVHOI6~leteF3)uJ6Hk%A6_>F7Z%*3Db#I229xE%00 zYNw8%dF&BlO&^L!QjU(tFEAK0m%q3lw(g|1w=>YVaTTKx$+zcpn;e|e#orGXwjZ;xc}{r9Mt}ZDW_-{C+D)s=ELGkfYRuFNQ^Y@%=k10CFoVEt zQk$x-_X#i$`rQS{Wc>>qxuEi3ik2Gursq6B3Ksw-7y)>-JM^BdCFX%eeis~<2u9=) zDuk-SfT~_=F7R~B>6p0)9Qg|z+(%R7(oTb7Wc1P{)==ku#&`{Sc1HdAuX{Wb%1g7t8;Zh< zD4!-GExZXD>LRrl_Ml;P9Of6w&5%b>{*!?K*SjK2V2}`SK2zGL`~2@rDOXfygf7(} zQ<)WM?iRW~z+XOXoj^WFrepmSH-kg#H>ehma6h1mQ2rPjw^8_72s<4QrF|%d3VFfo z{z(2DtIs%p*|eFRPcFiZFii~3e%y#ybC&qRI>!6Xe1ba<(VlUv%ff->t*?{T;0x!Ur&uD_;*ybBVSs!Ow8m8{Mavu46MUCXxIC%|+9^RGv@=4E-l3OYCF){%yZ z2!gU1P=Pi;RTh-3AUGO?@noeQ1U%~nfxay0(ph@=z9j3GHC+q%i&5GalCGS(%#&1> zl~$A%lAuGG$T}_9yIB^?JM?Z**PxEzOp+aSq4DE_EC{5ttbDLD?2K56LNCH3wCjZmvk}rU%axnz_oq? zheGPBmcODmO>T$Pk8I<#R?3|AFF>+%?^aIpHYHZIj?OhUk%U@EH%qKF`CGUC3)9Br AUH||9 literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/metrics.cpython-310.pyc b/vllm/spec_decode/__pycache__/metrics.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9518ee31e0843c7940171a0a2680b4759bf8cbdb GIT binary patch literal 5017 zcma)ATW{RP73T1^TxxZ(e0Q9LY2(ye+ftM?O_8{19Jvipv}#bx2+9|N;>_-H#mjcc zm9oOBC~Bi`LErl%J?5c6`v>wv_}YgeuYD>4*zR|RyQ_<(rd;rlJTpAQIrE+GoY{PS z-qmod{q5`C+N+xOPkK0hH1O~-&SD3GYn&yTPggx*sqX8k;Tx>ho4%>qE#Jb`NbI!X zH&mOMIBC;wGVN21TikxCaa)*umpAZpzO?)~p?5XW>gd-se;$3B+(jSviS93O_DE~b zeS@*JjkaDn>tT|FTZyQg-wet)&%&h5s>_cCB6=vIoQqGwLOcr7K_Vm?uEs?$h_fIS zX)Z@WxD&?e?TG9hKMXv4jI;OzLTEnYny){`Y<)wR!s7aUO&QIy%AW8F z%**3yV&1N@vf_F;s>@u4oiYfcNDRs_i$oy9QdAd;QBjIC5S>mO#UhJFRjck<=G!7G zs)flA!Q^?dwjlUSkqy&;hMg!j+KtKyf-Fo$5L9juq&XiZbl(bs-w(rN($WY5o<~qy zog@#-YB31HEX&J~7O8;Vh_mu5Oga(!AROL)uJ_9f^ap{bZp0R4E&lcL$z+Ub$>pGQv)tK z;V>x+-wmY6Rc(SUXynhSgxngu*9_Luq>Iy%3|FF~`c-n?9E z(K?4wuJEg9xhO92Yp_~dZbE;Z-*{?1Vg6;V19CU3%MXfC7EN%ro+pWj%3SXKVO@c3 zw--k}FUkk8C_E|pXvEU2mZL!&g~`1mD0xL4KT}vL#LXbd#ZDXvugtym&mMbu=7nCC zXZI!!_BL~gQHoNEFs-q=@h^B2$EA;}i^o#eZ?IX~78^h2_i|8S4K1#whfRSTqu0Ic1-0S@lAVCUZajC z&9JhAdZHFWHQPq`U@1mH?J0x{SgMo6-Cn8mlBNsSkT-q1MCj(SGLt;(R<8wNw<|># z@g7jHo{r#GPqm+l_*YIfpP+|Z0f1323^-Hq6+TA+>DandHv%B4e5nV`uWeOkA(D=o zxx9{9$QvYXl9&a*yiC&ruf!RGGx!Ofiq{})^AfXl+rX)t%w^6CQzuE!3vBPTGXcIb zHB*_&v$hoFP-c-1gVTfV=~U>sf}0vWK#hK2sI>)B9%)~(jkYCUN4xwUL}f-p9{Tnp z1!w6&)z)R3n(vS}HJ>Flo92Hqzd>eGLP zo3YkM!W}Ysf$sKexI=ruFe&{ITtI-uiHRLD`R1g*R=oa8?N5O9fvF}88;>-34+b&i zH*mqSPS^S+6jW~nMK2#F{QNGG7O;9%!S@Upg~Uy5qxRn0)O8LV!AF7E5m|ZKm9MAa zXiJjfW-Yd0EC4U2zW3hQUj7bym67GUC)|DlF*G&hMz(Fp2GK3yr_ZjD zR^PV33dY{%Zx!ym-JU@1mR>Hwj3x4TTzD}!O{GGwcB40t;JKPZ!?9W znqR zQ@doyibjpBJ}_A7eS;8o$e5irxg zm(g+py0$Uj)D$ALvE8O^<~*TJv({{uHQtGSBj1k&IT{nSmM#g%^73j~_np6vpu_Q7%=%2Z&^< zz@r@C0UuFLC2J|_lo{=@=E$X#QEhqDZzeKv@NkYDbu9fw33fl@?OLLmQ(Z70> znx^SP1^H=+z6%p6&ccIOu$KW*3S*tw&uznD_6ysxU)=nUJJ+I+e)k;k)^mC(VW~xt zKZ50mxN~|maWg~jAMr5O541y|h?w&bG5}}R*xg6k?^}f8eUu1&dmk7+XpF5fJJk2> zvBlZ8DQ}@TuwK#j8<|-;Tz|s$8)I9wyfZe(Hd>rni&Hj17>%(*@2JwlCX9wZMR={l zL#3ETzL#%NWmazi3M!wA61#@FZsXp$fO{aj#RoI^5rDQo@IWyu-X``DM{uSJY?#38 z4%Kjk=nuVNfjtKbIt~EJG|sT^P}_(T=E&sHw&Dnk1b{7aP zhQL_Y!$y9h^g=0S5Sa4qF*4m$8MN>O?KcUiCr2ot$5t1+nz9#`UM~c+w*&}wiglnn z&+g7fpfa`aZckgW3Gx#3zT>U6YpmW`BR)o-*`{YJ&xSBh@p6QGg}rYmK`MG{n>E;| zw(&2P*!m_4JZjH_Z!G-pAE(!WkO8zo+zH@`SrvbpDeuGDKX4WV=_S`;j_H`^E%y1C zZR;EIDlD|G$a!4k0*OTuOC)IJC6Q%iZPIS6hWHOjdy&K{2|~Pl527-+@;p&>#!>!} zRK!anc1a+vT8g|Pq=8<}B~9bl)D-Gz42+gM3%LIV|I(sP|{IF&IW=?HR=iAJL#R nr?VzoIrr;-Pd*~h6tpjLv?U5@*3w(LYhd=21E00tTsHm-0>wj# literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/mlp_speculator_worker.cpython-310.pyc b/vllm/spec_decode/__pycache__/mlp_speculator_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e4f97dacce9983929396b73221477424f6c6c08 GIT binary patch literal 2933 zcmZuzOK%*<5uWauot>SXT`tLp5|bbv2oZucWD-8*fgrZyI0g)oHbemd8VMT1?OyIs z^I~_8XcDs*OFrs9kjEVS7xF{qkc&^g<`xH*Q`NH+$wag4Vn3>@>+w}}m!nbW!1w+) z|5&K8<9v&Q+g}d`A41d5fe1&C%*hEOIb}3=nVWmeBREg99_zt>H}ms8>)Sam3-SRQ z*nTey^C27N5sU2D&qn!}jcvc5t>qIoA+P=15XC^%~vieJ@o=F7~9~_j$-roKfs;o*a zRX6{~M9Ud!dHWl};6rG-0wNtogu|#H%$43{?+Sb-ec{T!@MIwS+m7hTfehtv+mX?n zZaecp_@e*PdqLPp1ab^(YunC>&pVmG7qQhs+f7;yz37=8bJFPn!W(iIG0XCt_T5oLY5wkY2#gyzq-rb(qHx)=fu7A zV6AsWmVxmv`;FK54bBI+b~0#!D}On#{qSVi^nf=y88x&S2=dBXhGyJ^FGxe5QT^M` zoUcd|3OYwry2$}?oF{b=n}zJ0hfY096!;_2IyPl|EDNa;BjX$M<9Q8cNmvDo!Q1F; z{GM2AVHXI?0OoNyU+q$HX~FwAE(;k~QpH(vF4f~W$;#rm3rJdsbcRuWBM-KpEhI=& zaFm)YVgtuP$kO6}#Kwh$2!gUlpp;4qjj$HL_eK=J$AFm52E{vVAY|}|g0u2CovmPl zmi!q`$+Iff0KNFkrp;q8E7c%#rDd!aNtWS>PgN~%oG|s+AT1zK5g+Xq!0LHg0z4L} z5P%PDz?YhB6m`yF4TPBvIl5i9+ClE~e3kr9r)%sUS4pZRPv^WYbX`@YGE%hn5rM~; z>TkF?`n=sju~njAht;CWZuTWGZbS_xZp8-iBqeAT~ETy4N6HtKBaqN@Mys9(VB_xrmhuXd}nde0;Z z?0xUodA-Qr&(C(V^k}y_H;b~^eV%3cE{2&4IfLr43t{It?kW!Ng1=Lpx8u9u!Gra? zK*EVn1K0nN5}$g+`_ZEz35fSP@F?*JwLkwqlYT%i9^R=|T@~B8E7p2pMnUfNhhD-s z*n;s-=rlx+jB`Q+ft06$zM{v@u`_oMfi9uHbfH1P_>=Ru&z8NWcj7L6ptfMc+y_un z`j`HdvkV$v?L)s0{ecOZpcw$);J)*ME<>0PuN;^OzlK7-95(LdunBFMqL)!Kv^@{! zn!y7HN2l*to?yi@1PP-nyV3xka66vUd!YNcp+MKH{@yEh88uPkHXhnA?Hwp6O3sw1 zE%^7H?Nhjz%{kN!oSIJBo=w}fC-cgj+sSs2YQ!E)bg040v-O`R8K9Z<5$=E^b{{*K zdw3nd6)LWi&&|_vzpCtz&VEu=vJkB=zl6LJY~9MmdVkHs{4U{I z4=dYrq_#n08&vgE?EMVM&w;Rw|Ic9cOI&?~s}ZiE4OZ>ptSk(iX$SpPQnY?ofZ68l zoZ;R1HU&EJ4hsDVkl#VmzX9TTTf{@Kl7RYTOaZO{S%fMAcjAU5Boh+SF^LHMmp?&H z|8?kIy!$o)zrElbOgGwq^CHP5=WWRGCavN6M*FD$7j8IChZ?fH425y(O&0x4eKQL0MNs(8H$j z#J~a5dSE)Guc=Fa@O}FIxF7wS{wjo-EqDBbLS8V&2LM8G?vA&|-Kb=B-^5z7)0Lv` z`A{lbLD|E%tnAnuP+zCNHHeKYBgEIf&jc5d@h~T&7{ufB?Q)B=D literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/mqa_scorer.cpython-310.pyc b/vllm/spec_decode/__pycache__/mqa_scorer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a689b4112a182b841d1065281614c23689d1b44 GIT binary patch literal 2771 zcmZ`5%WmVyvAao8ltkIGCBI)Yp1gEeJDW=`i(r#Xut|U!Y}Pq75)ibyWmDoqX1C^r z<|W=eI!JCg+I!0(e-Pv+a^p3p`GNpJc9km9(#SIi2^OoWtLxoGrtNl!L}JQm%s7iJZpAjYDe_rViyiL7bzYBM?ouKfya`Y1Ga@Wu ze@TQb>(>_dMD2|9owuNk^zEYii&t_m)$&;?Wb}o+no6Y$?+lS-AfE<0C^qKQXIVO( zJd-*Q$X>mW+7F`0SMtgah3Z>H?`#6fqCkf)eaKP+^k?~U zfcO+1bqqifP6gpiP-zJ!ZDGmU&=Iz%eQ9$?I1o|2XgvGPPv`n&6_p=jSbiq2ipFH& zkdsM0&60ryn&;OOK(G)$he!Q|5pqFtOA}2E9kFYAO$FT|xoz0Uy0z~}UIVOFVvcqy znWyCU>F-!x&t2n=>Ul%Erh)s$O!a?F{y=p@H%;TVbw^NN4!CLVki2DDw;pKp;N6CI z+tf^b2TE!je1FA2X61sI_Se*`d`HqfUKk}asPZt8j0Sht&|i+Zkh z4H<2XwjtUb11)t!MeT;>J<}5o_}ByfZoUE5%8cckrZ?Klw*c0s>Z;5?Fwdr_JNc!W?1 z?^MPHm(yhItMHBN(_#x%UEpx=3_%!2 zq2jyK3FKA!V8|bU2AHm*a!h47&9e@IH3Ts5N!9~UIKgBh6Tw$}RgOsiWtxp;#@i1j zSy3BAX(D+Sbw_DB4lZRNd=;fy@wKv!b6eOCy!((6UpnDtdN%f zA{_+hWnHdPSz=Voj=u1`5kgtEJo41jNjqQtYpt*x&D9UPQ zE9mYjf_?O`hk?CcPu_mu7xz$Tu9L4VQXT-fXuINN5Sj8E@W4ig+D*4MoZV z{*`1WkdFWS>{Q2-(@8k_sIvXhkK^fO{Biv9Gz!m8C$IHonw-9fqWDz7citCr0AJVB z_$s*X*^|lZVzq4FMFo`yiq+o%*fmC7%4Uq(vnFd&cvElYunz4|m$|rx_pAx)9&?#p z($2TGy$+mC4c38&T$?r-=&=}e{_VA#QqrC|GuL)NuT$Qh<$>NF-J~lFEj>AG(z0- z{i|sZRVx|%QIHJ(Y0A@*{SYxs`Vu%rvH<{xrotiN5{B2e5bHP0Uioji`n=!fZLGox zG-dghRw>6ZRLvJ0jC`*Wc~_{DFoF3t3A%cec+e+nTiV*mgE literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/multi_step_worker.cpython-310.pyc b/vllm/spec_decode/__pycache__/multi_step_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ad453394e8571f26b0c54593a437ed0562a57a0 GIT binary patch literal 11509 zcmbVS-EZ8+b|;72)oQhpW!aJ?+i@5>PPF&h@^zagY2u{UvGd`4C~l;tC~s1N+L@Io zahK~MSGE=_;P%=@(uZ51eJg?%63~abK-<5d?f=l10tEWhrvOD;palxFvDM$1A-TJf zoD^M~qv3Go%$%7szwb<>rb}uYHU08q0M%wQmQFB(*;d;v}_JM3R7_bp_hgJ_tA$?Ulq&M4Z-EsJPt_xBbLVXY;rBWz_9_A`+jY7^F|F z-o1WX`pra>>bBcL)>(RbrGpMaKMA(QT^V(v*bn3M)R9tDHq#khL&)Eb`~1v@j9>Y z@kh2>Mvf-ccE?0zZ+ze2HBsf0VuDYJnwVHHxGg5dlsK_qi0LLwxZfB^ErM-)tXh5rWB<)l+W`8 zjOz^Ap3T~x!{2!t6EETWdH$Sgw}2WK#5^d@H*NkrcU0-wtn`I~k9oAY$SGoRATd&`65c5`^vBl_X0%iN9l=Ax?vKmB%-4i!+(hx#_EP}4wXAW z>_B{iZ9f$4#7QE@cXSI!JZOjxiZ=Yj2@=N-!{~lob?|EuZHTGUjO2Y^a;M|Rv51|Z zjn;9{UdN{f>R)kC-ff`(RD=^ZHU#fNx}6PQ6C#=o2r+j=JL;})IB~~s2pUD4L=qXa z@;MKZAZq)e^U>!^&guq6EJ3G%4x?=$H-yjCsMdBcfbD4OXm_EA6ufCz1)qj*&X?eZS*G9U-wdqIP`6iKMe8{FW07Up6+D6un!=Rdslz=t-Jcuo3!l zT{x|X2h9NN6164g)3#zS0*^EyZGJ^_G3Wtos`jzd60Jz?s3pkSJG?82(Ky@tv=m#h z12Ul6)w4TjuIG|wIH8J1gdk&Fv(&)uEPfVpHl7oB;_RYZPvp+Zg&@_#{ zLGTBuGnauUCD0Nwt$1G04ie9+Td5t3u$kJJgg7m4L15P7$7VY2d5sYA;dzgZ|GslA zX?3o3g3i^1<|?`RTC2O!y572fEezJKb#{`CsC{ia3|rS?z?H{E19*5%$)OiR$i4dt zkd`_-@)QPnfksQio-r#{Z~luEH@MH{)=*TnxGmKk!{$m;mC#JFXo6NI_)yHgk6&k9h2PTz5 zNlWT29gDRdbLWG0BkXeFt#`UK?6ufSL_3b8bU;#%xfpnQJ{Ae&=J*y%Xb{tgrdjXG zVJ^O};WOEiFVGk+BB|T*66I;u^%$=d#6DgCI6A>MFy4<{=Y`1Z|oU;b6@TAJ?#6wwa>m_yY_Br-PkQB3^HLQ z_CqEwQ*N)+xA#rtS>N{5=IGnl9_DVjy_i(`?4fzv`0~>3STYV28Oy#^l4`&55Ej4| zi*I}g1MQ*t3$tJ0_JXlnLH`r|O5fhB?K5Cad2g~`q0!u{|hD3Ik$=#vx2HCu%ArhN8=AOD9tH7*s%8IDj|8 zauJRYhOP@E05VNA^bjYY2htAFy1yY5)}y$w;pkmJDl^+ESszXtf<_u}4eC$&?HCpr z3Q!L=S|a4(p?P{}SXcZf!eO)?G%{^REWC$Nv`Aq?r;3iFf$|Etg&rM@Gv0tMrao6? zS7<#zjq9ddgb;j7nAoc~q83I&YDWs@1i4z|-t=^VoIl5=E_OZeeH3B8S>3F-_Qn}cj93hyNFoEkr zL$5{9Ipg`AjzGt4abItRI6XtVWSy*Kua%k09?3yk)^*dZeng zAZ3JJ#c<-ckyNMbl5Mhs@_}7qX4$k4s*F7?Q<+&gs8!1NTRxa#3|}hDW>lkwzfc)? z?_D^8O&Z#>ZqgTyV`>y&2(0_5C0r9DR*R=}(*n(0=)quK!)0N~mFg}iP=+-XQb zo}l9cJnk2Su`bFjRLvnu3gCulC}R=UHDEv){$y;AFj7#;7$W0azzPo51G2^o`fBf> zcUZdc9V3{ni668D*Yy>~2yKxPdr?yjh?MfXz?vGlXt8VNAYJ z43y}PAOP(`XYt*mK6L$!QD(AMBa}4A=gg0M`azj@7GVdVlNCYwWH48EZj;2oqZ%RC zh}tw|!z(;25t)V>~;_Xz-(qrYB3!dKrXUMd*u*T zj9U8@zLROk9MPNx%s2rB1#Q<}AE5{RMdI|VMII=a_g3YP0Rr{OwA6t96!KT`mR4wf zDQ=OLf<&}pd6G(B_8L)VM}cSo)NoQ^UZQ(#IU!mF#tT5%X4qA0!eT4iur2l7f|G%K6}p7k_CZ;f!u5a-xq4WK ze82vm%O%iT^OMF#p&943LWFXGc?FFMn@3|!V_3sbp3GNzV)D9FAPDB=BN&snjfWJA zv;1vTD@67^rQbD{>s_yVB0DT|M}XtujuL9}K;g(^KVeqY5*2iGE7+7ON|B>8Hn;Q} zNvu22ogZL|cutAO_^z4yey; z;(h=>^#Dia-dR+bvN9Z<{4&GPe6ccKie3qf>8BaZIn>}Y0064-tYyu5B_u;%cl4^~ z3S@Yl2HJ(l(XW`fAzqJ#(PQ{}!?~%UE`wzlbSU?= zxd3B`Tgcn0^v9GNDlMtfS5aEZOS%0^`;XW?@|;S_Xdf9I?Idyz)!^px!Hw0SbX%+G;-u`>^5YuFo4TLOUul8;o z!Hr`q@ueFSL=5A%k;zF7Wf5?f?|^7}O54X=+x^;=Pb+YjR9GTCIr^l&JX2d&vDQo|wFRFf+scZOX1bVXXX=Sydb038+NQ zsvZD36;S47_P>Me*Z;q4>y=>gr|4G?O2t+_q*pbvUqkLOo|qV`o@O)Uf9<{af9b73 zXqx{hPjW%rs~8gnhE#BX?UN{(y+64tBMQ-~eV|Sk5c71_amIiUaT_><(~TSf2iB1S zPT|0vTZ4~1d9&=d3qQln+25?PwCnDs~Oxi2ZHEBK3B?2w5|69*GZWyfT zd&-7fu2)m57j)e5Ag0~j>LhL!hm7T$m|u6Y10SW0WmhJWTg&v0HX3PLyC<^nfGUC+ z<&I_eke=11lt-}!LMxofWk?d&D}xnIXNdTy{pZf*6PA6*aMG<(UHy_;$h(!I-$|Xz zdOs|Pge(7qC!R;bAc(btGOMzhh18s4HIx09{oPT~JR1Cv%OEg<`4s#s_BMS@$Tx?l zyo-=OdFq?y9#h`p5Q>Krt2B2DCp(ZyF0P7ZQ;_Y1=vj^%)S^(nLoLYxh0Ramr2H6t zv;t|znbWX4^??XG&2FeklCs|Z_~TE8PN%R5)LE9&<9-VfKka$qG)rxY?5v>}!YWAk z6d-D=Q&f#cqH~qJ6|x-`DGmr?aAirHVZL#JNF05xzX^(v_WVGMY=lhgDf)%obiYm0 zfZ^xngE~N_wo0bu4J=VQJuJx-@r5?0wM*wH&UI8G(^_6mb18p=s@5VzyQ!)wQhK8Z z%u9+R{U@H74g#u^wl!l;nKf30nQxoDpB%=(A9B6Y6Vwq!reVSpFC)N6=vs&%$4eoC z3?gU^iJ+q+(S2J5gs!N12=w#;k`Tp`5=Ev&5hWyw^ku==vomO{;hg}Ml$PcH4eXs_ znr{^`mCTJ)>y;y#)?_Q0{r~>(QTvt5WFK~k2fjnr!l5|8u#HG&{P2V<4Z=V=U=j~} z160+49T1_gbC?^@IeOqoH$S_j!$%5f1C(idV6Ne~41_7?K#hlvKq($QIug}RyV=-S zg&9w)HlR~X+g)E`$6zlEMP-QKC?YzGdB-DDcQ`16hNJ!!?EyszC!+dd!(a%9hm9_D z5G13DcnqyxPE&CVtjXOwFpe!+@IgI2gT*6m#Id}P6LFm66#b9#G19R@FNHpZs|gev zIy^3FFMoT0A9nj)Y?A>X$ls*bPmsWdB_NQ$MagHBtWZKbUanGdXrsS{vPl4ROy+)d z&SvoVX{%Q&6e0dOV#8+dwIkb|W=dg4zAW(XkBQ4%1I-ablH&T4c$6XeXULEtxe4{r zHRH;M22Oz}5d6-rg>xngr%U!eJRc6#0#gX@w3lHBmG(;eP&Q|99@Q^XSQqC@Wqil^ z(gNxuU<}8f=$EtTC|=xpr7IP<|73cL}V#M3EJvDT2Y;zC2Qc8xBgXSj3m z(dSMl>>|cCe9=K~wBleRvw8>8zO1}8W%u;XuR;N3mJ^*3!SKOOB|VB`u->csoXa8O zs~L51K>%kf-6n1#1X=I;(glTDvoIil=&fEQ1A@1Db4$)5MbdDh1)FDU+l#{}iQnp- zQT54uQ&!R71FJ6~Hg*Z?nO4R8$(oiVPg=Z{Jb#K;cd|*rX(f!H?9S#E=ExA;IqhS3`Koz( z)!dn*d`g#k8Qmu78INwf$i&mDuO@}AuDnKD=M6NNE?3MM7-KcNf|C>qjlugvTCwQ3 zh5k{A{j_+j@)MLiwg08}@{#Rzd|x9v6RK;VGsz|(XM*l&Y%)C7I?hz(RD!culI8@L zyxa>84FZ-(c1dOjI2zC;wHAp_b|$Cd9*yEIOd;cg z>odpRo*|U&U4Ld*IdM&RKx@Qc%{h<3bbJxX?@8a%?xs$J+;A&4P z*A%R)t0arW|0wH{F2T6O^aWyii~_*6S$@KBoJbOFTa_m`FGL84N{CWGgtRgYrgFxHW;dEULQA7)?1JdZ~W&r_Fs)b$qm5{>g^Bxy;Vd%IIX zy9rE&)vO}xa*V!Nh#g9@B_;Va(xQaS%hXPPc0cysuLa6npK~&OPbEi6LrQ?Kc_8;=ZL3rveSVTD`|)6sT<5BndR<9 zx{@n-vIxW@HZ^XV^(^&#`w(D03+>6=5#xAbb< z_HB$Cac$`Mj_Nbx`q1@VMO$%W*z}u+K&u> ziR&9$*ZmT^YwKOJH19=OUOJzQ@+eKhxHL9IURn>vqgZtHvb7NoLCH_X`DmP@WBJ2< z(HrODiJbEZTmFXzW!Z<_!)%7&Fckkn>OUNIma!1G?gcPQ{Tf*v_+~&2DRa` z9NxfaQ`CiZP2(-$iiT+7ZC%soN3{Fwn$~xCn=fGHf?9Dtd$DixMZScQB{kCG9qh8q zSJ2bp9oX+u*;v0T!(pY+;pyCmHOe%y*L_doO_87-0P>( z8H$qh#WA)P)3}Gquh(rbZ0#Bxag%kN(ML7`Y_>Mlz9Dd zK9<7U6W%a9@Zf~h#QW7si1MlO4aQtP=Fnq#q5shNM97+XImY4>4&o$f840y61G)D9ud7{n8BOAS>;i zJ^K5FCYP}A@tw7NI9eM;qxW)ji~POs498o;o5Q`eINDqr9l#^W+QT>=u4RbeK-pBR zB?I_2*i+HEIyxwwAc&GE4}weJnb~Mu*47>4aAnS2tFuYvm1=$EuAp*PFpQF~7zp=! zWVP8dv)~8l$QIEQuobMQqs1xM(7w>uyG)WOas>@+)Sl5*FKMpSmmN^$r3}WKsuU$W z-BUJv6+;=Jk!`yA;p_8?ym0;nFO$bDyh_GfAaeMXL&9zLd+iVGq>@#gR7h>)W^NtZ zg+~3`^wzj}WMITm5n!z!TD>t3zt;n&rw>ID?GxZtg3?@h8DB~_m|%fuvQVY0Qd?>4 z+=|2DCJ%4FLu-kVsU3FbRaEZe;$hSitG872Ww$|8tGcYTt1;Qa+HSqH@>KS=2?OZSPZT3jNqkZB&C>q?jsy#xe zZ!Z$X%|{0K=96aO7R`pntzQB>Cat2$jU6WM7cRHBNqt3YhfUhaO5tuVaht|@4Ld9p z3-dF!JKW^Xkv?e`?d|0$y;v-6+s7*>n*0l>OUIXs=7~O8Dwd9~6bNESe3cF#WUe2v zE83(}bhfWypO?6av_ew4N3aQcUgl=e;f*6USuU*OSBhn_Rbd~$0v@k!U*CSMSOy=D zH;>ry>*#yqq_SpfV(MDb*u6!oxHVfAt^niKGYC5kW!FA zka;$!dPsWWu1v?Hk3}AWRXELtwC*EhNQGKn1;jTftA()?AwQrLB~&6sD$S8MG9)CB zQG!H^GUkSd#0O!X3IW`UdLd4+HwrVzna;!tDzR`f@QX9@)ELX zl!+kf2je6gk4CA?1uyFY8wm-gl~)KK24wtTIJJI2zAMc{?4ylf%F>}p3yvBI-yVci$CP$B8i|CLEJ=K~KTdj8 zJ?qyHD8W{mSAfj&)UWk{-TAk^SsIW{QZo0gLm|_wd#S?c*U%+BYF?+o<`$Gy)dBqB zw`Qs;TlovsCpn@ro)Pi=vh%!YCGX5{^dkxMR{egb$}p&&GE_Lkdf#sdSuO7>ueO^gE|*%h5Z`W(;5UldERaarLI&WVUXenP-=cHgcG$ z=7Q4`hnc6Q36B4E7m(Y2ruefCt8|-9TnxZ znLD#tS*N0r0`VF5$q&)@YrLu+wO!cm?C|=$&7SAE-lJM*_EdGzg(435Mdft-H3r5b z6fL>Qk;i|b=eVNM#S0_W9hKYVD;P6rj9W0XeJJKO2SDQnl%%_VWBdq}=PpxMWhURm#C5piHvXz(T$(%LAVZbNR=*+Nr`~5yR{hpM z07OUf_8&2tQGtk{*BDzobI%UHJ*U9x3t@V$B%Lw3P`;j9cc%8dsn)wTKs7n=mo`z^ z?oibikd;yQ`JHFF2-4c;eZ3U7o#pj3|KaS`Dfov{rc$m#2Sls7X{eCAsIt6)@!yiN z*Gbu$I?K9!$}FJBnPZrSbIMGj&Hr7zY`o9_-DSFSCE-v6LFoom;f`aXn?bNU4&&)a zEeLqp3j(=}O-lpWUtR$vNd{S?hJZoRO+^~iPyi^%FU`#~jpZHscBrA#q?DNRrBabD z6q1Tzx(a09MdN5dwwmoQ-92+^tic$2!VGq5+wAe>n)@I26Y8-o_FrbZk6Ei4Z=WhI z-Hyb+g|yN}C4tL(C6SWtTvU}}-=JIRYIPZ|3jI{B#^HgG+3FMxtK`*E6KCk7)b?Ml z`p&PktEw{Uhdty1($|0cxfvDm2syp?2AlXdjjYq$>(9;2@vobwcN=v9Qp&x^cBONx Y^4@JKlQJ@=qg~Y<)?@_#^s{38A2B~st^fc4 literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/proposer_worker_base.cpython-310.pyc b/vllm/spec_decode/__pycache__/proposer_worker_base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d326eb8f3076dd770d3dc042a54faac66f1ff5d GIT binary patch literal 2680 zcmai0TW{P%6t=xy?`Ch=q@r!wQrw~z7HC#gA0h-orB#J0rCUTQko+?8dQP%d#+S^D z6Ly~v&7a8we}Nz3SDtvT-~p*E=Zv$t&}^B+Cufdl#^1SpldRiqSvc2?8FHio92$FC2rs*^`M@3ftNIbhOyT~Gg%6jjP8n7(hk~2 zuZvF74Z246L@!wmmTl`hi#K@l(Be&4hVDx@=%Zzcx6smpKGYY zr6)f_RA_jXaS%ViUIEIOw{kR#P{;f5z0C4VfyC9TD zB3Nff2PY5lQ9q&pE3mm0INT0u;Bp7*yapb3p#k2S#p}?7C1^o=&04ddv+b-|+fDBA z#-aPt4!XQ+Lg5BI)Y`mr=myKYiNWZVt8;XxNM<*mrW)jS7=gc?Nk2cv?;jYjs$7A} zXO@uDOu}B-QYJ1;Kw92Wu*YJq*g>o(Y@DTqVmjM}l*K6%q0&rg$V*RxE@T?H=-zyg zZsYS9pI0=|*01)FJvzcbW7wWj>=QtA{E3vAq;x3|+p>#GWsk%%iX%&2#y|gT4t0_b z^Ekh!L%9R`-bPYPlKaWQP{iY5KE;fu!+jx=p~7r24iWk@oV&{^ci9+|vz|}OUge@> znaMqUQaLEx?(EjPL0mt-dK8gc^FjJKH@UrK9XYcrCk|raWT~GW7vaeGb7ad8aHG*H zaRsO44Fwt_8;ODg+sO-t4vaD4kg81Hz!k3yN@q5_2<3bBE`%)C2*$M&EZ0vo3cEC@ z?h-0B$qK=#Oycb++mQf#dJHd4b3p}08$Lua`*H!g|372k$UdGh8uZI1V`-QGV`Yoc zXA41khq1jP6vuBGjPWdDOxEbzODM`(oNDROv_XOhOH%boq%*5@$5|%iRhm;c3V~p;XHS8G`~nSPYD^Ua zRk4zn@Ccnau zoy;R@iuoY@T)H@>ORl1|NO~>FMzv+WUb)N^!5L3As14j$)rCEKun^GV{U#KHp8SXa_#|$TxJ|-v z2$9Pvk;uC=H4O7HX`hhzjKt?8PD|KmhEeX|3wjiBj$stned8gcc(vB+%MH&(etGNI zt+%VLq;;dgYFP`%k)%#D1Xp@`n#bu*1;d~^A-do;U4HA;Enby>Cf7wc1*zuum!ZI* zYZl_YUO*awAK8M6NQVRj!w-!ToFO^_L}k0~EmtyTNW^?qz;u^!^6BC1rkv zHQ*g@(d_(q~3h2}y@ceRNpfd+O0>JT8LW9bEl=NLx zLv5%((?ofw?kJg(sY7jF-B({2nRcKvH8X~I8!CsostlE(IyCkPthB-XC&Epn-ADTM zBPeE$k6M1)iZ~o4bw$6Bh9nxCCn5Qw7Zym+f3uVu0byWCxKW4a^!rDXv9T!fq-Dc9yS3n ztYO`vgYA!glzq{>OCBMY?7BgZBap2Q4wUFNq8d>_aPrBc-&@j($LQ?|KSN}ZtIRB) zjb@A#u`>X!icre!Fq4vbm?*k5^AK9IzilmL9#Ip zr5RaX8pO|lBYRNOXl$YcIcFM#O-&Hj!b0`6Dqf{FMa_(iSCBo-O(pJPZc2=s#^Bv~ z9`|De^0MH-{Al}_I>_;fFX{M0G%{r$>VQhLU2=OJEnPI2La2oiVy`D=9?w zBQNMNh??{WK3nPNXcDi{yyFfg-bUXel$2Cl2H%VNd|&!8>T^X9w`to`tllJIX*cc# zjFM=--G^;BXZ;b+ev|~R$IoPX6BM5uV_F-0I8V(#ZJ$%@Pa@zWY&2_u19jrZIH%|M z@y1C%E)-xCwwgK}eAt!p^WaQLx3KvaBq_lXna9+mQ(d1Y?UdepiSB{cNFqQSB9kP- zr3!sgq9;JZX=KW{Tt6vKQ`>H8-yY!+KLfeuDao!fo^cWS5qF|q=(xMCAGoc6J1vCm zv-C`9?|>3R8B42~T2+1hXz<=VX@7i=e35phFevsc-o45DspEcP^N*(*l2Io$v^>@j zq1vH#CVpKxsi!%1U#8&q-2e%s@0px)rml*gfT!Vbfb|cEEBxknx~tS zJLD_8xK@tttb#w!%~W1Joq$LAjg&uke3m+NhB>V`>H#=nfF}UO!EFH1iYul!o6%Bx zlF?EkNM3ayQGci(>SKh`3UFF1HWN-XHqih+6A`CjsGq#(h0(a`9;3@d5(g7*+?jp;k&k3W0Pa5Sc zd;AI{{!I2*L{+ZV;f4fD+7ZI^&3RTiy^lv}%lM)8B=n|m+nI)5kv0IFhU%eu20Z;# zxNRH@0pwpY=aRyn@VmR<=`ScnxET~~CPC4Sc`_cOdM43;mp&=cw#3c4URmtVZ zfwl!5{Kt;-4N{zhhXTPIx^OzJe#Xzn&tHQ8eQ6sX=D^RvJM$QxPx}=2#F*rstVG;D z1@8s2jbhu}MK0Eo{GEyHA7J6%i8ED`hNk60ei&#Rn zijw{TRaHR{nPtFR{pdoYx@4BkWwTws9Gp6jwle&%wDSct`D;P&zPXmfP(=|b g^>mhz^6CeLt9?i@Ak+LyN>y9a>PYU#KfLSz2g~jSOaK4? literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/spec_decode_worker.cpython-310.pyc b/vllm/spec_decode/__pycache__/spec_decode_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dae197d24fe340d459f465afe1805f9ebc55e453 GIT binary patch literal 31981 zcmeHwd5|2}dEZP=Pw(vP?7fTo7=Ra+1QrBAQY3hYAPx#7xS+TYDYr=tXQz92XLl}i z_X5ObHjxF&l4V1&t(-2$Uf9kdC#pE*#IBrnBA2TYC;!+^;&c+niE}A&>{QW~Wl_ZD z_xs-Ko}QTnNX!1ie*~z0`?~w}JHPkc-<#&}a6W~9C%*DemRnyqGXClrd`s zyMXVqwc+}RJyIXFN9DOy8>^4oIHvD(r4G5eV0$7{3o8`(>yO!wNq)Nac>N>xM*wrqQf5!>eEkLc1O76{6fkpI{V*AIs4rUH!}8|bI85q9L9AIu6^#M#hi1mb012+ z;@s~RRz8krhZare0p|#wz3Q0rshJ1AMmU&BheM9LSYE3IPPG!?cC=EiEW1vrVY?NzZX5J0L?B%sq%?-`V*IVw~j1i8!QufNV znp->5Y%Ern!twc*TUo1>gX&dRJwiLv3tqGARLXu(3fzX@^h&w|L)At#DAk%vORk4< zW3Rd^ZiUUwm+NST&xUpO(pu133((>;TR7`hnvVN?*?0Bp@L=che9){cqp@lwQ2ha_w}b;B8J=r!QVOTe^JdrE_zo3unXqXPWGYbjiK4 z=Hi=-l(9H7kDrcvIQlZ+tU4dKt@!=W+zVd09^X!=9=f^$t`+LuIbSC{ zd6!$=ukmX9a@oVQC5Jo!fUMSCFWgb_T+BCN3DCUYmzn}E;XeHw(=4l3TH^9WZ>OSF zXsp%aPs2TY>I(!qrJ%X$HahR}jjO@iC9m9Aa>KoTs|N60;4+mKuFE#!jF)O|1KqXs zNR4pM#im!DYXtVbO*5SehE@t~yvLJaTqC$fagE^`$Hh7(ob1iPjchv$ zjAO0;;K$q=kV>)c2Fp#S7%Z2AV%5iAx9Bzk?|Q3QZ3M-|rdL#ZsOkB|Ys=Nja&gUf z^F{nqLx0^Xw@}Y)@j_6nG#h^4tyKcQST6R?QE}00)}?;GgM#=8(9*0_7)~VKl+>HK zaIDVFTk>QNl`1uV#taMUnJN;Fwlt2@--J6GO8{i9cfCdQ)m#Ng)yr?+;YGDEGZ+s0 zvUf|@WOs#=4tKA< zN@}{p*+8MXQsR|@Xs-mU5V&3ySU6brTz|P)bHWEdRQYnPwOkIzq8>$iEgVlgDfz&A z{>)^U!7d8TO0#v{djLS{9cOYHNi@4^#zSkZ1&rw06KX_VGe51K^~}Hb1gm5+k=5>9ut?S`LgxOe94gDzE@SjFfF!Sj`I%njJk+f-ATc&BI)qm?-=Ae-_aK|+H zpYN@A%`xv~RM3TRaG#pS^&BqW$)r+l%1%2e+i=o$#x)(o%{m#^atGYpqOm*WnC_sH zb@T2J@_6bDxCJMNYY@-#xQ3k6;*?WxhTk#4>I@H#$p6v7(ZMmtb;jKh)Hv!)xMNaN z^wlJ497l~&%~QA^+?`sSaHgFd?_>c#i_T7G7nq(&XScKGos>Q0>~;3x@3b@I?r?|P zA@*+3aP~U~P->@h&>eD$xQ3k3Mawzl9LCdK?(WS!w^D{rJ)p zMk|XxA5f*x;{DD8GKzyJy%!}1QR;{@gPMxYgBa^W`1TO%kT#Aw$MEK`l$~|=qx^of zeB5~$Pw#Q>bxzz!%ezN7OS)CdIq5tqZQLjI8_r|S<0yYW%6~+ce}bck%RlLylJXCr z#8W5{_2Fsvi1Um)<2);+ceuM(9^@snl6OApJSSg06qi5Ex6bpDp20U~F^dKMU(6(C zrQn=%&Z9p^ofojiAH#J47ss1(UUXin|3TmBoJ}y0bRnkA6n&+L%sQEb80C4gN=QZ3t?7Xge>wHpR`~-V0 zaQ>?FUV4XB{0_%P=||j?SP4Ly^PSEcGM|sSk2!BTpL!<+xOp7;lJl12KZ3j}S#}nr zPh?XmMF}-|tdb$EgNFLW1R^U6zNt1q>V($KCum^ws_22g ztGIq#wg|#{Oh`x(96OlX>&1oZpueaF1Qz5?7)Ys4(Uqe`e+?A6O#17SPb&(7E(+2< zS`3`m8#~(TI z#1m-RktRREv^v5KLXAOQo#ad6rXqc0(P!0OC}YPiM!_3 zFahqN8O4I>HiBmB*ojW(l@VpRe=ENVVAj!qQ*KP43I;+gj3bOsB2uLXt0 zunNE^LZB)_7Hhb*qtWi~bjV*@XhJr}HY~b?*DChob?!~JsYXjy@o4`5WxlmQ6wjg> zyEo_b4OC;^DR3;pqj}_tyl0s7@OI=#g~9V4L9QYy zG-UX10vGSk;PQWjO0)DvzHLCjG9X|LNiKt2p*`f7tEP7#NISbARv98z4Y!9kjW*uh zGSaEWD-g%ByHgt@?U9wt#%MdeV&X5vC&${ITFI_h&cLeSJsAunYiP_S-skZCLXhiv zf1o|;4C48#0Yo(?|8%MW!7Q$E2yZ?Wq}@J2ouN`4&_HqpoSXnX8dW@9)Q z*&KC7g0c4SjWm1gq&CLeS!Wb=d^#AvfxAW?_hY#KLNL*`+5^t`O=DwRzMH^zm)wd`Ek z<$pDPzJt$$-RN~1qG+x?2$6KWop*NL%z{xT74W4(Nb3eSpIN>mr^f6!TtX!rz}t4Y#PzFMx< zD8LF1D@x5p)(mu1ff~h`U2lIIO(b}XBBbn+5)px~*s!o6=nfPN3fn5lO0gu}#b?*w z{5D$est*kkCMddcI)LB-G-#b&nw_x}n(3evBAjrNP#4LAdgjBuK-SG`CDNIa5^|L& z57E3Ovf|91Fw<(byn6|}(>^p(O6dUF?chUVc6&-|Q)Bs*;QLFR)`dJ<%WpF{7wjeWG zTSf*F^p<7h(uM&UddtGy7;|wR-{32jVp&y!Wz8)!1DV|57gmO;naQR3AEof#df&_< zO`Eq($na9$psa3g<;)zPZ<+c&pQXeuEtt|WYta9#Z>%5aMN6|?T9Wg?mEXa!a{S>q z*n28@0Ww<4>v2ZAs3BF3CN80dKYHx_Vrsx$!Xohs!(M~tRei+{Z`$FC_)6*g zG|=`oqniLX)hjv?!Duqj*HR=#Rlt=w#90*5C>$+7`>e1e7+s|;BnmHHEjOyQ8va72 z>Z?srxSme;qc?W_GWPW{8(0H9fKC@W;3ipR$_T}dTC%SR!I12zmI)R1*Tc{pb(dgA zl!?Oq6?JAMmUzL~D)|UQy@u7&v^}`iEIQT2MX;j~o%(7bmJ=COWA7sI8EWV?SH-ei zw9Ndg=k|20qn%f_t=ZxO+Yp8aNYh*}5Tvk;zMfQ}0H@)KlhfBaTu#QuI{u z!Xkh&;2K@_)*v+KG9pYQsXj!tNI{<*C1sB1?_jS2nZz2Eoo=v+IuU7+`@0}(Uc~zz zh`$E->>3X{Lb43;OqqR1m=~Jh!q_bZK$H$;PHBt4-V~Ls-p1V>V{M(;I1dXNrephF zBwB<{f-QDel^CPai}w#QnrG~cY7=Hrj1`6v1^<16Ol^~q+Jf2Bqdw|#s;I3)6)>Ml zf1oK3E78TD62yu%1zh%Zi!D>OWck*b2iCZE8t%eioq{k8vlQ_+1Y0sl!AoHEsUoZ?Yc<&G!7xj;s84gH+oC81nyS*C)i_Z+a{1EP zOQ(wWJodziM`z-m2#+-vX25^VOe+#jDK{Jdp9RLtvq#8q=^nzU7UmQA>|(814wM`$ zgnu7x(0AlRi>x?Dn-ZCWFoT5%4~xZ`I*QUlv$5uD18EgDQrh>0e>M2so`gADTe)E) zgP`76vWG~*{E|b&YLCd=!RW2-g@+Dzv^@7}wTb$d#hNbL2j=u~xDF^AS%f#jiR2?~ zei{l*->ofn=11{)q6O%%=QjGOCWG9L9hvBqVXB8I#5|P5TcH%9tWRZu^`Mm=0oRrn zK5qm(os|ZkcpH4#mSLE;b5=fs|HAEArj^d8*LTl@C?$o(*{IeE1<~jqClGpH#cO>5 z8E`zQO_*@O?W9(WIF}-irJqGPa}zw!iX{>>_@u789Mrvo8|fgwIYc8d^b!L>LEl5C zL7AH^;Tpy@;$&`;Qv<)XImULxnIyaZ=mmIfK-&aiLt$CPwvhS}Q*$pWiKv#q1M8>n zAbWiRY=}ZiZsh793dKmxq-q2fo?U$FG6mdiU*E1OrDMHHuU{zpvzuK&n7etVn zn;BQ{ls}3pAg~kWv0)q~ae8lIA?z^$f~D4)c4hHH6W#!;3HaKBV%}vYpI~yBNfk*r zRO&PtT58h5BU(&@22%^Rn#m|(7^1MyqT`I=Euj`WuRf%ux(7tPRlL6!aO6{H%xCjy zLf;6bmW&B|pox_70`JUp?smak-`59^GCVZ{T_Bl3l>|}oI-nfjq~=p!PR~gnFt_ZR zcY(=^OnM}$8lIhD&j{IOdVOzSyPbLNYL-p%n=`n4nq1-R00>=4uNW(tlG$}9aKAaA zcZ9W>6PTqGkSYH*`XA{GHWFm9pc`oDvmwaRE}#D)jOT@SZ4(e--xaTEi)O zw7me+1_8L2vQxXxabr8;f$PO2$8mjhjqCyj66?+n*U>dT=^dJTSk0_-6l!TYFQLj# zbyRYLPxGse@Uf(}LIRKL1$hbt7>!*kD>Khw5|(52Y7G@DE-tRBxj>6bq#AFa(~D&q zxJaIaiu?3$g84OF4 z$U!Ki70p}vE*7NRVNI*(k20{3A1%WIbuESuXb)vyA0_*n1RBkt_d zY*AEV9htk?&}JqM;_}?gpi&}HD-~u1#OwiGfxQb8R$Fu25>dHgN7UZ84=W~)*toiQ z9S&oz>xgp~IF@RoRRe?xZLs%ID|g3g?&sSNuIkHag*tl>W($2jPlP#wj4@S!8i{NM z(0O|O$v))yVQEMgYNa1ptk9v3oQRdI#T&*3{l|D`&}$62jO5IK9$;oW4J8l!yxPVh zk!!lVfR3Jx%(Jqb@TXe0M6v*)uN2n8bG3-5oD&>?`8mqtGlq6Lhu(ivunbsL%a~( zgfc0|n@zW@I?w)9uL4`aTjZ)sMK=)SHCJ=?G2{w*7bh>>1O!JJK&Opft|34_ z-r^?MflO}Df(}$^VhzQ~+|FgzkN0hcyHAgI4Rv{MBdLHTi2pGkZw9hYipu~$3@@jf zu!rH^Sb=+GbT8V)%=*U?yf9Z*K@1GZbT!bhsDX*_5lxCFWXaVvIz5|ITk%MG=dF$YuW{ZA$Y9Q(v4%c6o!?qN)Hhq?_mVAyRS_AkDe za-c?rCv0jH##(-Vi_2$vZ=m#$EMV>?@D|<|aF@A795%TA;yeq#f_)r*P4MJ4m6t5v2bT3gaZqAy4q~4Kbz=xtaOcN zKh-6TV{>Hx==H8Vv7uZLkN8pz(FX9E9`;wOE$vDTFZ5xrj?sh#7|9S(;L-DXHJqZi zZV8nopL)YAxT{1KceS(#xb_*_5NV{sgAp;lL*Kp6^7{-%6z;r3_d=6#3*jN?F=!Wy zDoTu5uNC^MdZ?!BfZQdEMSvyx5{+@a4RpZ!920iM`)(xmNGEiHZTGJHaMx1)X|DeB z_^dFHql9M^EHjl&gK3V zoCsfd?+PBkrawWo5;&U@0uk0+p@gD)(sI38{T?px`|#;6Mcou9-mUeA`?~o-fGr^P zMO4*A@ks56oX+6#_aSMgu}nnV#@W=H*^LZj>r5$i#S%f^P*~3UE)<+G70Vje3zz4< z{oJz>-uL6fU*=GVov4X`cd})EbN!M2;e6ok8qrL=4&&$=7O?=AK!){;$j~S$bRqgq!jfe#!$bc5{Z+2Je7AOgwOdE)tfQrom=WE6^NPU}KN^MxdK#&9F$=on* zq%l#m;Fbs8Fp%ap2F3Jwp*^@ch_8nnOIjXs2Atey;d(6X7aE3>gUgV$2@@x%*#KN= z=}rsR=yq-i6tWHH*_#8Tk5`TdDCZPza^DP>e-0oPZvG-g|3aiAR?t z_~k7n2a9V4sh6^+TsZ54e+3}Mb;V+_nCK#L9?l0OgJ62G>E60)9MoGKVt5*&9?T%% z{kczp++D+(hYeM*7f#W8mEkwq2--B%pXiBb}EMky@f2 ziIhQ_PJqmpyR8k^paszN^;odFp|E(QYw~sF+8))3P0{izH5Bn6DXz*YB{WdQc>WV3 zeO2OvdaTo3&C>Epi%c&h#D>EP9Q(>3ju5e+9{|lIm)g~n4_ND>Qo3j*ivliD3jnaX z>PZZM1LVIhszHoNBs7o9e-H^^KlSFg;*OdVz@bp`IbdYte0tUV0pkkzHSf=(IJlI- zs;{GP!ckcb;#UDYUwlARc#et)1Bdk)d_^n9l^*02=AY(Fz=qX&_ID#3Wu=VF#O-|A zy1o8lKaPs`Oytp}aS^zZI9M#fS;f^0m)A87oC03LBpyXZm_$7d6vu&N27z7-Fvvzb zv#6NHL9mUOIZa%`91*Yt7G|M!sw0L;25TGs5;V;cV(9%9)ZH_1?;SiN>GH?;mpmiz zviUwdQ*IZw)-U{##~tydT_fhg58{%AKY)xZ{1EIA6dHiAi3T0xW>zisOy+A_`1e0h zQ2xzi!(3!C3a6u7H0z2(9P51?!phWIjf_=}9}dfXWJCfFF@wrMiwPrkz7?13ov|Or z`ytNQ6f%I3LE{^d*9~;*?^#>k9A3>f*y*+{2Eb(KrF;`+;mqc~ttN!vX-_wxY>W1Er_Hh6LCB=_EKPVHTG0yL zyX?x};0C9Mmo;T(toNM9s)CQ0YBqlX8RidGxZ2xgocj#U;$muOb5`6Dm6q zq8Z!?#>Ppn8vcj`eL|du$1%iecp4+X)5)x*y|Y1wsmUtA+B@Dh18Z{t<#NHGFhcka zU&5~n;hz}@@lOk@l3vN)pf^{(k>eYP&k*ESvLap^{;R<-Y8Zj9R=SkO`%%>|(VUEl z@I6j(8^XH*A1?^-4tiFakTfUhWIMY=R%f70LFQE`*ULr5qQHMh+puX#PAG-^6#N;ih|AVG+ zw)i^j{9TeOzf_(yU9E~18t>w&=(j>;rt2JqeiFV{qWjd{SiO3zBKlLxJ&|RV=FLc| zWhKqAD4TdmU9b*aGYYbju~cDWQtG3>hg51wkxlb@({JVoD{^hr5Ztg<4T9 zC)CPPL|dyfV!D*}{vt|vKg6U*vLWGloM@D%QYgYRvaM{BkfRgyG@>g&*fG|cnNsTh zFQLXT1NY!?xXX9I`y1>BajW-}O#UX5FEb$$^&BQUnVd%w?uS_noP-Pf=Rzf2>+wUF z-uAJm=RB8{{2e6rWH+wF5-#{*E+#rk)2c?mz3vhDdOyuCcd0ME2nz>Iu(1pd@14jW zK=*V;XYS} zQV1n_`!z%prBq6>DpuJ_v7u-Xh<6lso4As3?}%OyIH&^ONxyhx8y(rR56_59Mom}Z zi?}-drC6^t2*G9*zt#-CU=D2>gUCa%$8+nNZ3mr|qYT8Hf??7?nb^308q0Y0m_QX30rV^YHQO*>2M=ljpl@X0A(J}+ ze~N4AG*q?e)OX-QDa${J0}}?klZeNG8UWEXQmfQA3e#9>Wf02eblXI2R$px|C2Ny! zrZ(X7yb0wr4q32-s-|w4N~O^E^-A(>`kEI*0~(SmS?whBRo5SW@-S0U>G@{*KPo4-L|;d)kF?W#!FPUYd} zf|A9xML>+8Q6ZhQn?cS-k^axIAr(~w0Yf?FHMi8iaXK>Xd{Py1R8`$2{A$)hdPwbK zU*9^LYHt1f8AV88N_Rj=)ize%f=~miSV*W!lD?)0rPF?bbQ4q)0hx7F#~wsfAWW}@ zL*mR?5@*htQMGx69vuNF%YL*c=+-A2L}4~)?7aoPEg`WB5If`NVF3t`b*XrcCJT&; z5h_t;q$Ei3y4F|=U9?A`g=qo%Nl!|3Kipf=Q3F~Sh^P|9jhUf@*uafwk3?ON@4av^ zy7!fD9k(s}!XggnhSe@7sdzQK(#=d5NT8#0CAMKreAw?`>mQyt^8IT@IG$ z#b4dC{Lk3)e~JXhjv$gLnjfARQz2s?{UaYTfGdN`Und&NDBC>jULcrlFyf+fw5WUB zur?4z(a!BmZScgF40VbORK%$)Iph$ih};16pr3`s9#0CWEi0wSvLloa&qk`LjnU7; zVUR(F_&XTnZy@0CbE!p;X~%pg2N3{PIEoAiI?O(uVj1Wx^9XSqZjU(D51Z{#349!D z4{a8J5KU)bXX=)*IV`%+G=d}z_5j*Y=?6}sLC9enrI#2^$&+Np5OO%i@_0J#;PSKr_Pjnk-iH8)hAI?B~5&@3W ztVh}(lsVYZp4gnmyJ_@rhXZ}<#ze5Q4Xsmq$BoSAQ>9(#!?^f243&3b0XqH-+FL7P z2ReZsB-0Vdc5qgVR&c&>{$rqKm}Uv1o*q_)T|@BGfA^R0lPRqGPU$8zj&Ym?sIm)G{C@ z7K%LpsvJv1?>(yZJODC)i0XDM)t8iFPmCp|_m}V$U3gu`dz0C`7I@$#4D0u2_$X6# z-u7-FmoWU02Zl0_mNHT1{TdNM9uKXQWfand+m^9@avm#pR!(PmU19=eyD?!x%82W+ zhu9V1MUXA%*6f`chbP>ixw=p~cKJj*zC9!qfg63nw@1~DtO5`D5+MsV&x}*8f9*DO zT!PeQcH6^A#Nc~>i=&)HVrNxFgK-sM2j{9DOvInYW%Lxn5l=^tC}&lMH+x?}5?bPU z>x9`#4KN|n5n}M*DE>1nLU|<2`paebY7bZ7H;8yXn0gVpXq$)%FWJ^o8KC3Aw+f7@ zix(nAk$%nWR3t)#t{C@uzkpYEr=viAzUYIzf4~<7p>og+!JpE%$7ts5egYN%?iHXN zQJJ0bm?K7CPxwUqac46AIok04C6j-})}|to=DrQfu|A2(`zqhZqqhf^eKFYCBhgl3 z+t6J76stJs1DbGrn>fI$hPnk$Ayq!BB!GH9hvX}S%mrlf@EbQ_egvxwXYmO-jAzVD zfn;ba4+56IZCR0m(fpQ?wcg9&03U<_LUfw~!%fpIU^~NL#5@lCnMm(TkEJKzmM&xr zI<57@(RkZJyc(@g)_=tQk-S!@YKs1GJ!w}vgUfeFjf@p~iGlhc2p-HY0^(u+0^@0T z`@zU&7#@y0oGAuJ^voM3En^!)8->PHI}?|Oi#e1|!(dalDG3`Js14c}*%*c22mL?B z+PTd2Y_RWL2J3zrB_~nRl9DGtAI92Kpdi!j z$xQ^jtMYsTMj2m9E+6McmoO!T|=)V*bu7yna z`*w6yBoNpxN&hufZBqUOc%%feb0FvUK(jdWz|IvQzerFCgf$AfL(j1&Iehd}fSzZ6v@fMLczl=YKBc_|T z1U$I7@370xTD^Wf(q9X1(*ezo^RohZzlhQzqDJ>nKT*X?G#N%Q2HtLb<~_tjXu;1i zcazEAW^h1A6g$=dCmY7zSuuU9tud91j>twK#~ez-uQ zF+fcXVwK*fP7r=Vkb$Cm5RL)Z0TJ__6H-JINRDymV4vo==oTxy5pGGvhv5f~#v^BO7KSLZh0b7>_z7l6TW&<#R zlwL*Xjf74YZlHe?w8fjQ1VhdTxEK7o8=; zK2Lq%3_Nq+?8A>hv!k>ovP&^w{dBvzP_MvZFJkDOAg0LU=Rfo5e_`^QOnNZ#zrnLl z5zI!AfgLSPi;g^&JyT$ z!C7n~q(qRv_*O6&16p3%pSbX{uc?_%$boTn6!ZW_RTy{RQ+$7xhRvnLs z-{MSzKemPd3(zhqD%SC={}8PlrNcdl3qWbRk*Q^&Uz8rvA5TN^JoFzZ(V>IIBZ&6a zzCt{`CF(+tk1+=suT=6IJk)^}Bp~8yCAdhlZyj6`)gZI5eGPbZOwaE&@$NwfQL!#% zjq21Nk5(%sx*_tk)=(rAuzgzvepZMytKe8Uz-4&>(Pf?5`*=xW0O}Xs-GM~>N5D%u zsU>I>rc&B(BrTkzU@7hVBLXHOZUo5)_7Z`NJ{{kOt^F=BKYTjUFhxGskFYeGwheEW zU?gKtpF4Kh`)_Ck4kx$pN1@RF&K&())NS0SFQQN0|HSjYx+3+;yNB^shCjTK25J-c z86v{ZLr+Itot_G#uZCy>r8nY6)aeUm=)d;&=KDJ{1h1&AZ@%Ai=8tR()!wuZ~U;V@W5*Oir^Z9Qv`9Dl#!G8M= zWq&6s2788>N|AEPWWFoLA70TlO3Z6C4u3tnZe}GDC=mL-ysM5Yo zX&*i+8M0#L?Ypo|+kmR~>um4eA?c#31w0@~oWbSKA)zmpigkeyZo;MocRU7YFx*0m z{-y+l!5LQm4#K6C;bQsrfcza2`%}I>B(&2kUqtmWlDH%7Bt(1&stIBl*Y#}^P3T8K ztyPO1#MF(r{y!;gLDSHoSAXJ2X^%B6661BC82MKRvPk0oM%Nz?Ag;;2TH z_dqx2aaZo`;*V{Dy;&9Y$xK1vuIj*cLhf{t&4I$7kH91Z(p)XhKfs zA)G_thQkkqZE*!s6k{73$Or16x*}dkDoALBTs(>l9Sxw3oszg=5Dw~L5$VR*VFUVL z9FvN82D&ET-7~2x84;e1_0Ptj{3>E|uWHYND0mCI6&3{&gpP28^MaZPA7@-bq#G#$ z+~wCF;>y5N04k#)&uM-apfznJ&eXZks)J4CN}|B3zh z4J2Judk_y|s=bICQf*Oo@6}Wr6AhI${Hr!Bgp$BF6oD^T1e{Epl_jB%XBrC>-5K<* z!qOG+{eYCGh~DmbE4(z=$Dmc zRU-Wt7+#IF#=#R9QR(myb9GnPuZ>p4~HIm3KFwc?DA}PX5R{4|aU=L0!Q z2!fme+t{{Vr5(Ebp+q;%^GRG1CnuH*oT7v|gqL(1d#V%0>wOxfq*uD9(rbA44|Q79 z$My)L{|<_OgB|8-8lT>i+Rp2w^J${9k-p7<#*ZJ4Ca>W(dXzsrinL%>iiRqxaiUu>Zz@Ar^d-`R&}w>jV_IW}1ik|+r@ zQc%{J(awOr2eoIgbm~|mV>_4GMBMTn2PVx0*p@g;V-EdLDiqn2(RM7ujsAaB;1nj`@MASt7LLj0-sEQP{@uT#?VK7y(gPd7JZ4!CK0{*TIarJTPB< zOG4UT{LU4{9m9D;aHJ+Sr&~&+c@00Q;>Oz2gUJx&{8SPmw1Q*jvqq4P(t8uxu)L?lu9_> z9*>9Q1Vp9WRX>qLwhBRNf#(q?geF*+nl~U1L0-L2AtwhEd-pMSg2_0OA`|MveyhkmXXE7nc^1j66&oTKvCbZW|0Iztpd&hYD z15CcaO4)eH52~^+Ne<$(c4P5>k*fqHn^dY;!`68MJuI}%{UiB?-zwbe1@=X}D{s7Jq z-$1z5?;CmJ>&8&w_l#w7$?CuPb$*4Tn&m4a@s)+IEPQ3<^Iy-um^+xu?AT*X<13!V zvxPSq6JABeZ7WlF-#Q1s4SH+HI~vi%2!`0A{B2`Io=c<=G*@RMGg)D4Q9VNt2od{_N zl0gs+4zdX3peyn-T|zsMDR5zo7ZGtI&ioKKmT@XqopD5}VyQ|W;|XlEq&zTG>`)(X zqkZAzo4F>NX0O(KY;<)xk*ZXfs-rR1hI5{F{!9ymv@^#k}ZH(6c^N=`5{p7rq_mT7po@G#cNZb?{JM9xPuMZ20=ZGJv!AXxGZ^d(o ziR^LtQ1irN#ME5z*IF28q-RvfR;R0axsp7~^FaNn@HG-*|ryNguV3|uXplPf9Z=$XWX-}s*(e}9zMy9WnBvGS9Ja;b{mKG0WI{eDXH0}BUPXxBD{pUw8c` zOH|h(*45v+&QCbXnW}n${lt9K`IEEaHxyNM53AbW*RK944RQZfzp!y%Um4|0690q* ze$WEZ#pJo{hx*FuA0a6Pos)~A9^B6_?(O?RtF5~hS1m9y7F zygkZ9n6X)YIU!#v!FX27$H)0%Xa5T|nul3|aFAbT?lO~?nUJp%Y^c!LZQc@bB+yLE zgoU}MF$yrb&jML0pVmUEkiHkLLSyu~mjB=(jG|F63+XB3qNmZ{d((w10vF){JN9tu U&m{j_&vfxMJHg?Bw;I{{f2DgLW&i*H literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/target_model_runner.cpython-310.pyc b/vllm/spec_decode/__pycache__/target_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dcefa2f62c7b6240729c9f9a81071689d73d97f1 GIT binary patch literal 2675 zcmbtW%WoS+7~dVQ*KfzpJ8daT)dEJXqh8>G5LHpCiZ-p90McGY>z#2t&AvJ_PUOm* zTB#@g1q8<)_!sy?_R5KSd9{V|eY18jZ8)*fewms5Ucc`(PFt zG`yv#6}SDiSv%2E-0?ec*YDca4U3hT``BVG?>;K|%dE0z^{PLEMXN`0=ax`9uinYD zNRuGy)$+#mARO`QX)+WCdGj-VB*OWwek%>w>T-DSdeKhXG}|bV3>`g`9aUl*Y@~>F;7B%L#E^GHrD|QbO3%| zG$wB**;wzS^2XgSz7l%07sOd4l7l&4UO0AdoFrU^hpk$A|N7h@JYY0!GZRC)?Nj-x<}BknOCrVL`^X``oL?$&g4PXL_cUJ!;HikS61C`7Ogh9nOZ zxWa@A23U`$1hGI;=vqjKcPyd^0(v-0825(guf*u^ekSx%FAk;0oE#|A`zO>UMqK?1Bd^|&{I3FQ5r^pQj|Wmo`1fr z<7_(<*_EO-uY44bNAcD8csmk1Pw^0TRTj z&I56mT$}vIU;Wv`GyAU(wf-EZv_U-IQ2YTsz}Zisn$7U15#CtsBO&!Th$v4EM8fkm z7)CA@{CSu>yh=HU1a$C($~%u^X=R(QD^EvUhUmp zS-uT?`AQ}^)H|PH7I^ze)ZlmLW9-CDe?uLLjGA+-&1_85u?7K9_6~^1cagk@pymG4UD~CM1^kxuG@e|B;L3(F*XSYX8aCdtf zSXK$l2}<)iMO1M4!n#T61BmVHONCOFhLp-G$V>c1Bb!Jth7toST_hMzSpkxll$N1J| pR*bR!VgnT+Swe;A%R(Sdsv)m_TqNof>=1?4I@V>A`5$ol%w(sTq!9xA9Bqxhx~w?QmG2AIq9B5V(XEw2fIsB4_829c4pAiGwA8> z>mKCowx{8`@Tb4@KV8$be^KM`WufslZg~R*)0m!UsUGO55g4f%n5h+5I`tb#BWU2) zOzhMN9MxwfZrTi*sTX*v-AHEAR?te@L0h%$WHvn(98+~CnM;oc$93%kjk&COOJhym zzG(*Y%vjet-aX8&b*##|63e1;KN%Epo<&JzuJfX5TpJD&ezl`l?e!={ReUlm2E(Fb zR15Fj;N4-tKh7CXKI7Mi9K)B^sgia12O=L1KITQlq9TIy;`#v6NmRr;{8Nz+av3GE zT6m(D3(PfNOK|adF1ES&MI)Qti{^5tf0kad7ICo z_ZY^_@MCB}rtYijJrkO2Hph-bevTh!^K9Xkc2kF;7cqv$En&VkUs%?l=R`e*II(oco2~QK1a;xaAJnY%uoX@#`EhT_n{~&SZJELKZyO zfitWAds6bOh+)1xnq$wG+wnlnRvV_)6lKic%+U=XZPtd$a*;-f&v-YdH}>O9jVdCs z$&1?V<9B#>CJE~*0r+Pk>r^C(Ka)IzN^Q0k?H(kM-o zCezlnul1{-Iwu^@=7jZhf+(Y!2@fXiG%8E-q$g(aL>!}HjtUa|S`#Pn^UZr}MLJj; z#DkT33sx?t!+v@p-CaxKjkUpE(a*EBog_)u$OkYL_KmL^n9yXc%^ky$2bp?kYkP8HY8i4_NYzUhWO0Jl~G`JYu0t@IwQ zh|)Wzcn-T1cy&A^K#0@mc)w#;o_enuHmwQPNTOOyqrD9t9*v<@`>>3xo(Ey=bCHkz zGfHSEJct33vYMX^E<}_`P%vj5E0~|E82cNO)$Ftnl?^FgruDY61HH1-66EgU1r*hZ zP!19xU}Nvd#NtI7))!+QksX!o#7k78#i+Gb%TF}~DHY#`P|!ZIBf(Vs31Esd5U;!gi&e*_wo&$y z;s>O#Iq8*(Tk7HzjhzR%!#*I8smPDeFTX@#d!Px&xNqq6XPLI+f;Q@wq1zAahN>Ih zecK!NKB%QF-EweS_Eg%?Ti+PXS_<@Y-?H~#JVHNDuA?>>neGUo%%Fo_X@MH@T76azKaa%rcPvY$GNMG znnV+z2vE#Wlb1j)#z~FZ+nRX01i_RR2+JWYWdk)uOQgB7OQOZOwfk8 z|MzwBzJnO;Zy+%5=Dq@|{OJ_I7gi6`1bEh`_)d7463rwuPRCq)V$9Rz{a4r!X-Dp} zSVkKO808R!Om*erv79YVhv^Yg>KxFG&|&zioo3bK;1*8FU~qz1W~v#8sXef=Bd}P; zDl6kRipq)$E-II(ijw!r-iZ<<_klwWKnS83*qNyjq#&cfLl_t5s33$?dK}C@$#&ur zo&^p)fTp04@!fFyF^?BNgp8z{{M?#>o9c7AXOd&j8?J8Io<55{PyM;NVUp8Y5B6U^ z;;v8UlrEjRCKpk`)KUM9MhRDhTOo!C#33XS2D!+@b@6L6W8~n5JHnCSnsB>*>7Z_m z>n;)xXIVr4kDH!o@QkR+aEvKL4ThNAcpL>CtmF^4<)OT)Ahh6f4h&pu)%8wRIG{ z3EAQ-iY|rY$wevh6h`;aC{5+quNNA)-9U+4e9NR15Q!o{br&E~&n%RUqtDQ??;^RJ zK0h?Ja40Rjt0T39>z7ReXK9YS!bPHJ-!#62-jRzG(%EX3`X=DA<*`QX{D0Ix0~C=i z8lx6Qx5~y_0LIpgl5>yB{azcjp&v;mWRPqcN-tK_Ds`086=59)A`nSU*Ml^4`<)&b z^e~0vIE4ceJc1_Wd-Dl5seO|b}h!o>h5 z!@%h!c~rc1Q7n-vg_{;?flE%_8zxEBko{=DgC=zlOUR(1emb*tVDM?63JjD1)i9g7 zk65KXmG~?k!C6=@NxXvRl^zBs=%g(80T}Cu6)Zj84=u`~Kh;LhL-aHzJ;;k7;aLPNgqlZ|y+}}dSOds-)G}PuDfHOmQFvy0 z#{TJ}z&etEig~P~bBa!US(NfHth|uUaKnV^tuVYkjFRyaCk$EM4MRcMRAtISRA!tN z!bDRnQ=u|q3TG;%75K!c2|5T1I?M?Iq>7|fi;fL(7YB}h&IHkluu)XjMxH0)O;iK> zn(_o?utP>v=A0OOyhxJ8Tajm_pQ;Q9$WQz_l*zj@$*v_ zTCXB}U!IKc)Cl8S+cv&29OJ+9?(Bcm9G!*Ah985?>sL{4tKLz)k6P>*I#RFJN8dW% z&c$jH?QtPj$43M8f^M6i&mGph#l;*;UbI5mEMoV8b{|5sYK;i%Z literal 0 HcmV?d00001 diff --git a/vllm/spec_decode/__pycache__/util.cpython-310.pyc b/vllm/spec_decode/__pycache__/util.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a4ce6b0685131eef46a6960c6a2d5118e830b1a GIT binary patch literal 7933 zcmeHM&5zthb|;H$Hk<1C(CB+C+qAuQrtMLWy>=p7Q3QMCM3y6K4Ucy1rL7R0u9|L+ z_|Yn|XEcKzz#Hkf3J@fR93>!FtPVjA!2&tu5adr-AOL~nl1r3BE_=z!UQK?l*zD<^ zv5-rET*4U^i^Zx}ud3e1@4ZfEW*iOITYvKpy_>IU+P~4uu6y;^_H6Y{j~ZSBeay&-k9bE^TP&?b< z-dsHI&BsT*qq-&*ykp!FH8Foi<8?e2&S>i8s<;?m#KgCbqt9fU}yq_s~#~N(>sr8gCG~?lx(Fzv%bKqlSo6!cUlGsqI@i}7{yqLGzP=zBe~tgnJ?*aUy?90A zOT2YYZX zKb7d1k6yFl#8mH3L>}-U4~p5Uc{Ki1CT{y(BIw7XK7K@8T2kZ#nRFnnbjfvgqzW}g-O@#2Qr8=m#1!$ z=5B{Zt*6orl5Mvdx$Kr}bKg{-@?MZDQrHT08g2D zZ<%ryeZ6_i=1=nIEgarnuQ}P{ktP=~^vT7QJnpab!~VHEkX@0VJ0A~v@mulMN))cG z^tba~nyhR_QM{7%MaSo&lX9^#$it}J-!6`0gnw}IInd3BFVnVl`ZM*WZZdfsO~veJ z4Sp_?ES1GkwMfDJJiR3nKjc}5p-e(>b$;`U9N( zLtVY$)Q9@IaYoy%4QptpuO?@l4b6wft~Jz#)-CfvO|@GzYPUYD4{Pg8Ny#TyM6S-Y z+Un6sjSgCHSMiy(-OrSmW~#Fcl3N*$nvajVJj%y(l8^UrwdzGJ>jiyiT%O8K&$A#d zNmqD{L6YJ0#f~T}ai+n&kQ*CTeds_3i2chuR+I`<)WVz&RURndVUsSl(U(3 zf;Cn5j%FefogBMavF|OGMA3Bk+o?l)@2iz4$ta6sn7LS_Q=aZy(SHfwv$H6g^?7FN zOt+pohNYW(j4_?B)- zDs|jSP`-}}ghLy$q0aUDEYy`S=nw~duQ zu9XV!k_jz!xjcNgt1QWs_DhisqP(&VAINTYK`Ap?ifUQzQkdke3+|1|Y*f;;FJn`N zU~=r#PE8UTZH7w+M^Q;_n%>sxc+0N6f2_T0E2X;*kF!;aJ6;T|f~+ zJZ_%(itXyS$zL$84Ri@x0--Tf{rJ#GAUqHtb_xjTKKl!Hn{0%6y$b^BlBe*H09{&3 zmxRJvX2IHTdX5s)FyZ2kSBh*U0$y`cf)DDMbCE;TI&i0#J^{6C9)+bjHJh1FW_4)Q zQ_I?Uy>w9HJ@ZrLvD+6T*nud!)ZQlo_dr|~)}Rj`EQ*GJ2@#U2#!lF8S<=O4NjfZF zrs5SUzDEU3eUQDA-#zp1qH9LJ#5@L-2b1QWV>K=NiRs8U&|FyLi$Y$^0WSb4j5+Vn zkyIBMdeUK~ze}j#Ahe+l4K|0hyS3eVuIF&+Xah95Z*=vc#o2vjw1a=1)Y>VxUV@GP zSJ?6dChZ1V6y|V4en1&3SGyWWmFz>;)9;I0=6FXMV$MlYAc( zf$g!H^+dMOrI>;6_tPwh{0P8j3gZx;V|4okUb^^Ohg}*yjT|_Kl%aH3H&4T6nYe2} zGnPCvWQjgB@Lu1whjrkgnsR71YNkTKb5J5f;TwJD?|ua27UtXT`XK2Lwui$AayJ8p z&4FNC1uRfza3A=%)e~?iQhAkX0WKEchM972J3^)|P%a>E+x;9Msh18S4wxIKn*yW4 z3{*P8-Mn_~6E_?5`>DhToM4UdE{>s*lr5khtt` zXfcXC$L6s*$G=3k>_ZghQA@{}KV_!%joCmTiO|xtCfe*KtZh@DWhR?$WnZHOF4)|Y z@6)93Qt><$3cs*y;LxHq?CT<4}?mi3=U| zBlzyzui=YM6k11i{^KJN!E3rmmGx2%w@g+vPCLjtz+Nc#eC5+0RW_46=<1X`-3s%b z%Lj41trT0uc*tSgAP2n_B2~2}Moyd4;7>nu&#$~!df#52_p=KtD?IIF?QWWOBhiL# z(mR5QBwyhxZ^=J=r!U{W%Cr7wTkoC!V(n`DlXuov-@E$xPX-@<{Pssz@@wZm;8*|n zirjj4MZoC3(h~tk((Dg|15xfD|I_MH-*;Yfac8pTh^;3(BXkf!}uLxK;X z+p6|RzpjLApZz$|4QvwP+V`ZqQ>)WoazVEMvIfbkK!?fY>zn# zBw>vx-73sp*zFZ2WqagD)QtwTno1Q*YhoY;R~Se&mTJrr;F}8I1rgA3x{jRi9TgdK z54Hzk|J&a$>Hr})2IPmMH0ic#2Q)>^Q&^?299T?o=9$~}D{!Y^v0XR!mW&0Y{%n-M zNKY*jT8TfVH@}rP&~Iu#$&``BONjt&;7=9&*ySM#GG}~zF1bNzkb>WcFn8qNYAz@NsM&73|!<^m9~I3 z5Q|D6Eh{>J*W0SCA7mpCv)>mmD9*ZVh#^%~GPaLuFr~YcqxGn38yj_0WFK5c7ZmCNuNqqel)G71Be6(&}k71pgS zdL9%shX_%Euq*zP1cTIreZANu%>4qB6ohqfPhI4s-^ZP~D1_$eT=QVWJ(y=o0D2}j zc@57Rw_sk8n!OPsFS7IQ4}uJ+FQ*s6sPt3HSXL;RAB!Lx(7w2r$=ae{X(Ok^7}BN@ z17r+T8a&q?Xb)kxd<1c;rICR0n>4&|;BjT{QyrkmM2>d-;082&A7)la->-pmMuy9Q z4(-g{AU@mUuTu_P_FjJqV)Fm%8u6Kl#4&K}wQ* zOw2g;3Gz#P&}l|8JGN%(w$U_It1{)Sm((5<&PW#Ui-@F6Ei9}IQXsKUnxw6k1Rx~w zEeS>_MetsxpH14}P()ar=nP0w$c2+k{h6F#`vvg;VEg;qL)HK*{XHDl@l?I{2|5Jt*$7J6+ zHTI2TIZxQK`5UA}#@_(hM&H;ItImGIZ0En&Yl SpeculativeScores: + """Score the proposed tokens via the scorer model. + + This converts each input sequence to a set of k+1 target sequences. The + target sequences have the unique continuations to be scored and a + unique sequence ID that is different from all input sequence ids. + + If a speculative sequence length would exceed the max model length, then + no speculation is produced for that sequence. + + Args: + execute_model_req: The execution request. + proposals: The speculative proposals to score. + Returns: + SpeculativeScores: The scores of each speculative token, along with + which sequences were ignored during scoring. + """ + + # TODO(cade) perform this on GPU to remove blocking call. + proposal_lens_list = proposals.proposal_lens.tolist() + proposal_token_ids_list = proposals.proposal_token_ids.tolist() + + # Filter the list to ignore invalid proposals. + proposal_token_ids_list_without_skips = [ + proposals for proposals in proposal_token_ids_list + if VLLM_INVALID_TOKEN_ID not in proposals + ] + + (spec_indices, non_spec_indices, target_seq_group_metadata_list, + num_scoring_tokens) = self._expand_batch( + seq_group_metadata_list=execute_model_req.seq_group_metadata_list, + proposal_token_ids_list=proposal_token_ids_list_without_skips, + proposal_lens_list=proposal_lens_list, + ) + + target_sampler_output = self._scorer_worker.execute_model( + execute_model_req=execute_model_req.clone( + seq_group_metadata_list=target_seq_group_metadata_list)) + assert len(target_sampler_output) == 1, "expected single-step output" + target_sampler_output = target_sampler_output[0] + + if not non_spec_indices: + # All sequence groups in batch have spec decoding enabled + contracted = self._contract_batch_all_spec( + target_sampler_output=target_sampler_output, + proposals=proposals, + ) + else: + # Batch has a mix of spec decode enabled and disabled seq groups + contracted = self._contract_batch( + contracted_bs=len(execute_model_req.seq_group_metadata_list), + target_sampler_output=target_sampler_output, + proposals=proposals, + num_scoring_tokens=num_scoring_tokens, + non_spec_indices=non_spec_indices, + spec_indices=spec_indices, + k=execute_model_req.num_lookahead_slots, + ) + + all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted + return SpeculativeScores( + probs=all_probs, + token_ids=all_tokens, + logprobs=spec_logprobs, + hidden_states=all_hidden_states, + ) + + def _expand_batch( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + proposal_token_ids_list: List[List[TokenId]], + proposal_lens_list: List[int], + ) -> Tuple[List[int], List[int], List[SequenceGroupMetadata], int]: + """Given the input sequences and potentially multiple corresponding + proposal tokens, create a new batch where each sequence has a single + query token. + """ + + # vLLM currently only supports proposal lens equal to zero or the batch + # proposal len. This adds some complexity (splitting the batch into spec + # and non spec sequences) and should be removed in the future. It can be + # done by supporting per-sequence proposal lens. + (spec_seqs, spec_indices), (non_spec_seqs, non_spec_indices) = \ + split_batch_by_proposal_len( + seq_group_metadata_list, proposal_lens_list) + + target_seq_group_metadata_list = self._create_scoring_model_input( + seq_group_metadata_list=spec_seqs, + proposal_token_ids=proposal_token_ids_list, + # NOTE: We determine the seq ids in the expanded batch using the + # full seq_group_metadata_list, instead of only spec_seqs. + target_seq_ids_iter=self._create_target_seq_id_iterator( + seq_ids=get_all_seq_ids(seq_group_metadata_list)), + ) + + num_scoring_tokens = len(target_seq_group_metadata_list) + target_seq_group_metadata_list.extend(non_spec_seqs) + + return (spec_indices, non_spec_indices, target_seq_group_metadata_list, + num_scoring_tokens) + + def _contract_batch( + self, contracted_bs: int, target_sampler_output: SamplerOutput, + proposals: SpeculativeProposals, num_scoring_tokens: int, + non_spec_indices: List[int], spec_indices: List[int], k: int + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, + Optional[torch.Tensor]]: + """Contract the expanded batch back into its original size. + This maps the scores of speculative tokens back to their original + sequences. + + contracted_bs is the original batch size, and the batch size that the + target_sampler_output will be contracted to. + """ + (target_token_ids, target_probs, target_logprobs, target_hidden_states, + non_spec_target_token_ids, non_spec_target_probs, + non_spec_target_logprobs, + non_spec_target_hidden_states) = self._split_scoring_output( + target_sampler_output, num_scoring_tokens) + + # Map distinct sequences used to score each token + # of shape [batch_size * k + 1] back to [batch_size, k + 1]. + expanded_batch_size, k = proposals.proposal_token_ids.shape + + # The number of tokens in the expanded batch used for speculation is + # equal to the total expanded batch size minus the number of samples for + # non-speculative sequences. + non_spec_expanded_bs = len(non_spec_target_token_ids) + spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs + + target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1) + target_probs = target_probs.reshape(*target_token_ids.shape, + self._vocab_size) + target_logprobs = target_logprobs.reshape(target_probs.shape) + + if target_hidden_states is not None: + target_hidden_states = target_hidden_states.reshape( + *target_token_ids.shape, target_hidden_states.shape[-1]) + + all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1), + fill_value=-1) + all_probs = target_probs.new_zeros(*all_tokens.shape, self._vocab_size) + all_logprobs = target_logprobs.new_full(size=all_probs.shape, + fill_value=-float("inf")) + + if target_sampler_output.hidden_states is not None: + all_hidden_states = target_hidden_states.new_zeros( + size=(contracted_bs, k + 1, target_hidden_states.shape[-1])) + else: + all_hidden_states = None + + if non_spec_indices: + all_tokens[non_spec_indices, :1] = \ + non_spec_target_token_ids.unsqueeze(1) + all_probs[non_spec_indices, :1, :] = \ + non_spec_target_probs.unsqueeze(1) + all_logprobs[non_spec_indices, :1, :] = \ + non_spec_target_logprobs.unsqueeze(1) + if all_hidden_states is not None: + assert non_spec_target_hidden_states is not None + all_hidden_states[non_spec_indices, :1, :] = \ + non_spec_target_hidden_states.unsqueeze(1) + + if spec_indices: + all_tokens[spec_indices] = target_token_ids + all_probs[spec_indices] = target_probs + all_logprobs[spec_indices] = target_logprobs + if all_hidden_states is not None: + all_hidden_states[spec_indices] = target_hidden_states + + return all_tokens, all_probs, all_logprobs, all_hidden_states + + def _contract_batch_all_spec( + self, + target_sampler_output: SamplerOutput, + proposals: SpeculativeProposals, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, + Optional[torch.Tensor]]: + """Contract the expanded batch back into its original size. + This maps the scores of speculative tokens back to their original + sequences. + + It assumes all sequences in the batch were previously expanded. + """ + + # Map distinct sequences used to score each token + # of shape [batch_size * k + 1] back to [batch_size, k + 1]. + contracted_bs, k = proposals.proposal_token_ids.shape + + # Reshape tensors to original batch size + target_token_ids = target_sampler_output.sampled_token_ids.reshape( + contracted_bs, k + 1) + target_probs = target_sampler_output.sampled_token_probs.reshape( + *target_token_ids.shape, self._vocab_size) + target_logprobs = target_sampler_output.logprobs.reshape( + target_probs.shape) + target_hidden_states = target_sampler_output.hidden_states + if target_hidden_states is not None: + target_hidden_states = target_hidden_states.reshape( + *target_token_ids.shape, target_hidden_states.shape[-1]) + + return (target_token_ids, target_probs, target_logprobs, + target_hidden_states) + + def _create_scoring_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k] + target_seq_ids_iter: Iterator[TargetSeqId], + ) -> List[SequenceGroupMetadata]: + """Given the original input sequences and proposed tokens from the draft + model, create a list of target sequences that can be used for scoring. + + target_seq_ids_iter provides sequence ids for the expanded batch, + fulfilling the requirement that no seq id in the expanded batch is equal + to the seq id in the original batch. + """ + + if not seq_group_metadata_list: + return [] + + target_seq_group_metadata = list( + chain.from_iterable( + self._create_target_seq_group_metadata( + seq_group_metadata, + proposal_token_ids, + i, + target_seq_ids_iter, + ) for i, seq_group_metadata in enumerate( + seq_group_metadata_list))) + + return target_seq_group_metadata + + def _create_target_seq_group_metadata( + self, + input_seq_group_metadata: SequenceGroupMetadata, + proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k] + batch_index: int, + target_seq_ids_iter: Iterator[TargetSeqId], + ) -> List[SequenceGroupMetadata]: + """Given an input sequence group metadata and a list of draft tokens, + create a list of target SequenceGroupMetadata, one for each + token id that needs to be scored. + + Naive speculative decoding requires K target model scores, one for each + draft model token. However one can add a bonus token such that if each + token is accepted, then a final token may be sampled from the model. + This function creates K+1 target SequenceGroupMetadata to take + advantage of the bonus token. + """ + assert not input_seq_group_metadata.is_prompt, ( + "Speculating on " + "prompts not yet supported") + assert len(input_seq_group_metadata.seq_data) == 1, ( + "Beam search " + "not supported in speculative decoding") + input_seq_id = next(iter(input_seq_group_metadata.seq_data.keys())) + + token_ids_to_score = self._get_token_ids_to_score( + proposal_token_ids[batch_index]) + + # Use simpler sampling parameters apart from for final token + # (in particular don't do seeded sampling) since those sampled tokens + # aren't used. + # We don't replace the sampling_params in the greedy case because + # this also controls whether the probs get modified in the sampler + # (see use of _modify_greedy_probs_inplace there). + sampling_params = input_seq_group_metadata.sampling_params + non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \ + if sampling_params.temperature else sampling_params + + target_seq_group_metadata_list: List[SequenceGroupMetadata] = [] + last_index = len(token_ids_to_score) - 1 + for i, token_ids in enumerate(token_ids_to_score): + target_sampling_params = sampling_params if i == last_index \ + else non_bonus_sampling_params + target_seq_group_metadata_list.append( + self._create_single_target_seq_group_metadata( + input_seq_group_metadata, + input_seq_id, + next(target_seq_ids_iter), + token_ids, + sampling_params=target_sampling_params, + )) + + return target_seq_group_metadata_list + + @staticmethod + def _create_single_target_seq_group_metadata( + seq_group_metadata: SequenceGroupMetadata, + seq_id: SeqId, + target_seq_id: TargetSeqId, + token_ids: List[TokenId], + sampling_params: SamplingParams, + ) -> SequenceGroupMetadata: + """Create a single target SequenceGroupMetadata. + + Args: + seq_group_metadata: The metadata for the input sequence. + seq_id: The input sequence ID. + target_seq_id: The corresponding target sequence ID. + token_ids: The list of token ids that are to be appended to the + input sequence. + """ + seq_data = seq_group_metadata.seq_data[seq_id] + prompt_token_ids = seq_data.prompt_token_ids_array + new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids] + + new_seq_data_dict = { + target_seq_id: + SequenceData( + prompt_token_ids, + _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE, + new_output_token_ids), + ), + } + # This is a hack. Technically, spec decoding should compute + # num_lookahead slots at one shot, but instead, it expands the batch + # and evaluate one by one right now. context_len is seq_len - 1 because + # the kv cache is filled by a previous batch in the batch expansion. + for data in new_seq_data_dict.values(): + data.update_num_computed_tokens(data.get_len() - 1) + + return SequenceGroupMetadata( + request_id=seq_group_metadata.request_id, + is_prompt=seq_group_metadata.is_prompt, + seq_data=new_seq_data_dict, + sampling_params=sampling_params, + block_tables={ + target_seq_id: seq_group_metadata.block_tables[seq_id], + }, + lora_request=None, + token_chunk_size=1, + ) + + @staticmethod + def _split_scoring_output( + sampler_output: SamplerOutput, num_scoring_tokens: int + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, + Optional[torch.Tensor], torch.Tensor, torch.Tensor, + torch.Tensor, Optional[torch.Tensor]]: + """Split the target model output into speculative and non-speculative + output. + """ + + # vLLM currently only supports proposal lens equal to zero or the batch + # proposal len. This adds some complexity (splitting the batch into spec + # and non spec sequences) and should be removed in the future. It can be + # done by supporting per-sequence proposal lens. + # + # First samples are from speculative scoring, latter samples are non- + # speculative samples. + split_sizes = (num_scoring_tokens, + sampler_output.sampled_token_ids.numel() - + num_scoring_tokens) + (spec_probs, non_spec_probs + ) = sampler_output.sampled_token_probs.split(split_sizes) + (spec_sampled_tokens, non_spec_sampled_tokens + ) = sampler_output.sampled_token_ids.flatten().split(split_sizes) + ( + spec_logprobs, + non_spec_logprobs, + ) = sampler_output.logprobs.split(split_sizes) + + if sampler_output.hidden_states is not None: + ( + spec_hidden_states, + non_spec_hidden_states, + ) = sampler_output.hidden_states.split(split_sizes) + else: + spec_hidden_states, non_spec_hidden_states = None, None + + return (spec_sampled_tokens, spec_probs, spec_logprobs, + spec_hidden_states, non_spec_sampled_tokens, non_spec_probs, + non_spec_logprobs, non_spec_hidden_states) + + @staticmethod + def _create_target_seq_id_iterator( + seq_ids: List[SeqId]) -> Iterator[TargetSeqId]: + """Create an iterator for creating target sequence ids. + Target sequence ids are distinct from sequence ids because we create a + distinct target sequence id for each proposal token to be scored. + + This implementation increments a counter starting at 1 + max of all + provided input sequence ids. + """ + return count(start=max(seq_ids) + 1) + + @staticmethod + def _get_token_ids_to_score( + full_spec_token_ids: List[TokenId] # shape: [k] + ) -> List[List[TokenId]]: + """Given an int tensor of proposal token ids, return a list of + token ids that should be scored. + + Returns k+1 output lists. The additional one is used for generating the + bonus token. + + Example: + Input: [0, 1, 2, 3] (k=4) + Output: (k+1 lists) + [] + [0] + [0, 1] + [0, 1, 2] + [0, 1, 2, 3] + """ + empty_token_ids: List[TokenId] = [] + + token_ids_to_score = [empty_token_ids] + token_ids_to_score.extend(full_spec_token_ids[:i + 1] + for i in range(len(full_spec_token_ids))) + return token_ids_to_score diff --git a/vllm/spec_decode/draft_model_runner.py b/vllm/spec_decode/draft_model_runner.py new file mode 100644 index 00000000..aaf6ec5f --- /dev/null +++ b/vllm/spec_decode/draft_model_runner.py @@ -0,0 +1,327 @@ +from typing import List, Optional + +import torch + +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.sampler import SamplerOutput + +try: + try: + from vllm.attention.backends.flash_attn import FlashAttentionMetadata + except (ModuleNotFoundError, ImportError): + # vllm_flash_attn is not installed, try the ROCm FA metadata + from vllm.attention.backends.rocm_flash_attn import ( + ROCmFlashAttentionMetadata as FlashAttentionMetadata) +except (ModuleNotFoundError, ImportError) as err: + raise RuntimeError( + "Draft model speculative decoding currently only supports" + "CUDA and ROCm flash attention backend.") from err + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.logger import init_logger +from vllm.multimodal import MultiModalInputs +from vllm.sequence import ExecuteModelRequest, IntermediateTensors +from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata, + ModelRunner) + +logger = init_logger(__name__) + +# A flag to enable debug prints for the updated input tensors +# before each step. +debug_advance_input = False +# A flag to allow GPU advance step for draft model runner. +# Set to False for debugging. +allow_gpu_advance_step = True + + +class TP1DraftModelRunner(ModelRunner): + """Specialized model runner for speculative decoding draft model. + Since the draft model always execute k forward passes consecutively to + generate k speculative tokens in a single speculative decoding step, + we could get rid of most CPU-GPU synchronization and data transfer + overheads by keeping model input and output tensors on GPU all the time. + + TODOs: + 1. Currently supports only flash-attn, add support for other attn_backends. + 2. Support TP > 1 (this requires some designs because we do not expect + any broadcasting inside execute_model). + """ + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + return_hidden_states: bool = False, + observability_config: Optional[ObservabilityConfig] = None, + ): + if return_hidden_states: + raise ValueError( + "return_hidden_states is not supported for TP1DraftModelRunner." + ) + + super().__init__( + model_config=model_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + device_config=device_config, + cache_config=cache_config, + load_config=load_config, + lora_config=lora_config, + kv_cache_dtype=kv_cache_dtype, + is_driver_worker=is_driver_worker, + prompt_adapter_config=prompt_adapter_config, + return_hidden_states=return_hidden_states, + observability_config=observability_config, + ) + + def _update_sampling_metadata(self, sampling_metadata, num_seqs, + num_queries): + + assert sampling_metadata.num_prompts == 0 + assert len(sampling_metadata.seq_groups) == num_queries + assert sampling_metadata.selected_token_indices.shape == ( + num_queries, ) + # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501 + + # Verify that all sequences are decodes + for i in range(num_queries): + seq_group = sampling_metadata.seq_groups[i] + + assert seq_group.is_prompt is False # No prompt + assert seq_group.prompt_logprob_indices == [] # No prompt + assert seq_group.sample_indices == [i] # Simple + + def _gpu_advance_step( + self, model_input: ModelInputForGPUWithSamplingMetadata, + last_output: SamplerOutput + ) -> ModelInputForGPUWithSamplingMetadata: + # Currently, we expect "decode mode" only + assert not model_input.is_prompt + + # Get num_seqs + num_seqs = len(model_input.seq_lens) + num_queries = len(model_input.query_lens) + + # Get output tokens GPU tensor + sampled_token_ids = last_output.sampled_token_ids + assert sampled_token_ids is not None + + # Update attn_metadata + attn_metadata = model_input.attn_metadata + assert isinstance(attn_metadata, FlashAttentionMetadata) + + attn_metadata.advance_step(model_input, sampled_token_ids, + self.block_size, num_seqs, num_queries) + + # Update sampling_metadata + sampling_metadata = model_input.sampling_metadata + self._update_sampling_metadata(sampling_metadata, num_seqs, + num_queries) + + # Create new input + new_model_input = self._model_input_cls( + input_tokens=model_input.input_tokens, + input_positions=model_input.input_positions, + attn_metadata=attn_metadata, + seq_lens=attn_metadata.seq_lens, + query_lens=model_input.query_lens, + lora_mapping=model_input.lora_mapping, + lora_requests=model_input.lora_requests, + multi_modal_kwargs=model_input.multi_modal_kwargs, + sampling_metadata=model_input.sampling_metadata, + is_prompt=False, + ) + + # Ensure we skip CPU samples + assert new_model_input.sampling_metadata.skip_sampler_cpu_output is True + # We can reuse sampling tensors since every decode iteration is the same + new_model_input.sampling_metadata.reuse_sampling_tensors = True + + if debug_advance_input: + logger.debug("NEW INPUT: ") + logger.debug(" input_tokens = %s", new_model_input.input_tokens) + logger.debug(" input_positions = %s", + new_model_input.input_positions) + logger.debug(" seq_lens = %d", new_model_input.seq_lens) + logger.debug(" query_lens = %d", new_model_input.query_lens) + logger.debug(" attn_metadata:") + logger.debug(" seq_lens_tensor: %s", + attn_metadata.seq_lens_tensor) + logger.debug(" slot_mapping: %s", attn_metadata.slot_mapping) + logger.debug(" block_tables: %s", attn_metadata.block_tables) + + return new_model_input + + def supports_gpu_multi_step(self, execute_model_req: ExecuteModelRequest): + """Determines if draft_model_runner GPU multi-step can be used. + Currently required conditions are: + 1. Only decodes + 2. Only flash-attn + 3. No LORA + 4. No prompt_adapter_config + """ + if not allow_gpu_advance_step: + return False + + # We allow multi-step GPU only in decode mode + for seq_group in execute_model_req.seq_group_metadata_list: + if seq_group.is_prompt: + return False + + # TODO: Add support for other attn backends + if self.attn_backend.get_name() != "flash-attn": + return False + + # TODO: Add support for LORA + if self.lora_config: + return False + + # TODO: Add soft-tuning prompt adapter support + return not self.prompt_adapter_config + + @torch.inference_mode() + def execute_model( + self, + model_input: ModelInputForGPUWithSamplingMetadata, + kv_caches: List[torch.Tensor], + previous_hidden_states: Optional[torch.Tensor] = None, + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + """Executes num_steps forward passes with advacement of input tensors + on the GPU. Look at supports_gpu_multi_step(..) for pre-conditions. + + Optimizations used: + 1. Input tensors are updated on the GPU directly + 2. Skips GPU=>CPU serialization of sampler outputs (we don't need + them since we do batch expansion later that uses GPU outputs) + 3. Reuses sampling tensors (since we run only decodes and they have + a repeating sampling logic) + """ + + # When num_steps == 1, we execute the fallback here for the GPU + # advance_step, which runs prepare_inputs on CPU and for each spec + # iteration invokes this function only once + # (Look at multi-step-worker code) + is_fallback = num_steps == 1 + if not is_fallback: + # Since we do not broadcast data inside execute_model anymore, + # we need to figure out the best way to support TP > 1 in this + # case, because we will at least need to broadcast the sampled + # tokens to all workers. + if not self.is_driver_worker: + raise ValueError("TP1DraftModelRunner only supports TP=1.") + + # Sanity + if self.lora_config is not None: + raise ValueError("TP1DraftModelRunner has no support for LORA") + if self.prompt_adapter_config is not None: + raise ValueError("TP1DraftModelRunner has no support for " + "prompt_adapter_config") + if model_input.multi_modal_kwargs: + raise ValueError( + "TP1DraftModelRunner has no support for multi_modal_kwargs" + ) + else: + if self.lora_config: + assert model_input.lora_requests is not None + assert model_input.lora_mapping is not None + self.set_active_loras(model_input.lora_requests, + model_input.lora_mapping) + + if self.prompt_adapter_config: + assert model_input.prompt_adapter_requests is not None + assert model_input.prompt_adapter_mapping is not None + self.set_active_prompt_adapters( + model_input.prompt_adapter_requests, + model_input.prompt_adapter_mapping) + + self.attn_state.begin_forward(model_input) + + # Detect exec mode + assert model_input.attn_metadata is not None + use_cuda_graph = False + if model_input.attn_metadata.num_prefills > 0: + # In this case, execute_model(..) was called directly + if num_steps > 1: + raise ValueError( + "execute_model(..) of draft_model_runner can be called " + "directly only with a single-step prefill") + else: + # We can skip CPU samples for spec token generation. + # (We do allow CPU samples for num_steps == 1 to support the + # fallback case, where supports_gpu_multi_step(..) does not pass) + model_input.sampling_metadata.skip_sampler_cpu_output = ( + not is_fallback) + + # Attn attr defines if we use cuda graphs + use_cuda_graph = model_input.attn_metadata.use_cuda_graph + + # Get model + if use_cuda_graph: + graph_batch_size = model_input.input_tokens.shape[0] + model_executable = (self.graph_runners[model_input.virtual_engine] + [graph_batch_size]) + + if previous_hidden_states is not None: + hidden_states = torch.cat([ + previous_hidden_states, + torch.empty([ + graph_batch_size - previous_hidden_states.shape[0], + *previous_hidden_states.shape[1:] + ], + dtype=previous_hidden_states.dtype, + device=previous_hidden_states.device) + ]) + else: + hidden_states = None + else: + model_executable = self.model + hidden_states = previous_hidden_states + + outputs: List[SamplerOutput] = [] + for step in range(num_steps): + multi_modal_kwargs = model_input.multi_modal_kwargs or {} + + kwargs = {"previous_hidden_states": hidden_states} \ + if previous_hidden_states is not None else {} + + # Run model + with set_forward_context(model_input.attn_metadata): + hidden_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(multi_modal_kwargs, + device=self.device), + **kwargs, + ) + + # Compute the logits. + logits = self.model.compute_logits(hidden_states, + model_input.sampling_metadata) + + # Sample the next token. + outputs.append( + self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + )) + + # Prepare inputs for the next step + if step != num_steps - 1: + model_input = self._gpu_advance_step(model_input, outputs[-1]) + + return outputs diff --git a/vllm/spec_decode/interfaces.py b/vllm/spec_decode/interfaces.py new file mode 100644 index 00000000..029f5646 --- /dev/null +++ b/vllm/spec_decode/interfaces.py @@ -0,0 +1,90 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Optional, Set + +import torch + +from vllm.sequence import ExecuteModelRequest +from vllm.worker.worker_base import WorkerBase + + +@dataclass +class SpeculativeProposals: + """Datastructure used to represent proposal tokens from some proposer. It + also tracks how many speculative tokens each sequence has. + """ + + # Speculative proposal tokens. + proposal_token_ids: torch.Tensor + + # Probabilities of the proposal tokens according to the proposer. + proposal_probs: torch.Tensor + + # The valid length of each proposal; can be zero. + proposal_lens: torch.Tensor + + # A flag to mark that there's no available proposals + no_proposals: bool = False + + def __repr__(self): + return (f"SpeculativeProposals(" + f"proposal_token_ids={self.proposal_token_ids}, " + f"proposal_probs={self.proposal_probs.shape}, " + f"proposal_lens={self.proposal_lens})") + + +@dataclass +class SpeculativeScores: + """Datastructure used to represent the scores of speculative tokens + according to the scoring model. + """ + + # Probabilities of the speculative tokens according to the scoring model. + probs: torch.Tensor + + # Log-probabilities of the speculative tokens according to the scoring + # model. These values can be used to generate Logprob objects that are + # returned to the user. + logprobs: torch.Tensor + + # Token ids sampled from the scoring model. Used for speculative bonus + # tokens and also non-speculative normal decoding. + token_ids: torch.Tensor + + # Optional last hidden states from the scoring model. + hidden_states: Optional[torch.Tensor] = None + + def __repr__(self): + return (f"SpeculativeScores(" + f"probs={self.probs.shape}, " + f"token_ids={self.token_ids.shape})") + + +class SpeculativeProposer(ABC): + + @abstractmethod + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + # If set, this contains all sequence IDs that were assigned + # bonus tokens in their last forward pass. + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> SpeculativeProposals: + raise NotImplementedError + + +class SpeculativeScorer(ABC): + + def __init__(self, scorer_worker: WorkerBase, device: str, + vocab_size: int): + self._scorer_worker = scorer_worker + self._device = device + self._vocab_size = vocab_size + + @abstractmethod + def score_proposals( + self, + execute_model_req: ExecuteModelRequest, + proposals: SpeculativeProposals, + ) -> SpeculativeScores: + raise NotImplementedError diff --git a/vllm/spec_decode/medusa_worker.py b/vllm/spec_decode/medusa_worker.py new file mode 100644 index 00000000..0d233f39 --- /dev/null +++ b/vllm/spec_decode/medusa_worker.py @@ -0,0 +1,136 @@ +import weakref +from typing import List, Optional, Set, Tuple + +import torch + +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata +from vllm.spec_decode.interfaces import SpeculativeProposals +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase +from vllm.spec_decode.top1_proposer import Top1Proposer +from vllm.worker.worker import Worker + + +class MedusaWorker(NonLLMProposerWorkerBase, Worker): + """Worker for Medusa. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Lazy initialization list. + self._proposer: Top1Proposer + + def init_device(self): + super().init_device() + + self._proposer = Top1Proposer( + weakref.proxy(self), # type: ignore[arg-type] + self.device, + self.vocab_size, + max_proposal_len=self.max_model_len, + ) + + def set_include_gpu_probs_tensor(self): + pass + + def set_should_modify_greedy_probs_inplace(self): + pass + + @torch.inference_mode() + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + # Unused parameter. + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> Tuple[List[SamplerOutput], bool]: + """Run the model forward pass to generate sample_len future tokens. + Returns the list of sampler output, one per layer, along with indicator + of whether torch tensor in sampler output need to be transposed in + latter sampler_output_to_torch logic. + + For medusa worker, this indicator shall be False. + """ + self._raise_if_unsupported(execute_model_req) + + seq_group_metadata_list = execute_model_req.seq_group_metadata_list + + seq_lens, query_lens = self._prepare_input_tensors( + seq_group_metadata_list) + + generators = self.model_runner.get_generators( + execute_model_req.finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, seq_lens, query_lens, self.device, + self.model_runner.pin_memory, generators) + + model_outputs = self.model_runner.model.generate_proposals( + previous_hidden_states=execute_model_req.previous_hidden_states. + hidden_states, + sampling_metadata=sampling_metadata) + + return model_outputs, False + + def _prepare_input_tensors( + self, + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], + ) -> Tuple[List[int], List[int]]: + if not seq_group_metadata_list: + return [], [] + + seq_lens: List[int] = [] + query_lens: List[int] = [] + + for seq_group_metadata in seq_group_metadata_list: + is_prompt = seq_group_metadata.is_prompt + + for seq_data in seq_group_metadata.seq_data.values(): + seq_data_len = seq_data.get_len() + if is_prompt: + context_len = seq_data.get_num_computed_tokens() + seq_len = min( + seq_data_len, + context_len + seq_group_metadata.token_chunk_size) + seq_lens.append(seq_len) + query_lens.append(seq_len - context_len) + else: + seq_lens.append(seq_data_len) + query_lens.append(1) + + return seq_lens, query_lens + + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> SpeculativeProposals: + """Produce speculations given an input batch of sequences. The number of + speculative tokens per sequence is determined by max_proposal_len. + """ + + return self._proposer.get_spec_proposals( + execute_model_req, seq_ids_with_bonus_token_in_last_step) + + def _raise_if_unsupported( + self, + execute_model_req: ExecuteModelRequest, + ) -> None: + """MedusaWorker does not yet implement support for cache swap + operations or beam search. + """ + if any([ + execute_model_req.blocks_to_swap_in, + execute_model_req.blocks_to_swap_out, + execute_model_req.blocks_to_copy + ]): + raise NotImplementedError( + "MedusaWorker does not support cache operations") + + if any( + len(seq_group_metadata.seq_data.keys()) != 1 + for seq_group_metadata in + execute_model_req.seq_group_metadata_list): + raise NotImplementedError( + "MedusaWorker does not support beam search.") diff --git a/vllm/spec_decode/metrics.py b/vllm/spec_decode/metrics.py new file mode 100644 index 00000000..89ccaba7 --- /dev/null +++ b/vllm/spec_decode/metrics.py @@ -0,0 +1,196 @@ +import time +from typing import Callable, Optional + +import msgspec +import torch + +from vllm.model_executor.layers.spec_decode_base_sampler import ( + SpecDecodeBaseSampler) +from vllm.utils import is_pin_memory_available + + +class SpecDecodeWorkerMetrics( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """Dataclass holding metrics emitted from the spec decode worker. + """ + + # The empirical acceptance rate of the proposal method on a per-token basis. + # This is useful for evaluating how well the proposal method aligns with the + # scoring method. + draft_acceptance_rate: float + + # The empirical efficiency, measured as the number of tokens emitted by the + # system divided by the number of tokens that could be emitted by the system + # if the proposal method were perfect. + system_efficiency: float + + # The number of speculative tokens produced by the proposal method. + draft_tokens: int + + # The number of tokens emitted by the entire system. + emitted_tokens: int + + # The number of tokens accepted by the scoring model and verification + # routine, e.g. Llama2-70B and lossless rejection sampling. + # + # NOTE: Any token accepted by the verification routine is considered + # accepted (regardless of if the speculative prefix is also accepted). The + # user will usually see less accepted tokens. This metric is helpful when + # evaluating alignment of the proposal method with the scoring model. + accepted_tokens: int + + # The number of speculative tokens per sequence. + num_spec_tokens: int + + +Timer = Callable[[], float] + + +class AsyncMetricsCollector: + """Class which copies rejection/typical-acceptance sampler metrics + from the device to CPU on a non-default Torch stream. + """ + + def __init__(self, + spec_decode_sampler: SpecDecodeBaseSampler, + timer: Optional[Timer] = None, + collect_interval_s: float = 5.0): + self.spec_decode_sampler = spec_decode_sampler + self._timer = time.time if timer is None else timer + + self._rank: Optional[int] = None + + # We don't have a device set yet. + self._copy_stream: Optional[torch.cuda.Stream] = None + + self._in_flight_copy: Optional[torch.cuda.Event] = None + + pin_memory = is_pin_memory_available() + self._aggregate_num_accepted_tokens = torch.tensor( + 0, dtype=torch.long, device="cpu", pin_memory=pin_memory) + self._aggregate_num_emitted_tokens = torch.tensor( + 0, dtype=torch.long, device="cpu", pin_memory=pin_memory) + self._aggregate_num_draft_tokens = 0 + + self._rejsample_metrics_collect_interval_s = collect_interval_s + self._last_metrics_collect_time = self._timer() + + def init_gpu_tensors(self, rank: int) -> None: + self._rank = rank + self._copy_stream = torch.cuda.Stream() + + def maybe_collect_rejsample_metrics( + self, k: int) -> Optional[SpecDecodeWorkerMetrics]: + + # If a copy was initiated in the previous call, collect and return. + if self._in_flight_copy is not None: + ready_event = self._in_flight_copy + self._in_flight_copy = None + return self._collect_rejsample_metrics(k, ready_event) + + # Otherwise, check if we should start a new copy. + if self._should_collect_rejsample_metrics(self._timer()): + assert self._in_flight_copy is None + self._in_flight_copy = self._copy_rejsample_metrics_async() + + return None + + def _should_collect_rejsample_metrics(self, now: float) -> bool: + """Return whether or not this iteration should print sampling + metrics. + """ + if self._rank != 0: + return False + + return now - self._last_metrics_collect_time >= self._rejsample_metrics_collect_interval_s # noqa: E501 + + def _copy_rejsample_metrics_async(self) -> torch.cuda.Event: + """Copy rejection/typical-acceptance sampling metrics + (number of accepted tokens, etc) to CPU asynchronously. + + Returns a CUDA event recording when the copy is complete. + """ + assert self._copy_stream is not None + self._copy_stream.wait_stream(torch.cuda.current_stream()) + + with torch.cuda.stream(self._copy_stream): + self._aggregate_num_accepted_tokens.copy_( + self.spec_decode_sampler.num_accepted_tokens, + non_blocking=True) + self._aggregate_num_emitted_tokens.copy_( + self.spec_decode_sampler.num_emitted_tokens, non_blocking=True) + # Number of draft tokens is calculated on CPU, so no copy is + # required. + self._aggregate_num_draft_tokens = ( + self.spec_decode_sampler.num_draft_tokens) + + aggregate_metrics_ready = torch.cuda.Event() + aggregate_metrics_ready.record(self._copy_stream) + + return aggregate_metrics_ready + + def _collect_rejsample_metrics( + self, k: int, + ready_event: torch.cuda.Event) -> SpecDecodeWorkerMetrics: + """Create metrics object from statistics copied asynchronously. + + Args: + k: int. The number of speculative tokens; used to determine system + efficiency. + ready_event: torch.cuda.Event. The CUDA event recording when the + async GPU->CPU copy is complete. + """ + + ready_event.synchronize() + + # update time of last collection + self._last_metrics_collect_time = self._timer() + + accepted_tokens = self._aggregate_num_accepted_tokens.item() + emitted_tokens = self._aggregate_num_emitted_tokens.item() + draft_tokens = self._aggregate_num_draft_tokens + + max_num_emitted_tokens = self.get_max_num_emitted_tokens( + draft_tokens, k) + + if draft_tokens > 0: + draft_acceptance_rate = accepted_tokens / draft_tokens + else: + draft_acceptance_rate = float("nan") + + if max_num_emitted_tokens > 0: + system_efficiency = emitted_tokens / max_num_emitted_tokens + else: + system_efficiency = float("nan") + + return SpecDecodeWorkerMetrics( + num_spec_tokens=k, + draft_acceptance_rate=draft_acceptance_rate, + system_efficiency=system_efficiency, + accepted_tokens=accepted_tokens, + draft_tokens=draft_tokens, + emitted_tokens=emitted_tokens, + ) + + @staticmethod + def get_max_num_emitted_tokens(draft_tokens: int, k: int) -> int: + """Calculate the number of emitted tokens, assuming all tokens are + accepted. + + This is equal to the number of sequences that have been speculated on, + times (speculation len + 1). The +1 comes from the bonus token. + """ + # Determine the number of sequences that have been speculated on. Since + # the batch size can be variable, we divide by k. + assert draft_tokens % k == 0 + total_num_spec_seqs = draft_tokens // k + + # A single sequence may emit k accepted tokens and one bonus token in + # the best case. + num_emitted_per_seq_if_all_accepted = k + 1 + + # The max num of emitted tokens is the number of speculated sequences + # times the max emitted per seq. + return total_num_spec_seqs * num_emitted_per_seq_if_all_accepted diff --git a/vllm/spec_decode/mlp_speculator_worker.py b/vllm/spec_decode/mlp_speculator_worker.py new file mode 100644 index 00000000..fc41bb82 --- /dev/null +++ b/vllm/spec_decode/mlp_speculator_worker.py @@ -0,0 +1,91 @@ +from typing import List, Optional, Set, Tuple + +import torch + +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata +from vllm.spec_decode.multi_step_worker import MultiStepWorker +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase + + +class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker): + """Worker for MLPSpeculator models. + + Not currently compatible with LoRA or chunked prefill. + """ + + @torch.inference_mode() + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + # Unused parameter. MLPSpeculatorWorker does not use the KV Cache and + # therefore does not need this parameter. + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> Tuple[List[SamplerOutput], bool]: + """Run the model forward pass to generate sample_len future tokens. + Returns the list of sampler output, one per layer, along with indicator + of whether torch tensor in sampler output need to be transposed in + latter sampler_output_to_torch logic. + + For mlp spec worker, this indicator shall be True. + """ + self._raise_if_unsupported(execute_model_req) + + seq_group_metadata_list = execute_model_req.seq_group_metadata_list + + (input_tokens, seq_lens, + query_lens) = self._prepare_input_tensors(seq_group_metadata_list) + + generators = self.model_runner.get_generators( + execute_model_req.finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, seq_lens, query_lens, self.device, + self.model_runner.pin_memory, generators) + + model_outputs = self.model_runner.model.generate_proposals( + input_ids=input_tokens, + previous_hidden_states=execute_model_req.previous_hidden_states. + hidden_states, + num_predict_tokens=sample_len, + sampling_metadata=sampling_metadata) + + assert len(model_outputs) == sample_len + + return model_outputs, True + + def _prepare_input_tensors( + self, + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], + ) -> Tuple[torch.Tensor, List[int], List[int]]: + if not seq_group_metadata_list: + return torch.empty(0, device=self.device), [], [] + + input_tokens: List[int] = [] + seq_lens: List[int] = [] + query_lens: List[int] = [] + + for seq_group_metadata in seq_group_metadata_list: + is_prompt = seq_group_metadata.is_prompt + + for seq_data in seq_group_metadata.seq_data.values(): + seq_data_len = seq_data.get_len() + if is_prompt: + context_len = seq_data.get_num_computed_tokens() + seq_len = min( + seq_data_len, + context_len + seq_group_metadata.token_chunk_size) + tokens = seq_data.get_token_ids()[context_len:seq_len] + seq_lens.append(seq_len) + input_tokens.extend(tokens) + query_lens.append(seq_len - context_len) + else: + seq_lens.append(seq_data_len) + input_tokens.append(seq_data.get_last_token_id()) + query_lens.append(1) + + input_tokens_tensor = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) + return input_tokens_tensor, seq_lens, query_lens diff --git a/vllm/spec_decode/mqa_scorer.py b/vllm/spec_decode/mqa_scorer.py new file mode 100644 index 00000000..f35a8a0a --- /dev/null +++ b/vllm/spec_decode/mqa_scorer.py @@ -0,0 +1,106 @@ +from vllm.sequence import (ExecuteModelRequest, SequenceData, + SequenceGroupMetadata, get_all_seq_ids) +from vllm.spec_decode.interfaces import (SpeculativeProposals, + SpeculativeScorer, SpeculativeScores) + +SeqId = int +TargetSeqId = int + + +class MQAScorer(SpeculativeScorer): + + def score_proposals( + self, + execute_model_req: ExecuteModelRequest, + proposals: SpeculativeProposals, + ) -> SpeculativeScores: + target_seq_group_metadata_list = [] + target_seq_id_start = max( + get_all_seq_ids(execute_model_req.seq_group_metadata_list)) + 1 + all_proposal_tokens = proposals.proposal_token_ids.tolist() + all_proposal_lengths = proposals.proposal_lens.tolist() + for i, seq_group_metadata in enumerate( + execute_model_req.seq_group_metadata_list): + seq_data_dict = seq_group_metadata.seq_data + assert len(seq_data_dict) == 1 + seq_id = next(iter(seq_data_dict.keys())) + + seq_data: SequenceData = seq_data_dict[seq_id] + prompt_token_ids = seq_data.get_prompt_token_ids() + output_token_ids = seq_data.get_output_token_ids() + proposal_token_ids = all_proposal_tokens[ + i][:all_proposal_lengths[i]] + new_output_token_ids = [*output_token_ids, *proposal_token_ids] + + target_seq_id = target_seq_id_start + i + new_seq_data = SequenceData.from_seqs( + prompt_token_ids=prompt_token_ids, + output_token_ids=new_output_token_ids, + ) + new_seq_data.update_num_computed_tokens( + len(prompt_token_ids) + len(output_token_ids) - 1) + + # Ensure that the new sequence has at least one token + # because we only use mqa scorer in the decoding stage. + assert len(output_token_ids) >= 1 + new_seq_data_dict = {target_seq_id: new_seq_data} + + new_seq_group_metadata = SequenceGroupMetadata( + request_id=seq_group_metadata.request_id, + is_prompt=seq_group_metadata.is_prompt, + seq_data=new_seq_data_dict, + sampling_params=seq_group_metadata.sampling_params, + block_tables={ + target_seq_id: seq_group_metadata.block_tables[seq_id], + }, + lora_request=None, + token_chunk_size=1, + ) + target_seq_group_metadata_list.append(new_seq_group_metadata) + + target_sampler_output = self._scorer_worker.execute_model( + execute_model_req=execute_model_req.clone( + seq_group_metadata_list=target_seq_group_metadata_list)) + + target_sampler_output = target_sampler_output[0] + + k = execute_model_req.num_lookahead_slots + bs = len(execute_model_req.seq_group_metadata_list) + target_token_ids = target_sampler_output.sampled_token_ids + target_probs = target_sampler_output.sampled_token_probs + target_logprobs = target_sampler_output.logprobs + # If all requests have the same number of query tokens, we can avoid + # the for loop to build output for better performance. + if min(all_proposal_lengths) == k: + bs, _ = proposals.proposal_token_ids.shape + all_tokens = target_token_ids.reshape(bs, k + 1) + all_probs = target_probs.reshape(bs, k + 1, self._vocab_size) + all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size) + else: + all_tokens = target_token_ids.new_full(size=(bs, k + 1), + fill_value=-1) + all_probs = target_probs.new_zeros(*all_tokens.shape, + self._vocab_size) + all_logprobs = target_logprobs.new_full(size=all_probs.shape, + fill_value=-float("inf")) + target_token_ids = target_token_ids.flatten() + start_loc = 0 + for i, proposed_len in enumerate(all_proposal_lengths): + output_len = proposed_len + 1 + end_loc = start_loc + output_len + all_tokens[ + i, :output_len] = target_token_ids[start_loc:end_loc] + all_probs[i, :output_len] = target_probs[start_loc:end_loc] + all_logprobs[ + i, :output_len] = target_logprobs[start_loc:end_loc] + start_loc = end_loc + + hidden_states = None + if target_sampler_output.hidden_states is not None: + hidden_states = target_sampler_output.hidden_states.reshape( + bs, (k + 1), -1) + + return SpeculativeScores(probs=all_probs, + token_ids=all_tokens, + logprobs=all_logprobs, + hidden_states=hidden_states) diff --git a/vllm/spec_decode/multi_step_worker.py b/vllm/spec_decode/multi_step_worker.py new file mode 100644 index 00000000..4b53fbe0 --- /dev/null +++ b/vllm/spec_decode/multi_step_worker.py @@ -0,0 +1,366 @@ +import copy +import weakref +from typing import Dict, List, Set, Tuple + +import torch + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData, + SequenceGroupMetadata) +from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner +from vllm.spec_decode.interfaces import (SpeculativeProposals, + SpeculativeProposer) +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase +from vllm.spec_decode.top1_proposer import Top1Proposer +from vllm.worker.worker import Worker + + +class MultiStepWorker(Worker, ProposerWorkerBase): + """The MultiStepWorker is equivalent to a Worker except that it allows + multiple forward passes in a single call, assuming the scheduler has + allocated enough space to store the additional KV. This reduces overhead + by invoking the scheduler less. + + The MultiStepWorker does not support cache swap operations, or beam search. + Cache swap operations do not require large modifications. On the other hand, + beam search requires memory allocations during sequence forks and thus + requires more thought for MultiStepWorker support. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Lazy initialization list. + self._proposer: SpeculativeProposer + + def init_device(self) -> None: + super().init_device() + + self._proposer = Top1Proposer( + weakref.proxy(self), # type: ignore[arg-type] + self.device, + self.vocab_size, + max_proposal_len=self.max_model_len, + ) + + def set_include_gpu_probs_tensor(self) -> None: + # Need include_gpu_probs_tensor for MultiStepWorker + self.model_runner.model.sampler.include_gpu_probs_tensor = True + + def set_should_modify_greedy_probs_inplace(self) -> None: + self.model_runner.model.sampler.should_modify_greedy_probs_inplace = ( + True) + + @torch.inference_mode() + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> Tuple[List[SamplerOutput], bool]: + """Run the model forward pass sample_len times. Returns the list of + sampler output, one per model forward pass, along with indicator of + whether torch tensor in sampler output need to be transposed in latter + sampler_output_to_torch logic. + + For multi step worker, this indicator shall be True. + """ + self._raise_if_unsupported(execute_model_req) + # Expand the batch for sequences with a bonus token. + # Perform a forward pass on the expanded batch and filter the + # response to retain only the original sequences' responses. + expanded_request, indices_of_seq_with_bonus_tokens =\ + self._expand_execute_model_request( + execute_model_req, seq_ids_with_bonus_token_in_last_step) + + # Run model sample_len times. + model_outputs: List[SamplerOutput] = [] + if isinstance( + self.model_runner, TP1DraftModelRunner + ) and self.model_runner.supports_gpu_multi_step(expanded_request): + # Here we run the draft_model_runner with multi-step prepare + # on the GPU directly + expanded_request.num_steps = sample_len + model_outputs = self.execute_model( + execute_model_req=expanded_request) + else: + # Here we run multi-step directly, with every step prepared + # on the CPU. + # TODO: Remove this branch once DraftModelRunner supports TP>1 + # and other restrictions that are part of DraftModelRunner's + # supports_gpu_multi_step(..) + for _ in range(sample_len): + model_output: List[SamplerOutput] = super().execute_model( + execute_model_req=expanded_request) + assert (len(model_output) == 1 + ), "composing multistep workers not supported" + model_output = model_output[0] + + self._append_new_tokens( + model_output, expanded_request.seq_group_metadata_list) + model_outputs.append(model_output) + + filtered_model_outputs = self._filter_model_output( + model_outputs, indices_of_seq_with_bonus_tokens) + return filtered_model_outputs, True + + @staticmethod + def _expand_execute_model_request( + execute_model_req: ExecuteModelRequest, + seq_with_bonus_token_in_last_step: set, + ) -> Tuple[ExecuteModelRequest, List[int]]: + """ + Expands the execute model request based on sequences with bonus + tokens. + + For each sequence with a bonus token, this method creates a new + sequence without the bonus token and adds it to the execute model + request. The original sequence groups are also retained. The indices + of the original sequence groups are returned for further processing. + + Args: + execute_model_req (ExecuteModelRequest): The original execute + model request. + seq_with_bonus_token_in_last_step (set): Set of sequence IDs that + contain bonus tokens. + + Returns: + Tuple[ExecuteModelRequest, List[int]]: The updated execute model + request with expanded sequences and a list of indices corresponding + to the original sequence groups. + """ + updated_seq_group_metadata_list: List[SequenceGroupMetadata] = [] + updated_execute_model_req = execute_model_req.clone( + updated_seq_group_metadata_list) + indices_of_original_sequence_groups = [] + for seq_group in execute_model_req.seq_group_metadata_list: + seq_group_has_bonus_tokens = False + for seq_id, _ in seq_group.seq_data.items(): + # Identify sequences with bonus tokens in the sequence group. + if seq_id in seq_with_bonus_token_in_last_step: + seq_group_has_bonus_tokens = True + break + if seq_group_has_bonus_tokens: + #Create new sequences without the last bonus token. These new + # sequence have the same sequence id as the original sequence. + # We create a new sequence group and add them there. + updated_seq_group_without_bonus_token = \ + MultiStepWorker._copy_seq_metadata_excluding_last_token( + seq_group, seq_with_bonus_token_in_last_step) + updated_seq_group_metadata_list.append( + updated_seq_group_without_bonus_token) + # Add the original sequence group. + updated_seq_group_metadata_list.append( + MultiStepWorker._shallow_copy_seq_group_metadata(seq_group)) + # Record the index of the original sequence group. + indices_of_original_sequence_groups.append( + len(updated_seq_group_metadata_list) - 1) + + updated_execute_model_req.seq_group_metadata_list =\ + updated_seq_group_metadata_list + + if isinstance(updated_execute_model_req.previous_hidden_states, + HiddenStates): + updated_execute_model_req.previous_hidden_states\ + .expand_with_bonus_tokens(seq_with_bonus_token_in_last_step) + + return updated_execute_model_req, indices_of_original_sequence_groups + + @staticmethod + def _filter_model_output( + expanded_batch_outputs: List[SamplerOutput], + output_indices_to_retain: List[int]) -> List[SamplerOutput]: + """ + Filters the model output to include only the specified sequence + outputs. This method contracts the expanded batch output from the + model to retain the outputs of only those sequences indicated by the + provided indices. + + Args: + expanded_batch_output (List[SamplerOutput]): The expanded output + batch from the model. + output_indices_to_retain (List[int]): Indices of the model outputs + to retain. + + Returns: + List[SamplerOutput]: A list containing the filtered model + outputs for the specified indices. + """ + return [ + SamplerOutput( + outputs=[ + expanded_batch_output.outputs[i] + for i in output_indices_to_retain + ] if len(expanded_batch_output.outputs) > 0 else [], + sampled_token_probs=( + expanded_batch_output. + sampled_token_probs[output_indices_to_retain] + if expanded_batch_output.sampled_token_probs is not None + else None), + logprobs=( + expanded_batch_output.logprobs[output_indices_to_retain] + if expanded_batch_output.logprobs is not None else None), + sampled_token_ids=(expanded_batch_output. + sampled_token_ids[output_indices_to_retain] + if expanded_batch_output.sampled_token_ids + is not None else None)) + for expanded_batch_output in expanded_batch_outputs + ] + + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + seq_ids_with_bonus_token_in_last_step: set, + ) -> SpeculativeProposals: + """Produce speculations given an input batch of sequences. The number of + speculative tokens per sequence is determined by max_proposal_len. + """ + return self._proposer.get_spec_proposals( + execute_model_req, seq_ids_with_bonus_token_in_last_step) + + @staticmethod + def _append_new_tokens( + model_output: List[SamplerOutput], + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None: + """Given model output from a single run, append the tokens to the + sequences. This is normally done outside of the worker, but it is + required if the worker is to perform multiple forward passes. + """ + for seq_group_metadata, sequence_group_outputs in zip( + seq_group_metadata_list, model_output): + seq_group_metadata.is_prompt = False + + for seq_output in sequence_group_outputs.samples: + # NOTE: Beam search is not supported, so we can assume that + # parent_seq_id == seq_id. + seq = seq_group_metadata.seq_data[seq_output.parent_seq_id] + + token_id = seq_output.output_token + token_logprob = seq_output.logprobs[token_id] + + seq.append_token_id(token_id, token_logprob.logprob) + seq.update_num_computed_tokens(1) + + @staticmethod + def _shallow_copy_seq_group_metadata( + seq_group_metadata: SequenceGroupMetadata, ) -> SequenceGroupMetadata: + """Copy input data structures to remove side-effects when input data + structures are shared with other modules. + + Helpful when the vLLM scheduler runs in the same process as the worker. + The alternative is deep-copying (or other form of deep copy); this has + performance downsides. + """ + # Shallow-copy the SequenceGroupMetadata. This allows us to + # append tokens and change is_prompt without external side-effects. + # We must shallow-copy seq_group_metadata as is_prompt could change. + new_seq_group_metadata = copy.copy(seq_group_metadata) + + # We must shallow-copy seq_data as we will append token ids + new_seq_data: Dict[int, SequenceData] = {} + for seq_id, old_seq_data in seq_group_metadata.seq_data.items(): + new_seq_data[seq_id] = copy.copy(old_seq_data) + new_seq_data[seq_id].output_token_ids =\ + old_seq_data.output_token_ids[:] + + new_seq_group_metadata.seq_data = new_seq_data + return new_seq_group_metadata + + @staticmethod + def _copy_seq_metadata_excluding_last_token( + seq_group_metadata: SequenceGroupMetadata, + seq_ids_to_copy: Set[int], + ) -> SequenceGroupMetadata: + """ + Creates a shallow copy of the given SequenceGroupMetadata, retaining + only the sequence IDs specified in seq_ids_to_copy. For each of these + sequence IDs, all output_token_ids except the last one are copied. + Sequence IDs not in seq_ids_to_copy are excluded from the copy. + + Parameters: + seq_group_metadata (SequenceGroupMetadata): The original sequence + group metadata. + seq_ids_to_copy (Set[int]): The set of sequence IDs to include in the + copy. + + Returns: + SequenceGroupMetadata: A shallow copy of the sequence group metadata + with the specified modifications. + """ + # Shallow-copy the SequenceGroupMetadata. + new_seq_group_metadata = copy.copy(seq_group_metadata) + # Shallow-copy seq_data and modify the output_token_ids. + new_seq_data: Dict[int, SequenceData] = {} + for seq_id, old_seq_data in seq_group_metadata.seq_data.items(): + if (seq_id in seq_ids_to_copy): + new_seq_data[seq_id] = copy.copy(old_seq_data) + # Copy all the output token ids except the last. + # Also reduce num_computed_tokens by 1 since we are not + # including the last output token. + # NOTE: num_computed_tokens is not directly used by the + # speculative decoding workers, as it is only relevant for + # chunked prefill, which is disabled for speculative decoding. + # However, to maintain consistency in num_computed_tokens, + # we update it here. + new_seq_data[seq_id].output_token_ids =\ + old_seq_data.output_token_ids[:-1] + new_seq_data[seq_id].update_num_computed_tokens(-1) + new_seq_group_metadata.seq_data = new_seq_data + return new_seq_group_metadata + + def _assert_enough_kv_space( + self, seq_group_metadata_list: List[SequenceGroupMetadata], + num_steps: int) -> None: + """Assert there are enough physical blocks per sequence to store the + current KV plus additional KV from num_steps tokens. + """ + assert self.model_runner.block_size is not None + for seq_group_metadata in seq_group_metadata_list: + # Only one seq_id is guaranteed because there is no beam search. + seq_id = list(seq_group_metadata.seq_data.keys())[0] + seq = seq_group_metadata.seq_data[seq_id] + + # After num_steps, the seq len will be the current seq len + # plus one token per step. + final_seq_len = seq.get_len() + num_steps + + # We will have final_seq_len - 1 KV because vLLM saves KV for a + # token in the iteration after the token was generated. + required_num_kv_slots = final_seq_len - 1 + + # The allocated number of kv slots is the number of allocated blocks + # times the number of slots of block. + number_physical_blocks = len( + seq_group_metadata.block_tables[seq_id]) + allocated_kv_slots = (number_physical_blocks * + self.model_runner.block_size) + + if required_num_kv_slots > allocated_kv_slots: + request_id = seq_group_metadata.request_id + raise ValueError( + "The worker attempted to run " + f"{num_steps} times but found insufficient KV space for " + f"{request_id=} {seq_id=}. ({allocated_kv_slots=} " + f"{required_num_kv_slots=}).") + + def _raise_if_unsupported( + self, + execute_model_req: ExecuteModelRequest, + ) -> None: + """MultiStepWorker does not yet implement support for cache swap + operations or beam search. + """ + if any([ + execute_model_req.blocks_to_swap_in, + execute_model_req.blocks_to_swap_out, + execute_model_req.blocks_to_copy + ]): + raise NotImplementedError( + "MultiStepWorker does not support cache operations") + + if any( + len(seq_group_metadata.seq_data.keys()) != 1 + for seq_group_metadata in + execute_model_req.seq_group_metadata_list): + raise NotImplementedError( + "MultiStepWorker does not support beam search.") diff --git a/vllm/spec_decode/ngram_worker.py b/vllm/spec_decode/ngram_worker.py new file mode 100644 index 00000000..36e5e177 --- /dev/null +++ b/vllm/spec_decode/ngram_worker.py @@ -0,0 +1,169 @@ +import weakref +from typing import List, Optional, Set, Tuple + +import torch + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.spec_decode.interfaces import SpeculativeProposals +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase +from vllm.spec_decode.top1_proposer import Top1Proposer + + +class NGramWorker(NonLLMProposerWorkerBase): + """NGramWorker provides a light drafter without need for model. + + Current NGramWorker only implements prompt lookup decoding, + and in future we may also do RAG type drafter and other scenarios + which don't rely on LLM model to give proposals. + """ + + def __init__(self, *args, **kwargs): + # Get local_rank/vocab_size from kwargs attribute + self.local_rank = kwargs["local_rank"] + self.vocab_size = kwargs["model_config"].get_vocab_size() + + # Lazy initialization list. + self._proposer: Top1Proposer + + def set_ngram_window_size(self, ngram_prompt_lookup_min: int, + ngram_prompt_lookup_max: int): + # Search valid candidate window between + # ngram_prompt_lookup_min/ngram_prompt_lookup_max + self.ngram_prompt_lookup_max = ngram_prompt_lookup_max + self.ngram_prompt_lookup_min = ngram_prompt_lookup_min + + def init_device(self): + self.device = torch.device(f"cuda:{self.local_rank}") + self.load_model = lambda *args, **kwargs: None + + # Current NGramWorker only supports Top1Proposer + self._proposer = Top1Proposer( + weakref.proxy(self), # type: ignore[arg-type] + device=self.device, + vocab_size=self.vocab_size, + ) + + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + # Unused parameter. NGramWorker does not use the KV Cache and + # therefore does not need this parameter. + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> Tuple[Optional[List[Optional[SamplerOutput]]], bool]: + """NGram match algo to pick proposal candidate. Returns the list of + sampler output, one per SequenceGroupMetadata. + + For ngram worker, we already done needed transposed internal, so the + indicator pass to sampler_output_to_torch shall be False. + """ + self._raise_if_unsupported(execute_model_req) + + has_spec_out = False + token_id_list: List[Optional[torch.Tensor]] = [] + token_prob_list: List[Optional[torch.Tensor]] = [] + for idx, seq_group_metadata in enumerate( + execute_model_req.seq_group_metadata_list): + seq_data = next(iter(seq_group_metadata.seq_data.values())) + + input_ids = torch.as_tensor(seq_data.get_token_ids(), + dtype=torch.long, + device=self.device) + input_length = seq_data.get_len() + + for ngram_size in range( + min(self.ngram_prompt_lookup_max, input_length - 1), + self.ngram_prompt_lookup_min - 1, + -1, + ): + ngram_tensor = input_ids[-ngram_size:] + if ngram_size == 1: + # Do not match itself and do not use unfold and all + matches = (input_ids[:-1] == ngram_tensor) + else: + windows = input_ids.unfold(dimension=0, + size=ngram_size, + step=1) + # Do not match itself + matches = (windows[:-1] == ngram_tensor).all(dim=-1) + + # first_match includes "values" (bool), indicating whether + # the match is found, and "indices", indicating the index + # of the first match. + # Note that "first_match.values.item()" triggers GPU-CPU + # sync so it is a bit inefficient, but we have not found + # a better way to do this. + first_match = matches.max(dim=-1) + if first_match.values.item(): + proposal_start_idx = first_match.indices.add_(ngram_size) + spec_indices = ( + proposal_start_idx).repeat(sample_len) + torch.arange( + sample_len, device=self.device) + spec_indices.clamp_(max=input_ids.shape[-1] - 1) + res = input_ids.gather(dim=-1, index=spec_indices) + token_id_list.append(res) + token_prob_list.append( + torch.nn.functional.one_hot( + res, + num_classes=self.vocab_size).to(torch.float32)) + has_spec_out = True + break + else: + token_id_list.append(None) + token_prob_list.append(None) + + if not has_spec_out: + return None, False + + outputs: List[Optional[SamplerOutput]] = [] + for idx in range(len(execute_model_req.seq_group_metadata_list)): + if token_id_list[idx] is None: + outputs.append(None) + else: + outputs.append( + SamplerOutput( + outputs=None, + sampled_token_probs=token_prob_list[idx], + logprobs=torch.zeros((sample_len, self.vocab_size), + dtype=torch.float32, + device=self.device), + sampled_token_ids=token_id_list[idx], + )) + + return outputs, False + + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + # Unused parameter. NGramWorker does not use the KV Cache and + # therefore does not need this parameter. + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> SpeculativeProposals: + """Produce speculations given an input batch of sequences. The number of + speculative tokens per sequence is determined by max_proposal_len. + """ + return self._proposer.get_spec_proposals( + execute_model_req, seq_ids_with_bonus_token_in_last_step) + + def _raise_if_unsupported( + self, + execute_model_req: ExecuteModelRequest, + ) -> None: + """NGramWorker does not yet implement support for cache swap + operations or beam search. + """ + if any([ + execute_model_req.blocks_to_swap_in, + execute_model_req.blocks_to_swap_out, + execute_model_req.blocks_to_copy + ]): + raise NotImplementedError( + "NGramWorker does not support cache operations") + + if any( + len(seq_group_metadata.seq_data.keys()) != 1 + for seq_group_metadata in + execute_model_req.seq_group_metadata_list): + raise NotImplementedError( + "NGramWorker does not support beam search.") diff --git a/vllm/spec_decode/proposer_worker_base.py b/vllm/spec_decode/proposer_worker_base.py new file mode 100644 index 00000000..28a53759 --- /dev/null +++ b/vllm/spec_decode/proposer_worker_base.py @@ -0,0 +1,56 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, Set, Tuple + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.spec_decode.interfaces import SpeculativeProposer +from vllm.worker.worker_base import LoraNotSupportedWorkerBase + + +class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer): + """Interface for proposer workers""" + + @abstractmethod + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + # A set containing all sequence IDs that were assigned bonus tokens + # in their last forward pass. This set is used to backfill the KV cache + # with the key-value pairs of the penultimate token in the sequences. + # This parameter is only used by the MultiStepWorker, which relies on + # the KV cache for token generation. It is not used by workers that + # do not utilize the KV cache. + seq_ids_with_bonus_token_in_last_step: Set[int] + ) -> Tuple[Optional[List[SamplerOutput]], bool]: + raise NotImplementedError + + def set_include_gpu_probs_tensor(self) -> None: + """Implementation optional""" + pass + + def set_should_modify_greedy_probs_inplace(self) -> None: + """Implementation optional""" + pass + + +class NonLLMProposerWorkerBase(ProposerWorkerBase, ABC): + """Proposer worker which does not use a model with kvcache""" + + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + """get_spec_proposals is used to get the proposals""" + return [] + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """This is never called on the proposer, only the target model""" + raise NotImplementedError + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + pass + + def get_cache_block_size_bytes(self) -> int: + return 0 diff --git a/vllm/spec_decode/smaller_tp_proposer_worker.py b/vllm/spec_decode/smaller_tp_proposer_worker.py new file mode 100644 index 00000000..8896b7db --- /dev/null +++ b/vllm/spec_decode/smaller_tp_proposer_worker.py @@ -0,0 +1,161 @@ +from typing import List, Optional, Set, Tuple + +import torch + +from vllm.distributed.parallel_state import (get_tp_group, + init_model_parallel_group, + patch_tensor_parallel_group) +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.spec_decode.interfaces import SpeculativeProposals +from vllm.spec_decode.multi_step_worker import MultiStepWorker +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase + +logger = init_logger(__name__) + + +class SmallerTpProposerWorker(ProposerWorkerBase): + """Class which allows a speculative draft model to run with smaller tensor + parallel degree than target model. + This reduces the communication overhead of small draft models. + + To implement this feature, this class differs behavior based on is_dummy + flag, where dummy means worker that does not participate draft generation. + Participating workers use a smaller tp group by patching vLLM's tensor + parallel group temporarily during forward passes of draft models. + """ + + @classmethod + def maybe_wrap_worker(cls, worker, draft_tensor_parallel_size: int, + target_tensor_parallel_size: int): + """Wrap the worker in a SmallerTpProposerWorker if necessary. + """ + if draft_tensor_parallel_size == target_tensor_parallel_size: + return worker + + # gpu ranks that will generate draft tokens together + draft_ranks = list(range(draft_tensor_parallel_size)) + + logger.info("Wrapping {%s} in {%s}", type(worker), cls) + return cls(worker, draft_ranks) + + def __init__(self, worker: MultiStepWorker, draft_ranks: List[int]): + """Create a SmallerTpProposerWorker. + + Args: + worker (MultiStepWorker): an actual worker wrapped with this class + draft_ranks (List[int]): if this value is given, only the GPU ranks + written in this value participate in draft generation + """ + self._worker = worker + self._draft_ranks = draft_ranks + + # init during init_device + self._is_dummy = False + self._tp_group = None + + def _patch_tensor_parallel_group(self): + """Temporarily patch the global tp group state with its own tp group + state. + """ + return patch_tensor_parallel_group(self._tp_group) + + def init_device(self) -> None: + self._is_dummy = get_tp_group().rank not in self._draft_ranks + + # dummy workers do nothing + if self._is_dummy: + return + + # creates tp process group containing only a subset of gpu ranks + local_rank = get_tp_group().local_rank + tp_backend = torch.distributed.get_backend(get_tp_group().device_group) + self._tp_group = init_model_parallel_group([self._draft_ranks], + local_rank, tp_backend) + + with self._patch_tensor_parallel_group(): + self._worker.init_device() + + def set_include_gpu_probs_tensor(self) -> None: + if self._is_dummy: + return + + # Need include_gpu_probs_tensor for multi_step_worker + self._worker.set_include_gpu_probs_tensor() + + def set_should_modify_greedy_probs_inplace(self) -> None: + if self._is_dummy: + return + + self._worker.set_should_modify_greedy_probs_inplace() + + def load_model(self) -> None: + if self._is_dummy: + return + + with self._patch_tensor_parallel_group(): + self._worker.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + if self._is_dummy: + # this case is not used now + return -1, -1 + + with self._patch_tensor_parallel_group(): + return self._worker.determine_num_available_blocks() + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + if self._is_dummy: + return + + with self._patch_tensor_parallel_group(): + self._worker.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + def sampler_output( + self, + execute_model_req: ExecuteModelRequest, + sample_len: int, + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> Tuple[List[SamplerOutput], bool]: + # Do not check _is_dummy, as it's always called by get_spec_proposals + return self._worker.sampler_output( + execute_model_req, sample_len, + seq_ids_with_bonus_token_in_last_step) + + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> SpeculativeProposals: + """Produce speculations given an input batch of sequences. The number of + speculative tokens per sequence is determined by max_proposal_len. + """ + if self._is_dummy: + return SpeculativeProposals(None, None, None) + + with self._patch_tensor_parallel_group(): + return self._worker.get_spec_proposals( + execute_model_req, seq_ids_with_bonus_token_in_last_step) + + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + if self._is_dummy: + return [] + + with self._patch_tensor_parallel_group(): + return self._worker.execute_model(execute_model_req) + + def get_cache_block_size_bytes(self) -> int: + if self._is_dummy: + # by returning zero, target worker can use the entire kv cache space + return 0 + + return self._worker.get_cache_block_size_bytes() + + @property + def vocab_size(self) -> int: + return self._worker.vocab_size diff --git a/vllm/spec_decode/spec_decode_worker.py b/vllm/spec_decode/spec_decode_worker.py new file mode 100644 index 00000000..50d2767a --- /dev/null +++ b/vllm/spec_decode/spec_decode_worker.py @@ -0,0 +1,1074 @@ +from collections import defaultdict +from functools import cached_property +from typing import Any, Dict, List, Optional, Set, Tuple, Type + +import torch + +from vllm.config import ParallelConfig, SpeculativeConfig +from vllm.distributed.communication_op import broadcast_tensor_dict +from vllm.logger import init_logger +from vllm.model_executor.layers.rejection_sampler import RejectionSampler +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.layers.spec_decode_base_sampler import ( + SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler) +from vllm.model_executor.layers.typical_acceptance_sampler import ( + TypicalAcceptanceSampler) +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, + CompletionSequenceGroupOutput, ExecuteModelRequest, + HiddenStates, SequenceGroupMetadata, + get_all_seq_ids_and_request_ids) +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer +from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner +from vllm.spec_decode.interfaces import (SpeculativeProposals, + SpeculativeScorer, SpeculativeScores) +from vllm.spec_decode.medusa_worker import MedusaWorker +from vllm.spec_decode.metrics import AsyncMetricsCollector +from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker +from vllm.spec_decode.mqa_scorer import MQAScorer +from vllm.spec_decode.multi_step_worker import MultiStepWorker +from vllm.spec_decode.ngram_worker import NGramWorker +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase +from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker +from vllm.spec_decode.target_model_runner import TargetModelRunner +from vllm.spec_decode.util import (Timer, create_logprobs_output, + create_sequence_group_output, + get_all_num_logprobs, + get_sampled_token_logprobs, nvtx_range, + split_batch_by_proposal_len) +from vllm.worker.worker import Worker +from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase + +logger = init_logger(__name__) + + +def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker": + """Helper method that is the entrypoint for Executors which use + WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config. + """ + assert "speculative_config" in kwargs + speculative_config: SpeculativeConfig = kwargs.get("speculative_config") + assert speculative_config is not None + + draft_worker_kwargs = kwargs.copy() + + kwargs["model_runner_cls"] = TargetModelRunner + target_worker = Worker(*args, **kwargs) + # Set the disable_logprobs variable in the TargetModelRunner instance + # as per its value specified in the SpeculativeConfig. + target_worker.model_runner.disable_logprobs =\ + speculative_config.disable_logprobs + + # Override draft-model specific worker args. + draft_worker_kwargs.update( + model_config=speculative_config.draft_model_config, + parallel_config=speculative_config.draft_parallel_config, + ngram_prompt_lookup_max=speculative_config.ngram_prompt_lookup_max, + ngram_prompt_lookup_min=speculative_config.ngram_prompt_lookup_min, + # TODO allow draft-model specific load config. + #load_config=load_config, + ) + + spec_decode_worker = SpecDecodeWorker.create_worker( + scorer_worker=target_worker, + draft_worker_kwargs=draft_worker_kwargs, + disable_mqa_scorer=speculative_config.speculative_disable_mqa_scorer, + disable_by_batch_size=speculative_config. + speculative_disable_by_batch_size, + draft_token_acceptance_method=speculative_config. + draft_token_acceptance_method, + typical_acceptance_sampler_posterior_threshold=speculative_config. + typical_acceptance_sampler_posterior_threshold, + typical_acceptance_sampler_posterior_alpha=speculative_config. + typical_acceptance_sampler_posterior_alpha, + disable_logprobs=speculative_config.disable_logprobs, + disable_log_stats=speculative_config.disable_log_stats, + ) + + return spec_decode_worker + + +# Reminder: Please update docs/source/serving/compatibility_matrix.rst +# If the feature combo become valid +class SpecDecodeWorker(LoraNotSupportedWorkerBase): + """Worker which implements speculative decoding. + + Speculative decoding reduces decoding per-token latency by using a proposal + method, such as a small draft model, to speculate ahead of a larger LLM. The + probabilities of the speculative tokens are then determined by the larger + LLM, after which some verification routine determines which (if any) of the + speculative tokens are accepted by the larger LLM. + + See https://github.com/vllm-project/vllm/pull/2188 and + https://github.com/vllm-project/vllm/pull/3103 for more info. + + The current implementation has the following limitations: + * Only draft-model proposal is implemented (contributions for more forms are + welcome!). + * Only top-1 proposal and scoring are implemented. Tree-attention is left as + future work. + * All sequences in a batch must have the same proposal length, or zero. This + can be improved by having per-sequence speculation in the future. + * The scoring forward pass is done without an MQA kernel, which is + suboptimal especially as the batch size, proposal length, and sequence + lengths grow. Contributions to add a MQA scoring are welcome once + correctness tests pass. + More info here https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit. + """ + + @classmethod + def create_worker( + cls, + scorer_worker: Worker, + draft_worker_kwargs: Dict[str, Any], + disable_mqa_scorer: bool, + disable_by_batch_size: Optional[int], + draft_token_acceptance_method: str, + typical_acceptance_sampler_posterior_threshold: float, + typical_acceptance_sampler_posterior_alpha: float, + disable_logprobs: bool, + disable_log_stats: bool, + ) -> "SpecDecodeWorker": + + allow_zero_draft_token_step = True + ngram_prompt_lookup_max = ( + draft_worker_kwargs.pop("ngram_prompt_lookup_max")) + ngram_prompt_lookup_min = ( + draft_worker_kwargs.pop("ngram_prompt_lookup_min")) + if ngram_prompt_lookup_max > 0: + proposer_worker = NGramWorker(**draft_worker_kwargs) + proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min, + ngram_prompt_lookup_max) + else: + draft_parallel_config: ParallelConfig = draft_worker_kwargs[ + 'parallel_config'] + draft_tp = draft_parallel_config.tensor_parallel_size + target_tp = scorer_worker.parallel_config.tensor_parallel_size + + if draft_worker_kwargs[ + "model_config"].hf_config.model_type == "mlp_speculator": + proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs) + elif draft_worker_kwargs[ + "model_config"].hf_config.model_type == "medusa": + proposer_worker = MedusaWorker(**draft_worker_kwargs) + else: + if draft_tp == 1: + draft_worker_kwargs[ + "model_runner_cls"] = TP1DraftModelRunner + else: + if draft_worker_kwargs[ + "model_config"].hf_config.model_type == "eagle": + raise NotImplementedError( + "EAGLE does not support TP > 1 yet") + + allow_zero_draft_token_step = False + proposer_worker = MultiStepWorker(**draft_worker_kwargs) + + proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker( + proposer_worker, draft_tp, target_tp) + + logger.info("Configuring SpecDecodeWorker with proposer=%s", + type(proposer_worker)) + + spec_decode_sampler: SpecDecodeBaseSampler = None + if draft_token_acceptance_method == "rejection_sampler": + spec_decode_sampler = RejectionSampler() + elif draft_token_acceptance_method == "typical_acceptance_sampler": + spec_decode_sampler = TypicalAcceptanceSampler( + posterior_threshold=\ + typical_acceptance_sampler_posterior_threshold, + posterior_alpha=typical_acceptance_sampler_posterior_alpha, + ) + logger.info( + "[Speculative Decoding] Configuring" + " SpecDecodeWorker with sampler=%s", type(spec_decode_sampler)) + + if not disable_mqa_scorer: + if scorer_worker.model_runner.attn_backend.get_name( + ) != "flash-attn": + disable_mqa_scorer = True + logger.info( + "[Speculative Decoding] Disabling MQA scorer as the " + "MQA is only available with flash attn backend.") + + if "model_config" in draft_worker_kwargs and \ + draft_worker_kwargs["model_config"].max_model_len < \ + scorer_worker.model_config.max_model_len: + disable_mqa_scorer = True + logger.info( + "[Speculative Decoding] Disabling MQA scorer as the " + "draft model max_model_len is smaller than the target " + "model max_model_len.") + + if not scorer_worker.model_runner.model_config.enforce_eager: + disable_mqa_scorer = True + logger.info( + "[Speculative Decoding] Disabling MQA scorer as the " + "target model is not running in eager mode.") + + return SpecDecodeWorker( + proposer_worker, + scorer_worker, + disable_mqa_scorer=disable_mqa_scorer, + disable_logprobs=disable_logprobs, + disable_log_stats=disable_log_stats, + disable_by_batch_size=disable_by_batch_size, + spec_decode_sampler=spec_decode_sampler, + allow_zero_draft_token_step=allow_zero_draft_token_step) + + def __init__( + self, + proposer_worker: ProposerWorkerBase, + scorer_worker: WorkerBase, + spec_decode_sampler: SpecDecodeBaseSampler, + disable_mqa_scorer: bool = False, + disable_logprobs: bool = False, + disable_log_stats: bool = False, + metrics_collector: Optional[AsyncMetricsCollector] = None, + disable_by_batch_size: Optional[int] = None, + allow_zero_draft_token_step: Optional[bool] = True, + ): + """ + Create a SpecDecodeWorker. + + Args: + proposer_worker: A worker that can produce speculative tokens for + sequences. + scorer_worker: A worker that produces probabilities of speculative + tokens according to some base model. Typically a vanilla vLLM + Worker. + spec_decode_sampler: A Torch module used to perform acceptance + sampling of the draft tokens in the verification step of + speculative decoding. Currently we support two different + types of sampler namely RejectionSampler and + TypicalAcceptanceSampler. 'spec_decode_sampler' is either an + instance of RejectionSampler or TypicalAcceptanceSampler. + disable_mqa_scorer: If set to True, disable the MQA scorer and use + the BatchExpansionTop1Scorer instead. + disable_logprobs: If set to True, token log probabilities will + not be output in both the draft worker and the target worker. + If set to False, log probabilities will be output by both. + disable_log_stats: If set to True, disable periodic printing of + speculative stage times. + disable_by_batch_size: If the batch size is larger than this, + disable speculative decoding for new incoming requests. + metrics_collector: Helper class for collecting metrics; can be set + for testing purposes. + allow_zero_draft_token_step: whether to allow a step where the draft + model generates no draft token; should disallow when the tp of + draft model is larger than 1 (TODO: #5814) + """ + self.proposer_worker = proposer_worker + self.scorer_worker = scorer_worker + scorer_runner = getattr(self.scorer_worker, "model_runner", None) + self.generators = scorer_runner.get_generators( + ) if scorer_runner else None + self.disable_by_batch_size = disable_by_batch_size or float("inf") + self.spec_decode_sampler = spec_decode_sampler + self._allow_zero_draft_token_step = allow_zero_draft_token_step + self._metrics = AsyncMetricsCollector( + self.spec_decode_sampler + ) if metrics_collector is None else metrics_collector + # Tracks the sequence IDs that received a bonus token ID in + # their last forward pass. Needed only if KV cache is being + # used for token generation such as in the case of MultiStepWorker. + self._seq_with_bonus_token_in_last_step: Set[int] = set() + # Tracks the currently active request ids and the sequence IDs + # corresponding to them + self._request_id_seq_id_mapping: Dict[str, Set[int]] = defaultdict(set) + # Tracks if the proposer worker uses the KV cache or not. + + self.probs_dtype = self.spec_decode_sampler.probs_dtype + self.token_id_dtype = self.spec_decode_sampler.token_id_dtype + # Lazy initialization. + self.scorer: SpeculativeScorer + self.disable_mqa_scorer = disable_mqa_scorer + + # Hidden states from target model to pass to proposer + # in the subsequent step. + self.previous_hidden_states: Optional[HiddenStates] = None + self._disable_logprobs = disable_logprobs + self._disable_log_stats = disable_log_stats + + def init_device(self) -> None: + """Initialize both scorer and proposer models. + """ + # The scorer worker model is initialized first in case the proposer + # model has a smaller TP degree than the target worker. + self.scorer_worker.init_device() + self.proposer_worker.init_device() + + # NOTE(cade): load_model is not part of the WorkerBase interface. + self.scorer_worker.load_model() + self.proposer_worker.load_model() + + self._metrics.init_gpu_tensors(self.rank) + self.spec_decode_sampler.init_gpu_tensors(self.rank) + + scorer_cls: Type[SpeculativeScorer] + if self.disable_mqa_scorer: + scorer_cls = BatchExpansionTop1Scorer + logger.info("[Speculative Decoding] Use batch " + "expansion for scoring proposals.") + else: + scorer_cls = MQAScorer + logger.info( + "[Speculative Decoding] Use MQA scorer for scoring proposals.") + + self.scorer = scorer_cls(scorer_worker=self.scorer_worker, + device=self.device, + vocab_size=self._vocab_size) + + self._configure_model_sampler_for_spec_decode() + + def load_model(self, *args, **kwargs): + pass + + def _configure_model_sampler_for_spec_decode(self): + """Configure model sampler to emit GPU tensors. This allows spec decode + to keep data on device without transferring to CPU and serializing, + which significantly reduces overhead of sampling during verification. + + NOTE(cade): This breaks abstraction boundaries pretty badly. The better + design is to have the "move to CPU and serialize" sampling decision be + done outside of the model/sampler; this way the "last-mile" worker + object which interfaces with the scheduler can serialize and incur the + performance hit as necessary. This allows us to run the worker several + iterations in a row without incurring the "move to CPU and serialize" + performance penalty. + + Since this requires a large change to vLLM, we defer it to later and + temporarily accept this broken abstraction boundary. + + NOTE(cade): This will require a special check if the proposer worker + does not have a sampler (e.g. ngram speculation). + """ + (self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor + ) = True + (self.scorer_worker.model_runner.model.sampler. + should_modify_greedy_probs_inplace) = True + self.proposer_worker.set_include_gpu_probs_tensor() + self.proposer_worker.set_should_modify_greedy_probs_inplace() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of cache blocks to use. + + This is done by profiling the scorer model (which is typically the + larger of the two). Then the total memory which would be used by the + scorer cache is divided evenly between the proposer and scorer model KV, + such that the number of blocks is equal in both KV caches. + """ + num_gpu_blocks, num_cpu_blocks = ( + self.scorer_worker.determine_num_available_blocks()) + + scorer_cache_block_size_bytes = ( + self.scorer_worker.get_cache_block_size_bytes()) + proposer_cache_block_size_bytes = ( + self.proposer_worker.get_cache_block_size_bytes()) + + new_num_gpu_blocks = split_num_cache_blocks_evenly( + scorer_cache_block_size_bytes, proposer_cache_block_size_bytes, + num_gpu_blocks) + return new_num_gpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the cache engine of the scorer and proposer workers. + """ + self.scorer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks) + self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks, + num_cpu_blocks=num_cpu_blocks) + + @torch.inference_mode() + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> List[SamplerOutput]: + """Perform speculative decoding on the input batch. + """ + if self.rank != self._driver_rank: + self._run_non_driver_rank() + return [] + + if execute_model_req is None: + # This signals that there's no more requests to process for now. + # All workers are running infinite loop with broadcast_tensor_dict, + # and it stops the loop when the driver broadcasts an empty input. + # Send an empty input to notify all other workers to stop their + # execution loop. + broadcast_tensor_dict({}, src=0) + return [] + + self._track_finished_requests(execute_model_req) + disable_all_speculation = self._should_disable_all_speculation( + execute_model_req) + num_lookahead_slots = execute_model_req.num_lookahead_slots + + # Speculative decoding is disabled in the following cases: + # 1. Prefill phase: Speculative decoding is not + # used during the prefill phase. + # 2. Auto-disable enabled: The running queue size exceeds + # the specified threshold. + # 3. No request: There are no requests in the batch, or + # none of the requests in the batch have spec decoding enabled. + # In any of these cases, the proposer and scorer workers + # are called normally. + no_spec = num_lookahead_slots == 0 or disable_all_speculation or all( + sgm.num_speculative_tokens == 0 + for sgm in execute_model_req.seq_group_metadata_list) + + # Broadcast how many lookahead slots are scheduled for this step, and + # whether all speculation is disabled, to all non-driver workers. + + # This is required as if the number of draft model runs changes + # dynamically, the non-driver workers won't know unless we perform a + # communication to inform them. + + # no_spec is used to signal non-driver worker about prefill vs decode + # stage. This is needed to ensure that order of execution of proposer + # and scorer is same in both driver and non-driver workers (i.e., + # scorer -> proposer for prefill and proposer -> scorer in decode). This + # order is needed to support models like EAGLE that take scorer states + # as inputs. + broadcast_dict = dict( + num_lookahead_slots=num_lookahead_slots, + no_spec=no_spec, + disable_all_speculation=disable_all_speculation, + ) + broadcast_tensor_dict(broadcast_dict, src=self._driver_rank) + + assert execute_model_req.seq_group_metadata_list is not None, ( + "speculative decoding requires non-None seq_group_metadata_list") + + self._maybe_disable_speculative_tokens( + disable_all_speculation, execute_model_req.seq_group_metadata_list) + + if no_spec: + return self._run_no_spec(execute_model_req, + skip_proposer=disable_all_speculation) + return self._run_speculative_decoding_step(execute_model_req, + num_lookahead_slots) + + @torch.inference_mode() + def start_worker_execution_loop(self) -> None: + """Execute model loop to perform speculative decoding + in parallel worker.""" + while self._run_non_driver_rank(): + pass + + def _should_disable_all_speculation( + self, execute_model_req: ExecuteModelRequest) -> bool: + # When the batch size is too large, disable speculative decoding + # to stop trading off throughput for latency. + return (execute_model_req.running_queue_size >= + self.disable_by_batch_size) + + def _maybe_disable_speculative_tokens( + self, disable_all_speculation: bool, + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None: + if not disable_all_speculation: + return + + for seq_group_metadata in seq_group_metadata_list: + # Once num_speculative_tokens is set to 0, the spec decode + # of this request will be disabled forever. + # TODO(comaniac): We currently store spec decoding specific + # state in the global data structure, but we should maintain + # this state within spec decode worker. + seq_group_metadata.num_speculative_tokens = 0 + + def _serialize_sampler_output_no_logprobs( + self, execute_model_req: ExecuteModelRequest, + sampler_output: SamplerOutput) -> SamplerOutput: + """ + Creates and returns a `SamplerOutput` with only the token IDs being + serialized to CPU and populated in `CompletionSequenceGroupOutput`. + All other parameters in `CompletionSequenceGroupOutput` related to log + probabilities are skipped. + + Args: + execute_model_req (ExecuteModelRequest): The model request that + was executed. + sampler_output (SamplerOutput): The output from the sampler with + only GPU tensors populated. + + Returns: + SamplerOutput: A new `SamplerOutput` instance containing a list of + `CompletionSequenceGroupOutput` objects with only token IDs + populated. + """ + seq_output_prompt_logprobs = [ + seq.is_prompt and seq.sampling_params.prompt_logprobs is not None + and seq.sampling_params.prompt_logprobs > 0 + for seq in execute_model_req.seq_group_metadata_list + ] + # ignore slots for prompt tokens that are filled with INVALID_TOKEN_ID + sampled_token_ids_list = (sampler_output.sampled_token_ids[torch.where( + # subtracting is faster than testing for equality + sampler_output.sampled_token_ids - VLLM_INVALID_TOKEN_ID)[0]] \ + if any(seq_output_prompt_logprobs) else \ + sampler_output.sampled_token_ids).tolist() + + seq_data_entries = ( + (seq_id, seq_data) for sg in \ + execute_model_req.seq_group_metadata_list \ + for seq_id, seq_data in sg.seq_data.items() + ) + completion_seq_group_output_list: List[ + CompletionSequenceGroupOutput] = [] + for index, ((seq_id, seq_data), needs_prompt_logprobs) in \ + enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)): + if needs_prompt_logprobs: + prompt_token_ids = seq_data.get_prompt_token_ids() + prompt_logprobs = [ + create_logprobs_output( + token_id=p_token_id, + token_id_logprob_rank=-1, + token_id_logprob=0.0, + topk_token_ids=[], + topk_logprobs=[], + ) + # no prompt logprobs for the first token + for p_token_id in prompt_token_ids[1:] + ] + else: + prompt_logprobs = None + + completion_seq_group_output_list.append( + create_sequence_group_output( + token_id=sampled_token_ids_list[index][0], + token_id_logprob_rank=-1, + token_id_logprob=0.0, + seq_id=seq_id, + topk_token_ids=[], + topk_logprobs=[], + prompt_logprobs=prompt_logprobs)) + return SamplerOutput(outputs=completion_seq_group_output_list) + + @nvtx_range("spec_decode_worker._run_no_spec") + def _run_no_spec(self, execute_model_req: ExecuteModelRequest, + skip_proposer: bool) -> List[SamplerOutput]: + """Run a single generation step without any speculation. The input is + sent to the proposer and scorer model so that the KV cache is consistent + between the two. When skip_proposer is True, the proposer model is + not called, meaning that the kv-cache in proposer for requests is not + updated, so they cannot enable spec decode in the rest decoding. + """ + + sampler_output = self.scorer_worker.execute_model(execute_model_req) + assert len(sampler_output) == 1 + sampler_output = sampler_output[0] + + # Store hidden states from target model execution. + hidden_states = sampler_output.hidden_states + if hidden_states is not None: + # remove hidden_states for prompt tokens + if any(seq.is_prompt + for seq in execute_model_req.seq_group_metadata_list): + hidden_states = hidden_states[ + torch.where(sampler_output.sampled_token_ids - + VLLM_INVALID_TOKEN_ID)[0]] + if self.previous_hidden_states is None: + self.previous_hidden_states = HiddenStates( + hidden_states, execute_model_req.seq_group_metadata_list) + else: + self.previous_hidden_states.update( + hidden_states, execute_model_req.seq_group_metadata_list) + + if not skip_proposer: + # We prepare the prefill hidden states here so that there no + # additional complexity in worker for spec_decode vs non_spec_decode + # flow and execute_model doesn't need additional modifications. + execute_model_req.previous_hidden_states = \ + prepare_prefill_hidden_states( + sampler_output.prefill_hidden_states) + + self.proposer_worker.execute_model(execute_model_req) + + sampler_output_to_return = (self._serialize_sampler_output_no_logprobs( + execute_model_req=execute_model_req, sampler_output=sampler_output) + if self._disable_logprobs else + sampler_output) + + # Clear device tensors from sampler output. This reduces communication + # overhead when the engine runs in a different process than the workers. + sampler_output.sampled_token_probs = None + sampler_output.sampled_token_ids = None + sampler_output.logprobs = None + return [sampler_output_to_return] + + def _run_non_driver_rank(self) -> bool: + """Run proposer and verifier model in non-driver workers. This is used + for both speculation cases (num_lookahead_slots>0) and non-speculation + cases (e.g. prefill). + + Returns True if there are remaining sequences to process. + """ + assert self.rank != self._driver_rank + + data = broadcast_tensor_dict(src=self._driver_rank) + if not data: + return False + num_lookahead_slots = data["num_lookahead_slots"] + + # In case of prefill, scorer_worker has to be run before proposer so + # that the hidden states can be propagated to proposer when needed. + if data["no_spec"]: + self.scorer_worker.execute_model() + + if not data["disable_all_speculation"]: + # Even if num_lookahead_slots is zero, we want to run the + # proposer model as it may have KV. + # + # We run the proposer once per lookahead slot. In the future we + # should delegate how many times it runs to the proposer. + for _ in range(max(num_lookahead_slots, 1)): + self.proposer_worker.execute_model() + + if not data["no_spec"]: + self.scorer_worker.execute_model() + + return True + + @nvtx_range("spec_decode_worker._run_speculative_decoding_step") + def _run_speculative_decoding_step( + self, execute_model_req: ExecuteModelRequest, + num_lookahead_slots: int) -> List[SamplerOutput]: + """Execute a single step of speculative decoding. + + This invokes the proposer worker to get k speculative tokens for each + sequence, then scores each speculative token using the scoring worker. + + Returns a list of SamplerOutput, each containing a single token per + sequence. + """ + assert num_lookahead_slots == execute_model_req.num_lookahead_slots + + # Pass last hidden states from target model to proposer + execute_model_req.previous_hidden_states = self.previous_hidden_states + self.previous_hidden_states = None + + with Timer() as proposal_timer: + # Generate proposals using draft worker. + proposals = self.proposer_worker.get_spec_proposals( + execute_model_req, self._seq_with_bonus_token_in_last_step) + + if not self._allow_zero_draft_token_step and proposals.no_proposals: + #TODO: Fix it #5814 + raise RuntimeError("Cannot handle cases where distributed draft " + "workers generate no tokens") + + execute_model_req.previous_hidden_states = None + + with Timer() as scoring_timer: + proposal_scores = self.scorer.score_proposals( + execute_model_req, + proposals, + ) + + with Timer() as verification_timer: + accepted_token_ids, target_logprobs = self._verify_tokens( + execute_model_req.seq_group_metadata_list, proposal_scores, + proposals, execute_model_req.num_lookahead_slots) + + stage_times = (proposal_timer.elapsed_time_ms / num_lookahead_slots, + scoring_timer.elapsed_time_ms, + verification_timer.elapsed_time_ms) + + return self._create_output_sampler_list( + execute_model_req.seq_group_metadata_list, + accepted_token_ids, + target_logprobs=target_logprobs, + k=execute_model_req.num_lookahead_slots, + stage_times=stage_times) + + @nvtx_range("spec_decode_worker._verify_tokens") + def _verify_tokens( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + proposal_scores: SpeculativeScores, + proposals: SpeculativeProposals, + max_proposal_len: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Determine which speculative tokens are accepted using the + probabilities of each token according to the proposer and scorer models. + + Returns a tuple of Tensors, one for the accepted token ids and one for + the logprobs according to the scoring model. + """ + proposal_lens_list = proposals.proposal_lens.tolist() + + # vLLM currently only supports proposal lens equal to zero or the batch + # proposal len. This adds some complexity (splitting the batch into spec + # and non spec sequences) and should be removed in the future. It can be + # done by supporting per-sequence proposal lens. + (_, spec_indices), (_, non_spec_indices) = split_batch_by_proposal_len( + seq_group_metadata_list, proposal_lens_list) + original_indices = spec_indices + non_spec_indices + + # Get probabilities of target model, including bonus tokens. + proposal_verifier_probs = proposal_scores.probs[spec_indices] + + # Get non-speculative sampled tokens from target model. + non_spec_token_ids = proposal_scores.token_ids[non_spec_indices] + + # Get bonus tokens from target model. + bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:] + + # Get probabilities according to proposal method. + proposal_probs = proposals.proposal_probs[spec_indices] + + # Get proposed tokens. + proposal_token_ids = proposals.proposal_token_ids[spec_indices] + + # Sampler arguments + sampler_extra_kwargs: Dict[str, Any] = {} + if self.generators and isinstance(self.spec_decode_sampler, + SpecDecodeStochasticBaseSampler): + sampler_extra_kwargs["seeded_seqs"] = { + idx: self.generators[sgm.request_id] + for idx, sgm in enumerate(seq_group_metadata_list) + if sgm.sampling_params.seed is not None + } + + accepted_token_ids = self.spec_decode_sampler( + target_with_bonus_probs=proposal_verifier_probs, + bonus_token_ids=bonus_token_ids, + draft_probs=proposal_probs, + draft_token_ids=proposal_token_ids, + **sampler_extra_kwargs, + ) + # Append output tokens from non-speculative sequences to + # the accepted token ids tensor. + non_spec_token_ids = non_spec_token_ids.expand(-1, max_proposal_len + + 1).clone() + non_spec_token_ids[:, 1:] = -1 + accepted_token_ids = torch.cat( + [accepted_token_ids, non_spec_token_ids]) + logprobs = proposal_scores.logprobs + # Rearrange so that results are in the order of the original seq group + # metadata. + accepted_token_ids[original_indices] = accepted_token_ids.clone() + + hidden_states = proposal_scores.hidden_states + if hidden_states is not None: + # Contract hidden states based on accepted tokens + hs_size = hidden_states.shape[-1] + + accepted_index = accepted_token_ids + 1 # Convert -1 to 0 + accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) + index = accepted_index[:, None, None].expand(-1, 1, hs_size) + second_last_token_hidden_states = hidden_states[:, -2] # b x d + hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d + # Store hidden states from target model for subsequent decode step + self.previous_hidden_states = HiddenStates( + hidden_states, seq_group_metadata_list, + second_last_token_hidden_states) + + return accepted_token_ids, logprobs + + def _create_output_sampler_list( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1] + target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size] + k: int, + stage_times: Tuple[float, float, float], + ) -> List[SamplerOutput]: + """Given the accepted token ids, create a list of SamplerOutput. + + The output is padded with -1 tokens such that each sequence has + the same number of outputs. + """ + batch_size, num_steps = accepted_token_ids.shape + accepted_token_ids_by_step = accepted_token_ids.transpose(0, 1) + if self._disable_logprobs: + # We are skipping the logprobs. Hence don't serialize the + # logprobs related tensors from the GPU. Instead create + # empty/dummy lists. + (accepted_token_id_ranks_by_step, + accepted_token_id_logprobs_by_step, + topk_logprobs_by_step, topk_indices_by_step) =\ + self._create_dummy_logprob_lists( + batch_size, num_steps, + self.scorer_worker.model_config.max_logprobs) + else: + # Organize input tensors by step instead of by sequence. + target_logprobs_by_step = target_logprobs.transpose(0, 1) + # Serialize all tensors into Python lists. + (accepted_token_id_ranks_by_step, + accepted_token_id_logprobs_by_step, + topk_logprobs_by_step, topk_indices_by_step) =\ + self._create_logprob_lists_from_tensors( + target_logprobs_by_step, accepted_token_ids_by_step, + self.scorer_worker.model_config.max_logprobs) + + # Get the sequence ids and num_logprobs (sampling parameter) in the + # batch. + seq_ids, request_ids_seq_ids_mapping = get_all_seq_ids_and_request_ids( + seq_group_metadata_list) + + num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list) + + # Serialize tensor to CPU Python list. + accepted_token_ids_by_step = accepted_token_ids_by_step.tolist() + + # Construct the output on a per-step, per-sequence basis. + sampler_output_list: List[SamplerOutput] = [] + for step_index in range(num_steps): + if all(token_id == -1 + for token_id in accepted_token_ids_by_step[step_index]): + break + + step_output_token_ids: List[CompletionSequenceGroupOutput] = [] + for sequence_index in range(batch_size): + # Each sequence may have a different num_logprobs; retrieve it. + num_logprobs = num_logprobs_per_seq[sequence_index] + step_output_token_ids.append( + create_sequence_group_output( + token_id=accepted_token_ids_by_step[step_index] + [sequence_index], + token_id_logprob_rank=accepted_token_id_ranks_by_step[ + step_index][sequence_index], + token_id_logprob=accepted_token_id_logprobs_by_step[ + step_index][sequence_index], + seq_id=seq_ids[sequence_index], + topk_token_ids=topk_indices_by_step[step_index] + [sequence_index][:num_logprobs], + topk_logprobs=topk_logprobs_by_step[step_index] + [sequence_index][:num_logprobs], + )) + sampler_output_list.append( + SamplerOutput(outputs=step_output_token_ids)) + + # Populate the data structures needed to keep track of sequences with + # bonus tokens. + self._track_sequences_with_bonus_tokens(seq_ids, + request_ids_seq_ids_mapping, + accepted_token_ids_by_step) + maybe_rejsample_metrics = ( + self._metrics.maybe_collect_rejsample_metrics(k)) + if maybe_rejsample_metrics is not None: + sampler_output_list[ + 0].spec_decode_worker_metrics = maybe_rejsample_metrics + + # Log time spent in each stage periodically. + # This is periodic because the rejection sampler emits metrics + # periodically. + self._maybe_log_stage_times(*stage_times) + + return sampler_output_list + + def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float, + scoring_time_ms: float, + verification_time_ms: float) -> None: + """Log the speculative stage times. If stat logging is disabled, do + nothing. + """ + if self._disable_log_stats: + return + + logger.info( + "SpecDecodeWorker stage times: " + "average_time_per_proposal_tok_ms=%.02f " + "scoring_time_ms=%.02f verification_time_ms=%.02f", + average_time_per_proposal_tok_ms, scoring_time_ms, + verification_time_ms) + + def _create_dummy_logprob_lists( + self, + batch_size: int, + num_steps: int, + num_top_k: int, + ) -> Tuple[List[List[int]], List[List[float]], + List[List[List[Optional[float]]]], + List[List[List[Optional[int]]]]]: + """ + Creates and returns four dummy lists representing token probabilities + and their ranks. + + This method initializes and returns: + - The ranks of the accepted tokens, shaped (num_steps, batch_size) + - The log probabilities of the accepted tokens, + shaped (num_steps, batch_size) + - The log probabilities of the top k tokens, + shaped (num_steps, batch_size, num_top_k) + - The token IDs of the top k tokens, + shaped (num_steps, batch_size, num_top_k) + + Args: + batch_size (int): The size of the batch. + num_steps (int): The number of steps in the sequence. + num_top_k (int): The number of top-k token log probabilities to + return. + + Returns: + A tuple containing four dummy lists as described above. + """ + accepted_token_id_ranks_by_step = [[-1] * batch_size + for _ in range(num_steps)] + accepted_token_id_logprobs_by_step = [[0.0] * batch_size + for _ in range(num_steps)] + topk_logprobs_by_step: List[List[List[Optional[float]]]] = [[ + [None] * num_top_k for _ in range(batch_size) + ] for _ in range(num_steps)] + topk_indices_by_step: List[List[List[Optional[int]]]] = [[ + [None] * num_top_k for _ in range(batch_size) + ] for _ in range(num_steps)] + return (accepted_token_id_ranks_by_step, + accepted_token_id_logprobs_by_step, topk_logprobs_by_step, + topk_indices_by_step) + + def _create_logprob_lists_from_tensors( + self, + target_logprobs_by_step: torch.Tensor, + accepted_token_ids_by_step: torch.Tensor, + num_top_k: int, + ) -> Tuple[List[List[int]], List[List[float]], + List[List[List[Optional[float]]]], + List[List[List[Optional[int]]]]]: + """ + Creates and returns four lists representing token probabilities and + their ranks. + + This method initializes and returns four lists containing: + - The ranks of the accepted tokens, shaped (num_steps, batch_size) + - The log probabilities of the accepted tokens, + shaped (num_steps, batch_size) + - The log probabilities of the top k tokens, + shaped (num_steps, batch_size, num_top_k) + - The token IDs of the top k tokens, + shaped (num_steps, batch_size, num_top_k) + + Args: + target_logprobs_by_step (torch.Tensor): Tensor representing the + log probabilities of the target model, + shaped (num_steps, batch_size, vocab_size) + accepted_token_ids_by_step (torch.Tensor): Tensor representing + the accepted token_ids, shaped (num_steps, batch_size) + num_top_k (int): The number of top-k token log probabilities to + return. + + Returns: + A tuple containing the lists as described above. + """ + # Serialize all tensors to CPU Python lists. + # Get the logprobs/rank of the accepted tokens. + (accepted_token_id_ranks_by_step_tensor, + accepted_token_id_logprobs_by_step_tensor + ) = get_sampled_token_logprobs( + logprob_tensor=target_logprobs_by_step, + sampled_token_ids=accepted_token_ids_by_step, + ) + # Get the top-k logprobs (which may or may not include the + # logprob of the accepted token). + (topk_logprobs_by_step_tensor, + topk_indices_by_step_tensor) = target_logprobs_by_step.topk( + k=num_top_k, + dim=-1, + ) + accepted_token_id_ranks_by_step = ( + accepted_token_id_ranks_by_step_tensor.tolist()) + accepted_token_id_logprobs_by_step = ( + accepted_token_id_logprobs_by_step_tensor.tolist()) + topk_logprobs_by_step = topk_logprobs_by_step_tensor.tolist() + topk_indices_by_step = topk_indices_by_step_tensor.tolist() + return (accepted_token_id_ranks_by_step, + accepted_token_id_logprobs_by_step, topk_logprobs_by_step, + topk_indices_by_step) + + def _track_finished_requests(self, execute_model_req: ExecuteModelRequest): + """ + Removes the finished requests and their associated sequence ids from + internal book keeping data structures. + """ + for finished_request in execute_model_req.finished_requests_ids: + for seq_id in self._request_id_seq_id_mapping[finished_request]: + self._seq_with_bonus_token_in_last_step.discard(seq_id) + del self._request_id_seq_id_mapping[finished_request] + + def _track_sequences_with_bonus_tokens( + self, seq_ids: List[int], + request_ids_seq_ids_mapping: Dict[str, Set[int]], + accepted_token_ids_by_step: List[List[int]]): + """ + Updates the internal data structures which keep track of sequences + which have been assigned bonus tokens in their last forward pass. + """ + for seq_index, seq_id in enumerate(seq_ids): + last_token_id = accepted_token_ids_by_step[-1][seq_index] + if last_token_id == -1: + self._seq_with_bonus_token_in_last_step.discard(seq_id) + else: + self._seq_with_bonus_token_in_last_step.add(seq_id) + for request_id, sequences in request_ids_seq_ids_mapping.items(): + self._request_id_seq_id_mapping[request_id].update(sequences) + + @cached_property + def _vocab_size(self) -> int: + """Get the vocab size of the model and make sure it's consistent between + draft and target workers. + """ + vocab_sizes = [ + worker.vocab_size + for worker in [self.proposer_worker, self.scorer_worker] + ] + assert all(vocab_sizes[0] == vocab_size for vocab_size in vocab_sizes) + return vocab_sizes[0] + + @property + def rank(self): + return self.scorer_worker.rank + + @property + def device(self): + return self.scorer_worker.device + + @property + def _driver_rank(self) -> int: + return 0 + + def get_cache_block_size_bytes(self): + """Return the size of a cache block in bytes. + + This function is only used to compose workers within a SpecDecodeWorker. + We leave composing a SpecDecodeWorker within a SpecDecodeWorker + undefined for now, although it could be implemented in the future. + See https://arxiv.org/abs/2308.04623. + """ + raise NotImplementedError + + +def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int, + proposer_cache_block_size_bytes: int, + total_num_gpu_blocks: int) -> int: + """Given total_num_gpu_blocks, the number of GPU blocks that could be + allocate to the target model, this function calculates how many blocks + should be given to the draft and target model. + + Note that usually the block size, in bytes, of each model is different, + as it's a function of number of KV/layer, number of heads, and hidden + dimension size. + + Since the target and draft models allocate the same number of blocks, we + simply calculate the number of blocks where if allocated by both models, + the total memory usage from KV cache is no larger than the number of + blocks allocatable by the target model alone. + """ + new_num_gpu_blocks = int( + total_num_gpu_blocks * scorer_cache_block_size_bytes / + (proposer_cache_block_size_bytes + scorer_cache_block_size_bytes)) + + return new_num_gpu_blocks + + +def prepare_prefill_hidden_states( + prefill_hidden_states: torch.Tensor) -> HiddenStates: + # For prefill step in proposer, we run the model for N-1 tokens + # because Nth token will be processed in the first decode step. For + # N-1 tokens, the input should be 0:N-1 hidden states which should + # be concatanated with 1:N token (since output of scorer has to be + # the input for proposer). Therefore, we shift the hidden states to + # align n-1th hidden state with nth token. + return HiddenStates(prefill_hidden_states.roll( + shifts=1, dims=0)) if prefill_hidden_states is not None else None diff --git a/vllm/spec_decode/target_model_runner.py b/vllm/spec_decode/target_model_runner.py new file mode 100644 index 00000000..2bb7af7d --- /dev/null +++ b/vllm/spec_decode/target_model_runner.py @@ -0,0 +1,69 @@ +from typing import List, Optional + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.sequence import SequenceGroupMetadata +from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata, + ModelRunner) + + +class TargetModelRunner(ModelRunner): + """Specialized model runner for speculative decoding target model. + In speculative decoding, the log probabilities selected finally may not + be the same ones as selected by the target model sampling. This means + that the time spent in the log probability calculation of the target model + is time wasted, since we calculate log probabilities after deciding which + tokens are accepted. For this reason disabling log probabilities in the + target model will make decode faster. The model runner sets the + SamplingMetadata parameters according to whether log probabilities are + requested or not. + """ + + def __init__(self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + return_hidden_states: bool = False, + observability_config: Optional[ObservabilityConfig] = None): + # An internal boolean member variable to indicate if token log + # probabilities are needed or not. + self.disable_logprobs = True + super().__init__( + model_config=model_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + device_config=device_config, + cache_config=cache_config, + load_config=load_config, + lora_config=lora_config, + kv_cache_dtype=kv_cache_dtype, + is_driver_worker=is_driver_worker, + prompt_adapter_config=prompt_adapter_config, + return_hidden_states=return_hidden_states, + observability_config=observability_config, + ) + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForGPUWithSamplingMetadata: + model_input: ModelInputForGPUWithSamplingMetadata = super( + ).prepare_model_input(seq_group_metadata_list, virtual_engine, + finished_requests_ids) + # If token log probabilities is disabled then skip generating sampler + # CPU output. We directly serialize the GPU sampled_token_id tensors + # as needed. If log probabilities is enabled then synchronize all the + # sampling related tensors which includes the logprobs tensors. + model_input.sampling_metadata.skip_sampler_cpu_output = ( + self.disable_logprobs) + return model_input diff --git a/vllm/spec_decode/top1_proposer.py b/vllm/spec_decode/top1_proposer.py new file mode 100644 index 00000000..f6a52a51 --- /dev/null +++ b/vllm/spec_decode/top1_proposer.py @@ -0,0 +1,272 @@ +from typing import List, Optional, Set, Tuple + +import torch + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata +from vllm.spec_decode.interfaces import (SpeculativeProposals, + SpeculativeProposer) +from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase +from vllm.spec_decode.util import sampler_output_to_torch + + +class Top1Proposer(SpeculativeProposer): + """Helper class which separates out sequences which would exceed the max + model length when speculated upon. + + This allows combinations of models such as JackFram/llama-68m draft with + meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of + 2048 while Llama2-13b has max_position_embeddings of 4096. + + We treat the sequences which exceed the proposal draft model length as + "non-spec sequences". Essentially they skip the draft model and go through + normal decoding in the target model. + + Currently, only proposal_lens of 0 and k are supported, where k is a global + batch proposal length. In the future vLLM should support per-sequence + proposal lengths. + """ + + def __init__( + self, + worker: ProposerWorkerBase, + device: str, + vocab_size: int, + max_proposal_len: Optional[int] = None, + ): + self._worker = worker + self._device = device + self.max_proposal_len = max_proposal_len + self._vocab_size = vocab_size + + def get_spec_proposals( + self, + execute_model_req: ExecuteModelRequest, + seq_ids_with_bonus_token_in_last_step: Set[int], + ) -> SpeculativeProposals: + """Get speculative proposals given the input batch. + + Sequences which would exceed the max model length are skipped during + speculation. + """ + proposal_len = execute_model_req.num_lookahead_slots + seq_group_metadata_list = execute_model_req.seq_group_metadata_list + + # Split speculative- and non-speculative- sequences. + ( + proposal_lens, + nonzero_proposal_len_seqs, + nonzero_proposal_len_indices, + ) = self._split_by_proposal_len(seq_group_metadata_list, proposal_len) + + if nonzero_proposal_len_seqs: + # Speculate tokens using the draft worker for the speculative + # sequences. + # If sampler_transposed is true, then maybe_sampler_output's + # token_ids is like [batch] format in proposal_len size list, + # while if it is false, the format would be [proposal_len] + # in batch size list + hidden_states = execute_model_req.previous_hidden_states + if hidden_states is not None: + hidden_states.prune(nonzero_proposal_len_seqs) + nonzero_execute_model_req = ExecuteModelRequest( + seq_group_metadata_list=nonzero_proposal_len_seqs, + num_lookahead_slots=proposal_len, + previous_hidden_states=hidden_states, + ) + maybe_sampler_output, transposed = self._worker.sampler_output( + execute_model_req=nonzero_execute_model_req, + sample_len=proposal_len, + seq_ids_with_bonus_token_in_last_step=\ + seq_ids_with_bonus_token_in_last_step, + ) + ( + proposal_lens, + maybe_sampler_output, + nonzero_proposal_len_indices, + ) = self._remove_no_proposal_seqs(proposal_lens, + maybe_sampler_output, + nonzero_proposal_len_indices, + transposed) + else: + # If no sequences can be speculated, set sampler output to None. + maybe_sampler_output = None + transposed = False + + # Combine speculative- and non-speculative sequences into the same + # representation. + proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs( + batch_size=len(seq_group_metadata_list), + proposal_len=proposal_len, + maybe_sampler_output=maybe_sampler_output, + proposal_lens=proposal_lens, + nonzero_proposal_len_indices=nonzero_proposal_len_indices, + sampler_transposed=transposed, + ) + + proposals = SpeculativeProposals( + proposal_token_ids=proposal_tokens, + proposal_probs=proposal_probs, + proposal_lens=proposal_lens, + no_proposals=maybe_sampler_output is None) + + return proposals + + def _split_by_proposal_len( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + proposal_len: int, + ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]: + """Split sequences by two groups: + 1. Sequences with non-zero proposal length. + 2. Sequences with zero proposal length (due to disabled speculation + or exceed the maximum model length). + """ + + proposal_lens: List[int] = [] + nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = [] + nonzero_proposal_len_indices: List[int] = [] + for i, seq_group_metadata in enumerate(seq_group_metadata_list): + # The speculative decoding for this request has been disabled + # (e.g. due to high traffic). + if seq_group_metadata.num_speculative_tokens == 0: + proposal_lens.append(0) + continue + + seq_data = next(iter(seq_group_metadata.seq_data.values())) + seq_len = seq_data.get_len() + + # Currently only proposal lens of 0 or the global batch proposal len + # are supported. + # If max_proposal_len is defined, then we shall not exceed this + # quota for nonzero_proposal + new_k = 0 + if (self.max_proposal_len is None + or seq_len + proposal_len < self.max_proposal_len): + new_k = proposal_len + nonzero_proposal_len_seqs.append(seq_group_metadata) + nonzero_proposal_len_indices.append(i) + proposal_lens.append(new_k) + seq_group_metadata.num_speculative_tokens = new_k + + return ( + proposal_lens, + nonzero_proposal_len_seqs, + nonzero_proposal_len_indices, + ) + + @staticmethod + def _remove_no_proposal_seqs(proposal_lens, maybe_sampler_output, + nonzero_proposal_len_indices, transposed): + """Remove sequences from nonzero_proposal_len_indices and reset + their proposal_len to 0 the draft worker does not provide a proposal + (maybe_sampler_output=None). This can avoid scoring overheads. + """ + + # If maybe_sampler_output is None, then the draft worker did not + # provide a proposal for any sequence and thus no action needed. + # Also we do not support transposed maybe_sampler_output for now + # because it seems not straightforward for draft workers outputting + # transposed sampler outputs to handle the case of no proposal. + if maybe_sampler_output is None or transposed: + return (proposal_lens, maybe_sampler_output, + nonzero_proposal_len_indices) + + new_proposal_lens: List[int] = [] + new_nonzero_proposal_len_indices: List[int] = [] + new_maybe_sampler_output: List[SamplerOutput] = [] + nonzero_proposal_len_idx_ptr = 0 + seq_idx = 0 + while seq_idx < len( + proposal_lens) and nonzero_proposal_len_idx_ptr < len( + nonzero_proposal_len_indices): + if seq_idx < nonzero_proposal_len_indices[ + nonzero_proposal_len_idx_ptr]: + # Sequence is not in the original nonzero_proposal_len_indices, + # meaning that it has a proposal length of 0 before sending to + # the draft worker. + assert proposal_lens[seq_idx] == 0 + new_proposal_lens.append(0) + else: + # Sequence is in the original nonzero_proposal_len_indices + if maybe_sampler_output[nonzero_proposal_len_idx_ptr] is None: + # but does not have a proposal from the draft worker. + new_proposal_lens.append(0) + else: + # and has a proposal from the draft worker. Add it to the + # new nonzero proposal list and keep the sampler output. + new_proposal_lens.append(proposal_lens[seq_idx]) + new_nonzero_proposal_len_indices.append(seq_idx) + new_maybe_sampler_output.append( + maybe_sampler_output[nonzero_proposal_len_idx_ptr]) + nonzero_proposal_len_idx_ptr += 1 + seq_idx += 1 + + # The remaining sequences should have proposal length of 0. + new_proposal_lens.extend(proposal_lens[seq_idx:]) + + # We assume sampler_output will not be a list of all Nones. + # In this case this function should not be called. + assert new_maybe_sampler_output + return (new_proposal_lens, new_maybe_sampler_output, + new_nonzero_proposal_len_indices) + + def _merge_outputs( + self, + batch_size: int, + proposal_len: int, + maybe_sampler_output: Optional[List[SamplerOutput]], + proposal_lens: List[int], + nonzero_proposal_len_indices: List[int], + sampler_transposed: bool, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """After speculations are produced, merge the speculation results with + the skipped sequences. + """ + if maybe_sampler_output is None: + # If no speculative tokens, the sampler output will be None. + # In this case we return empty proposals. + proposal_tokens = torch.tensor(-1, + dtype=torch.long, + device=self._device).expand( + batch_size, proposal_len) + proposal_probs = torch.tensor(0, + dtype=torch.float32, + device=self._device).expand( + batch_size, proposal_len, + self._vocab_size) + proposal_lens_tensor = torch.tensor(0, + dtype=torch.long, + device=self._device).expand( + len(proposal_lens)) + return proposal_tokens, proposal_probs, proposal_lens_tensor + + sampler_output = maybe_sampler_output + proposal_tokens, proposal_probs, *_ = sampler_output_to_torch( + sampler_output, sampler_transposed) + + # Now, reformat the output GPU tensors such that each sequence has + # a proposal. the proposal can be empty, e.g. [-1, -1, -1] + + entire_proposal_tokens = proposal_tokens.new_full( + size=(batch_size, *proposal_tokens.shape[1:]), + fill_value=-1, + ) + entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens + entire_proposal_probs = proposal_probs.new_zeros( + batch_size, + *proposal_probs.shape[1:], + ) + entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs + + proposal_tokens, proposal_probs = ( + entire_proposal_tokens, + entire_proposal_probs, + ) + + proposal_lens_tensor = torch.zeros(batch_size, + dtype=torch.long, + device=self._device) + proposal_lens_tensor[nonzero_proposal_len_indices] = proposal_len + + return proposal_tokens, proposal_probs, proposal_lens_tensor diff --git a/vllm/spec_decode/util.py b/vllm/spec_decode/util.py new file mode 100644 index 00000000..193ef870 --- /dev/null +++ b/vllm/spec_decode/util.py @@ -0,0 +1,268 @@ +import time +from contextlib import contextmanager +from typing import Dict, List, Optional, Sequence, Tuple + +import torch + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob, + PromptLogprobs, SequenceGroupMetadata, + SequenceOutput) + +SeqId = int + + +def get_all_num_logprobs( + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]: + """Given a list of SequenceGroupMetadata, create a list of all num_logprobs. + + If the sampling params do not call for any logprobs, return 0 for that + sequence. + """ + + all_num_logprobs: List[int] = [] + for seq_group_metadata in seq_group_metadata_list: + num_logprobs = seq_group_metadata.sampling_params.logprobs + if num_logprobs is None: + num_logprobs = 0 + all_num_logprobs.append(num_logprobs) + + return all_num_logprobs + + +def get_sampled_token_logprobs( + # shape [num_steps, batch_size, vocab_size] + logprob_tensor: torch.Tensor, + sampled_token_ids: torch.Tensor, # shape [num_steps, batch_size] +) -> Tuple[torch.Tensor, torch.Tensor]: + """Get the logprobs for the sampled tokens. Returns the ranks and logprobs. + """ + num_steps, batch_size, vocab_size = logprob_tensor.shape + + selected_logprobs = logprob_tensor[torch.arange(num_steps).unsqueeze(1), + torch.arange(batch_size), + sampled_token_ids, ] + expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand( + -1, -1, vocab_size) + sampled_token_ids_ranks = (logprob_tensor > + expanded_selected_logprobs).sum(-1).add_(1) + + return sampled_token_ids_ranks, selected_logprobs + + +def create_logprobs_output( + token_id: int, + token_id_logprob_rank: int, + token_id_logprob: float, + topk_token_ids: List[Optional[int]], + topk_logprobs: List[Optional[float]], +) -> Dict[int, Logprob]: + """Create a Logprob Dict for a token given the sampling results. + + Args: + token_id (int): The sampled token for the sequence. + token_id_logprob_rank (int): The logprob rank of the sampled token. + token_id_logprob (float): The logprob value of the sampled token. + topk_token_ids (List[Optional[int]]): The list of top-k token ids. + topk_logprobs (List[Optional[float]]): The list of top-k logprobs. + """ + # vLLM logprobs always include the sampled token. In addition, the user may + # request topk-logprobs (where top-k varies per user up to max_logprobs). + logprobs: Dict[int, Logprob] = { + token_id: Logprob( + logprob=token_id_logprob, + rank=token_id_logprob_rank, + ), + } + logprobs.update({ + topk_token_id: Logprob( + logprob=topk_logprob if topk_logprob is not None else 0.0, + rank=topk_index + 1, + ) + for topk_index, (topk_token_id, topk_logprob) \ + in enumerate(zip(topk_token_ids, topk_logprobs)) \ + if topk_token_id is not None + }) + + return logprobs + + +def create_sequence_group_output( + token_id: int, + token_id_logprob_rank: int, + token_id_logprob: float, + seq_id: SeqId, + topk_token_ids: List[Optional[int]], + topk_logprobs: List[Optional[float]], + prompt_logprobs: Optional[PromptLogprobs] = None, +) -> CompletionSequenceGroupOutput: + """Create a SequenceGroupOutput given the sampling results. + + Args: + token_id (int): The sampled token for the sequence. + token_id_logprob_rank (int): The logprob rank of the sampled token. + token_id_logprob (float): The logprob value of the sampled token. + seq_id (int): The sequence id. + topk_token_ids (List[Optional[int]]): The list of top-k token ids. + topk_logprobs (List[Optional[float]]): The list of top-k logprobs. + """ + + logprobs = create_logprobs_output( + token_id, + token_id_logprob_rank, + token_id_logprob, + topk_token_ids, + topk_logprobs, + ) + + return CompletionSequenceGroupOutput( + samples=[ + SequenceOutput(parent_seq_id=seq_id, + output_token=token_id, + logprobs=logprobs) + ], + prompt_logprobs=prompt_logprobs, + ) + + +def split_batch_by_proposal_len( + seq_group_metadata_list: List[SequenceGroupMetadata], + proposal_lens: List[int], +) -> Tuple[Tuple[List[SequenceGroupMetadata], List[int]], Tuple[ + List[SequenceGroupMetadata], List[int]]]: + """Utility function that splits a batch based on whether the proposal len is + zero or not. We should remove this once vLLM supports per-sequence proposal + lens in a batch. + """ + + nonzero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], []) + zero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], []) + for i, (seq_group, proposal_len) in enumerate( + zip(seq_group_metadata_list, proposal_lens)): + seq_groups, indices = nonzero_lists if proposal_len else zero_lists + seq_groups.append(seq_group) + indices.append(i) + return nonzero_lists, zero_lists + + +def sampler_output_to_torch( + sampler_output_list: Sequence[SamplerOutput], sampler_transposed: bool +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Utility function which converts a list of SamplerOutput to tensors. + + sampler_transposed here is used as the indicator for whether + we need do additional tensor transpose logic here. + + Returns: + sampled_token_ids: torch.Tensor + shape: [batch_size, len(sampler_output_list)] + + sampled_token_probs: torch.Tensor + shape: [batch_size, len(sampler_output_list), vocab_size] + """ + + # shape: [batch_size, num_sampler_output, vocab_size] + sampled_token_probs = torch.stack( + [ + sampler_output.sampled_token_probs + for sampler_output in sampler_output_list + ], + dim=0, + ) + + # shape: [batch_size, num_sampler_output, vocab_size] + sampled_token_logprobs = torch.stack( + [sampler_output.logprobs for sampler_output in sampler_output_list], + dim=0, + ) + + # shape: [batch_size, num_sampler_output] + sampled_token_ids = torch.stack( + [ + sampler_output.sampled_token_ids.flatten() + for sampler_output in sampler_output_list + ], + dim=0, + ) + + if sampler_transposed: + sampled_token_probs = sampled_token_probs.transpose(0, 1) + sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1) + sampled_token_ids = sampled_token_ids.transpose(0, 1) + + if sampler_output_list[0].hidden_states is not None: + # shape: [batch_size, num_sampler_output, hidden_dim] + sampled_hidden_states = torch.stack( + [ + sampler_output.hidden_states + for sampler_output in sampler_output_list + ], + dim=0, + ) + + if sampler_transposed: + sampled_hidden_states = sampled_hidden_states.transpose(0, 1) + else: + sampled_hidden_states = None + + return (sampled_token_ids, sampled_token_probs, sampled_token_logprobs, + sampled_hidden_states) + + +def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int, + vocab_size: int, device: str) -> None: + """Helper method which mocks out the GPU tensors in SamplerOutput with dummy + values. This will be removed in PR 7/9. + https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer + """ + values = [ + sampler_output.sampled_token_probs, sampler_output.sampled_token_ids + ] + assert all(v is None for v in values) or not any(v is None for v in values) + if not any(v is None for v in values): + # Do nothing if the tensors are already created (usually in unit tests). + return + + # Softmax to ensure valid probs. + sampler_output.sampled_token_probs = torch.nn.functional.softmax( + torch.rand(batch_size, vocab_size, dtype=torch.float32, device=device), + dim=-1) + + sampler_output.sampled_token_ids = torch.randint(low=10, + high=100, + size=(batch_size, ), + dtype=torch.long, + device=device) + + +@contextmanager +def nvtx_range(msg, *args, **kwargs): + """ + Context manager / decorator that pushes an NVTX range at the beginning + of its scope, and pops it at the end. If extra arguments are given, + they are passed as arguments to msg.format(). + + If running with cuda graphs, you must enable nsys cuda graph profiling. + + Arguments: + msg (string): message to associate with the range + """ + torch.cuda.nvtx.range_push(msg.format(*args, **kwargs)) + try: + yield + finally: + torch.cuda.nvtx.range_pop() + + +class Timer: + """Basic timer context manager for measuring CPU time. + """ + + def __enter__(self): + self.start_time = time.time() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_time = time.time() + self.elapsed_time_s = self.end_time - self.start_time + self.elapsed_time_ms = self.elapsed_time_s * 1000 diff --git a/vllm/tracing.py b/vllm/tracing.py new file mode 100644 index 00000000..50068d8c --- /dev/null +++ b/vllm/tracing.py @@ -0,0 +1,119 @@ +import os +from typing import Mapping, Optional + +from vllm.logger import init_logger +from vllm.utils import run_once + +TRACE_HEADERS = ["traceparent", "tracestate"] + +logger = init_logger(__name__) + +_is_otel_imported = False +otel_import_error_traceback: Optional[str] = None +try: + from opentelemetry.context.context import Context + from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL) + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.semconv_ai import SpanAttributes as BaseSpanAttributes + from opentelemetry.trace import SpanKind, Tracer, set_tracer_provider + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator) + _is_otel_imported = True +except ImportError: + # Capture and format traceback to provide detailed context for the import + # error. Only the string representation of the error is retained to avoid + # memory leaks. + # See https://github.com/vllm-project/vllm/pull/7266#discussion_r1707395458 + import traceback + otel_import_error_traceback = traceback.format_exc() + + class Context: # type: ignore + pass + + class BaseSpanAttributes: # type: ignore + pass + + class SpanKind: # type: ignore + pass + + class Tracer: # type: ignore + pass + + +def is_otel_available() -> bool: + return _is_otel_imported + + +def init_tracer(instrumenting_module_name: str, + otlp_traces_endpoint: str) -> Optional[Tracer]: + if not is_otel_available(): + raise ValueError( + "OpenTelemetry is not available. Unable to initialize " + "a tracer. Ensure OpenTelemetry packages are installed. " + f"Original error:\n{otel_import_error_traceback}") + trace_provider = TracerProvider() + + span_exporter = get_span_exporter(otlp_traces_endpoint) + trace_provider.add_span_processor(BatchSpanProcessor(span_exporter)) + set_tracer_provider(trace_provider) + + tracer = trace_provider.get_tracer(instrumenting_module_name) + return tracer + + +def get_span_exporter(endpoint): + protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc") + if protocol == "grpc": + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter) + elif protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter) # type: ignore + else: + raise ValueError( + f"Unsupported OTLP protocol '{protocol}' is configured") + + return OTLPSpanExporter(endpoint=endpoint) + + +def extract_trace_context( + headers: Optional[Mapping[str, str]]) -> Optional[Context]: + if is_otel_available(): + headers = headers or {} + return TraceContextTextMapPropagator().extract(headers) + else: + return None + + +def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + + return {h: headers[h] for h in TRACE_HEADERS if h in headers} + + +class SpanAttributes(BaseSpanAttributes): + # The following span attribute names are added here because they are missing + # from the Semantic Conventions for LLM. + LLM_REQUEST_ID = "gen_ai.request.id" + LLM_REQUEST_N = "gen_ai.request.n" + LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences" + LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue" + LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token" + LLM_LATENCY_E2E = "gen_ai.latency.e2e" + LLM_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler" + # Time taken in the forward pass for this across all workers + LLM_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward" + # Time taken in the model execute function. This will include model + # forward, block/sync across workers, cpu-gpu sync time and sampling time. + LLM_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute" + + +def contains_trace_headers(headers: Mapping[str, str]) -> bool: + return any(h in headers for h in TRACE_HEADERS) + + +@run_once +def log_tracing_disabled_warning() -> None: + logger.warning( + "Received a request with trace context but tracing is disabled") diff --git a/vllm/transformers_utils/__init__.py b/vllm/transformers_utils/__init__.py new file mode 100644 index 00000000..74ca3962 --- /dev/null +++ b/vllm/transformers_utils/__init__.py @@ -0,0 +1,17 @@ +from vllm.envs import VLLM_USE_MODELSCOPE + +if VLLM_USE_MODELSCOPE: + # Patch here, before each import happens + import modelscope + from packaging import version + + # patch_hub begins from modelscope>=1.18.1 + if version.parse(modelscope.__version__) <= version.parse('1.18.0'): + raise ImportError( + 'Using vLLM with ModelScope needs modelscope>=1.18.1, please ' + 'install by `pip install modelscope>=1.18.1`') + + from modelscope.utils.hf_util import patch_hub + + # Patch hub to download models from modelscope to speed up. + patch_hub() diff --git a/vllm/transformers_utils/__pycache__/__init__.cpython-310.pyc b/vllm/transformers_utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..628a03dc9e1375fb0dff4f3fec41759adbc58318 GIT binary patch literal 542 zcmZWlO-~y!5VgI|$A)mKgnBf`NU+PE+5<(E+6W;fP${B^6_VLxP1a!VTDB7uj_^my zu?PMFKV+|*xc1mfI~!F62}|?w#`9*LrrO->A$ZBdV{u3jdTPMyM*(~Wb7$aklw*Z7 zPBBIT+a16}k(59TRG?`}p-z9a1i5VA<0i}LJ-)(rTXgc)9lpiT*;ZX0rXh5OcL>9T z$=U{;XV`!SxPJG2G#c~M$&imvz79u|FDKuI$58Jqg>|wjZ`w(ee2n&P=BG}UbGC$b zb}fCu##JuVWL6o$N+EK`bW?Cm;o#Gnlbxs z|3Z!yUMd$A7raSqbNZjih7i{7_hIa{i9zhH&+J_I-4A+E=tN(~N={?5@3I)_6!n&@*2~t&B58U3lB~6@l}L$>NXepPWjo%qn^QGpQ~elM z_mCpfv)L>StbK5DSRh$oy@>>D4i1n@kYf-a$bFAFGy>$1OLE$5;$(MiS<3fU_mGqy zK!D6({;BG!`m5@%zrO#`VWUx1@HzX}f4}^YAtd1*@Ravx2AlMXTr=mf@GIl3%vUe#NTDHr1LbQey8|(*5)PtBF3D~ z|DVPGw^2WX|JU$8!OwwTtJ`9eZ0h}@^*W8IXPLLPv~`6CTsXw^QW$jIrL=WJ@K`u* zz*+VLD><;lW83weC2sdb*x^wW3cDlwqxIE4Uz@Y%F5b8? zJAcKVzc@F&fKIhb*XL(uvpTA~-oA!Tw-=^uIe6jH^&8XF%MIKZ%UoLGqTNaxmzJIQ z%C)(CY1PY%;$B{^O<%lnZ91>ir|&so!1LPJY!Gu1ynT&E4~M>aadu(YdycbyTWO@-CIR8@KXebDsNQEViW8n137lxNPUmjknf#@H}SDWLds> zCok0&LeClUYu@%C;ngtT3-&#N)9Bo_mzMfnn@$pssu%lln2r9d)^mj8N0Td27yz>5 zg408GF4CLRS7sM(-Tbb7{q53cmp+7bXJld?*S+*$ItWpF8ItKmu!uX;93W0iwkDwdJS!zxC! zU#r~DcWRa2BgxpSUfTNdx;v!sJ3lHaZ{Qd01K|q6x1}-FDliS(DOg3OJ9{EY0F&zIIz?v~Jt)^&Ky7(J$VOtkc_CQ%s;zFV%Y64k(R2b;7p6GFLQxk`nwxDcO zVk2u!3as#veh2+^B`&R$S1RjgRfWFah1pwB)*##YcEKtM-V1G)rDak{l2uw3eASHr z7Pmgp1njVq7US?P4`$j`0mG}LP!I`0B{!0aEXZu9Rg4Lu*a@t!MU6R7UO_{20Hm(9RK1|7nrb|%7IaP1$JCZ;6tpkIYczN?7X2u; z1!*}4@G+ikzClGi&Sa9-McCtZ)N!!*rL-z*@iLE{6deg%ZPE=V?v%iE_!&cig!DW3 zMgIoX&kl7^PRe2|R_U-;w4}t;3rb!q43xx(^_AjXP0S|c=mZ^XTuRXEewk?xR61U9 z3OyYw_j#ZVo!SqD=sq!;zQ~jYTXRBbgzlfVFBZ<<5ib08r zA7jR!tADJ%qYWy9YEr@M#shUwOKNdzRS_a?CAIy^pq|uMMxlCg3>32wyuL9KkI^%5 z%6OGNE@Zq0jXO#(1;uQv|GBCRnyRufnyArFSUKL6G&fc7{RyiKM&j{gWK&I=Kh*F8 zKn$ASZ?36pnPh5M>tDsYY0gaz^OdllYGQ2dr!0PvG*ZvyhBF&{c)_B&}pLnCS*)JG+144U9;u5UdN;CW>DkCq~P!MUBM%BJ?xutX!3io zZ!tHanmPA=<^h;=M>hUt#=#$>p5!mcZjyiAdD5Ki?s(`%rek(+Jwo5ix&>iuuJhP* zR>6qwF0Gd)yPDni8QgbsCsdliQQ?U7Gc-fw;&z!4k7HaS^WCr9yl>wUef}=7ga=`U z!t4WEmL&goCnjX~iR_|I(CvKJ?0U}9a0QpH%}#C{yd8Xlc+Z;~2QE3mGqDMnH|cD4 zh~G@r$+Y2!&axZx4uG0Rg792?horp!c8|f0!b(m?Cb?MVNkJk!DcXm`G3qAqQI{}k zz@GCUmf$ZYs8Pa9^XkmdHj>+v7O)C2iRN@yZkKj8izyl?k+k}@<6*yXFSp|=_qDSN6toEroK(;YxHTWJT)>u zE!Ewi8>S^p>BO;Uk4c>Vk```DN{}T>jk{~GrIAH;g5;bIVVzc+sR-?vR}Eb&|3Le+ z+N`ThO)pf@R!3VE=Ef)(YW>l0ZfemVHVy3ux+dPiLObLG4L@Wq$V3$Ii~boTQ6im! zg$;wj)_cl81!#@{G}Uzkn)Hpe`>^r^S+MEoxu$HKlYR9Ml@FBr#Uz6>4Ph=q;DH8+ z(!}$LwxaK7Q@!#4J{M-Fe5m5TNFnh6aK;RR!5=9nJ_YGQKRw4jz*PK7YMsJCAx;}z0!Jhyi zT%F0$B~yYSVoA-Qju96w?Cu6P7uQXx4^=b&+;QV&;LVpT_!$Wq88;LUvCFY8AL8dQEw8);Ik`}{nUN`7l1cLx;Vfyex#oJFN$|MHx7fLo z!w@E;p0frI-CPUBT{w}n8OFN~+-UQZ1LR)|37w>qWjcUZ@C$=2ULh7BFahIH`;57^ z>~@yTMZulB(UyE>D&dN52c-P z%i)`jp@(g5u^Y-S%IAwXa8-tzWYn$scLRX!vHEohZvdembKhg3Uk?k+4OKUY#qwLw~tV6 zNh&hiK9>c~&zs9}+>6dnO)X(#{l&>n=udUBBkA#Bi4N!W*jfBGarXH{xH>^L{dl+v zz;qcFK9QYZG%*yIJi-Z|BajQ+$e)0tF|oaI*ic*o(eK3BM&|_q)5hwhuY>3`Yp}VMm<~Zzy;4rj}K1eyu5=L|9x9&N?Lu=D^Ckl!e)v z-+2NrX^9j=?ZPi2|M~-Ug-{R9^aHgE%vw-BQRk8UD3YCv(jx2oy{N4U8+|{~V2id@ zJvB0cW!5DNCE)S1K4KQ>?qql9moybw_%U^3+IQ9$DJ;mKYiNgu0_XPPx-AiE@`4uv zY@*jDzlP(+-`tjg&l4y(lv{pxq{ZA~oF^yt3f);gvK zgDy-NfBioW2nSQ<_c}H?1W4bE?{xBpN|DxP<+aMZmA4==$e%Cvmjp$(*1j#VWpdvV zmbI5c?8(e|4aFq%APcd@_lU=c4~%s4>=g7|aN<^zW3nR>bU*+jNH zD4$iL(`#@fvIGH)zs`!xAhXXZtopI>p~7mhvE>fbS>yfshiX#1Ut&!*@<2<9ta(9^ z^%fhI^|8FZi;c_r?!3N-?UnUCd3|4A-R zPf;r~)CNXUgwyfV$J(ICjtu)UGh5dO-~&mNk`f}aqx0efg!qY?h4-qkYRfKTJg_5p z7FJ#OksTm}w|OtJb_IRkhWnY_WqTQSDY8cCVLrxl)?va^*$RZ&b?$Nh2@E9squi0iz&BAKg@{#T zgO0oK7ZaH=LIB%B0ILlOK*s_CbnO9Z5DAon0`m*Xd$kyzri-i+1^Nr%0WoRwQuUFM zISEeF28?Q7M95xa579Qx zihJmf_bEz5QAbtPk&=NQjjyh4w7(jmY@_Lu3@>A}RoekYR@H%bvns*R(j(XFTSHl* z!D_}X*5WU6M)oqAv3Ah|*mAbxDP9aY_w0zjXCpJ_Dxa!~wU?gs!brL#JG&PAkXvJO z?meu*n?Y9DLH60ZHYa9iWoVGe${h_>GfODh@PAa$w; z!O2V-#j#ttILfJ|Iz5-iZH-DWhIw?5AeJ^-+#$XaFxRL=fGMtnJj5@$2BPbAt&C8L z0;GDOt{!@<=@eLHU*og!N~?-29lqtqEg&NOH_@-EH8q4!$lf7$M*$kzBsMP~swGrWme(i%XCxJ}Es(p>2p3{zRY1R$MQx zQnf+tg>8w6Tl7?JS`uj`4pUf)7c^i0I^`VcJH>jsVPGp&bzBvHNL)yMPh4aX7JEgRM~d`R*poeaNP-|hxEDXsF4nOcAwZzGC3w1&7LX6&Vr}cewn)Zu zk?hd%D4X+`M=sO3lXoj2o) zRuSHUsy)_zBQ|NGcR||IunDk5;txm7qkSTntz5_s`Z1*zLG`aKnq}+EjNpE5%pHOhjXVX)8N35s6agb!N)tP|b8hfI% ze5)cxXmPXDTFJb^U^SBWGWYnh%#;|V9!F%4olG*n5fK#SVQ>?cBR3dBJohlm^|@)VIHL`)(_L9D~yG-0-OI!Y_c*hU@TZTA$CCHe(ORe>X_9s(MTt1WU?HThMq7U~sUYod-U7qVOX0N7B_ z$9R462rpB<&~@#hUerF<4ec|%r2SSeYroMe+OPGh_CI<}`)}a=r+P#CFTJV#r#_nTY7Y(YHNaN`U#(L8^%!rS(_Ysf YQ9lFys(STdv#E{hUx3Gskb%7 literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/__pycache__/detokenizer.cpython-310.pyc b/vllm/transformers_utils/__pycache__/detokenizer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6812464ac3296e27ac13323b5f5aee495d50715d GIT binary patch literal 7323 zcmb7J&2!vFcE{IXFc{2mNQx3=%U&q!BpPQ$#Yt_pqV;a9wInZXExEN~oGO$PcsSid zf||jM8W?>rm|9olR8GF{k;)-eu1YF@L6ScrRXO0AlP<~C_F86suN%P3(2}L4ZOUAJ2|gw8q=6Q)JA%skBq<=nSnX7 z0!ycUV^|y6fjz1R^^p@eqejpexq*vsCbNdkkr#OKxi)N#+Cf{_-qV=P>Yr+?{=^7Y zn8OWj^3Fx=qBbyCgSnqt!76KVk9qile%9h^tc`nxcLpx&uvLs$XKTT^#yb0+9C;2S zH;R!f+*0%S2HzN%pz5#aa`fHPE`iUcak#FFzxRb*czU;9}v;Z=tm+9%EyF9$F0;7%rJxEAP43k;?QCNsDN z$>X#j*BYyFo7ucRuvncr=y6zs<5alIaVEUQ+XI8n3i4|57EaQGd|RxIkt;YMn{{{x z^IHR(t+F-rt}+jjTF;v|d0Mi3{NrDW@q>u*#2@kWaLf`v9s7*;$Bg^wA@|4AbTUo- z@xbr-BXkb^D8`7CKT5aqm5NSyKvL-&*im`WGu`jvPDsNuJ<~!cDwxoBy1H0FU)RX3 zgbxSu<#SD}qVKEsucf2OwMjI&n)bv2Pp|%HG(8->HF|h$80}x1JO-cfwFkrD=vpdz zaWWW-5f@1~O`~CQjmhY) zdk@yI(Qr5xy|9`Wc=BBkTS?kAL4NL=4(Pma=F@35h zjrX;`gPczEr@*L-+N_qA8%+744GxP7MubycqW>$Nu&m3mV8n|8DP2AprHfu3` z)=sU=c%t9b{`$j@jM>VpldWXt1#PyPb&@~7zm;`PYELzM{WE-Bo2_T-N1#5{M3Swg z^?N3M>rBs9X|-(aK*!8;sgpU`xvZTvGB>l18n|6%eOCX-nr+O^&o0b1v-8<{XlHAg zovmKf?irwRVb;txXI{23Yu*1z)|@pmZ{}ugxz?|<)~pR)+9%Dd^;Flj`&*23qt^RJ z2<4`Ww<9BfIYJfy(mCxhZcgO*(V9 zYHGz%W&dX;Ei1rve^)93aZLOVB9WxP2F4$iT@}Tp#ffXdhrJYLKn&941a>YlM!pnQ z=wBgw=^swxdz?W&d=L$X-8oYg)xzwS1w+y(WLoj@FiH>QdJ5l|ve0Bsaz_xXKRdvZ!y#}4NEk*e$(!KjfwH$GZ%Ni36^4;F1@E!)Wj(nUO~Pct`%!OLPIxVu z^!hvr_j&q|^Z3l?t{1E%g%b)V(hCI+87$5gi&dsejEM7KH*fHG3IOb-Jg-G5hu3Nq z8&oR_n?`NUfe}0D3qIm;ifug>m+@Kr9*wbq>O5xMiwbEj(1%TG$iQ=JB0$>&AAyvI z6c4cxbPA2Cdc-A~^*w5QG{J=!ltVBl!X;@18}knfatk{1Zb>r{uM?SzYJm!iqPRq; z;NlsjmKZ392kqrqxuf1Ambay#c}3*av-=21Wt0?iH|7t^BXs05_7tqr)AF8HsB2+2 zXTkb)bvl@}gt4SwLnsE}d-6AE>=jS+8M^-b_#g5XZ4u_=+%!JawRu&#ox}U0;1g$SGt3@;10Z5EAui7@coxrJ!^%{ z``cOT2+jrk9A{qY!95&j&CHW-p)FT$f5+-@K5#%QS*M@{K2KViO@I%FLVEyba2vX% z9ksLij}Qih?Q|upKT*!$+!<#8eNj+ULx7SZG4A3Spyvz+(w33Zn~HUSQy*zzaOu=Ysq|{}2e)msER#52@%LWVA3sfQsO_rp+7bT8JaX;9-)L(WkUJ_!OXK>@3v1Ryt&!iYM(UZaNcF@eN;QCL!J z6j}=k^@&LG1{z1R^tQ2P+7|7`K9#Wh+%Vne_6OKGWYRvBGsIgM-G`l57x6$WNU$U& zOOZjyc-8|%=`$lUmJ>5;X9f=oV zJdU|i2#z@Z@jqdj%)xM2_=6|-M*I|g%FxAY-}T#FIurKi;;onz%9#_cQCn${fN@q9fnxGIVYt zO|Y`+S-kV2IhiTmWBLi^Q4+?C@~La?e5L#1$6&E*Q_Pa)Yh*2FLa`MJrz9|i561o8 ze!f{ki83~Vkt7wU2jq5dGC^#VTSwz4?lzP){{`sJV*p=2X75bnfKIS0`DqxR*8=SbsrEdgMX=UelKRo2`J@OgOu=Pw@Nn?=_4o~SO{HF zrjP@p<3-Q7Gft^;(M$b!occKPBFguBg8PMdl^;?En{4^J@$j+qamBhcTP{dbVvbln z>50eEIlp}Qm(!P|o)Plwe6A~PAz+|tRC(M>9xLx|?`-dF-?)Q1)w{pC^^03Odw9Ni zOIEA4gByF>yF20T?b~;6?TJ4Hq1+tx9!YcV+Da|yc-2DWZe$3qe9N-Fi9f;D|HPe8 z%xSec&>#C72Z5xcTc?ic>T<-M^l&l)pL?i2;b(af`jQ4oY`c!lWx5Q@00IEg{4SolQ02fK+4U5i(tl}1)S&@-Bi zIsr7WkwW{#+gam4pEY8W)#x`fvIa7I7wi5!bMdYn)skHRi2W394!?C|{Lo&~YJ{iF zvPbDTdc1fO)d6ij3ObEy+-KITS@st;QTBV_)q4Vq$(rC;PyQkEP8tmGB|gg{4TRR7wVW!HR2D;@Dcq9X zh%k!aZF}H_UB0rxd3C=el_x1wj_rA?yZk6g}Xc1AW4a!7FNPk38Q$RH|oQO(%UXZHDMD2etG6~Q)mmLH%Dk(u_ z<2gca-Lo#47Nr%ZE?VOg8wm(iINGRpAOJeG43|Q5xHFZO*k2p={P~r-SL+xZU9N+F zN8IijxA(f6xt%_q!0=?Iu#h-V+!VCvpAK7PuA;I;nMcSJLS`N^-;mWWnS;>(I8<^% zd4$-dh63;0+8>XHQoiC2bzY~2{IN<<=Z;!F$s)h=w#tHjO3_2|F`7;7lHr-2-Zr&g zpMQ;ZXPnxG@eTI;+^~$Vjhg#g>kY%Uo-cmsSnk)xWk~VMve&J~8@~mWuaNG2x#qOL F_&>d+xkCT| literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/__pycache__/processor.cpython-310.pyc b/vllm/transformers_utils/__pycache__/processor.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92197beff8e5c6d37f751a3af58fec32e3b90b3c GIT binary patch literal 2054 zcmdT_OK&4Z5T5RN*^VFCEU-L;)Dj0U!nPI;NC;LN5Ya~R!fHVv`Ql96ZD*EwST#Kk zj>ZVFocRTu5SKmh7x*D_g$oh~q}))JrD_~IlaL_x$VgMukLr4S^>t0$Xw(dRZvOQ1 z!S8j$_=P6rr;3x?_$C^KG$ad*kPTSK2b>v_OY?gsEoDkuG3h8vx{8e}vLdVBIs^O4 z{hHjsedo&kx@_RSE1RDf?bdJT*=Y0J+>YkCHS!X@(`NZK4QZl>pu$+I;V70WZ%!c| zsU(SE7>jzBr`Gh>QF2z(aFkGvjD8 zvmRKPF<}}1f!#Nxl`)@X=8x<?fF~GSh%RYk+GyB?L|d9S>++i+;2F( z2+zX($~|OA`xO{fuMZE%=g5dbTaXy z$&NQtU7#!wipF-T<9n;zaiO`M-1WcoqXL4XUw1q|P*Uhv1mu!jtetkn?)dzQ@Dq$` zgc!%+nVyJZE#^uKZ-&;q{XhvnDh`$c6B@iI!8Aezi3t3C@L>Li7{yWIO9diPT4N4k z?;j<6?^8(C9+~X-?+U~rO-PYK-RpD;_0F>59jf@A7zf^D6~PDn-R?*2N?tF!2deVm zSZG;6dxOtl{*KrF~xV~E4y6cp8GpjL^A|VyV{X+>nzm(e z&Y#pAi&wtp$F&wbgX&k_45;8P@^pL2(@6NCHz_ga>FK&fbloK@_f}CqV)@qlUZT#g zE_wUMCA_^5!e#*wXpm@<_!p9rVJObD zXtYV<8VRE2Qe4MTex0aw&Sea*k;=0qUPN(@HgpS!f7DQuH&K)us^+BzYL*(Pt<3oS5Ur!zrwwt2y&^7ko~LVza0C4p1Zvyx0*#A&iRuM1gH~aG1!W9v=hFLphSFu& zT(Hqc>j{Bdui_Ouck!H)x(hGi7Ii2`&!<>C;HzkIGtrP+`*9o;x3R#sP%o9F2%xA- iq*mGw zW_Bo>>?$Z?19lEBdMuCxv4i&DLk|Ud=&6VP8H?W9ORnvuaX!@dW@(DDodRx2?Ci{& znKy6VypP{(T&$!`I|Td+BBe{?&t(* zdSHYFr=aRaV1`Af7+Q{{+9W7BB^Xl(%HfzZ7FL`}SaqtZ&kSnexHGQm#b6?wbS72Z z3Z}woXIj-u!MtMGI1tV`b425l{NQ!s9O4$Q@G75a>2sPh&nkSL zRr!3&U^O;=!*C7*rNXL=0A+$r0_7<-#ikj7|GrNQHUsmIaJ@z6G`61~VYA$32Y8Vm zf%e?sjr=O?AUkx!be?AOd>U3L^7%Opcphd?0nej+jvobSo?&D~s~`CSHF~~IGUK8r z*Wq1W=5jq|tGy1d)6BZik$xO`K~}gNL8Y!|)wjDcUX3?-#H?zwZz?&XzJ?(&77oI1C1^1{VaKpyiWU%EluZgYXnbMd9OFY))gTuqu- z_7f?*V7LMd8E>xh=BC?jcUx}D4>*v@p|{ChFX=_itR#mbnu5!&h?-y~!ykoO0UjNm zy(>Yz-*=nace7+bimZxg0tZNvOn z`;?}{HB*vmZXwk`#fJ9`EPR?vdnl28)tBpb(}Nu_yCdQb7t-fRqg=M(x4P~pFd_)- zjwhSzwp`~UQ{%M_-juc<*`2^^^1+;UdN$)NuN%l?HYh4eTCoVZ81NCi8Hb&&4z|6o%d`{a1b;II6Gzs++%*!24QT^PW*P{xBR9TNjnMRt1P}6 zHI(d%O{*I<(Q@)#Cde$|BYU+$K)VGd!G-NLn>Ew> zt8r4D%h1&Jm<^f`KGX303+1~7M|_W*zXwHTBs^$=8O{xvx}Z5X>F}oS1+LE$f%ER4 zS(IUCvEz3Zq$fa@h1bLGdiYX!busYQ7CSxgzi4qQ2*SnDqjbB{50b?p??$JWjX%V{ zj$)x{8?^uC6Ya9`5U20lwU`9 zb%EfzuY!a5!JLGRt)%FXXP9m0dTCSvSCX#dY8BdcmN_8U&V}IDm>d zgvDVjj$nbdbI%a_v3?K>$nctY3JYacO55WPx#IKColHPsX_VFo)$?aU7taCFG4tW| zVx}it=F6%TaqUcp{bwcDZ3bSFxUR5)Sf^qfUTS*2?z7lbi|>1!c5o{R8h^QDP`Xp5 z7MZ6LlO{EYKjzq+s2GT zZjtNs7I9%T#H?cf1f*vO%x=eHcBnG63d0aNJJP-aRE*4S;xm3lh1NJ=5Lt)TkTfvI zS4PK)8IVZ40L4&fjBJ<`F|gq&B`2UrA?$XEn8TE|p>677juDCCkq!DWpsY{zZA0oC zMrvRXzD)_l#FYA4s{fAs8ig;M?;m?eLI``^kVEExOc%wFK)RhyEF@&ZwVvIdvAu}p z8E}o;kY{4SS>p>-Cq&tDBE9Bjz_)l%FDUslLjqb+2bx(|JrRK_lFYywj2Lb+Gho>y zGdH0F-g=m{?@`B!h2KWC=9aK)d$WpO(a=@Z}|!2TDV zhhk_UxR)9{h|Dj*ll%s3MuKCbQ3oWX+eMSDbXKWvEs$3^qTJl?lTvc6s#`bSlU6H^KB*4SSF%VxmRyW};BF3fut0%n(!v;_kI!5ZqRskF1 zj1dY-&~V1VC!(f@uvA6L%m{c(I;G(i;yzh)N@0H0@O-!;=GFc$&#FiTE*wYgpl2&Q z4e<-G0W4wzS9>rtS1iJA0QgrH7KZv)UTGg43_E%S=YgiW2(M!S3?($dO%Y#X^kV?o zvC$AOg6_pJbpT3h5I}~`*+0f&o`+C=4TQzR8-&zm7q~_er1ir*wl?}N2)I%C0%r8$ zu9DyyZ{oFJ?-c`YQ~iG|dGJ#E!$`4rNW(tc+DpM@J-Z1pCk{sxkEhgwtBiLcN8Mut zT*xEf0!Bc{jDgpFAi>FVXCWz4XX50#Cr_VSUVU(1bS4LsSB9j%e_3(%9x|-_!UKoj zUYBswL3@NDkG&b4e6XEdMo(RSMl|I|a2N7eqyDrg!C^ZCOHxj_-d$@v;(7v8Lq=oB zbl@oKnW-!w$;Q`nXKsNryD$^H4wja1PfMKInI-qk<#*f*OH1d@o)zMi#X;7NT@D#;bP6-%TunvJn4+LL>#&u zxN_$o1q33Xta7Ov;iWQPrGXr5q?aKfDdCztPnMX~goRyZ{xKjc4l=sJSM{2B6GuEI zM+t?$U&db4j5CrEQ9n-6Pt?Z(h%t$)gd8}d8;iC|{u$v}kqhAhGRV>3E*hvJ8XS7_ zC0HQ&6%@u;#i;0zmDS%QCN=LHMT=Hc+t{%v{E4L-q(;hQzg{5*_P`ADE6XfH-P|#r zr+-t_9ZHRFvCsIL7Rq1MiiS}kIHO!JsP!xQ&)Q@SxRn8H!k8W4O2rBYkyV0Nc#!MW z=aONrh8~6HhPk>0gcQDpbv0o!xVQGl^N${o)d*%az-(0IGyOe4Zz});CGEWo8O?x< zHYxQx@Yw?}AW4n8kTIFI4dZ@+Z{q{V@GFBFX0bXw-(Nl-kC=BW?m|=pPkk;qHgMB5 z58NeBL_@{Gh};eA@1MGc*cM{-8!)F1hzhsrl+;$0!Iy?31VRyU5egW#iuGMvQ2735 zNK=Qod!kc#qKeya^*_R}1TP|%Qt?4zShX(=JS<{-TZ2P{L-3j^xU3WSEr7qTLX zgKG!tIrsrA3nHm6oMDv<|P0_ z3`?Z&6l2ecBr2E&Jr5*R;WPwJdB`JcjO&n5k(a3LH?Gocugi~j7)w7rTO(9{NH|_Tt7Lf z_;Q_zGP4!WN+EeBOg7KX&Mr=~Uhe4hIvlN5UHR4#y}0vzJ2xJEN*-#j+{MDo{9zZF zt}MkzN1K+{j%q$Z5{$7B*7F9z*Caq!S93& zeONs2J74=H{5?x@dc68A+vOm_#z6V2}lRSeyb&@~# P^E7^lDN1pQ9|wN`PxzYj literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py new file mode 100644 index 00000000..bdc60686 --- /dev/null +++ b/vllm/transformers_utils/config.py @@ -0,0 +1,360 @@ +import enum +import json +from pathlib import Path +from typing import Any, Dict, Optional, Type, Union + +import huggingface_hub +from huggingface_hub import (file_exists, hf_hub_download, + try_to_load_from_cache) +from transformers import GenerationConfig, PretrainedConfig +from transformers.models.auto.image_processing_auto import ( + get_image_processor_config) +from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES) +from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME + +from vllm.envs import VLLM_USE_MODELSCOPE +from vllm.logger import init_logger +# yapf conflicts with isort for this block +# yapf: disable +from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig, + EAGLEConfig, ExaoneConfig, + InternVLChatConfig, JAISConfig, + MedusaConfig, MllamaConfig, + MLPSpeculatorConfig, MPTConfig, + NemotronConfig, NVLM_D_Config, + Qwen2VLConfig, RWConfig, + SolarConfig, UltravoxConfig) +# yapf: enable +from vllm.transformers_utils.utils import check_gguf_file + +if VLLM_USE_MODELSCOPE: + from modelscope import AutoConfig +else: + from transformers import AutoConfig + +MISTRAL_CONFIG_NAME = "params.json" + +logger = init_logger(__name__) + +_CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = { + "mllama": MllamaConfig +} + +_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { + "chatglm": ChatGLMConfig, + "dbrx": DbrxConfig, + "mpt": MPTConfig, + "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct) + "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct) + "jais": JAISConfig, + "mlp_speculator": MLPSpeculatorConfig, + "medusa": MedusaConfig, + "eagle": EAGLEConfig, + "exaone": ExaoneConfig, + "internvl_chat": InternVLChatConfig, + "nemotron": NemotronConfig, + "NVLM_D": NVLM_D_Config, + "solar": SolarConfig, + "ultravox": UltravoxConfig, + "qwen2_vl": Qwen2VLConfig, + **_CONFIG_REGISTRY_OVERRIDE_HF +} + + +class ConfigFormat(str, enum.Enum): + AUTO = "auto" + HF = "hf" + MISTRAL = "mistral" + + +def file_or_path_exists(model: Union[str, Path], config_name, revision, + token) -> bool: + if Path(model).exists(): + return (Path(model) / config_name).is_file() + + # Offline mode support: Check if config file is cached already + cached_filepath = try_to_load_from_cache(repo_id=model, + filename=config_name, + revision=revision) + if isinstance(cached_filepath, str): + # The config file exists in cache- we can continue trying to load + return True + + # NB: file_exists will only check for the existence of the config file on + # hf_hub. This will fail in offline mode. + try: + return file_exists(model, config_name, revision=revision, token=token) + except huggingface_hub.errors.OfflineModeIsEnabled: + # Don't raise in offline mode, all we know is that we don't have this + # file cached. + return False + + +def get_config( + model: Union[str, Path], + trust_remote_code: bool, + revision: Optional[str] = None, + code_revision: Optional[str] = None, + rope_scaling: Optional[dict] = None, + rope_theta: Optional[float] = None, + config_format: ConfigFormat = ConfigFormat.AUTO, + **kwargs, +) -> PretrainedConfig: + # Separate model folder from file path for GGUF models + + is_gguf = check_gguf_file(model) + if is_gguf: + kwargs["gguf_file"] = Path(model).name + model = Path(model).parent + + if config_format == ConfigFormat.AUTO: + if is_gguf or file_or_path_exists(model, + HF_CONFIG_NAME, + revision=revision, + token=kwargs.get("token")): + config_format = ConfigFormat.HF + elif file_or_path_exists(model, + MISTRAL_CONFIG_NAME, + revision=revision, + token=kwargs.get("token")): + config_format = ConfigFormat.MISTRAL + else: + # If we're in offline mode and found no valid config format, then + # raise an offline mode error to indicate to the user that they + # don't have files cached and may need to go online. + # This is conveniently triggered by calling file_exists(). + file_exists(model, + HF_CONFIG_NAME, + revision=revision, + token=kwargs.get("token")) + + raise ValueError(f"No supported config format found in {model}") + + if config_format == ConfigFormat.HF: + config_dict, _ = PretrainedConfig.get_config_dict( + model, revision=revision, code_revision=code_revision, **kwargs) + + # Use custom model class if it's in our registry + model_type = config_dict.get("model_type") + if model_type in _CONFIG_REGISTRY: + config_class = _CONFIG_REGISTRY[model_type] + config = config_class.from_pretrained(model, + revision=revision, + code_revision=code_revision) + else: + try: + config = AutoConfig.from_pretrained( + model, + trust_remote_code=trust_remote_code, + revision=revision, + code_revision=code_revision, + **kwargs, + ) + except ValueError as e: + if (not trust_remote_code + and "requires you to execute the configuration file" + in str(e)): + err_msg = ( + "Failed to load the model config. If the model " + "is a custom model not yet available in the " + "HuggingFace transformers library, consider setting " + "`trust_remote_code=True` in LLM or using the " + "`--trust-remote-code` flag in the CLI.") + raise RuntimeError(err_msg) from e + else: + raise e + + elif config_format == ConfigFormat.MISTRAL: + config = load_params_config(model, revision) + else: + raise ValueError(f"Unsupported config format: {config_format}") + + # Special architecture mapping check for GGUF models + if is_gguf: + if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: + raise RuntimeError( + f"Can't get gguf config for {config.model_type}.") + model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type] + config.update({"architectures": [model_type]}) + + for key, value in [ + ("rope_scaling", rope_scaling), + ("rope_theta", rope_theta), + ]: + if value is not None: + logger.info( + "Updating %s from %r to %r", + key, + getattr(config, key, None), + value, + ) + config.update({key: value}) + + return config +def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None: + """Try to register HF model configuration class to serialize by value + With trust_remote_code, the config class is typically an instance of a + custom class imported from the HF modules cache. The class will not be + importable in spawned workers by default (and won't exist at all on + other nodes), which breaks serialization of the config. + In this function we tell the cloudpickle serialization library to pass + instances of these generated classes by value instead of by reference, + i.e. the class definition is serialized along with its data so that the + class module does not need to be importable on the receiving end. This + registration only works if the modules cache has already been + initialized. + See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs + """ + if not trust_remote_code: + return + + try: + import transformers_modules + except ImportError: + logger.debug("Could not import transformers_modules used for remote" + " code. If remote code is not needed remove" + " `--trust-remote-code`.") + return + + try: + import cloudpickle + cloudpickle.register_pickle_by_value(transformers_modules) + + # ray vendors its own version of cloudpickle + from vllm.executor.ray_utils import ray + if ray: + ray.cloudpickle.register_pickle_by_value(transformers_modules) + + # multiprocessing uses pickle to serialize arguments when using spawn + # Here we get pickle to use cloudpickle to serialize ModelConfig objects + # that contain instances of the custom config class to avoid + # serialization problems if the generated module (and model) has a `.` + # in its name + import multiprocessing + import pickle + + from vllm.config import ModelConfig + + def _reduce_modelconfig(mc: ModelConfig): + return (pickle.loads, (cloudpickle.dumps(mc), )) + + multiprocessing.reducer.register(ModelConfig, _reduce_modelconfig) + + except Exception as e: + logger.warning( + "Unable to register remote classes used by" + " trust_remote_code with by-value serialization. This may" + " lead to a later error. If remote code is not needed" + " remove `--trust-remote-code`", + exc_info=e) + +def load_params_config(model, revision) -> PretrainedConfig: + # This function loads a params.json config which + # should be used when loading models in mistral format + + config_file_name = "params.json" + + config_path = Path(model) / config_file_name + + if not config_path.is_file(): + config_path = Path( + hf_hub_download(model, config_file_name, revision=revision)) + + with open(config_path, "r") as file: + config_dict = json.load(file) + + config_mapping = { + "dim": "hidden_size", + "norm_eps": "rms_norm_eps", + "n_kv_heads": "num_key_value_heads", + "n_layers": "num_hidden_layers", + "n_heads": "num_attention_heads", + "hidden_dim": "intermediate_size", + } + + def recurse_elems(elem: Any): + if isinstance(elem, dict): + config_dict = {} + for key, value in elem.items(): + key = config_mapping.get(key, key) + config_dict[key] = recurse_elems(value) + return PretrainedConfig(**config_dict) + else: + return elem + + config_dict["model_type"] = config_dict.get("model_type", "transformer") + config_dict["hidden_act"] = config_dict.get("activation", "silu") + config_dict["tie_word_embeddings"] = config_dict.get( + "tie_embeddings", False) + config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000) + config_dict["max_position_embeddings"] = config_dict.get( + "max_position_embeddings", 128_000) + + if config_dict.get("moe") is not None: + config_dict["architectures"] = ["MixtralForCausalLM"] + else: + config_dict["architectures"] = ["MistralForCausalLM"] + + if config_dict.get("vision_encoder") is not None: + multimodal_config = config_dict.pop("vision_encoder") + + config_dict = { + "text_config": config_dict, + "vision_config": multimodal_config + } + config_dict["architectures"] = ["PixtralForConditionalGeneration"] + config_dict["model_type"] = "pixtral" + + config = recurse_elems(config_dict) + return config + + +def get_hf_image_processor_config( + model: Union[str, Path], + revision: Optional[str] = None, + **kwargs, +) -> Dict[str, Any]: + # ModelScope does not provide an interface for image_processor + if VLLM_USE_MODELSCOPE: + return dict() + # Separate model folder from file path for GGUF models + if check_gguf_file(model): + model = Path(model).parent + return get_image_processor_config(model, revision=revision, **kwargs) + + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + if hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + return config.text_config + else: + return config + + +def try_get_generation_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, +) -> Optional[GenerationConfig]: + try: + return GenerationConfig.from_pretrained( + model, + revision=revision, + ) + except OSError: # Not found + try: + config = get_config( + model, + trust_remote_code=trust_remote_code, + revision=revision, + ) + return GenerationConfig.from_model_config(config) + except OSError: # Not found + return None diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py new file mode 100644 index 00000000..8d6385d4 --- /dev/null +++ b/vllm/transformers_utils/configs/__init__.py @@ -0,0 +1,40 @@ +from vllm.transformers_utils.configs.chatglm import ChatGLMConfig +from vllm.transformers_utils.configs.dbrx import DbrxConfig +from vllm.transformers_utils.configs.eagle import EAGLEConfig +from vllm.transformers_utils.configs.exaone import ExaoneConfig +# RWConfig is for the original tiiuae/falcon-40b(-instruct) and +# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the +# `FalconConfig` class from the official HuggingFace transformers library. +from vllm.transformers_utils.configs.falcon import RWConfig +from vllm.transformers_utils.configs.internvl import InternVLChatConfig +from vllm.transformers_utils.configs.jais import JAISConfig +from vllm.transformers_utils.configs.medusa import MedusaConfig +from vllm.transformers_utils.configs.mllama import MllamaConfig +from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig +from vllm.transformers_utils.configs.mpt import MPTConfig +from vllm.transformers_utils.configs.nemotron import NemotronConfig +from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config +from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig, + Qwen2VLVisionConfig) +from vllm.transformers_utils.configs.solar import SolarConfig +from vllm.transformers_utils.configs.ultravox import UltravoxConfig + +__all__ = [ + "ChatGLMConfig", + "DbrxConfig", + "MPTConfig", + "RWConfig", + "InternVLChatConfig", + "JAISConfig", + "MedusaConfig", + "EAGLEConfig", + "ExaoneConfig", + "MllamaConfig", + "MLPSpeculatorConfig", + "NemotronConfig", + "NVLM_D_Config", + "SolarConfig", + "UltravoxConfig", + "Qwen2VLConfig", + "Qwen2VLVisionConfig", +] diff --git a/vllm/transformers_utils/configs/__pycache__/__init__.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52a9ae47b6fe78741ea1561776cc7183c637d02b GIT binary patch literal 1522 zcma)5%Tm-p6iw#Aypnl>h#(?9SuiQwxUnoLpeU>vL`SizH(h~*nOOHGv6C6#W8AoN z;V<|Nu4n7YUvOpjbqrLwXp_pxIbAoM`#2rhcGZyA*4J;NPjiOxLy7BGPKp=uNxn<) z49^4uO=g;k=OBl9mM5Nv0v1`3cmd|H#7e}AP{sw^r2`sdkO}q?sY_JCL z3N*3BTEwf+#t!QcufaSnum$24+`vV)NZf{-xWtx-*Wnf}vt?XiE2JB68&}yX@h05C zHMV9NL&Ix*HrZXTJuo&pKV?u&MsB*_>+eM2Sum6f)!h?uk*V6=_I__Klh)pa z8-;$R%WvLiynPU+z6g(dYR7D~`f~eVkSVM0d*j5trU6{M(w%}loeDI6ciK{C^w0k0$V{{K}$hfK|?`P!r{g`do%&SuKZb; zoJ9hCk??UEK+-*>eC1nqa0o~Vu=~1bPY#UIWDj(qzB>e;l4cLJu{3HbP9J>cf=r%T z#2)K*4#+4Ipl0gp```u%wOpSDlD%j&{}D{yqrtJRE@y<8Cz&&8(CnV}qc|n&=vxxb zJWb}&ryfoKxko0`yXm7k46@sq2lYNl)X5Qf)jjtv=oTH@RV;QWgr=9T9a{Cef2 z7&~zgZ>Fvo`swCV9FK4d&m9O(ocJ;wMWLgb>HK>roY|ppI1j=g<-8kTPRq|Q^2Xpl aH|6%px&+IxOu7rR_a7y(X<9kCmirC$s?V?h literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/arctic.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/arctic.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..492608d0015ed0ecb1428291a5f6902ebbe689c3 GIT binary patch literal 8223 zcmbtZO^_SMb)LcB0$A)1m&-p>k&UcdMHd#}5rg@w9-=h8p?WB)(T8^&*`F!_{F_!K|+S0Jv@F zXi~Z8mx6Mq98@|LRWA8+LA6s=?`6Lh)H`+4;Pqe5^Tw~t&VpNU=RC8s$cwiPx9Zkj z7;f!(;knsaQZ;pV9yRl-W*IFS?gGjSy!yP@Ip!`tGMY;eJ}Wf%>MIF5eeC7Pc31_qrMZb#s2S zyX%F!=&m?Cz2A@Hf&8f5?hn<6p6&3K6SW_O(O%EDAM>^xy{+jkYhEa0F?8Z~EI4lm zV9?eQw!V;2=#YGqPZ5Pr@spQ9xY048=E9Gl*iMm`c-bv{Y`Dcvh0o!=g!gi%%4>M9 z;C&A2sHV$W8+RjN>xmrDcxlB)g(VJ_sZdu1+aL7pTj`4I+rqVWY{{)X?(Oztxz((v zRm%$PfLm5tw=Af3=+k?{vc4SJzWyTSX|SRf#_5t}*QnUuYC{<`@yAPukCv~?ZJNBkHYp7-w&WWJCwah z1YAgK7<<0VlqlO;&8@+{sA1p}_{l0r-DngV=CY}3U#{hAXnI5~llGOCJ2hpADl}e$ z#_Q>6t?>s#JB+=jN&}hZNs_ekrM2V5vNIQQTW}etRS^wC7k+g>)=n1!>L;bGnjQR% z>gu~_Gq9xhl*1Nc33Y-jCyo($gUAYz<3!Gaq(!)yI7h|r5E*M(zn!AG(;#o+Cux;y z1t}F#{kK$9MkPJ+??6T>A~$pkoswJJH9O^6saCF4YIFFj)@pdK*XC;tyf4%iYfH6d zydA^)O6|BRFW`Ntwn{DBb3ZqFrhCFY`JzV7?4IJ}#`Rv|jNzVkjb2H8KjXgfq5&+J zcTeZvdj<9Fto!DRMyK&}qhxdzV1~EShBkv%z5OBdQ)hU#`<`U@7x(!zHZX+)ERL9r zBf$@85cQbNe$v(BZW6k5KX0)+F%37A+@-#>zR7-e%$Xf3j=FsYN4Dccf{aa#Nd`IK zj@R?h8%{qAco@rz%;h~VRE^L=Z;vo}P)~y644x#Qm z&+~n@?{OcjnDheAx5acvEt4HH%#>t%d;8vvdl!VX1J@3{=10tSUAEUp11(eE)_TlX zWP`O@^|}&4t3HEX7VUfi>7o0u&xIGWo`?cya~5a2pF?jhbL^1q@UdE*D2zqqE2%W^ zXc!NMv23x2+=k}+T(NW`N9jv9Ocwi2&vKJDLsOBTFH^i>|NR3AR@+5NX9kue< zg)X42dy%~x4Jeq{{=3RjTDvgkmCNhv>&?6?b^kGSA41dnQ2A3n((I&9wAc+YH`lrh zhrtdP)Xw#KJucu7Fw0{e(r0KsQ;gsPfCoOpad!5{^Cy<{KvL3<<+-xU28eHHw%6w& zy8Ax4%#0h4r`z{jmxo82?|bV%`2NgvZ+P%v3FV9?s>fqvadkjwjrl>N960XMf;k%3 zOYdE|@>;f=ckjQPCsb@@^Yra~#E>Icc4nSiV=HA7%m(s8<)YhpfMPs!02LyiGO@Sg z7`B6+tUlcL|0Te9D5i%Z*XFj9HZd1sE+ddd_ES`;#!hn6fvG*ufCw-1b*D_x{M8{aTglQT* z6cz%r*(RXTW1kU*j9IV?o1bkoFEZ~4j4r)=&>6uo>Vu6Yiy(y9^T7J{gN;$-XtBS* z;+&cm5bp^Wv4;16q+BEWGN;d-$Abv#hS1tpqvUlLwdqy5z@QX`rOKE}?JQR~#@CrB zcc=IQOag9)17v54H(~~;f4_C<%H{XoZ=u*EgEPbn1X(1MdAz)+1KKF| zKpEW+yZI{17P#%iSlyUu7gJ!}-Y~@8APQfB;@$Jo^M~iV-|}h>{^1(5g1n7kT<=L1 zWwJ5Zh0*B53M`?&=4jQqdGRUh5?iO z%OPNgGoOdMDh+^{$OxJD2b9$_4*wc2Ju+x3+c|wtGUdgWXg&NCD9(* zj<_La4F}|vnTT23%58$22APqMI4CS2uNrbXS-bQ~4(E3`;Eq!8 zwWzl?^6>N8qjQe@58-&28|QO6&qMc>ykZagaqJzeDX@5*XWX)V`BK*y`vg|JN*GnR z(3v$OUTlI_9y%b2o9jSC18!lY*gnb_`38L6yQOvd(yX zHW1N{z2o^n&`F4*M2CL`ts&V5CI3HUHBGzzVUCz5YT0On<08W1$Ea&+pXKb z(M95pJqT?;vTa~Djs|OkF1q1NCfuC@hk-3{dW0_HT9Vwy?#Lg>|5~|c6V0u05V5o}*k!gZF6Tjspv9H3t!P<#D97Npeg#x;iE8 zh2|^d)lAj)pIkiK3Lh^1SYx`=h>KFtg((T9IBqzI;Lu_ zzxl8KxVC*@;`2cP{@(m+lJ{&_OwQ5&@te<$FV4Azq>(Hni^-B}{%t9Fi(27Y zz`)^MajRKM8(zJV8xyQV9O5gl@=Fgn1(O;3!vkd{L$@CO!9D6M-~ z)|C7^JT1{3jylFp8@PoSpSGuEiIW3CR~DpS%W8fH>yrCDLANO4JtFTD`96^ki2Q)a zM?|g=`BNev6S+#HMWjt+oya93bX6d*?hWw?kxz+i5cwgI>qPDlxk2P6ky}J=6ZwqD zU68c$cn`*c$+S%wsms_ z_0>Yb)PJSl7s~bjd3r*F?sc7{kJOaTYuvNc-B?9$5FEm+(L=WYbnB;ph$avlSjuK% z_Dn4F0xjWZ#TZ}|7XbjrlcH;$F+wvbJ}aTN^hGHyqyC+^qJRNq1q~%YR#7c`1$}zy z&WTNopVlZ&D1ij&AT7s391tnxi9bh$GJmt279C%TM^tg2nn8GpGn2HUXCxlbmt3zu zLh+yQQ&%6Wg_C9jdMcSuPfqD&%-SJD9zCRwV)m04@j^QIrTHu4nTa`?G-EaU)wrZf z&!HdJ{1~8(ITp5*?9H;C-dr5zlT&jMG=8%n?%}_*NSBZ^vUl*|pYhZ8TIM2dwIKQG zl;qi{&2!>BT8g)cyhG%VMr2F;@%YR4=4B|cdlFaAKu)4xN&Fe<_+umy{%h! zZd>^9*_|KXv^F>HtE*4#VkEv^4N70*b&*y-hMjP;dyO1k(!FA-yj)l|>GHH*C>2Zp OTPQD^h03zII`@B#i+1S% literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/chatglm.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/chatglm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8baa1b961ad824451626c64513be3a9b99bb34b GIT binary patch literal 1782 zcmb7^&2J+$6u>=`WRl58lQeDmwOtCkd_-a!EeIhYR3WXDy+DOTPjkSU#7;6>&qwW< zE~$J#+Le|Qe*s4>d*Fz~i9f{GMXGw_hBz#|H(sRN}+Y*zk;3@TT!iwrE&`C9T+ z`a#6mqd1xdvk5NTY^WHU`WC>6XE6dwEU$!WS@|d1E4vlfajS04b=|t#aGUV8+%>nI z|LtYpff>rW?z-D%n=i=JVm-F;-1RCiNSSyJnAJC}M{_@Y{N-1LF!2|N6WpQ8__xU89hC}x&!uKk=KY0;QLPy zC_Bg(YJ^tBByos6z5kVwSwWVcw&xd+71KrV>-4WKPeDA6|1%4zEo zj7z!%fL?0GqlAZ0QzoO|Jplx()2;F0iZDf$O>l z?CA~QhTa5j>OQcqw}4xE8@R1^fIGSa+)ekMktcVU6%f7qv-Km~4Mrg7_VG(H*q7*x zS3i#@{+O!Zf}3vQLnIl!A!;mxo3FR0QwVMm?Of0}J*SYbDB?ny9_GSNQy!&3TqHr6 zy5eV4@MxCKy{*vyjwZ1R@^v1LIb%UIQ)X?clgT)Vv($9oXjzH!n-Ik^q&!i9h$B;# zp+bTAz)upfppvVAWxk-37z~=^hrnrRaWV3p?Kg8;2*6=F4tzxgJP2Q!CJN5LEfktw zzO=M~PBR6L-M)+~OJxP3Y4!_Kii>*67sc6BO?=QlGdrv8RK?R2;}Jx3ntc3`*+%}C z52kaL`4G|zKlf-Hq^6NbPM3j&q)9#vAh0riQh06JL6mYSxPK1G1ZPY|WeJz2M&Y<1 zrF5`o%8H9=eg$L~ESKwu9%2KriRdG?5Zj0y1ilD~PeJ1AlKY6ei1!h1Ar2524*53X z9mGAveZ;$n_W;H@ed^1ZdTAM#(uweuqEx~|C4V0e(=Zt(LGmDl9Xx&TX_(E!hvCzq z2*$%?kY~uv_ zyx@!mRm@W4hn!MVr!hb3xNZ!o3z;DjAHkl+(~pa$ikr8PEwA zVj0vtvhqVT;t){+puy@0P$Ol_wr$IKZ9BI0|GxfZuIqSE@|ri(A?@p9ilZ^J#R$(g Swa*LRj?i(1wWwLzDE$iu>DS@_ literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/dbrx.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/dbrx.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f014576ae9581b7e046af7a57164411b21f1dc20 GIT binary patch literal 9406 zcmd^F&6C{5bq6q*4`61$m!ud`G9uZf9fjH@B~o$}i_(hZ%8a9xLNRhO3WLQ?_v}E6 z0Y(kXlFNXpl*uall=8use7K5qrBcO1@+ait{{rPpl7ouJ*op1XvY6lN#tdeLOL7nP zCBYJJfJT43e(&|`-+RqyX2#ZVo&M*4?f%otn)V;`GJctOc?WkKf-tS6F`XHK-ZNT8 zuhc5(^j-?gUb$74&t_2RRa;eEeDz`OO;pJPURs-)ONu zzR%#@Vm97w`990)A8F0Tjin3Q;=0q0!j8W)5MJU(;p%-bqBV`Iv>xtfwGaAK=mkwZ zoBvSoM0kG4*?HLlWz`_s+3~}jHZ>Z*>iJ9j7kJ=WOUK*{rngF#VU;Y?DzioKYO*RX zbMrXySz#95ZC>rz-_<%gtFy-KnHJ{AX1Gn|Y?jS+Dr}xD+%|9Nt@?L0Q)@LakEKk` zV?9ZDNZcy!ymw2QrXKUDIqkrUW2X}dN6vfHVAZxAT%T-F@u+m`s%o(6TuK~2b_OwL zP7*nO7$+VE@)GWN!!ez}+vg&d{dnzmBp9SZwUaJ);y!Qt9Usj-u`}p_%lM?jc*hTA zCDc%@@wwgg6W&e+g6B+4&LGt!hmZxCQ!5ZI~JQ_Z3&HNOG@QDboH^hLDoZTo?r z>`O(Gc&C-HIvThg_P*V+vjJq!p~79h4@#2W z^MU~AGTS5IR6We#fL2nx+QOCioW#7ek5FvLVu*7p#q!us^$#&a|In>NCPS{pr zrK+l-(6N9w?-Tu8uIcan_?^ryG@8|hLeA`BLYZ9*mQ{AI!S3UBS+-ZCiXZZQM->jsSw7jQ@7t-<_Jl{o& z&yFS4w0sL5bhji z(Y4S~+`e^BYg$=3h6Y4db6u>}#C3%QRt33qL7a;QNLK1c{j3!5Fta}xMq7n-6cCnDBba->eQNoITecd3{$2F7$fbjfC6xl5SSP*MSHH>9TEr3@<7mg?|h8 z&r*<7yQ8HBsVRk`Q@x`@)ds2h&yAa9z z({CB#EWYT+sSguMKAS-K%q+_8V#~z0YFaA#mx>p2;2|?fG+AAmuaee-@Bdjefa4`M zN#59lOOvj#1uvH)HOq!n}&yu9(WV=H7Uq@m50*Gyzh5@@Vbo=3~ zS<`E!nr=LtHK^1snO`hcZoGJqMNReBG?ix&HC(tr zFUY@CT^Ab^ip$c63GDJ%f^eIH%um{UR+nq4$FW*iR*E4UU@tA3$roFj2SL8z)Qpc) zOPaQ*fGpyUUj?aYrfw`5I{lfyG7RI(O3C;&X8MIuHhyJPW*->K@+*X4JT#2uFJ=2j zz!0rNy5heAIS32^6jiLURb|fr2${SLKx6?B*|_Vt8z`%yY=%Hk{yfKPY@S=4MG2M` z*doA^&FgH5H>jK~vm*dYN7*q6mS!Yan#D>up3RVUHa0fZO7|{dm8!?bz$8?i07!U0 z3^eL+&aePrhin9I#(_lw7aVgy3qh0rt9qiE)i~VUZUmnl5@>Zx)QTUYyz}6M5Oc? zo@xZ8s83QCSbb!CFNokdWmjIf?|6*)Dr%sSt^o@dxVJzfH1Dp@7+hTJ-{8&)mhi;f ziFsAQkrm)JiGgvu3VDL=cZ*JY5qANBTz(zfHt^(uofa5rSS`nGJ^wl(Z*tJXFA|tf z_Mb$3clWS81DXM%4T0g|2KdxRR3Tj62mjK?Jt7uW!$A!nq95u1aNP${9J|-Jztc?~ z*FRNp@+!26k=zg0c}Rgk0Da~v$On4tOosJyi-vV!xZ#jW2{|Kip@l){qsmarE`mCL zC#0~-S=o-FU_xBq2Hr)iL>QSAkvGlt>l96k-pSJCejckuLvVW(MiFbkjwvxaAsatN@Y(UlN=t%(HX+*KMVOX&m#F+U9tf4+(eG-6#+fz%ZlL^xuipfK zI~bj(Wt0;(*W7Aa1+AquP>Yo$YPVB6wE*Z|O|3h4mZ<%uq@G$p?F8uFNgAx2S`(8 z&AK>7&xdVI=v@$`77{lQZdc%e_W4h7$6p1p8^A_6;vORJiav$Ck525xBh;-h+H=$~ zspGN0T0D=hh-xJE6JNoLGBbti#P3oEgxSOj5y@!5EkUC1;eI-*Gt6HvmTx>a$?G(p zt8I$2G|KBl2-IzgH-?WJ%{lRV^m3BOD@0a_NbF|P4kb>}`x;1Q(i$$noH$Ki2#!e* zCtjncVk!l6JA)4&;g0_hL_)V`6m%O>^A{CBIAaXLRVu~Tv~D%cTmpA^uknfT+o=kW#6)Gd` z1W@yYT-WQT&zzY`I$iLwk;O9SN;$3^^l)T#fU3eB6*CTDQGy)g*uFT*COu`?Ghd$? zbaRw7DzZ!}F-7|w_uAd*B-;?tTq&{;4z@)Yk0^H1(9QFjPm{x$>6Bl=P(IbXqa3I` zanzo79pUq5E)p_NRlsYnzVZ6BFbH>|55G6)Ip~LyVo1>Pa0f^^SBJFTVxD_Z+uP1j z>{CR4`qZh{&b~S&{*S5sAn?Tg1k$A?tTF|w&IQUABf}_>Sk7F5W0~LSa6te_<_xHs zmkiUtf+LEL@evt`?fpjpJs74Ou54M z>-GBqrOE-=XtISts$@0L7i^Lvr_$S2`Ewa}NN$iMD3pRF^IPDK)&sFMT97KXEruOS zEy4O|YnxdAQ<_)t9j$iget5v3c5ezqq_)W+zl3!^I%7sPyPYQ{r64mnb?Y&1hBN82i&D9dqLlTZY?mS@*T z5JCRb$KEQCV&T3<#Y!iA68A%%;1uWGZwPe*C zHD$FSZpi9mdwJ7rEn&wi&cW2tS*t#lL~1Qijl4iXhSo9a10lLhNx37x9p6Wcpxvhs z@;=3K_qChD6E3`MUdbc(AH2Qs`QQC5UH9J3>|&$O>SOy~W{ox*;h?ox2g$6Fugn%k zegJD*dZTRlpaq{TAG82eoK9iy@FK29ar#g2lGbE&Zc)6BXj=~#5+ki8B^;O>OEh_^ zR->a9`K=@)ay_+Vn->#PwkgXtmTYr6F=ZPoZ&MCSsfD&>8TD7>nH-K-P*Rg;a+Zwx zZ5j2~Wz^q5oUgevX&rPnZGg_DGobV7?41S5A>1`?&ZTqsKL?AfMCWtK63+jY5jW}S zoUf;|>I^VHwabHwQw}=ecxapW#W{xkrTU*C z=jD0gC@Xy2^HY`r%YnJDQP4 ziZml-$2TMrn=JSRKBTs+*t0?~=2o}KSd>)nTer_W_uQ`f{Rlz3|Lbq_p^wn-ZnN7M zY`%c5k6;i;;2gznCOIxhObQxP?DjNgg%^8;ANv?RM1l$L3JFj8moyFpevC%`AK(E^ zM#P5SE=^XINp8LGOIW~B^q@A?msL56NWXQt&#EtD)$WR48A1lZ9^7RunhLU;Gc1c*Z6|1=}YzseT9B}?+0?>UHEHH z;8UvZ!A#t2yfs^U@cN$d-D(YFWAyY8)`$mdv!?L?3&HmWkEIRNF;?4zL z(-l_V2|9z7z(>eLmuS&#&`^g4!~4b;Rx7u9*?Wby$lhv^0TSep=o*l~YKvql&|?q> zu;@+VexlNOW@Kt=CAEzj0=YR`$~bBm{>Cx-8jbok%5+w0la#5nOk}B1PKI%3S}ET0 z?t!C1+hCsP#294*Esa~K5DZnvSw}Ysv7WA%QrVF6tjrAOHZT=;jz@uITISQ6ch*0B zmZ+JIdo94U6(v5kc=a0F4(Dl}XwA8D8U5>IY>MT0nJw=acqPr9Pm6k9+%KMu^X$oZ zd1mHSIbP*?F$PJO8X_p9(!4fVuE(u!J#I90cX_6U;L_D&Q*Avj#Lop9N``&&)Gi=pC9aq_U@aoKU}^kt(Hh$CWYic>oPHO z>%(oW%9H?v|C(do^{=7pPhbEAfJ&zrv`U;hKmqgul(Yc#+?+zrG1Pjz51jBe&W3Q? znX0O2xo4K9xmLHpsd@*7xVvN6(Ay3lz@6Q1-re9I!J+QKzlI=D!C?q6USgJvT zq|!Eg1U#N(r&8T^=KZZ&Qkmyl#k3*4?>xG=l>3*te}w)Bh7d7KD8zFOiQM>P@0#2u z7*c*s*ume8?I=_qgQfMMaV(h3_l#Sohl~z6cc#-$`u)2ANKkTOPf7C^7Fhi6 z=h~Z^sv7ug{QfVU-z*u%KWQ=eDPi#je!&j`zHit@!>|i>(J6F`jbeeWN1TyvsZmnb zC1iB|EP!6)s}*Nhv6UAD(QGVC#4`f#K%WskpO z)G802x_004T+X_l%^lYC+?HrZ(h7y=9)r43E5xO%q8Z}v>}|#!pSP*)W zpR&PgF^g@fZCeW0-p`;LH)wgX%Vkj41aFa6_v-9sNMxdb+k{iNL1?+5utE+bH*W8; z??HtJo;HHQW;L6hv~idA6MKQro1!Ivz>@8#%iSP~_B)9`oED&AcSW+R%Ifv%U3K{VhB8v59x)#4Hk0u$ZiE^URG5(P@`OCI1M9BKbZ=_zHI)MpAi&`Q_181tbk z>^juV9P^G+N{gN zj%Smi2ctM;WLP$p+>&U{D0PHwbN4YUzqx+l)gi-Q6^P|PxrqpmQoD1viOeCHV;v!x zFbfetF711sliuws4sKv38z0I4HZKhEu#@@`(-Ng*g{!FU_VXG`7|Rp4!Vn$>8O#oX zFJQ|_0KA+Az{~jnIKTN&1~ODkqoN?>RR5WxEOc8z&pNFEBXapUdnwC>f}dq^QU97y5^T!y+Tb&2CuGCFH^c=Mj`A&F9v zL4gF8{=JnP4&@td(kYom1&Xq;G7ak+1+;=tM$IJeb-CqIFoAXs9TJ?9+>-)bjpZa3 zrXr~v4vAC|tF!BAb+CD82`A7Tq6+2k{Q&O;NI*W9KU(gDp&x9mty%KEIH-HFz2@7k zwaxSE7wa46H!obQV_73dV5pLK&GqugvKC-P*XX3Ckdl-Uhdrb3{L%|iSxmnfA~su& zXHj{2Sahz}H!lvw0oZTCd9Ai8Oi7Zk)6Av%BxnvL3=Ivq(c~CMb+W042{N6)rX2l| zO|8#kn? z_f6jmAS_C4pZ8RiU)^<{(tD790jKCjT^8^UA~*?W4tLvXBtWiH*-Mv}u4s2c#*whI z8b?{N&m1|&?vXF3NkkLNledu$0*`tvO7rM%kwWf90rDHQlR>_KFY|gE;d*gnlfr3Z zee(kSRgCxPghxI%AGhTm&K@MW!f~?Y*6yrYXH(IlmN@Mp2ixGxIaFBaN$`={MvQEj z{T#IU(nMDTgVhC7BwIzV~K9_G~eQZ7E4ER-xsjo zY@$2NG`NqNo=?2F$xGXIS4Mm|)uK5;2x69!RCWitpQMGr$wwr0JB;Zuq-JAHH5&ce zMsARWFq&Y(lL|!YRB;s4`rOkCZ@CB-j9SD(JA@}pAJy>1!0`?#aahfJYWAX`vM&w>M{19^w)AjKYN5~N%a`@kK6tLe&`G1L;J5*Y zR%#wwNsb<_D7BqL@glpK#+!0%rVpy}#~_)8Qo`B@=@G`I9F67LBT&&aGju} zHxh+ZqUGERrxM$4A|L~02g>0m&+bl;$>C&|^gDHZS8Z;TxjG4K%)mNffAk3AIT|~X zSyxA3T%}w3`q=nY!8kU?4da;M5N{_gAtOYe;}W00zn)we#^=BI;!mIa>u-O&{P|B7 z|8V0U4}Sgnt}I~7aS@=26S?fOh^sPw6Zi!__JqcV#@&K_rZ?N0>&@GR-;VSaepmQq zp6B$?93P znn~V!POWC`Ikoe7wVF>TFtu9fRqVxkMfLoJo^kh_{IgA3d&V~2!C^BFOwaDrM&ikR zFEDi-6m~qx2UiQB;N~GtC--@rzj`mP*@;4S^o8S!f{b4%m8fxk1O=BO$8~Q zq)YTLs*J?bv`VKGFNi0SYbwC;G`$R8H(Tg{J$X!C6VK8c)_&kQ5k?7gMZ58gLU|wQ zLNgFACRmV_P$5m-*<`r)(SEdn$}oQZ*EcjjAKW`YcOUThEDp(tV{@OQOhO<807g?e zh25VSwWW9z{VkXAxT#92X~xxlOA1o&x{EPJ8WmD%X0)b6J_ncFe0_&ee1xHq@wS1FDJ}S5tE}LpcwIC53Q2 z;(NY4OUow+JW1djfu{&OP2d><&k=Z@03+}M0sL2pAzuWjEh33XcfeQrc1l93kZj=~PHnNGpu{Q(?Ci)%x+b@l(|IsM%UJ-OrsW)6)+*%kuGQt_ z@}`N(6Cp~>c+xaKh%6_0GG>}MeK*ZURh1^yM>n2^GgCq_MH^imf;U9$sFUg@Z%n}@ zrP!xf?pnUmT0E6lrpX;Av5qp4e3e++B0y#0S^NSr`M6Okl-?^thsBcC^H;&MU#Pct zYSZ#EUXIJiHApg&%29(2+BT_@Ni(1=Ek|hEd~Ta^0ZCb~QHA4B5^8fPzpJn(i&0vP xVC;@(Ow)4FBdX3efP5s2mXG{UUy+dU_-k6tmnrcDWTx`$WD#HZ%3^VGEz!C|rq03SW)O#2 zx(Rb9+AUR3L|J+S-;*gdYab|Cs31zExKZzixN&;4R)Q{IA&F1HEfW;Xvt8=j=u)(V z^VsrRUfXj$&+B+yujlo>fj0zc4W0pLM$IG6dtGti5!s<)T}&SKeMh#%hB)((JfOY{ z>$Boqz4jgvoA@0F;i7RryxW}4cXsMv{9zvGg73*C-w)zaYST`694w_WPRbWD5XNhM z`QHB2@<(Zz@I@qqT)n3KbyNjKAydpp{f+i}sNCCcq$Bd9MUJjR*hdV(1(-kmZSmyO z?|%RJ+S89G-`@J^;PdyljwozAg;|fA#(|WUDh=0LV3$=EpjFv`cI5!Jsy3igxq$7e3+ProK(F!uJ5>j;TlE2Z z)u6WajzDVz(DbVz;GkLq99AR1wQ39qt9wjXdraLH=xJ270LR7p19ER79I)IGt;eid z|C)YADUc2-(w<|!WwI@W>O(LQ*(79Z%1m!N)4a&`AkC35J$XtxB-0Jk+0VjYo9pO6 z8Y|Co(_Iw>H453SFiU1!*OI51O1R8*6lbX!MQKz-L5y#56{Ne;c%_y+48nypgOwqn zvOFsbGkR9ql}e4!{20x5lQJ$M{!uB_QaPw;f!*!M*xON{O+OD*5XUk`ex&AHD(`b4 z!%TqVK!uB_kReo9nzKlk7-CKN^PjYInFAMH4qH*dIfQO&8SkhGp5Y9_NwDe_6we}@ zLpYCca#nQ##fu0pAmC}$iwG|vyo|63V4S`Cf!ft-4TM8#JkG;7(3*3FJpAMKtVr@% z9_5z`cuf|U-c8CyawWMxgTQC`vRGv4Y(I{Z8C(ac-hs?ZrFmIIv7R+ohn_XfcR63G zSHULw)>i?XGd(t@&hrmL`I)xE9ME(aqxRp7b^baS9ryI*<0OUHN5=!#*mXBx>K*{l z6eqm^1U#hkDQnV*bUCC7@54IK2HpVCxOfk65hZs((rB#5PPska{f}F8UAx7GN4Gt2 zsiq)IX_NXlVf}A!j>;gj%e$~u`*PXA@Fc`$Z#q_O{1WXd4*_rcxx!09VT@aKH#J)W zR3dE6Z^D3hhak+SU1U`nB$9K};XKJi8RNRgfzLtQ?D!qf1AmKUF8zyeO=4H_Xcrnv z0;z$OJ5tFslzty5uA=R-kURObn3pw{UPZRUndTq1bL~}O zYNc+755dxbufPZKgv5Y4v$4Q)a+S&-TmHrG`Ne*I?>P?wA3@vx{AKo_gU~l4+-wdA zJ22!1fFg<$l;RL$^b}Ex+UJPc44>JdMV(h@+`0lQG{$=PQZOYVo-w+cX9xT+HVZeK z+1NC>17Ik`6u~Aqw5b(3o(~uz8*cx z6cgE-=ex5=HD`tauXXc;#GrgV6&TLC`UUMfOrU*zvZt;2_EoKKgCg$N`UV5|7&p5x z$B}uxJBuiMVND?hz=ixyvVn%ox z9Wo;MDbqumA7?U3i-ZvoDV}RrF_0p~^oTC9DCO}@`@rn3W@WmcMLf}dA<3KxC{Av7u6gp`UWa?&^@C)bs?RV;+5ALE;R02mEi%fV(0ts!n(ZTxrl@#)=vi+Srh zx#M8k6mnh7!W8IKv<&z5gsEWx?-&UeFz%R|fLt&_v`9{vizMiTP3YUSEeLS+$5gbi7Us66G!9JnlwX?2Y`qr2^j?F z1z=eUe5o^&_R>=?ZPS@#rk5PrTMxbT)MNhw+&byh^{u^iI%(bC+vNv9$(E;$C_Vy< z-M8<(eedn>E&QpeiiW?H-~6F%ENI%7bd&xo;N}f{gYzJ^R@ZFZ&bfLgSI_BmopgQgM7VY7jJ0Q@txUsE?vFB8ot+bT9L3q$M+tA zr&iUInd^dw!g4%rugd3W8+ZLy%kf$b>XZH(>)iw)F1S|LAyv-S>v=m@FWC8(ULT|R z?V>$ymp(1ji{)~;Ql2PJ;xkpAE+3GenexH%EIx%e1jg+%qE}Gug>-?7(DQI1##SwYi7ku1m|9^_Rg| z&*=JrLwW}LD!#IO_T>@1*U34odrl|nuz=r*xYyv!yXH>BM-rXq6E?4cQ~%0229-IY%>br@1$IiHNyh0wQ7s(+{#&o zr6+;u_w4e^uU2a*oq1=&-dEjMzWeHkwwE0QM<9K|3AbtaXpzB;+Z0oQbnCWb+ps$V zFyujcQrFss>ic=u%Gr@g*9N}Q<1G15kjdHCh2;h0+~5uCBg#d3(l&+7|1qZ(hVVJ4 z%!pIKD5AL4XtUu1rI!+CU}IX*eMECS?-}f|=6ig^B6}}`V%Ox{J_8}QEMdc0U~%LQ zlbD$BFxMrw$_R-x1qXYXg;?8wc?P-0po7&2ox3t?7|qDT4eYRB2BCOLdNAi(+>PeV z5uq;)Sg}5Qr8KxGMYE%1@T;D^MRv&hmL^M;n+sk7Je5BG+p0%T83M$t&B)daZZDgj#*du8`g&7QV5VcAX=dkg)Gvy zr!*P(=)jcZ5ZP#1?ECAYjGM0SyJ_F0h4`8zmKZ|Hnw4N^A9yn|;n!lG>|}#58O|kb zV|WNzgLko>(x>)Yk`;dD+#YMvX8=dS-al|4nQ3SS%XN0tM%jyn!FphG2R^`KE)1j) zB-Ll}i{;VmM2XTCS0rmDXb*4SC1l8RdC$sx2;e z3~bxNX{2*$fE|Jxa?Mm9+PgO3KTiwGbrJE=pu9nhQmL%Z4I8ZO+;t>S!gX**gbOPB zN{CRc4ObIpHyTKYWWWDKgdeuCvmp;6kUXDuygu3r+J5BP40tyZD%ArUhU_-qUb<_! z5f3uP8~7PaL3|_{Px%~#I4KQWDjcu$6wq$6b{L-=S?Wsv-Hh{wc95Sz1Zm`R; zeYN127B)w63bm_0(c+m2%oVnz(l-#zJ@y^8l1XO3uy9*=oe_0`0Yd2udx}js_Sr?1 zh9MKfitkpD=^QKzRA~4epeyN@{rxZ_?t<-yXOI`|s1a5(PYCEL8B=bS>b(^B%@7ER zW|28yp{j73ta318h8tld)^YA(uK*fsID|!puGmL%w1+7~VuagWa?Jg9ESFmXCDpD? z>v|6xTJ8|#^Z_Y{Ebv=%9!X@wnK#LU&G`YRp;(=$Yiv1o*fx}T_s=scwdIjq*NLtJ z)d~wR5Ocy-ij|y=mIX-{aSU4zl8D^+S1q>zRE`9cF{4HK!Zu`FEH1;-de>%kk&G|0 zn6lr~?3ZkE+^GO~0!qnQ`&|~72c6&z_qH>$W`$j1tC3-uoLi)qCOyN^n zB~@%0mfLMxu+YAH(z7m;;>%E{J7G3$EttJI)w7=V7uYUoPRpj08=0wmM!i)|yK~z> za2d#IgnIYO$9rb38&AwonLQ!39l+^=kZ`u&%q;lsZG)`=o=W#caib*8>|$_iQ|?pD z%Gzm*yYA3sWW-60$@dq`%T6N;=SHH3o>KxwbxIXBj!$SDZ8anVU88w;*dmeArEAXj zos2p|C#|v7NKnxV%;$Slt3}yzR;BkY%4;cr;zHwIWLNt@mz3@uhGpD+Fh{19T}ZKBs6K_G%esXnfAMqr`~B+=|Iz;P z$G`jMUoJlUXzsUf{dwnSudhG4j-gj@2CNXwLv`f)_tzUIdWXOCa}wVoz7zNc=&!|k ztQlII1J&a^Xf7^*=HoHYLRa786JEJGLfJ|b5SBP4nj*V4_1;xdz^(N6RC5Bq%y47lZxEdWI>(G?k;$f zgDIS*%Fkq$@?Ey7OUi@lDtRIGc(0-*IV$*`Wr4t{yCx2kGDn|)kdp%=7&V!rhT(>% z&_yyc3QLm1dqR(7Dzif;#{sE;k^4I~#hPQwLOB`3*2zUuGGu9N7+8`5&V!ob7)iay ziiqR%fV^9fTZ`w2JWu2sL|!1mhywZuvopX z$=X2isJ97Zx$^`>|^I5WnTyy7&mrccDzuf5-^H8yFa{ z6|9ts)sl?6q=I5zzO`%cq(GKT@_?yH1xpq2H-+#;l1IEHW3nns@?cagh%*qQKATxU zS@YNDvTZ#B9#*YA*xS!XGO`s}0Y7mXRza34{HO4Ex-hLz>oa-* zZKcO01Y%B6)#q{HaVfvM4I}lPm(t;xm2j97tH(u|#Gu#Bg0^fyctDahF-4ctS#p4G zXF%51#X-6za^;GcrE6N~A-S@)R;^IBC9rK6NyVU3fC&4JkxUxKoyc-~PsR;Yk`2SC zS0sE=Ry|2aL)cvnr-jp1+N_Ya<@>t^uTNpm6V65ya-(B)Wpb0IOs0A{w1Fb7mq(Sg zmq<2ga!N^**n_u0N?JiLG>dw!m)nY8<;p6{Jv*KL>&&Vku4~n@cmuz6LjD8v+(U!C z7q==s3ySahY^Iyc+X9$Ot5@KC2EA(MLR>&YQtaK(EGZYFZ9*!Ql)k5Idy#@Apye#g MOy(x_N^UOyU-fLhz5oCK literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/medusa.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/medusa.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad40fe5b9c0987369b66ae86b9ed2272cc6bda67 GIT binary patch literal 2089 zcmaJ?TW=dh6rP!#UGJ{#*h$kiX&Z@)xEQH1uOCIHv z&+@oudyd!gTyaf!k_#WG4b%bZ0<8e`q%HX-nXEh{6Ds;*^`sj(vL)7}D?RD&wkK?p zhzo*DxL6k#PxvtnIW9`8Vez_P{)VT`1|=dy5I0#Hc!K;jD$9; zHQU@vq64Y4@zUt~u!zd3HoVM>nOQH@JR3))6yXacV;6p7u{>Zft8E}X1*m96LQ*k6 zT3LXs;(%6V1MJp?QPwF+RY zI4grT`mN<&n8-}6q1e{*LMr2iVVuQf7!EwcwM-`eudR9O%pOE)N1syTh2c1fv<^dc z0kl53Gc40$Sj5Hk5^l)y`bX(}n%+ns43qf&usA5Ec{bcnl5_|aoasrfQb**Uxyr^2lrI(*8)o}UAYA}dWEV0%p^zJfx%rkISx2md%>T?DL8*U6zS}zD zf>wN=sBOqCe!mH)R>dE|En(dxd#i#s=Sy-M<)EEc)(h>}`y*pP;RbeH#C1vorzzh$ zwF5ioL1e=wTA_%?rQu1e0cY|+8>`3*<4sha)@K9TSmQ*iYv7C800`R4FX^mKTmx6( z*!1eBFCzB_2e4>Y3S~d6(Zg#m#s8dwG4}3aA&+z$NMoSudB;Wy zg|!u@R9#;1OJ-fbW{%2IW;hR<4*vj^VD>mWeC;I@UUB)q&2T^iruiN@r6c(Hx{Ka^ zg$?VdHp%l@gt-aJd?qvf;lc=q9lriABhKGe@4@w#Dh};Osw0BM-pm)A0P0=XssX@& zH9^7G=Mq+H`~>86Bz!h#DRfA|zY|e7$P`X6g_;%4u+d-*yT0y((D)&C#ypW>XgXoI zH;D_46{7S$=rj)a7iT+-bLdLYS)NP^ fOSFEl6m^_*wSC6D*#z%XtX_@5c1a(8blv(J0lV$M literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/mllama.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/mllama.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2470257abc34b69291394e9be81d611408c374d GIT binary patch literal 1245 zcmZuwJ#P~+7`E>tP0|EPXe&`M+?pXh6(J!Zija_CY8T3Ki*-Fajf?Lt*iK8UFcqmA zzk;O$KY)cFaw`*mfuZ7keHR)imi)$keec)vJSqGAE`hc4_50-7Bjg8OmRA6m7qHD^ zFr09zNKO-vlGlWL-2X(lFT7*_!cTng1Uv*!2%Z2u0dKt_!cZ7He<4oma$}TyOp6=1xBcG z)p~cD>LYVSs}5rhC&o0^{IwU`d>T*XbY$TRVMkB$*(Be|Ph%wyDFQnOg;#@^!&lUv~4ds9=5RiBFzP3RhO~6U)0Qmb!2L*5o^QPXkw2)q`!v-HB#J?q^t`H@;5xD7zwbWSCmdkw!ev(>5v zb+Z~)Hx^XwN+Szn(;^d9z-4CLFgk=H4w9j-SMWSQ(?;WvhkLC^<4A*ODTuNtdz`h3$_hC0+d%@Emb`Z2G!Gzc54;ryvg_njU{Mq;if)l2-2_N_QP2{!NZZUxltNOw z*uc3o0dnh)D0<1Ef1#KD3tfBKe~_k+bSSxg)SFg<2BI{a`R1Dk^M1dJ;JWqG&$DkM zgno64^(%nIC-9h?AUNV)iZU3pVxi7sq zh=QmUwWCfHMp4v_y8H^?5iRb+(}SlE&jryI9nrdut|NXC)&mjpOCp>MUZ9D`-{F^^ z_v7dV3Q*hyxL2#oU){NPzYs~8GMnoLSbQN%3u=~MJUGH;@Fn`rn|pIay*UPn=RQb(9)JwyEs(8w8)Uow(wTSQJ)DOiBint39_>IP z&^-Fl``&|U@PNXA{Q}OrWO3bA(n^*BQG_XS4!PZ>>Zf{Y_LnQSf5qB9k;-oTbN}s3 zkSymSZ8V9?$A;1(HUGC8)CA{JO$lTL_=ghRFtg0q+YQw!tL-G8tkCU|+Y@s0204*w zO2(r8u|T%B;b&P6Z}+20z^a}3G|!KiD-uH{xvq__N_4~QxuvyFTt_|xTEE#)cDx=g zoiz1GY)+CKc;VoBqVtRhmduEAPEKY*IRKfX4-&FvY+1j=Y9TfQcFqm}d~I%}IMz&o zeW~iG+_oEyGeruW^D+?zAQeXs)i!npiEL&JG39u-F(u5l-^05PKHnz~X3~&?X_g5K z!NDRawHC^zr=(P+5qxv!dOx0@rB#NKnM^rZhUGAmLc5kw_e$TPEr{aNEWVfHZBGu~RZ*#|{^llZkLd)?Imnm>hht?H&dYrv~7= z9N2%A$J&H@3mI}k9@J#iy;bYiq5Olt8Po)v!2GOBcPoy~5LVef9QxXg(la+pwHvT6 z(XdzfsZiRDR^4`@=S1I$3r<`F5nq84pv(4L%ybP01JdqDt6HXn{8u4`;Vmtt!)_HA zkxu^G?3#i0(FucL`O>Q*N|TfsL#e(DreA+OvRN@IWU*^OL)hI9vvQW*%1%b99FK}q zJImGRI8Cz=9KFi8z7^Wg(#q6~pvB=BHlxkcZm&4i*TA+5(!2{I{2=rOUVH1lhM&BD z&Vb)rh8gM(Y}cU^?ue{OJ*-JSI2i6YLKO5NrB#5eO$7%`4e~RLZkMy z5OJSbtL3<~f@Z9!e3)h0ow^!>r~=nLHS(&zU?~x4x?rlNOJ8@|?>XVlX_rv548xF6%s_h&O+f^G7LTA=cYEW@+ iopD_km2=tvamj1m9GWWps0sgLN0V{52B?Rx_L0608s+ePJnHL^^!A`R$A^- zvrEY$=pc;^q$tp$Pkm?{zy|t~hxQ+&=tE!n7xpy;3ZrS8CTW_cwcYQ`Qj{$Bf)?l! zdv-2oZfCx8<{O5?!vzh`!q0xO{QXBX?PJR9e2U1tfH(Rv#3hYuw#IeNeBIXZ&G@X9 zu`?~hHdmbdziCtX|sE}2hUpH>?|#L!P0rR zA?BiZ(-&4NVW37%Kd&|P*jvL1!pccQS≥UvGn3FIDgwjZWgP9PT<6wZIKjI zvdv}KPSRqp)%NY|+N$FRw%PE#wzIanZVyLcGj6$8oT%aYBFVRh9$CIqlZwf{_NW(lG1LPe#&ThWO?zAw zac|x6B1ekI;~m$>7Pji>3CdB-?eRgb6L?Jw=J;OAHV5^2m0H3MBx!fMX4+78-yxpv zr=u^B5E{^_*^KLU1~|%agJ*AKY@?6GJifplekG9=IX=wGebkMJAt0~Z9OoGU%;Gih zj^Q20JArpn z4?Pq02gFG8srR&|&Y$2U9zWc0faHc3 z!C%s`30g@u4rOCG84*|7ZlLOvgC<4g0btEAu3tkW=b0Zc#A%~9v98HidlTF^gQ%@?7m>@J z(R$iS{^lh4&MhWaIL*kU9yn&TV!p5O{0GX*sGoSxHEm^z7d)*u@l*YWx{lU2r*CN( zK?6pMi`~n88(A<=r-eXj!TqfPaxE{if^fs~mSA_mEjQrShUfcMU071I!gaxG)}=^T z?ag?ZAQdhFE9(gKUJ%7Vd+j!ZZ9mkLLKM4=RaGa)P-GAJVH7#dK$ej^-(66Flg5iX zgdi&n4q1_iV{E^5c^9lMTXAT8tzZ67o82q-puME+l;i~RQ4)`lut@A9@i>Y7Bo2^x zg2a;~W=I?)F-zhp5)`}SArf;Ci3#ikm_(EqluD}S1i6qpz}`%&lFX8blWf#!3z?Ws z8dsg9;;ds)+>HTKb!^Q}CPpOu=Kp!hvPP4ihk)Jlj??g6SlW?CQ1bOF^Kq*^-}c&b zF}7U9bH`hq<<>%LV;<+%e7cSE>%QNb$B7$6bQmB&Ii1+^qxp0h(LBOgt-UEvqC+xN zv>kVH?wCeSr+gW&fl@=y-D50wr)re+Zgqgc+Q6W>zK4&b-|h1> z2Yvw5G%}dS){{ko9Q;MR5p8yl-PCSr=d`y<-(goXR}DD1zOKs(+#C*nT3f>xPN6S? zab&$%w4&LpJdK9(1rlc=5`8t%*Oeht@jMldg9yaC4{wUP5s|q{#>6VwJBHqUbjOS^{t1NLe16mOK>81F8RJ5vs-D5O7of6PDvzkOg4nQPPJ#ihX5@i5 z&)GHoT|F+)34C4k#k14&Mf6%JH8bb5YluPeS!m`+6Qw#I!ue;yOs}965ic!yU;9Ay zrSgSoP1;*Eya-)89_FP%55uTmxvzc&_2ohR2DQO}=&BK2M|~OV(B=0q zk1Ey0h^;G?c=Q^RHz>u&u*w^#9jDy;j8A-!>GzSnS>=;lyTuT1HTiSs?14@eI#UCk zvFIa|=^V3-K8nXzCipbp%O9dBity=4v|qq-KOyt73ybM!fJz?n$o`}@ZwPmHch&O}?ti!5wK@thTI&tID>}(U^a#_;M zyVXmpUx(y77S0B4rJX_1Mgyp z)YReEJ)WL@|9$IWn&iuBa=R}i|HE8frrs{7ds^*5>NSAaw($;%D-FLBdFx^}HKcXW z3${x3*%dEXccL{JJAIUPN7ZibAbV%J7gCB5V_@pAtbTa$-jD!BlWuvkc9;C3FzypL;`-*#VvuUyV$xxPmsJmSLPO|8*z0jY+5_c zYDc8Q!JEP@-N-smpgy~WJ8ONRcDQzoOh4%SWC5gLD{Cnym?LDJIg{$n%?06Hz2o_O z&Wl1nHPn4=K;fTWj^lQ8a(*7^r zp}5;Pr3Tb6DZD|*c}dDpCK)%_+~$SJ*kTx8*!Eeaqv&8#Kqw{KnsDNj(( z`$-%nag2nT*xm9)FG=xHSQpBi0L^>dmP=p}GH(o(H$-udGMOS$W z6(Qc}1_Ve22$RXSWrY#ds50{|GeD>Al`{&IMA9(Y-6<+zMBFHko`=|9&3D+na=xNh^eS|_d;fK;NAUaxroT8lPF$Hh z({Yl5LwDkiPwA55tl>Y3w8k!|fTV78_Ame`y?O_()^giY?Yr7?+U+Fc1z_P#KXl{7 zsE461>7bLZksu~0vn1#MRLnI65ScU??vvz(@clkfPECJ?l!@d?x;xUQM?ZluHJE~o zs^{)7QyC#O#lMlpC(2Ze-G%Y*V1xe7vg~gx$G#@>p_i{%p8bUt*q5xx{>)13i}Eo0 zoR!&UY=r%ZRoJI&l>L!a*&o;#`+aVleZnTnzhkGcP4^1uwZIB@tEQUgY&jWn93D2X z7Q!h05Zi1cnx$)3Mj6<{F6}^VBy&(#c1=bc#|;n&m7~-3VPI%A#tvxGy+Ro&F;AzP f_#Baih{|(dzf%AU0dDuN{AC3d_m`Hx8ZZ3=UuV;0 literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/nemotron.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/nemotron.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..693b3ca2f9e7a3cbf684cf8ed44033a8212d4060 GIT binary patch literal 7279 zcmcIp&2QY+5huAH+Fi@CWce#j+P)-htxcqr}Nl__IY$U)M2A3k=?$YLx zn@3WL1m;o)NPq%4_R_R)fL?Owspz$r{ue|K1yUf@t-Td(f%G@;k^80fM^SX8oh7;N z&CHuO^P73I!O^2N4X^dz{-L|@w5EMSH>0m2ZeGPZdd?<{^k%^^nnkD3 z(VL}ev0AE@t25O~m0q>#k?L&qXmt+P$Ex$yh3au!E#mV;b-{V;Gp(&VC!JHDR-0w# zv~%WDtvU0VR@9mmtl(@qm#;vnW<7aa(`pQ_8(lYK_>a1Ld|Ax)tPsxw7Dj>KQw9WW zW=S45TRHblGEeJPgS`_G(>UY~ad7=Gvi!)kBF-$I<)f7~B@4^81L43@G@Nt^``mWh zF6LXJ6Zg0uh0Dz0ZP%BBFhXsGCG4&naXX3yZ%9_}42020YhqQqZq$_mz{`rgC}g{? z=dnGPdl1D!x9576kc`H5S?Og(467#n@-?iy@LHoLhbzlwPQilh+fWLo^}AfS5o?Q} z$8NSJeWY~@2631Tt88=0Cbt=+!{aAs8Fum(K4)-6; z0MD&oyg0G^H(YpY2y5jFSNw(wOVfwSS!tOtF&91VPPhTO$L&qA&V(}8L_h&g+rG=)P8Xq% zd6^?XY!6uDt`NVWQVn*UlWz|^kGx^j`&L)kG)uW{wiWn8f2T2b_@23Ac`?7QH767v z1%5D4;l~DSGE2&c3%fxKblm1SppPl+_WB+tfM#!rAnqg1-itY6|7yO!$Ohwa-I}8C zVwX1GyIN-f zloPvd$k|)(UCq;7gMBDbCh&K-h*Sm|?)?}SyUXo6{Q%)l$;FafhR>Fj2Y1M(v@J4E z$Q>SIB0XxjLC6|mBw{;5#U8i(eh^5&1MbPy2Es*2VtI$yP%4bNGDHBgQgDN9z?JlNurwv9b9#F);+r_PMyV)QzHk_|odCB|dg{8iDAn_MP_X`3q~88tWH;T@BpU z$t#!}jd1}BKZu68e_18xELD@0r0Wc#TKRfwh$~s}SayVng&|rD>bKglFC+5-iLCXM z;9h+Gsn($=;ZVt6fd#oG7#O=dS+>oXvZb6o$%0Jmd3_qt90I|f^|9FxLRVrSM+7=l z06hW_uDv)3{x>L0tdHGZ++!jCDCWM+naBN(EOxM)a(h%D^$499hkND`J+=qr14DzN z+(OyrA&mBE#>=PPTMrffcDyB<1uUXQ;RWZk0XUJze2p_~}qUM}n`x*zF$x{IeO~fC!dQ(L3|3<`x2@!9i zM1K(T$mQlP&M)Rz6hCb9H!Lr_@9xtv$mO|f5BU!ESOk4;!Z{-AQM?;fI>K^dSrn7o z7V;NR+YRIr$=lu!uD!{I5{KK=w~)~goFa%&+?#EnA)NH#v9;EijI3*1*r*sR;7N8m z0A`_PptBJC8JVc-qI{D%ziaJqrbG_48Y(sLb939ZDANB|vm;QDj3u+Il$E+IlVZzE zMw--99mfXyLA6_dSOjLpuoF{00496wm4T+yly|V7lvN+an$z|%pD%k71ep>z1|mtd zZEM^00Oz9!+4K7UAG^hUldK^vOel2{xo|hPi8W448}1(>+YVqv{WAqJS0n{UfBeo5PM{aYTi(iT;kH09-~sP+Q@$)EsvzA$zy{D8jd|> zm(|#YdMXFO{>@6RXY&}2(9!Ua5gurNEocYY84Uz2tOJHN*1;OcR$7vk)PXjyY2RM| z#gkveAN~B5Z~xx?=JVhG<1bfk92hjR0J8Bdh^Br1$5+*b_UBi!3r(|6%kLC{5N!tU zBY1~@0!j2lGqt1&swW1hkrY4+NfES|lt4>Kx&K&l`d%4*KJ6>xlX6mav{)A`Df^Vo>Y&U){-NjN0M34*<=CqXmT8MPVzN~Z+}x>7qaWc zPZpC!j5wBP_vXLSbS*i7yM^Q==<(zf=wfu@u6FyRQ%a5|bIGw}-YI@rOiuqs|3XJk zQbVtfY|cuc2SO8IHjj?#d1d;C%`2i6| zg0v0euRPz2yS??^?yBc*uPTsT-SNEM>L|FHafIXEszQWtHAjL*e^0D{Ye?^(fRxXc z_55#MpT%|YyGpUBmyP@>ea;(2JU4XXyRuQq##ap7%fI5kjZ*Dj`=_@uAbwr;&l|Z# zZ5?v*7eNPg&tJzo{543T-9|SbQ9R@5*R=@Pa9fLvyM{Os0Tpg*chU8~QxH!?=&RHH zFSTD8clDpiara8-_v^ri{H`SLj6^4n;dy+PcXaV0=FTMgU?w{C_;v%bjHnDr8;Rjm z?&x9@G1AfpRg`OmB^PUu(QgJ**J6#2nVRzcn$s7o+k=L&idv zNuF(~+p2ooy8%Pr+;h+`Z$*23-jX@ErK&X?MmjhQq(%BBEvtuhqghr5+~$n4=UYA3 z-rpD(cn-~1%ut-s%u>?PgrfTG<~cR{+%h|t%|CZb3F<(3y#Dse{_}?zX7!^~cEHkN z@}$PDuyw_uJ|lfTt+~i+bb_>bTBJp%1srhG8f+T#H-!j<_&y}1B{{!d5DnZl7e;)GEfvbk7;5FpN?wMn+^Eyobpy&;kq!V&+A5kJecll1@iDp?$LNs z{BHl;q(_g*9wJWtlqiwhG7-vkqC%t!vb81XXD&h6Ps|cI3UWjFg*b-m4S`gvZHWbW zjURTjje3>v2s_D5Gp(5>DuLLe>sizMD7L)pn;Fx@PY9;jtjSVA+2?U8z;PTi-40-u zqfZN9lYMxb%g~qBNsneIV{^~TkVYAhssJIaNMTAK7`^~f(O{ns%eqm_qyFKaa|iRU z3gWU=uZqj~Y|f5Cl%QIod3^LyL!y=-kZ9I`bte4)$AzGu0Ml}ILmWqToIp}4oK{{| VhPpz46O!4B$7YRLy=I&!{0HilmWluX literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/nvlm_d.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/nvlm_d.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9cee42c46843292ce620a2d2ff1b060f9ff4aea GIT binary patch literal 409 zcmY*Vy-ve05I)C+Mx~Vy;sIDQ6sr>pLY0byklHRt9lThfcAH4H6C9_gGrS2CLtlXh z$;!klFmX->;G{d6{FDr&?bkE?2Q0?hT^aFu$ZD(7Z@Z+qQMYO2;6~;Bz*-* z6?vgNlD!8=j=m5B64IRATd$m5&1YHaXNBG5+ci%9Sy-K-SEm>ha3TR=2&XbaDB6sc ztNB9Qitaf-?#gA-Yhod6YLpPoKnPRFT8Dfn#8aK>Zo&uHS6cXesouf$@kATG7tEWI zm3cY#soN?)zA|-YCgz#x{E?M=pB0wvv^LDU)K;6q8RaTb`&?IS-7=||-DxA_Wx010 tflmU7AXE%7X*5904vy|BwdPx`Z6&;E7Q_xG*MTdjb= zb@Zp-MtO&jdsx_hIZ!waw>k&lWJtKhX<`j6_-iLLwTHHabxz`>wPDR7X9=%y_a@=4 zsBPFo7s_?+LD@6qI;UTfzQ4Zzi&c>wyL^5=&ACYCqdZ%TmnJeS;yjzetH_{l>DpHr z6yag^f)s@e<4o`mOwS0rY`1@V6ChH-O>ft%R102J)mFuz@Q9(4Nl-A=-@8t zH+4--#zJb3u{eth#&nH~aWT@qNUsRac%16SDCS&bOvUR$2b+a3Dzu*_V`jEW``ILA zBN1|?8*#>>QJ7^SQ92lhMKr30b`=!jFkz{XOQNdl6(Swy5-PVkLR5|t$)fDSm&+bv z4zUZd8?guR3}P>0A7Vd1yUVMgTvB%}?K2i7p;C;=8EE-;uu!Dqg>gJSTtE*|96p{- zM(NRXbs>qbER5HRQJyWVBuTme`pVQ|E+I@n2xAA9#vNjyziEgX2e9W=8xh3sb>ziAyokF0@W$CQ&94D&*mX!od4exJ@>e zoS%5OPkw!$#sy-RS?crUyy_|&_NYR zu?C{RRMM>#BXL$OLEjn9F(nFCDF!?djmscEjXcL8?ve7r2=i&)><)Mg}-WK*Bj$3Kgc%_DGwQQ+zYfJYpQI@H?v zYyVC6yOv|Q4=A1c&y0D(4{;*A)Q5nFe&}0Ial?k+^jmPZ!6j$>S$~%Ij5~IC7u?Y{ z?&p9?=5%Xw$t&NRl8Y3KZXS&6h5}rG7fnwiaHS!YW3WAPgOK~*&i#D$_w^r6-T!NJ z@4H|A`QzlJGxwnjxpDKzds70YXi8W!h%n%MP$t_JDK6E->re9CEH~##T938MZN9d!TzoEZAx; zIWrE+<7`Qp-S6+w0pwbyiZF|Wb})_1p33LplXb9?N8y#qHTR_9H*B1%7*EOV^WduK zPMj42?2E@?v1zj%W6zb1C*c}6o9h_Lu2VY*b zvF&p$i#4_qCKFMe1IDf~76>v-;OLNWRAfs+H)X0=2L8&#SZN>M7=_UYjux4Z1uI6P z2z6&GG7i=Gq=0eQtRk2!TatK$n!M}pk_YhleS|qDQ0ej`#2LgV2y=FPfKOQ!oRRNhjp=_9o(6+{P4==uAIUF9(kJS>@*>2?fz5ZxW1?BjPX1IOFIH(?ZPR7TZLXY pY;HeR&7oFl8%J&n<K3j_VjsM?a7Zcdy<##Oth+Y<+|3G+B$sQcTK_izRg```CiBA zM#2mo-`fUjt)VBouL>Rt)A6`{R{oB6Q?B3bI$qbJHlr6QMptkPgljDw%9U)rHDQ-p zWqYElwmSYkTqChr9G+ANrQo(e3^A zQyL@ph*oM&Okwq$kXvCScq)4+jar&xpt482#ady1zcwu{%z330f<4yO)S%xW4 zCc|8vSXx+EXyjd~`@7UVg84RKNAMUmm6|dol-kmDj;53=5A>q`Di_qnb~+s{F!3ZUXZlqNG7jvcfM{RZ%8G~4KL58Yjt+@t8c!{$B5 zwz>CYRyeuv>dRwBIOo7=1DGruzT(G*+DhxEGf6{&ow?|9o0iCvNnKoASa{`eVlQ91 z`f!=Bwvmq0H8;5ko?yPm1YQ|VSUSLzk>kmkxFJk0AZNp*+S~#_h=wl9a-)Kfu^+1*v2Px(G@X z%t$N*bhFEVMu)vkKrW&bby;9Fs6MB8a%Q+$ePLp`F-ppwoqG z%UNWZXJ5xkF0=z?u5%G8-0ze6bL0)o>Tzpr;A52$gqxB}{;p^U=`>w3HiH!ruWTl# zUd|cIX9%)_P(&7xr=SB+sLxGr;QJC_fHTqxzHpEcm~J}t6o%v~0hCE$`NE*tSjoWJ z=Ar4hff@k<0DBMp0yZ!QT&U1N&FMPc3&TNhy4f_vZD)Pn7v1K-?lhN9EW9+ocw*_~ zOY?Ya6vbTZFoY)X^ZYQ6y0a=ArbrKq6~-t1Yq+SCf3!tnYMtA>;7W z;YufW$Zbf|sq)T67DR)AFOW#UAK|+sI*o}%w^i+z(qj41W(ZqBtwG_Xzm_2ghf8LFdndJ`q0~P5wd00 zn&uK<4eujeoh>yi72$FtqP1Zm&XDm|vF0D;;WtFY$Nd`^M&L%SON)@qq`V|W8Wb0i z@CwGDHptFH!l$OhrXY`UumJJN20@0C{DQ_j`^Pvt*+mPuPQY(yhJU1M3i>szwP0CrBW9NKU{oql2iaF-u7~8ivU{uxsNE5Nx zx!He!2>w|_j63vsK>nkcBjc0>ADq+(2M@5sqepQ5Hn5u##1PS-!%bPaup3Ai;R6Fx zV9L1b1bv1zxmK7iIjMF;0LMm@$Sq62anZ%3B1>bOo=^z^h08L=9g z2UPogR_i%I5b@k}T_AU~3aj;-gH4pmR&V%MwhWr5US2w}G!l42hZ`Fxc#T5Dc^r1E zm~QY;@PH-2HABdp?{E%tYTalV^6;$Y|y+?7p*pLTb2y?G2@2ffrx}QL5|k;55tf4q$@4>6v}U%7;j7VHO8jM>l6Lz$48z88qRu?5L7Q zKNJ-Ba3mokB`HxxNKh6HC)pc(4x@r^9)DURwo zN4Ur2b$r+LHpfP<|XV#xN@7Oi4`h$lA4Pr`=UD#MFdOd7TCLKkWl0((t$gXfF=T&iUsup9yC*I zf)!1#pfMR`9M@;zY;q%rEVl{{YcNHj*YeeiNbV{7qP-&4i6|T}W)i zT`Vd2R4`$2cZ4=)#jsU{B!OraQJHJ?vJD1Fw&=KCV)X2QpV#wcLg!myvbRV`ZsMsSG)k z;|1}f@|G=1Occ~*NqnL4Lq{N`0I?|lmW^QK)?`2@53OC;wG8w9oM2n~P}8=x-9)|u z!f=pqB`H(wyscF=?b8qT|LVe@x4!qrr{DSeZ|?r{_rE@KV_Qd??Gnh!ry!d4?ccqr zZg9VSZ%WgwXXJZDp2Z)$({|t+%%E?q$C{zVRZu-HftKP4(22MVT8<|{C*z8(MVdGi zPkva5D|l0hr$DFT8fZ1HgVy3*p!IkM=#F>~=ybdnbZ4yHo%vYPwRj(%cE$TacgIhG z?uie8?u`$Ew=P!VgWg0J$2UJGefGtNYd&-_CFu8yN4 z4QDAkt~AQq(=g8Veh>khYumGQw=a9d@0cHL`w)>Ah%68xBX1uE>HhH#bp7p(?hk)J z*Y+EYaxy(yUM16`MN%?7TF)f=9*{wPqBWjQ>iOP%a_}KBJlT=%&=3$Ld+ytKPWI4V zVPRi5+4q3$L-lfOpE0TD+dss6Y=0q{9xc_AnUQQ9g;Y`-*3wBmPgv23lGij7NDisE zoIcX7tb_aop(vRdVx5HgWKaH_MvbJJ#erm|h_uPnYOrQ`#$tA}l--=jZcZju)h1j| zCXqsNkyH)2gJBqrXR!hi`vk-6ilaoHCqi>CzC`59M2->pA`$We(IA2~tcfE;$UOut zK7k`mO?-if^s70#YZ4(h5{pEZh@2pDlE_O$UM6x1B&n=z;N&d$L{DmlAy4uQLp+CH z{&lGt_6N;@Gnfm}f`@ab`cbdH*xzWn&MH+dK=$Uk>-L+Yh;Bq7HX)Hh5kdt`iA?i@ zP4OyNejnf997yH4ik^L{6J@hRYCdiJT7O1k=#|5KW*|9fk{#O1SUDNveV9E>%BNDS6USf)e=F;{colu8Vtv>XN4EHN1+q$E1(H_c zl3iWX#bvas#igtr>C>ohy(Tf8^6+<+x2n{*J?hu(;xT2Oxm*!57~|HJvDSHoVdsUD zI@?51gR@iD4rW=K%7(VgNZP7Y!1h!Z+e1m2en~3oWuw%ps5+-Ll_tVlD@B3Fa6*S3 ziu16(@jZFK_uetp``9cymi9k(Qwi!qd0hW^W9!s?4AWvsDm!GEl}u;YqT_EX5m&|%Nv?<*J=6bk5EO?6#hV*f#7?>=hmdzGVi#_&tO$xWD`F2l z(AB6BG*O9Y7)jkQkY7YD-A^0F`;qCUzf2hhc6tn>RhJ2fvd$4eI&IGxP8S=W97lA( zIC&Opak<{5ZIYcSu2|f4Q*2S@dySNnHl#&Lm=?SRQq{_OdA*{SV6)HunfdGP^Md$c zXyU8*+nOGQCqcPQYwzf*d5Ktpz@k-$TNv~|GPn>I@FJPhuAWwJ8_ SJORx@dEaztTCbP(Py7%1m`~&Y literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/configs/__pycache__/ultravox.cpython-310.pyc b/vllm/transformers_utils/configs/__pycache__/ultravox.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..103564d032f1def0e83d59c06f6fcc61afa7ea90 GIT binary patch literal 3413 zcmcgv&5ztj6)%_F-EL2JXC@z$1Q1vehwZTGNp_Jy2mzy@i6UhuQFe226uCXF>Y1AO zYs>abCUQ=@(w^WB;s6qtmEd3C|G-xcC^w|#Yj=U+SLL2=XEX_W!kv1lva8H@VMat^!!u&)(b)7%L%pMsRHV6I(=*Y>-4+*Uf=Hb`;L0UuCf*02uHtGj~+#Z3=Yy~x6@=M zc7wZJLL3**2YVuS@vqq9Zm7SNGEfY-VHD(epSpRGN`A(Kbmj(H$K&aV_N`RH022*` zcX+}ZWz#1U_imx=lsRWgsYvo7ND2`Y+zk?~=0u4br;JB#5QeE_BH2~b6}&v-p_mD* z59Dqc^Q6e%b{U_EL{GvDy-@24WVj~^9u}qK6Gy*lJU{U+-Py-ixSs5BDGGNc)7X7H zeIPk_5(#I`^yw3zV{RBE?hZczuhoI9LZ*>M)&Ne+A}fo0;y&a7d;}hrh8kx>=xa@A zeX~>oKZq6yy0Km|N4_a{^B?F*YR7_-iPVRA?4$eE^ihJ%dc0W{X@eM)Ada6*-*(;W z={%ONkC)4pirXdjG;TD_cin|~wQymk;LnP$V&%0)K3GJQ+mP)dU=%A#v%3?7Pj}LU zPu$xLw?T5GCQR=|K^)ABg;_H%}3wur@kCztW+0Lt%|l=NsH4_|B1L{>>!l^w4?lN2v^! znQ{-KCO+R6id}VJy#nzJDEHEbo2O1+wG}wocu_7+H=Q@yH?Ru5ZD zV>oE1dcD?6ZY5&9vL_frTIS+8uLtc!sr!q_t%t7!QoY_X(%R1YavMzRLpW>|1}IHP z2W>)H6T9ctQ^%k0pxu#LIl5uDz{<&(9?d#rZMfRQjHM~PZ?>u_viwhr-{fipKP_0!( z7b=eawgeGmY}C#!Ke_~nKTEByqZ$Wfd zyR>WI*GR*r18PxfILEeW8#buthC>~?PM=?X^h(`1Sr9bX#v@UM64_Hi zofERJgsSoQcTzY?SgQJc6eB-K4s2z5taUfS(?I43vcQ8Xu5vn;Qv)!$Y^rMQBcKZLM}Wmv9d zfD_}apY_>@(*KN1`tPnq|JCi#FKL&4L3{LbYSVwxKK%#y`8!>qe;WSEL}$y&2l3V7}U%|xe62T?#n}wDaS(hr5}z$%a4*#W(6WT z&W0-S=(S4z)a1-lCfC7)+(a@@SK;j{wIMeWNJ*0A zjp?zD&pkwSHjuv&%sHEU_Cw^wTv)DEY=-IuNXl@Z1)&!~?1C|Q2T55thEQhj;rb4c z$!6rry%Xqi87HARCUS)g`fE z-|P+3!|T8)H1|NE=xA1&+(li97gL+{dhM;e<~WfX3dhMCjuXaw6d-Rp&a;slOjoLo z!($J`O)qfMw0Lx-hAvf*U_r_jkX-M>4K6ou4*wNMuK7_Wv14-$9&Cx7CO<%;I-rK! zeIy?ud4S})|L1iq>#%@d8Ud*hoifH~gEG4On&~adbf)RA48~p=nt8>lOk-5PVwAlu zWbA4x*Fk;>dqEFmI=eJ<^ODtOyV-PiVGCzwRh`{yFXfus_at6BD(>9K#&|~-5g`b6 z6JL%_@pvgj>{TVfBJWIZ9tDbhU&(L3F`aoUNQDfBT!u$p`>sgT_gG;mCbLCa)MO?# H>4x?n-#BK` literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/tokenizer_group/__pycache__/ray_tokenizer_group.cpython-310.pyc b/vllm/transformers_utils/tokenizer_group/__pycache__/ray_tokenizer_group.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c3bbcc5f6f77c2093be461fbab0848be12b14af GIT binary patch literal 7366 zcmc&(-)|eqeczc~E>|Q)QMN2gXP<9Drd9}+8Eb!?st08U2G@UED&r~oT0Qb zxl7M3ts_CT&e=foas~Qcpc0Tg_@VzoU;3Pxk!*)Y^k5`EJ;xkb;v`3 zlA7V}cV@mb^PTVa^ZkBD@%((rz~{|>`PZ%AylNQ#MK9AI2QP1u>48hulm;W-4 z;hnd6$MnpO#Cj)w`h-?exTxB7u4!;N|Saq{i)#PZIt8=*hquU(c^-7rnJ0* zv;~(%>8#=;g#ig!qVU9m6z7C3OQI;t!jTm*hdvebDdD*+=UWv~5%Z8_RhV*NyVl}b z;+D54&w*0Gdrd5AKIgT}4R1*-i3?8*?}AvS5#pk*S7f78JwuZUOCN^?%PUPjAn z;&rsVIHTp7cmpk0L|uFXdwEG*f*#$-E&Ra5n}Je& zKYoYGyX6wV9|Tn)JfB`f6h>?R3|= zVfRLgW|`jjR;Rbsd9$;#)($t;y8U!3j@BNw+nqIRB1)hw9f^s1Y1mHIj)cBRSg=OI zI6-T4`&qfAVhCL;&=t&_xTq}CVK%dvb!eHKaaN(HgG#Z&O59;pKDatlsT)(gvdYn9 zp~%WI3O3pjkfwsH(g{BI>7Ww%AjxX<7DnA308d7n>6SWA5+GSLNpr^^I*bBJ870|9 zHDp5$t+NFCCu>;z1^b!d8$UC)nXp3oHAn+NkTt_$^?u1msz78lm*mUG{7o3ckL_Ks z+%xu#f8-DN-TZh7(y?VgyjE3s8Q%D2#{8!&vl7{E;v#~u72ZIJ8lN;yCO2+nXimBc1T3 z80O2*rG<#aP+DZpSPFke0(%1$w@HgRv!3yS{RgLLF$bD$F@6XdigUQaZC0`dSH51X z$Bvtho{sw+nuU&g2%UrWF=&{vJ;p!2p3B`-*U?3NgQ{y(>Ge)*U9VhSMZ@<{@>6DK znc0-;oLUj-hh7oIaw)9@NPpLO$|#mGVPL9(hMBXkqTy2%J$s1{UYae8367nbdm@cx zJ@+1DA@~z54YI6#@(1Snv0-DIxl8lXFl{r%U2;vv9y@L|2*$dIqO=g}T)m9$SwX8& zt^!(`qyp-7R6jsT2sCVibLY@L95l|@5O~fqn7Wzkyb0(%TWWub8Sc?kQM?m`Da58tj7;N%(6UwduB4}=k2tdj2Mc@(gEkzN@X$M< zP_e}5TV>XBix1Xj@A6D8iYCpfu?jasMAA6`WL15Dewg@}>!ECt5{*CfY@%rS{16W# z`LLw;T?65ZspjDVSdXdmXo19roMZ#~H7zSMm({3GUH2KuZ!Eo&63iv0VGiE>FED3j z2i>lW1U$fz=)R2xb&D#kbh^^uQ}Yq@YpFpFO#~`GL8s&#sw%H=I8s(AIK0NJ-&w`M z>t{&t1dH{|?!rrV)2yWOBmfASRmhk6Bf>GU&0)5A6fL$}z^S2jKoE>JDf-<;kl$m; zf)*p*_RM`lt)!OVIxZv%V!&NXn7f6cwQCR3K{d7aibHGL*`6bTA_la+H22wV5y{>6 zEh8=Uf69zq2dUqCX?eRMFmC7|el+#G^Th3IdAzE5JRI>@Ny~dxf%z!n6b@ElY%e5R zLkA|YFy7B+NFD{-b^g|g3iJ*{XL)b5g@AP<#!`}v+`F#So3Fq~}>Uuo2 zro;^%24S1Rv&Oxrrn(KyQEyZA9aLXke=TtZa$R>POt;+I$5P+xu96d&ph*zGIB@5{ zj#m8D_r@bBjY%Y&wQwm=rws(Iig&KNoj}pzC>6Rl&WCmC`YB~ZLE`qJEg7`at$ySE z`eIf=LeRYL=Op!)v__431qtKmXO%m7CZJ>ctdxi7gnAms)t{q(X6G``9WmicX6I>k zRzwUQVKLeZX9Y>Qz4|82@mHvx&gItsHW9RlNIDwQavUH;K>9AzjL8$Apgu-h7bW=# zDyM`HS_jhb7Nf6|hbITlVb!uZg(~)egU}m+%2W#i7JRJ(%d`=mFEQs4|NYV<{u}g8Xn5y zJxl??9|sSW8Srp|2HsN?z=IRK0>S-1AjC&4f&tilfD#?E=1HKS%~$WY@^D zkwnO30s}`zKOy~JVNMiBH)i0@Jw*;ly-U@5RDFQz{rbH6kX|O%tnE~r%#o2Qii*`b zRadF{gsO>cs!!?lgd?ux`tIw1=%3Lcp#XwEa^}hWYCJc++`MWBRdd?xj;Lpt+;92w z>hhGyz5I11moMSitqg^|CJG7Ue<-lh;R=FY9V4^B%?~6T1Y7`ew^ZZz2 zJ~h&HVd>XKhOG*6zLw-q%pcPfD!L`{NImPqL-6)EM z^N@73pmQ`>$10OdP|;DGI571$p#3}1KSMQmBWmMVYIj{3%Bf z^4E&;UqzD5p@b4lt?i2LkF>HWI?Wt>j;umI=^Szatt%+W7pNQqcwus0g3v05)ny0| z_Xa4ubWmI2W{o+fV>>nplXQZggQI>wrdZ>OJ*bDJoizqC`S1qB^ETv`va-v=WgDq&gddPbGhD(v`$MUE(rikMP{9 zhR%}ECfF_f--goi!LQzlyM33gFx{ZtcAGt=WRz0G+muOyQ1xM`;;2J3TyKUHE+R0E zHA;V|-CstY1XOppfm5g{wtlmbXC^O*_NPRy*m)6CgNQ=Q_B`(!9& z*rK2nGk4oTQ#KHnjxJ``>xGQRNmd*&&8#MNt}a4snceFO{D%gHqy;TNI|);y690{& zA)tgS(JHP-6;bO~)VFAWM3vcTU*A<|4J1YPsw(I=CU;$l(!po>qfR$gsg77xfx2Cx zZghp)IDP#c%0ypGdu6)yZH%tJ$b!6!nj$}!%^ohl8(@;(1Lvlb_s;`eMd_ipsQVL? zHbx#qTa0&A(@)RDJSK!3j^VHx0!6!NT4aL;Zjq!dKolit4Q12e zUxrk+78qdLqb_z{lr!ix@&0lEp^CJeF{eGy3MSnFvqlwi&k)%d>P_#A`|0 zYbPDAlh~e}biJGzQ&ou%j{H zwYVP@W!?T|UPf6O!~<4edXzm5(`Xv1Uu9W*CriiC5xTqgvj?{ygij`6Q4U1y+)hsy ziuKk}So(3M0>7M3u6N(<^Mm3v9o6eJvQWg@?fVaYeJ2I`o`QPktyjpt(+(vsHxb%gq-puct@1!VOw`u7$NO$!(1G zJ2dXf9oTUhGsU?M1z~2-6}b!D&HqVvPhN%Y){0G+u;VrP0Y23X z-ZGlgAKrG42v#>6yNgg1Zh0J(?kGsz!_b`+p>)TYaw+m|AcMRNRWYocg}xujx*Mj! zVI1n{*G>}r!A~cNUxZJJx=&wGnoq!uFg+@dN39D1w9uvm_yxuA3y6wUhHq3YNLF!3 zUI|FEggi0EEW+4d*6giy)siKndE<_{7b)} z*rM*A+dk6FRvYM6ny2e<7XzynMHr9un#~GG@7Xe!(0XO4EBN=X z-|v@6zMn_=tujzx{jDD)ljG#GyIP?_qvgw|(FnmA+376l8&zxReG5CVoh{8I5s7>!3yaib&wJFn&?;MAt$8H2(ABX4Mp3yxj+#rsg zhQmjVwM<>c)K%L=9;HVvElfrUK0Pmr4T>~=P&=9zsws*~h|#o#al|2_Fpy%?@7)bc zZ5fgyN+yY`6U1#2M5h~>ty6Bc@NmF1K?l6C^+PPTYVanN)TMO$2wK{DOHADrNZ(Pe zdto}tWGI`mT#P8<^M{g_@YO*742=@0`kXy8k$&qHKVf+n-tlK5HJksHkj8|m8GgK!W>0_c+SN_(Nrh$`EdeGg&$!uW5-t4}Zi=F_Iy1Z}$-(C12MoM!Ll ze()106jXfqSN+z^)@$#zwb5TnJR&Z^u3>(JM#U6hJ%R6#s1Ox9!P#zOaOM=RO!*)tN^&Jva02)?kEY6+REvisz9fFS6 zB8O-jg7t8k+h^RiK?UcmCwL2`31qOi{g$`I+eNzvs=NVVFv8zjCVyj@_8ab?_cfpQ z2>#j<_G|879{MC%V6(Q$Q##%nC&+WJOIMWP?7~D&dz-q4h&P|KEw}z^WO-~4M;iA)n;Cx&H}x<{i7zVKc<7X RARPOK!yUHE9O3Y7^WPR!KOg`A literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py b/vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py new file mode 100644 index 00000000..8f78ef65 --- /dev/null +++ b/vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py @@ -0,0 +1,66 @@ +from abc import ABC, abstractmethod +from typing import List, Optional + +from vllm.config import TokenizerPoolConfig +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.tokenizer import AnyTokenizer + + +class BaseTokenizerGroup(ABC): + """A group of tokenizers that can be used for LoRA adapters.""" + + @classmethod + @abstractmethod + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig], + **init_kwargs) -> "BaseTokenizerGroup": + pass + + @abstractmethod + def ping(self) -> bool: + """Check if the tokenizer group is alive.""" + pass + + @abstractmethod + def get_max_input_len( + self, + lora_request: Optional[LoRARequest] = None, + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + pass + + @abstractmethod + def encode(self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + """Encode a prompt using the tokenizer group.""" + pass + + @abstractmethod + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + """Encode a prompt using the tokenizer group.""" + pass + + @abstractmethod + def get_lora_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + """Get a tokenizer for a LoRA request.""" + pass + + @abstractmethod + async def get_lora_tokenizer_async( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + """Get a tokenizer for a LoRA request.""" + pass + + def check_health(self): + """Raise exception if the tokenizer group is unhealthy.""" + return diff --git a/vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py b/vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py new file mode 100644 index 00000000..9a999a0d --- /dev/null +++ b/vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py @@ -0,0 +1,240 @@ +import asyncio +import os +from typing import List, Optional + +try: + from ray.exceptions import ActorDiedError # type: ignore +except ImportError: + # For older versions of Ray + from ray.exceptions import RayActorError as ActorDiedError # type: ignore +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + +from vllm.config import TokenizerPoolConfig +from vllm.executor.ray_utils import ray +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.tokenizer import AnyTokenizer + +from .base_tokenizer_group import BaseTokenizerGroup +from .tokenizer_group import TokenizerGroup + +logger = init_logger(__name__) + + +class RayTokenizerGroupPool(BaseTokenizerGroup): + """A Ray-based pool of TokenizerGroups for async tokenization.""" + + # Class to use for workers making up the pool. + _worker_cls = TokenizerGroup + + @classmethod + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig], + **init_kwargs) -> "RayTokenizerGroupPool": + if not tokenizer_pool_config: + raise ValueError("tokenizer_pool_config must not be None.") + ray_actor_options = (tokenizer_pool_config.extra_config or { + "num_cpus": 0 + }) + ray_actor_options.setdefault( + "scheduling_strategy", + NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), soft=True)) + + # Carry over the env vars to the actors. + # This is necessary for API keys and such. + ray_actor_options.setdefault("runtime_env", {}) + _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"]) + + init_kwargs["num_actors"] = tokenizer_pool_config.pool_size + init_kwargs["ray_actor_options"] = ray_actor_options + + return cls(**init_kwargs) + + def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int, + max_input_length: Optional[int], num_actors: int, + ray_actor_options: dict, **tokenizer_config): + # Store a local copy of the TokenizerGroup for quick access + # to underlying HF tokenizers. + self._tokenizer_config = { + "tokenizer_id": tokenizer_id, + "enable_lora": enable_lora, + "max_num_seqs": max_num_seqs, + "max_input_length": max_input_length, + **tokenizer_config + } + self._local_tokenizer_group = self._worker_cls( + **self._tokenizer_config, ) + + self._ray_tokenizer_group_cls = ray.remote( + self._worker_cls).options(**ray_actor_options) # type: ignore + self.tokenizer_actors = [self._init_actor() for _ in range(num_actors)] + self._idle_actors: Optional[asyncio.Queue] = None + + # If set, actor is unhealthy. Will reraise on the next + # check_health call. + self._exception: Optional[ActorDiedError] = None + + def _init_actor(self) -> ray.ObjectRef: + return self._ray_tokenizer_group_cls.remote(**self._tokenizer_config) + + @property + def pool_size(self) -> int: + return len(self.tokenizer_actors) + + def ping(self): + return ray.get([ + actor.ping.remote() # type: ignore + for actor in self.tokenizer_actors + ]) + + def _ensure_queue_initialized(self): + if self._idle_actors is None: + self._idle_actors = asyncio.Queue() + for actor in self.tokenizer_actors: + self._idle_actors.put_nowait(actor) + + def _finalize_encode(self, actor: ray.ObjectRef, + original_actor: ray.ObjectRef, actor_is_alive: bool): + assert self._idle_actors is not None + # Cleanup the dead actor. + if not actor_is_alive or original_actor is not actor: + self.tokenizer_actors.remove(original_actor) + if actor_is_alive: + # Put the actor back in the queue. + # This is done in a finally block to ensure that the actor is + # always put back in the queue, even if an exception/cancellation + # is raised. + self._idle_actors.put_nowait(actor) + # Add back the new actor. + if original_actor is not actor: + self.tokenizer_actors.append(actor) + + def encode(self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + """Encode a prompt using the tokenizer group. + + We pick an idle actor and use it to encode the prompt. + The actor is then put back in the queue for future use. + This is blocking. + """ + self.check_health() + self._ensure_queue_initialized() + assert self._idle_actors is not None + + if self._idle_actors.empty(): + raise RuntimeError("No idle actors available.") + actor = self._idle_actors.get_nowait() + actor_is_alive = True + original_actor = actor + try: + ret = ray.get( + actor.encode.remote(request_id=request_id, + prompt=prompt, + lora_request=lora_request)) + except ActorDiedError as e: + # If the actor is dead, we first try to reinitialize it. + logger.warning("%s died with ActorDiedError, reinitializing.", + actor, + exc_info=e) + actor = self._init_actor() + try: + ret = ray.get( + actor.encode.remote(request_id=request_id, + prompt=prompt, + lora_request=lora_request)) + except ActorDiedError as e: + logger.error( + "%s died for second time in a row, marking " + "RayTokenizerGroupPool as unhealthy.", actor) + actor_is_alive = False + if not self._exception: + self._exception = e + self.check_health() + finally: + self._finalize_encode(actor, original_actor, actor_is_alive) + return ret + + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + """Encode a prompt using the tokenizer group. + + We pick an idle actor and use it to encode the prompt. + If there are no idle actors, we wait until one becomes + available. + The actor is then put back in the queue for future use. + This is non-blocking. + """ + self.check_health() + self._ensure_queue_initialized() + assert self._idle_actors is not None + + actor = await self._idle_actors.get() + actor_is_alive = True + original_actor = actor + try: + ret = await actor.encode.remote(request_id=request_id, + prompt=prompt, + lora_request=lora_request) + except ActorDiedError as e: + # If the actor is dead, we first try to reinitialize it. + logger.warning("%s died with ActorDiedError, reinitializing.", + actor, + exc_info=e) + actor = self._init_actor() + try: + ret = await actor.encode.remote(request_id=request_id, + prompt=prompt, + lora_request=lora_request) + except ActorDiedError as e: + logger.error( + "%s died for second time in a row, marking " + "RayTokenizerGroupPool as unhealthy.", actor) + actor_is_alive = False + if not self._exception: + self._exception = e + self.check_health() + finally: + self._finalize_encode(actor, original_actor, actor_is_alive) + return ret + + def get_max_input_len(self, + lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + return self._local_tokenizer_group.get_max_input_len(lora_request) + + def get_lora_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + return self._local_tokenizer_group.get_lora_tokenizer(lora_request) + + async def get_lora_tokenizer_async( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + return await self._local_tokenizer_group.get_lora_tokenizer_async( + lora_request) + + def check_health(self): + if self._exception: + raise RuntimeError( + "TokenizerGroupPool is unhealthy.") from self._exception + + +def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None: + """Copy over all current process environment variables to the runtime_env. + + The variables in runtime_env will take precedence over the current process + environment variables. + + runtime_env will be modified in place.""" + env_vars = os.environ.copy() + runtime_env.setdefault("env_vars", {}) + env_vars.update(runtime_env["env_vars"]) + runtime_env["env_vars"] = env_vars diff --git a/vllm/transformers_utils/tokenizer_group/tokenizer_group.py b/vllm/transformers_utils/tokenizer_group/tokenizer_group.py new file mode 100644 index 00000000..e516eeab --- /dev/null +++ b/vllm/transformers_utils/tokenizer_group/tokenizer_group.py @@ -0,0 +1,99 @@ +from typing import List, Optional + +from vllm.config import TokenizerPoolConfig +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.tokenizer import (AnyTokenizer, + get_lora_tokenizer, + get_lora_tokenizer_async, + get_tokenizer) +from vllm.utils import LRUCache + +from .base_tokenizer_group import BaseTokenizerGroup + + +class TokenizerGroup(BaseTokenizerGroup): + """A group of tokenizers that can be used for LoRA adapters.""" + + def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int, + max_input_length: Optional[int], **tokenizer_config): + self.tokenizer_id = tokenizer_id + self.tokenizer_config = tokenizer_config + self.enable_lora = enable_lora + self.max_input_length = max_input_length + self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config) + self.lora_tokenizers = LRUCache[AnyTokenizer]( + capacity=max_num_seqs if enable_lora else 0) + + @classmethod + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig], + **init_kwargs) -> "TokenizerGroup": + return cls(**init_kwargs) + + def ping(self) -> bool: + """Check if the tokenizer group is alive.""" + return True + + def get_max_input_len(self, + lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + return self.max_input_length + + def _raise_if_input_too_long(self, + encoded_tokens: List[int], + lora_request: Optional[LoRARequest] = None): + input_length = len(encoded_tokens) + if lora_request: + max_input_length = (lora_request.long_lora_max_len + or self.max_input_length) + else: + max_input_length = self.max_input_length + if max_input_length is not None and input_length > max_input_length: + raise ValueError("Input too long.", input_length, max_input_length) + + def encode(self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + tokenizer = self.get_lora_tokenizer(lora_request) + ret = tokenizer.encode(prompt) + self._raise_if_input_too_long(ret, lora_request) + return ret + + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None) -> List[int]: + tokenizer = await self.get_lora_tokenizer_async(lora_request) + ret = tokenizer.encode(prompt) + self._raise_if_input_too_long(ret, lora_request) + return ret + + def get_lora_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + if not lora_request or not self.enable_lora: + return self.tokenizer + if lora_request.lora_int_id not in self.lora_tokenizers: + tokenizer = (get_lora_tokenizer( + lora_request, **self.tokenizer_config) or self.tokenizer) + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) + return tokenizer + else: + return self.lora_tokenizers[lora_request.lora_int_id] + + async def get_lora_tokenizer_async( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + if not lora_request or not self.enable_lora: + return self.tokenizer + if lora_request.lora_int_id not in self.lora_tokenizers: + tokenizer = (await get_lora_tokenizer_async( + lora_request, **self.tokenizer_config) or self.tokenizer) + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) + return tokenizer + else: + return self.lora_tokenizers[lora_request.lora_int_id] diff --git a/vllm/transformers_utils/tokenizers/__init__.py b/vllm/transformers_utils/tokenizers/__init__.py new file mode 100644 index 00000000..5f437d41 --- /dev/null +++ b/vllm/transformers_utils/tokenizers/__init__.py @@ -0,0 +1,3 @@ +from .mistral import MistralTokenizer + +__all__ = ["MistralTokenizer"] diff --git a/vllm/transformers_utils/tokenizers/__pycache__/__init__.cpython-310.pyc b/vllm/transformers_utils/tokenizers/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e6b3a2cac5965c0dc5df971374e1d54b792715b GIT binary patch literal 245 zcmd1j<>g`k0>icYGTealV-N=!FabFZKwK;aBvKes7;_kM8KW2(8B&&M4u=4F<|$LkeT-r}&y%}*)KNwotxuoz?)4+9qy085KN At^fc4 literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/tokenizers/__pycache__/mistral.cpython-310.pyc b/vllm/transformers_utils/tokenizers/__pycache__/mistral.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3399cc7fe7be6f37b66c89366641d0567398798 GIT binary patch literal 8555 zcmbtZO>7*;mF}+onI8^+G?XZjwB&K5SROkX%5faqiYzEAOO}(ER!lmHJF))*SPL$ zPWr1m4DZZ00@E>dYBPK*upL{~O+OQ4ootYEazWn72V>4yP;d&txHGQ$Eq@}IbS8r- zXG*=>{;{Cw6jeQg`m}Rg)wBMIV8)qI^_+h)IOUuQW}R7G6Ir>4IrIKpFz?Lc-F#Ey zW4v%rVi*DSiwT$99-A&GjX%T>KRK(iY2llH=95 zTJx(>gctKxHC`**$@ud7x30U_ZeG9k?Y9=+OpI@aTZ#FGSBn$#J6;qg*>{?;*9fbA zV!aolQf7&Dv;NJdmrSnJ-L=+=%Nrksexu64^u)EbYJ9B`G<`vX-W7LRB8t&5^)`6P zs=wUW5TVx=5^Xa}B8){?6Sq836Z<_ZDK0fd&FfO%YWQAlE6Fd5js3B62Pb-4L{W8B z+^WiIP^0Y*{uy|=iYJ;wAv8zln#11HgdxmXO;}vNj6;T?j3j$KtTnh7t|mDzY_?+8 zSLLdRFT4`8)`AytC(b)TAsATbwM4}FgSliXP{y_hk z_K8;4IpfB?Y)5-T+tIi6n053Wotqn`JQo|>O5cB>ar?f8GSi_l`^(IZ{&9~sx54D4 zLum7Zv>l!2W;MurY_a|9hy5+xu0XKDzg^j^Jb$k9VXRL3c4a+kgy%*!VPWmlv^G4QYCU|niW*z+8Q*MRW2qb$>pK$6zJD&R^wR6Fg+pS zmgFz{kZlc0eB}&HK|-6;+UM(D$ld;OT^e1v>^EvvKe~cSZ_KA8@V0%U(F%E~eY&Jh zp`;c7#VW;XqST%9QmOrNrSx`nLzJSH6s1P!Z=n#SuC0{%la;)v)RZEEy_CGLl;{GSTSyFFgh}R|rR!2QWSPl%EZ-Sd>lIw>GD?L5?9`-^ z(DWgqpkWcoE3X$>L7&i>ZtgLbe_Sy3>seEt2g!(q5be<75&kB^Yuw1UyZJy)#dp003D{_2nuvggVE%5PsCVKN?j5AT7aXrXH zD?oIhmEe$oy6$l{Df*b)^C~NCsj*PwM>KN6rWWs$NMyMd|tJj zz>LJ@DSjF|o5`KQzrio@r?K{unC8#m`4Vv{ehTIUoe+ADD|d6_?t)_KhF4gP{C@UK8}7r>wRJw^OTl3i+>SNH8BcWJV}GIj66MEwhhenA3AU`o z18~$^;u2Q(8NskeaFn^tw)Jg>Vfx?hLy3Z z*NH7*wyoLB+_U%E7lvJF#dW>Vi(PjBj{4(|T&sK@Oa~@@;ovK+Lsn3HgDMLs+9%%& zqgJ!okg?$V*mS8>o=h@bcjOek5PrT@i}#Vz$@bdhb6}I?y@(pBaQt$Lm`|c`?6=-r zeCOTk&haR9{ponOnL7WR2wU)j)mS*zCLnhuv0e2mCx~Tkd8;WDfF)*(DqOi43BRr; zbn<;Aa|iaT1P#ZYitYl~L{Fix^{k#{c|11D1HSU6tvw;{wz6!>Dy(Z+g@$~ijukjK?r+4&S{Q+DWIoe%=+%nuR+hEc3w-9+M zE!1{RSO9y#wvF&?cZ~HwlNVz<&g^E%`@waaaW3uanDgYyjc~k63tNa@2RCr(X?8re zTj&@M^lhu7b*vqV7_G(M>kFTf9kxGsztJkKRX0URcyL`3Zm6;8aVS8w6gI*O;x7Ci zxh2~2rP7avcD`L|$Wk?g)7q^19xvUzQM%b$DUn=C9$!VHmq`!Y6^IB#CKjO_#2oY zB3mq2$CcAG4O6%G3XDM$&AlSSpU(D7)agBMWc7B>$uIZf!O9TR!NpR1+E+fzqfw+v z0h9vOOQ^)!jz(Cu3y6dwK=lY1JH|Rwk)RnH9eBiT3(yG_LHuU?+}O4|HdMtt2!^CH zbbsa8 zik#%PsryYT`Z2!>n&cI$W_U=X$gz{b-dHB z_0K#hjxr92?7Jx1XP;mh|F7962WQ9a?@oW}QJWhgJ#t>w4ryx+rYwPXN zEq94-BsQFH(2QXsoDUgol1J1P)>O(N$p_WD#CSDcs}a!m{%B={&Pq<_Q0ZvtjTye* zkmDVTYaR2axC{X5b-d|6r)Yp7btI=j4bZfat^q)*1Vh-jN=y!IQ)ar-PfbQkB1_S(PS_!p9Kl0T)f$Oed#Kiu`B_A8@^J&E$TS>s`K4 zW{v^tcMOHe5g~RiF)1qna(X0b)|Id6YUcn`2kyvP2(_wD0gA1kf_-)mym`V|oGM zD}e3w5&6~{;iiz?t0K7@7eP7#uEQd>1E#5h5}JY@n)D|ykPouRSwgO{zQewUKS^LiTJ18H15k z9I6GW4s`Tc?Y@5UK&;-?f;l(M$);h!S3T~Ajc(+q)&h+n%)xza&8x?$eESOV+rdRm zU{MLh(sU>)5>knP#HfZ_P7yc#VdrLZL2f58*Bf4lP+eggS)wyWCz`G(qKpdgAf@fX zK$B*mRV1g5(zbs>uR0I7A_~U$iWV;aI%D?Z0^)NjO*{X&X%-$M5H3HlKFCclJ^&S@ zwgeN`Xhupl^wd=X9#)ZZ2V}BcgG{=oxMnJ&lORq76{FTecbZ^Gk0tg0JD!NJIj`HW zzC#c`mC z&8z2J_p?$_Wkl=A_NMWCb_#{EJpcx2J+8fw_~7ldtMOydaez{jK2P=VjvWZs`}+M~ z#F)+(ka6iq6@h*d!xTZE2<4RA!B(k@LOFcs08D5PG_Gy*a88A3Eu#J3fBRSZ(q|En zHV=h}2B`jw4hweUePSFq6u?<_Q_;ZHg4Ql>_(%@g77>3FE^Q-?zt`xo%2M0{35X6hs} z?MIZIvJLi#8HGpe1f+;}IQ=~~fqa(Hd!|s|eB;pb7*^Hcd2c2;&b%{q5QPGiX|nM6szK{JWKVl{%3CZda}tR zL+M3)U+rcWks=1LNHPGa#GvGpp;vVpBnxM9t+l!evU;^9=rgy<52+-O^IU+WB0gqQ zoEkL3O4pv%4}IH?DqVs!z9qRS6>xg(M-QY=e28O3Y9f@}cFzCF*;2o(^61KuI`i}a zxdK*ls|k-80}UyxRv(d7Ft09fss5d()g5x&N%nI3rTYrqY@-s2td@m4Dd;7AiWP8k RD;QJEG!Wk5WMMH=e*{R*Aua#_ literal 0 HcmV?d00001 diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py new file mode 100644 index 00000000..b7e33ae5 --- /dev/null +++ b/vllm/transformers_utils/tokenizers/mistral.py @@ -0,0 +1,229 @@ +import os +import regex as re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from huggingface_hub import HfApi, hf_hub_download +# yapf: disable +from mistral_common.tokens.tokenizers.mistral import ChatCompletionRequest +from mistral_common.tokens.tokenizers.mistral import ( + MistralTokenizer as PublicMistralTokenizer) +# yapf: enable +from mistral_common.tokens.tokenizers.sentencepiece import ( + SentencePieceTokenizer) +from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy, + Tekkenizer) + +if TYPE_CHECKING: + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam + + +@dataclass +class Encoding: + input_ids: List[int] + + +def find_tokenizer_file(files: List[str]): + file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$") + + matched_files = [file for file in files if file_pattern.match(file)] + if len(matched_files) > 1: + raise OSError(f"Found {len(matched_files)} files matching the " + f"pattern: {file_pattern}. Make sure only one Mistral " + f"tokenizer is present in {files}.") + elif len(matched_files) == 0: + raise OSError(f"Found {len(matched_files)} files matching the " + f"pattern: {file_pattern}. Make sure that a Mistral " + f"tokenizer is present in {files}.") + + return matched_files[0] + + +class MistralTokenizer: + + def __init__(self, tokenizer: PublicMistralTokenizer) -> None: + self.mistral = tokenizer + self.instruct = tokenizer.instruct_tokenizer + + tokenizer_ = tokenizer.instruct_tokenizer.tokenizer + if isinstance(tokenizer_, Tekkenizer): + # Make sure special tokens will not raise + tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE + + self._vocab = { + token: idx + for idx, token in enumerate(tokenizer_.vocab()) + } + elif isinstance(tokenizer_, SentencePieceTokenizer): + self._vocab = { + token: idx + for idx, token in enumerate(tokenizer_.vocab()) + } + else: + raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}") + + self.tokenizer = tokenizer_ + + @classmethod + def from_pretrained(cls, + path_or_repo_id: str, + *, + revision: Optional[str] = None) -> "MistralTokenizer": + if not Path(path_or_repo_id).exists(): + assert len(path_or_repo_id.split("/")) == 2, ( + "You have either provided a non-existent path: " + "{path_or_repo_id} or an invalid HF Hub repo id.") + tokenizer_file = cls._download_mistral_tokenizer_from_hf( + path_or_repo_id, revision) + elif Path(path_or_repo_id).is_dir(): + tokenizer_file_name = find_tokenizer_file( + os.listdir(path_or_repo_id)) + tokenizer_file = str(Path(path_or_repo_id) / tokenizer_file_name) + else: + assert Path( + path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}" + + mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file) + return cls(mistral_tokenizer) + + @staticmethod + def _download_mistral_tokenizer_from_hf(tokenizer_name: str, + revision: Optional[str]) -> str: + api = HfApi() + repo_info = api.model_info(tokenizer_name) + files = [s.rfilename for s in repo_info.siblings] + + filename = find_tokenizer_file(files) + + tokenizer_file = hf_hub_download(tokenizer_name, + filename=filename, + revision=revision) + return tokenizer_file + + # the following attributes are set to fit VLLM's design + @property + def all_special_tokens_extended(self) -> List[str]: + return [] + + @property + def all_special_tokens(self) -> List[str]: + return [] + + @property + def all_special_ids(self) -> List[int]: + return [] + + @property + def bos_token_id(self) -> int: + return self.tokenizer.bos_id + + @property + def eos_token_id(self) -> int: + return self.tokenizer.eos_id + + @property + def is_fast(self) -> bool: + return True + + @property + def vocab_size(self) -> int: + return len(self._vocab) + + def __len__(self) -> int: + return self.vocab_size + + def __call__( + self, + prompt: str, + add_special_tokens: bool = False, + truncation: bool = False, + max_length: Optional[int] = None, + ): + # Mistral Tokenizers should not add special tokens + input_ids = self.encode(prompt) + + if truncation: + input_ids = input_ids[:max_length] + + return Encoding(input_ids=input_ids) + + def get_vocab(self) -> Dict[str, int]: + return self._vocab + + def get_added_vocab(self) -> Dict[str, int]: + # Mistral tokenizers have no added vocabulary + return {} + + def encode(self, prompt: str) -> List[int]: + # `encode` should only be used for prompt completion + # it should never be used for chat_completion. + # For chat completion use `apply_chat_template` + return self.tokenizer.encode(prompt, bos=True, eos=False) + + def apply_chat_template(self, + messages: List["ChatCompletionMessageParam"], + tools: Optional[Dict[str, Any]] = None, + **kwargs) -> List[int]: + + request = ChatCompletionRequest(messages=messages, + tools=tools) # type: ignore[type-var] + encoded = self.mistral.encode_chat_completion(request) + + # encode-decode to get clean prompt + return encoded.tokens + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + if isinstance(self.tokenizer, Tekkenizer): + tokens = [ + t for t in tokens + if t not in self.tokenizer._all_special_tokens + ] + + if any(isinstance(t, bytes) for t in tokens): + # we need to encode and decode all tokens again + shift = self.tokenizer.num_special_tokens + byte_tokens = [ + t.encode("utf-8") if not isinstance(t, bytes) else t + for t in tokens + ] + ids = [ + self.tokenizer._tekken_token2id_nospecial[t] + shift + for t in byte_tokens + ] + decoded = self.tokenizer.decode(ids) + else: + decoded = "".join(tokens) + else: + decoded = self.tokenizer.decode(tokens) # type: ignore[arg-type] + + return decoded + + def decode(self, ids: Union[List[int], int]) -> str: + if isinstance(ids, int): + ids = [ids] + return self.tokenizer.decode(ids) + + def convert_ids_to_tokens( + self, + ids: List[int], + skip_special_tokens: bool = True, + ) -> List[str]: + # TODO(Patrick) - potentially allow special tokens to not be skipped + assert ( + skip_special_tokens + ), "Skipping special tokens is not supported for Mistral tokenizers." + + assert isinstance(self.tokenizer, + (Tekkenizer, SentencePieceTokenizer)), type( + self.tokenizer) + + tokens = [self.tokenizer.id_to_piece(id) for id in ids] + + if any(t.strip() == "�" for t in tokens): + # if any stripped decoded token is undefined + # because it's invalid unicode then pass bytes + # See: https://github.com/vllm-project/vllm/pull/8640 + tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids] + + return tokens diff --git a/vllm/transformers_utils/utils.py b/vllm/transformers_utils/utils.py new file mode 100644 index 00000000..7a9041b0 --- /dev/null +++ b/vllm/transformers_utils/utils.py @@ -0,0 +1,16 @@ +from os import PathLike +from pathlib import Path +from typing import Union + + +def check_gguf_file(model: Union[str, PathLike]) -> bool: + """Check if the file is a GGUF model.""" + model = Path(model) + if not model.is_file(): + return False + elif model.suffix == ".gguf": + return True + + with open(model, "rb") as f: + header = f.read(4) + return header == b"GGUF" diff --git a/vllm/triton_utils/__init__.py b/vllm/triton_utils/__init__.py new file mode 100644 index 00000000..80f7a329 --- /dev/null +++ b/vllm/triton_utils/__init__.py @@ -0,0 +1,9 @@ +from vllm.triton_utils.importing import HAS_TRITON + +__all__ = ["HAS_TRITON"] + +from vllm.triton_utils.custom_cache_manager import ( + maybe_set_triton_cache_manager) +from vllm.triton_utils.libentry import libentry + +__all__ += ["maybe_set_triton_cache_manager", "libentry"] diff --git a/vllm/triton_utils/__pycache__/__init__.cpython-310.pyc b/vllm/triton_utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ddb7112f1770e107aef8ec0038e1eee53ab2db4 GIT binary patch literal 418 zcmZWm!Ab)$5S^skZ7D5?M+J{U58cpHK}76LFBNQ0xeVJ)*bOAROE#Y1MI znTSW80TJoGiFr&wQqdQ2P8Z-cgEdTtd+&!S*^M5SYur3PZk|?_{!mvY?Y0sd>Cjo_ zsuHs_DezG1VH8 zU!O{|wsDpDzw0ocU!02n0)*XfVB?S@z}J#FQ?pvtGnd+2y4k|Cg_)aI<~^+LTv3%Q k3}nYa8KNqcLrm&!H@r2KXtlia7l0-lWDGGK`*%ja0qQYxmjD0& literal 0 HcmV?d00001 diff --git a/vllm/triton_utils/__pycache__/custom_cache_manager.cpython-310.pyc b/vllm/triton_utils/__pycache__/custom_cache_manager.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6359ac79ad8b723b1e7088cb8ec946fe2b032eb2 GIT binary patch literal 2066 zcmbsqO>f&al$2yyahxnIvZCmKfmk;T1{!wZBoAUW=^%YMLqh^{*g*e@6`bdQwWC1`gV4L*~P@B2$~VWERyt^V@c_}2)b z-|gk{3gGexw%!835W^{o?Z21e%!|FukNqr&gRB*|u*Lgnn6=|JM!e0VQykB+Io4v~ zSrB&~Bi3fo8DbIlPW^bE;ce9G`~h;&uoqNIn5Fs(fU1&6WoT>)kqR{S5$v4)D3h zF!L_FmuLz;mPp-V{!489^MD0YeBog<#g>lY`|!T)cCU5vbekK(^CO{Tp7Gp}qeO{h zH|4}gVtAU89VLv+?X}dLBpm{4RB9tL;=ChS?R31nv-Nc6`ypLlTYs`apREno9&c<7 zPwpP2Y1TJR3N4LDb-!kBg43|yTX!-qhw`9$|xl_z!8ZhWZx zeQv6-iC!S`J=yc(uH&Sj%Eq+asQekJMTp*C&juzd28AeAOrn5&O z+>sCa5o~=607o%qD26o0J`0)8f-@WktOcdmuCBiwz2qDtw8d9MR;1jfRySe%R9{Wp z9mu~fm5s9*C#FepnTzKoZz!RF`N&8$Ap)2cPYh>dPb$KpzDc3vh-=*^J7ahj+LcJo zIRhdno8(9eMn*DCg%&c`Q; z6B?qrTUrZtlhGm1S-<1>hSEUQ)+T5${TMj*#0yI|A$2BJQmB9?i-V#})4`XY-~H;2 zV+WB=0kBvvb7{x~dPC%+w6vZG=mA&64;~zS(f(GdMisQ3tu4? z0#)|{y+S{M4gbd?v%uP?_y^pJhE5at1lma1i|u28lk9j*|p>TZ>}5gFf5N-Fpc1e&LN>V_qWW;4AC?7Vk#rUvi1 zXG-0Wrvz@g4!?)Uyo=CJ2&Zo*iM zD9w|MQ(AQ>%_J*Rd+t*DyiC&OrA;Z5BTCgrV8bOV?OasWRCNu&`>=HgAVL9N>^NM{ zQ*FQudtDd9c)4zlKJ>MXN#BK7*-w=VnhU%OX*};f2=#joL;i>F$eFA1$N@tP3nX!danpOklYZu8 z+~>gy&N|G`LLQa}yz{!pg12U>Giz%UemgxH;d*}sijlJ5OGkR3z!IVJoSVz%+qnU& zRjjm)L6w#Y!P#-V8ICklct1W%@;s2+kW zw6OvfNo7G8&n=Mko`jIe&e1f-)e#|zN=`@}?w+&m_4$megP((lMbZnTA; zi`2>@Eua`%YI1JJhq7ABiQJ^lDJ`~ktyS8%q_#bWH0EWxTLx5~6x%vFk(yP4pSmwc R<FJs6omsCxH;I!@;y?!MUB`9|F)J%p97ssm1Z)HvVNH6cYj%6K zr)OMMvu2kbC1Q=>WP!MF!p+%7`%z5&HxCj$s()CBNlO-v?Jg!7)p_gZz(DRlb3t% z$LnvWTJE&z^yHzwjrJCr+6Ks2X0-sj^kor2uI)VwUnjG=)`;m+#7U(OjJzBu+Z3ZxTCbZqagnAxe11E3IjTqgj zIMHYz{}sQ@Z&(klhj#AAs@3kNN{b&4WPT*-x6;0Dsex$6VG{3(+)jtRhdk%Y=W{lY zq#OhOb!|oW1}lSja88GEL+Eo~>kT)1ul4S%B=P#nU`KEE)0OQc>8)rP>welA>Nruf zC`hQwgPpv(n#4-C`@O-{6TtfqV#u{Pa~OZ@a5UzQcVC&PVY$_c(^$7!%d1Jh9VY53 zI+OGNuN)*vV_yyhQzhc8GGVWn^JzE_@CZMA6UPzy7mbKh%MgXX-PX7gVV6i zzFHv&FA8F#X^e$tK5CO8%(9OC3gNNGbLV)yEf4=yf$Wd^@-Q?c@1MlNa5;u z!D9*;fa{Wc0bjnwo}EYmtV#g%91a%0#w z=7>f`PFiIg+Hh@Na{s;H?fSD?Ib?0bOBwA0I{ABq&kRXn$r{-k*2qCCp}A;fvc5eS z^sUB^_{bYoG7lk{ZCmn8R=&@($^*t=*puGv1+#8r{nE^OX=dG>>04c=ThcDw5LrLt zOBU|=`ly;!yXA}n$9FSMeb63quWvC%xujB2qps_P?JO)qwj zc8|e2|2`cZhoAm2t94I|7P5uz$*dmnRSUgS=-H;Xgr2i%0cZU)hLvYNW21Wd>J-DX z!0;Dfc<%n`2PBUZko%F*VzwAt`g#2VRszV2?=hU_XutyBuV+VAt=oS}9sOcd+G9Uu z*au3dSV5)yEvSdQfpgKwcn&;zB;TxV_@%xFOqc<#e`fPpy~9ZwnGee6Z%{Huf5`$x z^N_$xnY~AHdLwe9G6G$_pctXAzE7+5G_9kfnVlWQ9vigT`sFhR=#J!?m$z+U(1~h8Ao{q>AH0%lmnGlsf)a`fqdan zzpX>Hd+ObE8<{-{zE@y{6m*@$Oc!}^Del4!<%ZHfuw zkO;YDq&j%B&fS*i4fIaKE55Tn+xHERyOfW^RoC&7W zBMttD!n(XP1vGp_q&Hs4D*`MF8EPRZd*^OA7>G2I&(iKw1W@)=Snce#(!!>poL<33+RG2-x=DNX2H73XVg5qN5pc2Od7+y8=(y)sIdzxl_H z0W<%k#+=i&zak-f!1h+^xLVds0 zx;+e&Vnvn-{XfRJ_qcr8hX)HB`lgH4fwb;M*`b|9-j z;qU7#X+((tzh?u5(`m2{89|NfeXS9>M0*Q*OAn-ZSX#?ekQ%23HsZGfJ2`|Bve=jN zM8H$JCYEAokYar&01DaPF4T2kQHTp7RlxhCITR{d`{3r=2SPT??h^@(4+k4Ej6_5_ zfnh4hCAheJe);@avhITNf@_f314Yrrs(2Cq8O2A^9|F!Q$72 ztu>GWF@w*4``M~Y?fFykZxa*P?z0od4pQ25Uy($ad=L9);) None: + """Set environment variable to tell Triton to use a + custom cache manager""" + cache_manger = os.environ.get("TRITON_CACHE_MANAGER", None) + if cache_manger is None: + manager = "vllm.triton_utils.custom_cache_manager:CustomCacheManager" + logger.info("Setting Triton cache manager to: %s", manager) + os.environ["TRITON_CACHE_MANAGER"] = manager + + +class CustomCacheManager(FileCacheManager): + """Re-implements Triton's cache manager, ensuring that a + unique cache directory is created for each process. This is + needed to avoid collisions when running with tp>1 and + using multi-processing as the distributed backend. + + Note this issue was fixed by triton-lang/triton/pull/4295, + but the fix is not yet included in triton==v3.0.0. However, + it should be included in the subsequent version. + """ + + def __init__(self, key, override=False, dump=False): + self.key = key + self.lock_path = None + if dump: + self.cache_dir = default_dump_dir() + self.cache_dir = os.path.join(self.cache_dir, self.key) + self.lock_path = os.path.join(self.cache_dir, "lock") + os.makedirs(self.cache_dir, exist_ok=True) + elif override: + self.cache_dir = default_override_dir() + self.cache_dir = os.path.join(self.cache_dir, self.key) + else: + # create cache directory if it doesn't exist + self.cache_dir = os.getenv("TRITON_CACHE_DIR", + "").strip() or default_cache_dir() + if self.cache_dir: + self.cache_dir = f"{self.cache_dir}_{os.getpid()}" + self.cache_dir = os.path.join(self.cache_dir, self.key) + self.lock_path = os.path.join(self.cache_dir, "lock") + os.makedirs(self.cache_dir, exist_ok=True) + else: + raise RuntimeError("Could not create or locate cache dir") diff --git a/vllm/triton_utils/importing.py b/vllm/triton_utils/importing.py new file mode 100644 index 00000000..760be677 --- /dev/null +++ b/vllm/triton_utils/importing.py @@ -0,0 +1,11 @@ +from importlib.util import find_spec + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +HAS_TRITON = False + +if not HAS_TRITON: + logger.info("Triton not installed; certain GPU-related functions" + " will not be available.") diff --git a/vllm/triton_utils/libentry.py b/vllm/triton_utils/libentry.py new file mode 100644 index 00000000..4335c7ad --- /dev/null +++ b/vllm/triton_utils/libentry.py @@ -0,0 +1,167 @@ +# Copied From https://github.com/FlagOpen/FlagGems + +import inspect + +import triton + + +class LibEntry(triton.KernelInterface): + + def __init__( + self, + fn, + ): + self.fn = fn + self.arg_names = fn.arg_names + self.divisibility = 16 + self.kernel_cache = dict() + fn = self.fn + while not isinstance(fn, triton.runtime.JITFunction): + fn = fn.fn + self.jit_function: triton.runtime.JITFunction = fn + self.specialize_indices = [ + p.num for p in self.jit_function.params + if not p.is_constexpr and not p.do_not_specialize + ] + self.do_not_specialize_indices = [ + p.num for p in self.jit_function.params + if not p.is_constexpr and p.do_not_specialize + ] + + def key(self, spec_args, dns_args, const_args): + spec_key = [(arg.dtype, arg.data_ptr() % + self.divisibility == 0) if hasattr(arg, "data_ptr") else + (type(arg), arg) for arg in spec_args] + dns_key = [ + arg.dtype if hasattr( + arg, "data_ptr") else type(arg) if not isinstance(arg, int) + else "i32" if arg >= -(2**31) and arg <= 2**31 - + 1 else "u64" if arg >= 2**63 and arg <= 2**64 - 1 else "i64" + for arg in dns_args + ] + # const args passed by position + return tuple(spec_key + dns_key + const_args) + + def run(self, *args, **kwargs): + grid = kwargs["grid"] + # collect all the arguments + spec_args = [] # specialize arguments + dns_args = [] # do not specialize arguments + const_args = [] # constexpr arguments + k_args = [] # kernel arguments + for i, arg in enumerate(args): + if i in self.specialize_indices: + k_args.append(arg) + spec_args.append(arg) + elif i in self.do_not_specialize_indices: + k_args.append(arg) + dns_args.append(arg) + else: + const_args.append(arg) + for p in self.jit_function.params[len(args):]: + if p.name in kwargs: + val = kwargs[p.name] + elif p.default is inspect._empty: + continue + else: + val = p.default + + if p.is_constexpr: + const_args.append(val) + elif p.do_not_specialize: + dns_args.append(val) + k_args.append(val) + else: + spec_args.append(val) + k_args.append(val) + + entry_key = self.key(spec_args, dns_args, const_args) + + if entry_key not in self.kernel_cache: + # compile the kernel also completes the related computations + kernel = self.fn.run(*args, **kwargs) + fn = self.fn + # collect constexpr arguments for grid computation + constexprs = {} + while not isinstance(fn, triton.runtime.JITFunction): + if isinstance(fn, triton.runtime.Autotuner): + config = fn.best_config + constexprs["num_warps"] = config.num_warps + constexprs["num_stages"] = config.num_stages + constexprs["num_ctas"] = config.num_ctas + constexprs = {**constexprs, **config.kwargs} + elif isinstance(fn, triton.runtime.Heuristics): + for v, heur in fn.values.items(): + constexprs[v] = heur({ + **dict(zip(fn.arg_names, args)), + **kwargs, + **constexprs, + }) + else: + raise RuntimeError("Invalid Runtime Function") + fn = fn.fn + # In vLLM, certain kernels like fused_moe_kernel get the + # best_config(as kwargs) from a configuration json file, rather + # than using Autotuner & Heuristics. Therefore, all their constexprs + # (tl.constexpr) are assigned values through the following loop. + for p in self.jit_function.params: + if p.is_constexpr and p.name not in constexprs: + constexprs[p.name] = p.default #default=inspect._empty + self.kernel_cache[entry_key] = (kernel, constexprs) + else: + # load kernel from cache directly + kernel, constexprs = self.kernel_cache[entry_key] + + if callable(grid): + # collect all arguments to the grid fn，ie: + # 1. args, + # 2. kwargs, + # 3. all all other captured arguments in CompiledKernel from + # Autotunner & Heuristics when kwargs & captured args conflict, + # captured args have higher priority + # 4. We must filter out captured args with default value firstly + constexprs = { + k: v + for k, v in constexprs.items() if v is not inspect._empty + } + meta = { + **dict(zip(self.arg_names, args)), + **kwargs, + **constexprs, + } + grid = grid(meta) + if isinstance(grid, tuple): + grid = grid + (1, 1) + elif isinstance(grid, list): + grid = grid + [1, 1] + kernel[grid[0:3]](*k_args) + # maintaining the same return type as the JITFunction.run + return kernel + + +def libentry(): + """ + Decorator for triton library entries. + Motivation: + The runtime overhead of Triton kernels is the reason for the lower + performance of small kernels, particularly evident with smaller models. + Using this decorator can reduce Triton runtime overhead. + How: + The `run` function of JITFunction needs to accomplish: + - Parameter binding using inspect + - KernelArg type wrapping + - Cache key calculation + When dealing with small size, these steps can become bottlenecks in + Triton runtime. Libentry simplifies these steps to reduce runtime + overhead, thereby improving the runtime expenses of small kernels. + NOTE: + When Triton is upgraded to version 3.0.0, libentry can be removed, + see: https://github.com/vllm-project/vllm/pull/5036#issuecomment-2243396245 + + + """ + + def decorator(fn): + return LibEntry(fn) + + return decorator diff --git a/vllm/usage/__init__.py b/vllm/usage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/usage/__pycache__/__init__.cpython-310.pyc b/vllm/usage/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ebb66338b879e3aec1c275882658fed4799782c GIT binary patch literal 151 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HOeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_era)Hda8bWd}dx|NqoFsLFFwD Uo80`A(wtN~keS6yK!Sw<09T13s{jB1 literal 0 HcmV?d00001 diff --git a/vllm/usage/__pycache__/usage_lib.cpython-310.pyc b/vllm/usage/__pycache__/usage_lib.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7990bfe8df7e93ed79373dc420a58af55f406f5 GIT binary patch literal 6690 zcmaJ_OLH67mF_nhAP7DrQlccwt%vQvwn)o%CX>X;h$3m3u>@*J%1*kH>NL?8L{kkk zbo+udQDD{7(2LomrlxADW|IXtyDa7}WHB|{)GVj!Zf7;WAXTYYn(y2OC_qvIRG+?& z)Aw=id3@)l)6*plzq|kO`{u{jHSK??GX5)|at}}TkGiHk(IPF<6V1=xy05Fc;T!60 z`lfnYzNOx_Z{uwwPOIP-bo$0j+*Z*qsxzFxmEGiO>zaQ}m5a$+ zt?T~Vy2dUac>WD$9^4$qUo;cx4JVq7=DxN3C~#av^Lt^ z7QG*bqKUV&+vF^as+KfY+sD#ckLyCZTOARn?J$w{lQs(Av%B3m`V@u5{UqHBlb|U? zC#a|GHmegadS>fg&RJUoog@^El+!d*L2$@;28JN$&33bJp9Pr+MYep9rEOV`(x9D+ zK=82sR9*}oJlwj!`Y_mjQrq2pv=OYY?ygGnDLa<-VVHE8bT|`T-mV+8o#BtlCZ07s zSsA3S3$3p~#)gq~=I!aItS#M@(DhGnB7njXV)kB59s7ILx2 zLa_|V@d}2n8hi#XJ`3_YjnCutrys9~R%fLXckYOg?=x}d2d!?ib+>i2lEix}onz5V z+bf4j(ppio=QKdya_3ksW=sUU+ZJ()1-W?C>M(Fd8!H?9B3^Yo=Rex)`G!A6MPRF4 zjb9S_fpMaNnkTxj4(x#=3TOI>(bqZ)eI1Wc)t39BVdgc`BuS65AF0ke-u+`Q zVvVqyh!4G3cv&;;CJ~98wZpxHMfo6iFbrqZco_4v-J;_-40%i}nOaH|XH>x$`N%Z& z9q+4$4<7~VTS0AWH`v`?UHkGUch$Uk?_catpVhX4Cp)VTHiDhq)!m)phj+YHuMyKc zZMD;8+G1W5b0~QVy&$2yT!|MqXiw~$I1|GS=M&%lV(Zby$}+5@5%1?`Kd$I%iM6T= zg&$i%uu)sR|8QenT988K&%L%c|H2#AEsZpjRtLr`oqQqk`oyT?LI@tO?tW1<`81Tq zue8&l#;LRQ=f;;9kPwu9=i~yukc)9P{zWi6!gtV@kwv&=M|bqH?iz+}y|4^^6^#?B zi@IjvS;LcE0}-UI`oJKiJ+RL7fqe#LzNnol+pX<`MPq;X!)n!$X55Bnf^{%57HPKA zBk5AHm$p&uin=UP6}xG5#+FV-%GLUp0+?qo@>1u#K|A{~c$B?e)A>!5Cl*~qgN;Wm z`d?Al#}fZtKS7X+%*cxD$oZ{xqW86R?N_%>3^*U-R6jBM=0HEw_(ymfXWGC#(@w0u zKCsS+ukuUd#6EHQcHcVG_)6a~G+m=-M|*bT*>qHh+*m&;^b2GB?ujGp17}d^yJtFo zkbeXIA4Wx3bg5RgdpB0H<7}m#gju!{wc?dHTIujK>ej``4fo!9onpacUhVbes@1#7W0OLFRX}X^O@8~J5~ipN{)!TMUTx$Zl8 zO%|TizN~G14HM%g&Px`sI#Os^q*xaS_KlD>ycFZojs&2`Io79aLmdWqn zg9+tE9a=X;YqwEg8sRfwGab0h046hm$8b22?H3})ccTK7&6VXR6wB9WqY&oIP~gGZ zR&96V=ex2duZVtv=oib~d)Z+SrzJTaV%8HiNOI3&W=StGjDoP~@+Hud-2l zuvyy}4Ogocr5gmKfI%QjLC}IuL(YLL2f?#$nB+Zt1?%S@5xGl*Vk7^A$frbIP}dM!8pW1LUYOry%2~%!2$%nAsSGy#mfQ05E<|} zNT*gO#TxynOo&E#(E}97^`FAg_q70t04UM`P$U7MNC7|{MGK0$iWU_uDLSQSSdZ#O0kU$)Fm|UV#BW%UVG3gmg z09aY46SwzeS)>Bcx5Zij1F{_QdK2!W4oJ#i-$T?Zi~AU$o&wZKmx^lC1uEqq1|7y> z(}!`L$@zSSiIK_#7DxVC7L+rAjk!QsfeL+2mUela0bW}fOB!Q}tsDj{C2q$8R)QU7 z9!S}8ObX4f{PQ-mr1ut-H4imldGtNsAQ4Tg^UZx4mQjSg^DuaNnAItgkA4&&&x7Q< zs1OD}(EEDd=$jok)*5D{N5*eV1VM`eB4t+4KC`x4HKmh7XK4XJXVQs6NVpB4LNfp* zafT=$hS=jM@7`* zV4QnmpR}U$eO^JOuMLoK&@RtRooWdB`QG>s(Oos<{N(!6HqQOhsEtlHZl~QLvZ`8d z+RrPS;eU^T{{>G*tEs@d_ijwAel&P~a=E}Jc0u_$nSwG#3#KX5y7^^c9GC+u(vdb; zs%)PrL+o1z1sJ4@cM+qc8}Q_$iY%GKP9nTGZq(5c14zK0w5W z2M{sK;JuX_$h`%>Pm@~|%Qzhjrx4HAiBQ;94vVj%Bnzz&o~O9C5fzf)!o zsG3KPYT-@#@6Sw%4)+wa2We5R{Hk$0!L0v=C;J-^7Y0t?hyEq}DLJX=6@+=B72W;L zEs$4qO>h_%@u0u=_Fpk4vfqiwMp4bgql}3pSXAZ|8M%rb7{WZoCDv~ZOqkNTqYOqGPt?qJEnj0ors)o#Cj|~?895TUO38hs^MF1Er6}s#!VXTvPC1Vc_P+{R`E9L~)J6GhK4^ZhamiDTwHZ z#x7LU0jVasOQ{w)#Up=)Y&1J=*Mat9N+GOHnhAMfBBdXKX_vESU8K92EU|CuOkLS! za&|j!-552=0)WFl!52GXd)g^kUX9p7*Xz z3VYI$|FV1&3?_gL@E8MASZ9zekO8DCY+;@12ac-iBv*m_eQqm!%5cMz zsRT|IP$Um57bisivz@J)x;@DgO1Rc8`F1$+HYb<`SJZ8Fq^-Q$xP+ztDn3(+{y!su zE<%{(qRRTs=e?_w0-bP`unGmLb5t?QzmE|(xj6mX1o4Z6ub1hPEstEYstzA9 zyvojUQ>>`w5vtGf7_YGTG7Ofkv^m{2^N7ZrRY4Nj@8_Ejg6*xXUB66-8P|UST_Gyn z;&o#05xGKSfj+#XJ{-RW%&nWJ#G)LdI?QXRN^37o6FCzEVH>x!Dj~~oi@q5V; literal 0 HcmV?d00001 diff --git a/vllm/usage/usage_lib.py b/vllm/usage/usage_lib.py new file mode 100644 index 00000000..9ae46ff4 --- /dev/null +++ b/vllm/usage/usage_lib.py @@ -0,0 +1,223 @@ +import datetime +import json +import logging +import os +import platform +import time +from enum import Enum +from pathlib import Path +from threading import Thread +from typing import Any, Dict, Optional, Union +from uuid import uuid4 + +import cpuinfo +import psutil +import requests +import torch + +import vllm.envs as envs +from vllm.connections import global_http_connection +from vllm.platforms import current_platform +from vllm.version import __version__ as VLLM_VERSION + +_config_home = envs.VLLM_CONFIG_ROOT +_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "usage_stats.json") +_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home, "do_not_track") +_USAGE_STATS_ENABLED = None +_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER + +_GLOBAL_RUNTIME_DATA: Dict[str, Union[str, int, bool]] = {} + + +def set_runtime_usage_data(key: str, value: Union[str, int, bool]) -> None: + """Set global usage data that will be sent with every usage heartbeat.""" + _GLOBAL_RUNTIME_DATA[key] = value + + +def is_usage_stats_enabled(): + """Determine whether or not we can send usage stats to the server. + The logic is as follows: + - By default, it should be enabled. + - Three environment variables can disable it: + - VLLM_DO_NOT_TRACK=1 + - DO_NOT_TRACK=1 + - VLLM_NO_USAGE_STATS=1 + - A file in the home directory can disable it if it exists: + - $HOME/.config/vllm/do_not_track + """ + global _USAGE_STATS_ENABLED + if _USAGE_STATS_ENABLED is None: + do_not_track = envs.VLLM_DO_NOT_TRACK + no_usage_stats = envs.VLLM_NO_USAGE_STATS + do_not_track_file = os.path.exists(_USAGE_STATS_DO_NOT_TRACK_PATH) + + _USAGE_STATS_ENABLED = not (do_not_track or no_usage_stats + or do_not_track_file) + return _USAGE_STATS_ENABLED + + +def _get_current_timestamp_ns() -> int: + return int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1e9) + + +def _detect_cloud_provider() -> str: + # Try detecting through vendor file + vendor_files = [ + "/sys/class/dmi/id/product_version", "/sys/class/dmi/id/bios_vendor", + "/sys/class/dmi/id/product_name", + "/sys/class/dmi/id/chassis_asset_tag", "/sys/class/dmi/id/sys_vendor" + ] + # Mapping of identifiable strings to cloud providers + cloud_identifiers = { + "amazon": "AWS", + "microsoft corporation": "AZURE", + "google": "GCP", + "oraclecloud": "OCI", + } + + for vendor_file in vendor_files: + path = Path(vendor_file) + if path.is_file(): + file_content = path.read_text().lower() + for identifier, provider in cloud_identifiers.items(): + if identifier in file_content: + return provider + + # Try detecting through environment variables + env_to_cloud_provider = { + "RUNPOD_DC_ID": "RUNPOD", + } + for env_var, provider in env_to_cloud_provider.items(): + if os.environ.get(env_var): + return provider + + return "UNKNOWN" + + +class UsageContext(str, Enum): + UNKNOWN_CONTEXT = "UNKNOWN_CONTEXT" + LLM_CLASS = "LLM_CLASS" + API_SERVER = "API_SERVER" + OPENAI_API_SERVER = "OPENAI_API_SERVER" + OPENAI_BATCH_RUNNER = "OPENAI_BATCH_RUNNER" + ENGINE_CONTEXT = "ENGINE_CONTEXT" + + +class UsageMessage: + """Collect platform information and send it to the usage stats server.""" + + def __init__(self) -> None: + # NOTE: vLLM's server _only_ support flat KV pair. + # Do not use nested fields. + + self.uuid = str(uuid4()) + + # Environment Information + self.provider: Optional[str] = None + self.num_cpu: Optional[int] = None + self.cpu_type: Optional[str] = None + self.cpu_family_model_stepping: Optional[str] = None + self.total_memory: Optional[int] = None + self.architecture: Optional[str] = None + self.platform: Optional[str] = None + self.gpu_count: Optional[int] = None + self.gpu_type: Optional[str] = None + self.gpu_memory_per_device: Optional[int] = None + + # vLLM Information + self.model_architecture: Optional[str] = None + self.vllm_version: Optional[str] = None + self.context: Optional[str] = None + + # Metadata + self.log_time: Optional[int] = None + self.source: Optional[str] = None + + def report_usage(self, + model_architecture: str, + usage_context: UsageContext, + extra_kvs: Optional[Dict[str, Any]] = None) -> None: + t = Thread(target=self._report_usage_worker, + args=(model_architecture, usage_context, extra_kvs or {}), + daemon=True) + t.start() + + def _report_usage_worker(self, model_architecture: str, + usage_context: UsageContext, + extra_kvs: Dict[str, Any]) -> None: + self._report_usage_once(model_architecture, usage_context, extra_kvs) + self._report_continous_usage() + + def _report_usage_once(self, model_architecture: str, + usage_context: UsageContext, + extra_kvs: Dict[str, Any]) -> None: + # Platform information + if current_platform.is_cuda_alike(): + device_property = torch.cuda.get_device_properties(0) + self.gpu_count = torch.cuda.device_count() + self.gpu_type = device_property.name + self.gpu_memory_per_device = device_property.total_memory + self.provider = _detect_cloud_provider() + self.architecture = platform.machine() + self.platform = platform.platform() + self.total_memory = psutil.virtual_memory().total + + info = cpuinfo.get_cpu_info() + self.num_cpu = info.get("count", None) + self.cpu_type = info.get("brand_raw", "") + self.cpu_family_model_stepping = ",".join([ + str(info.get("family", "")), + str(info.get("model", "")), + str(info.get("stepping", "")) + ]) + + # vLLM information + self.context = usage_context.value + self.vllm_version = VLLM_VERSION + self.model_architecture = model_architecture + + # Metadata + self.log_time = _get_current_timestamp_ns() + self.source = envs.VLLM_USAGE_SOURCE + + data = vars(self) + if extra_kvs: + data.update(extra_kvs) + + self._write_to_file(data) + self._send_to_server(data) + + def _report_continous_usage(self): + """Report usage every 10 minutes. + + This helps us to collect more data points for uptime of vLLM usages. + This function can also help send over performance metrics over time. + """ + while True: + time.sleep(600) + data = { + "uuid": self.uuid, + "log_time": _get_current_timestamp_ns(), + } + data.update(_GLOBAL_RUNTIME_DATA) + + self._write_to_file(data) + self._send_to_server(data) + + def _send_to_server(self, data: Dict[str, Any]) -> None: + try: + global_http_client = global_http_connection.get_sync_client() + global_http_client.post(_USAGE_STATS_SERVER, json=data) + except requests.exceptions.RequestException: + # silently ignore unless we are using debug log + logging.debug("Failed to send usage data to server") + + def _write_to_file(self, data: Dict[str, Any]) -> None: + os.makedirs(os.path.dirname(_USAGE_STATS_JSON_PATH), exist_ok=True) + Path(_USAGE_STATS_JSON_PATH).touch(exist_ok=True) + with open(_USAGE_STATS_JSON_PATH, "a") as f: + json.dump(data, f) + f.write("\n") + + +usage_message = UsageMessage() diff --git a/vllm/utils.py b/vllm/utils.py new file mode 100644 index 00000000..edbe2d03 --- /dev/null +++ b/vllm/utils.py @@ -0,0 +1,1445 @@ +import argparse +import asyncio +import contextlib +import datetime +import enum +import gc +import inspect +import ipaddress +import os +import random +import socket +import subprocess +import sys +import tempfile +import threading +import uuid +import warnings +import weakref +from asyncio import FIRST_COMPLETED, ensure_future +from functools import lru_cache, partial, wraps +from platform import uname +from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic, + Hashable, List, Literal, Optional, OrderedDict, Set, Tuple, + Type, TypeVar, Union, overload) +from uuid import uuid4 + +import numpy as np +import numpy.typing as npt +import psutil +import torch +import torch.types +import yaml +from packaging.version import Version +from typing_extensions import ParamSpec, TypeIs, assert_never + +import vllm.envs as envs +from vllm.logger import enable_trace_function_call, init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + +# Exception strings for non-implemented encoder/decoder scenarios + +# Reminder: Please update docs/source/serving/compatibility_matrix.rst +# If the feature combo become valid + +STR_NOT_IMPL_ENC_DEC_SWA = \ + "Sliding window attention for encoder/decoder models " + \ + "is not currently supported." + +STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE = \ + "Prefix caching for encoder/decoder models " + \ + "is not currently supported." + +STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL = \ + "Chunked prefill for encoder/decoder models " + \ + "is not currently supported." + +STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP = ( + "Models with logits_soft_cap " + "require FlashInfer backend, which is " + "currently not supported for encoder/decoder " + "models.") + +STR_NOT_IMPL_ENC_DEC_LORA = ("LoRA is currently not currently " + "supported with encoder/decoder " + "models.") + +STR_NOT_IMPL_ENC_DEC_PP = ("Pipeline parallelism is not " + "currently supported with " + "encoder/decoder models.") + +STR_NOT_IMPL_ENC_DEC_MM = ("Multimodal is not currently " + "supported with encoder/decoder " + "models.") + +STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not " + "currently supported with encoder/" + "decoder models.") + +STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend " + "currently supported with encoder/" + "decoder models.") + +STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not " + "currently supported with encoder/" + "decoder models.") + +STR_NOT_IMPL_ENC_DEC_CPU = ("CPU is not currently supported with " + "encoder/decoder models.") + +# Efficiently import all enc/dec error strings +# rather than having to import all of the above +STR_NOT_IMPL_ENC_DEC_ERR_STRS = { + "STR_NOT_IMPL_ENC_DEC_SWA": STR_NOT_IMPL_ENC_DEC_SWA, + "STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE": STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE, + "STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL": + STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL, + "STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP": STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP, + "STR_NOT_IMPL_ENC_DEC_LORA": STR_NOT_IMPL_ENC_DEC_LORA, + "STR_NOT_IMPL_ENC_DEC_PP": STR_NOT_IMPL_ENC_DEC_PP, + "STR_NOT_IMPL_ENC_DEC_MM": STR_NOT_IMPL_ENC_DEC_MM, + "STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC, + "STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND, + "STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER, + "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU +} + +# Constants related to forcing the attention backend selection + +# String name of register which may be set in order to +# force auto-selection of attention backend by Attention +# wrapper +STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND" + +# Possible string values of STR_BACKEND_ENV_VAR +# register, corresponding to possible backends +STR_FLASHINFER_ATTN_VAL: str = "FLASHINFER" +STR_TORCH_SDPA_ATTN_VAL: str = "TORCH_SDPA" +STR_ROCM_FLASH_ATTN_VAL: str = "ROCM_FLASH" +STR_XFORMERS_ATTN_VAL: str = "XFORMERS" +STR_FLASH_ATTN_VAL: str = "FLASH_ATTN" +STR_INVALID_VAL: str = "INVALID" + +GB_bytes = 1_000_000_000 +"""The number of bytes in one gigabyte (GB).""" + +GiB_bytes = 1 << 30 +"""The number of bytes in one gibibyte (GiB).""" + +STR_DTYPE_TO_TORCH_DTYPE = { + "half": torch.half, + "bfloat16": torch.bfloat16, + "float": torch.float, + "fp8": torch.uint8, + "fp8_e4m3": torch.uint8, + "fp8_e5m2": torch.uint8, +} + +TORCH_DTYPE_TO_NUMPY_DTYPE = { + torch.float16: np.float16, + torch.float32: np.float32, + torch.float64: np.float64, + torch.uint8: np.uint8, + torch.int32: np.int32, + torch.int64: np.int64, +} + +P = ParamSpec('P') +K = TypeVar("K") +T = TypeVar("T") +U = TypeVar("U") + + +class _Sentinel: + ... + + +ALL_PINNED_SENTINEL = _Sentinel() + + +class Device(enum.Enum): + GPU = enum.auto() + CPU = enum.auto() + + +class Counter: + + def __init__(self, start: int = 0) -> None: + self.counter = start + + def __next__(self) -> int: + i = self.counter + self.counter += 1 + return i + + def reset(self) -> None: + self.counter = 0 + + +class LRUCache(Generic[T]): + + def __init__(self, capacity: int): + self.cache: OrderedDict[Hashable, T] = OrderedDict() + self.pinned_items: Set[Hashable] = set() + self.capacity = capacity + + def __contains__(self, key: Hashable) -> bool: + return key in self.cache + + def __len__(self) -> int: + return len(self.cache) + + def __getitem__(self, key: Hashable) -> T: + value = self.cache[key] # Raise KeyError if not exists + self.cache.move_to_end(key) + return value + + def __setitem__(self, key: Hashable, value: T) -> None: + self.put(key, value) + + def __delitem__(self, key: Hashable) -> None: + self.pop(key) + + def touch(self, key: Hashable) -> None: + self.cache.move_to_end(key) + + def get(self, + key: Hashable, + default_value: Optional[T] = None) -> Optional[T]: + value: Optional[T] + if key in self.cache: + value = self.cache[key] + self.cache.move_to_end(key) + else: + value = default_value + return value + + def put(self, key: Hashable, value: T) -> None: + self.cache[key] = value + self.cache.move_to_end(key) + self._remove_old_if_needed() + + def pin(self, key: Hashable) -> None: + """ + Pins a key in the cache preventing it from being + evicted in the LRU order. + """ + if key not in self.cache: + raise ValueError(f"Cannot pin key: {key} not in cache.") + self.pinned_items.add(key) + + def _unpin(self, key: Hashable) -> None: + self.pinned_items.remove(key) + + def _on_remove(self, key: Hashable, value: Optional[T]): + pass + + def remove_oldest(self, remove_pinned=False): + if not self.cache: + return + + if not remove_pinned: + # pop the oldest item in the cache that is not pinned + lru_key = next( + (key for key in self.cache if key not in self.pinned_items), + ALL_PINNED_SENTINEL) + if lru_key is ALL_PINNED_SENTINEL: + raise RuntimeError("All items are pinned, " + "cannot remove oldest from the cache.") + else: + lru_key = next(iter(self.cache)) + self.pop(lru_key) + + def _remove_old_if_needed(self) -> None: + while len(self.cache) > self.capacity: + self.remove_oldest() + + def pop(self, + key: Hashable, + default_value: Optional[T] = None) -> Optional[T]: + run_on_remove = key in self.cache + value: Optional[T] = self.cache.pop(key, default_value) + # remove from pinned items + if key in self.pinned_items: + self._unpin(key) + if run_on_remove: + self._on_remove(key, value) + return value + + def clear(self): + while len(self.cache) > 0: + self.remove_oldest(remove_pinned=True) + self.cache.clear() + + +class PyObjectCache: + """Used to cache python objects to avoid object allocations + across scheduler iterations. + """ + + def __init__(self, obj_builder): + self._obj_builder = obj_builder + self._index = 0 + + self._obj_cache = [] + for _ in range(128): + self._obj_cache.append(self._obj_builder()) + + def _grow_cache(self): + # Double the size of the cache + num_objs = len(self._obj_cache) + for _ in range(num_objs): + self._obj_cache.append(self._obj_builder()) + + def get_object(self): + """Returns a pre-allocated cached object. If there is not enough + objects, then the cache size will double. + """ + if self._index >= len(self._obj_cache): + self._grow_cache() + assert self._index < len(self._obj_cache) + + obj = self._obj_cache[self._index] + self._index += 1 + + return obj + + def reset(self): + """Makes all cached-objects available for the next scheduler iteration. + """ + self._index = 0 + + +def is_hip() -> bool: + return torch.version.hip is not None + + +@lru_cache(maxsize=None) +def is_cpu() -> bool: + from importlib.metadata import PackageNotFoundError, version + try: + return "cpu" in version("vllm") + except PackageNotFoundError: + return False + + +@lru_cache(maxsize=None) +def is_openvino() -> bool: + from importlib.metadata import PackageNotFoundError, version + try: + return "openvino" in version("vllm") + except PackageNotFoundError: + return False + + +@lru_cache(maxsize=None) +def is_neuron() -> bool: + try: + import transformers_neuronx + except ImportError: + transformers_neuronx = None + return transformers_neuronx is not None + + +@lru_cache(maxsize=None) +def is_xpu() -> bool: + from importlib.metadata import PackageNotFoundError, version + try: + is_xpu_flag = "xpu" in version("vllm") + except PackageNotFoundError: + return False + # vllm is not build with xpu + if not is_xpu_flag: + return False + try: + import intel_extension_for_pytorch as ipex # noqa: F401 + _import_ipex = True + except ImportError as e: + logger.warning("Import Error for IPEX: %s", e.msg) + _import_ipex = False + # ipex dependency is not ready + if not _import_ipex: + logger.warning("not found ipex lib") + return False + return hasattr(torch, "xpu") and torch.xpu.is_available() + + +@lru_cache(maxsize=None) +def get_max_shared_memory_bytes(gpu: int = 0) -> int: + """Returns the maximum shared memory per thread block in bytes.""" + from vllm import _custom_ops as ops + max_shared_mem = ( + ops.get_max_shared_memory_per_block_device_attribute(gpu)) + # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py + # will fail + assert max_shared_mem > 0, "max_shared_mem can not be zero" + return int(max_shared_mem) + + +def get_cpu_memory() -> int: + """Returns the total CPU memory of the node in bytes.""" + return psutil.virtual_memory().total + + +def seed_everything(seed: int) -> None: + """ + Set the seed of each random module. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + + if current_platform.is_cuda_alike(): + torch.cuda.manual_seed_all(seed) + + if is_xpu(): + torch.xpu.manual_seed_all(seed) + + +def random_uuid() -> str: + return str(uuid.uuid4().hex) + + +@lru_cache(maxsize=None) +def get_vllm_instance_id() -> str: + """ + If the environment variable VLLM_INSTANCE_ID is set, return it. + Otherwise, return a random UUID. + Instance id represents an instance of the VLLM. All processes in the same + instance should have the same instance id. + """ + return envs.VLLM_INSTANCE_ID or f"vllm-instance-{random_uuid()}" + + +@lru_cache(maxsize=None) +def in_wsl() -> bool: + # Reference: https://github.com/microsoft/WSL/issues/4071 + return "microsoft" in " ".join(uname()).lower() + + +def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]: + """Take a blocking function, and run it on in an executor thread. + + This function prevents the blocking function from blocking the + asyncio event loop. + The code in this function needs to be thread safe. + """ + + def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future: + loop = asyncio.get_event_loop() + p_func = partial(func, *args, **kwargs) + return loop.run_in_executor(executor=None, func=p_func) + + return _async_wrapper + + +async def iterate_with_cancellation( + iterator: AsyncGenerator[T, None], + is_cancelled: Callable[[], Awaitable[bool]], +) -> AsyncGenerator[T, None]: + """Convert async iterator into one that polls the provided function + at least once per second to check for client cancellation. + """ + + # Can use anext() in python >= 3.10 + awaits = [ensure_future(iterator.__anext__())] + while True: + done, pending = await asyncio.wait(awaits, timeout=1) + if await is_cancelled(): + with contextlib.suppress(BaseException): + awaits[0].cancel() + await iterator.aclose() + raise asyncio.CancelledError("client cancelled") + if done: + try: + item = await awaits[0] + awaits[0] = ensure_future(iterator.__anext__()) + yield item + except StopAsyncIteration: + # we are done + return + + +async def merge_async_iterators( + *iterators: AsyncGenerator[T, None], + is_cancelled: Optional[Callable[[], Awaitable[bool]]] = None, +) -> AsyncGenerator[Tuple[int, T], None]: + """Merge multiple asynchronous iterators into a single iterator. + + This method handle the case where some iterators finish before others. + When it yields, it yields a tuple (i, item) where i is the index of the + iterator that yields the item. + + It also optionally polls a provided function at least once per second + to check for client cancellation. + """ + + # Can use anext() in python >= 3.10 + awaits = { + ensure_future(pair[1].__anext__()): pair + for pair in enumerate(iterators) + } + timeout = None if is_cancelled is None else 1 + try: + while awaits: + done, pending = await asyncio.wait(awaits.keys(), + return_when=FIRST_COMPLETED, + timeout=timeout) + if is_cancelled is not None and await is_cancelled(): + raise asyncio.CancelledError("client cancelled") + for d in done: + pair = awaits.pop(d) + try: + item = await d + i, it = pair + awaits[ensure_future(it.__anext__())] = pair + yield i, item + except StopAsyncIteration: + pass + finally: + # Cancel any remaining iterators + for f, (_, it) in awaits.items(): + with contextlib.suppress(BaseException): + f.cancel() + await it.aclose() + + +async def collect_from_async_generator( + iterator: AsyncGenerator[T, None]) -> List[T]: + """Collect all items from an async generator into a list.""" + items = [] + async for item in iterator: + items.append(item) + return items + + +def get_ip() -> str: + host_ip = envs.VLLM_HOST_IP + if host_ip: + return host_ip + + # IP is not set, try to get it from the network interface + + # try ipv4 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("8.8.8.8", 80)) # Doesn't need to be reachable + return s.getsockname()[0] + except Exception: + pass + + # try ipv6 + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + # Google's public DNS server, see + # https://developers.google.com/speed/public-dns/docs/using#addresses + s.connect(("2001:4860:4860::8888", 80)) # Doesn't need to be reachable + return s.getsockname()[0] + except Exception: + pass + + warnings.warn( + "Failed to get the IP address, using 0.0.0.0 by default." + "The value can be set by the environment variable" + " VLLM_HOST_IP or HOST_IP.", + stacklevel=2) + return "0.0.0.0" + + +def is_valid_ipv6_address(address: str) -> bool: + try: + ipaddress.IPv6Address(address) + return True + except ValueError: + return False + + +def get_distributed_init_method(ip: str, port: int) -> str: + # Brackets are not permitted in ipv4 addresses, + # see https://github.com/python/cpython/issues/103848 + return f"tcp://[{ip}]:{port}" if ":" in ip else f"tcp://{ip}:{port}" + + +def get_open_zmq_ipc_path() -> str: + base_rpc_path = envs.VLLM_RPC_BASE_PATH + return f"ipc://{base_rpc_path}/{uuid4()}" + + +def get_open_port() -> int: + port = envs.VLLM_PORT + if port is not None: + while True: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", port)) + return port + except OSError: + port += 1 # Increment port number if already in use + logger.info("Port %d is already in use, trying port %d", + port - 1, port) + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def find_process_using_port(port: int) -> Optional[psutil.Process]: + for conn in psutil.net_connections(): + if conn.laddr.port == port: + try: + return psutil.Process(conn.pid) + except psutil.NoSuchProcess: + return None + return None + + +def update_environment_variables(envs: Dict[str, str]): + for k, v in envs.items(): + if k in os.environ and os.environ[k] != v: + logger.warning( + "Overwriting environment variable %s " + "from '%s' to '%s'", k, os.environ[k], v) + os.environ[k] = v + + +def chunk_list(lst: List[T], chunk_size: int): + """Yield successive chunk_size chunks from lst.""" + for i in range(0, len(lst), chunk_size): + yield lst[i:i + chunk_size] + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) + + +def _generate_random_fp8( + tensor: torch.Tensor, + low: float, + high: float, +) -> None: + # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type, + # it may occur Inf or NaN if we directly use torch.randint + # to generate random data for fp8 data. + # For example, s.11111.00 in fp8e5m2 format represents Inf. + # | E4M3 | E5M2 + #-----|-------------|------------------- + # Inf | N/A | s.11111.00 + # NaN | s.1111.111 | s.11111.{01,10,11} + from vllm import _custom_ops as ops + tensor_tmp = torch.empty_like(tensor, dtype=torch.float16) + tensor_tmp.uniform_(low, high) + ops.convert_fp8(tensor, tensor_tmp) + del tensor_tmp + + +def get_kv_cache_torch_dtype( + cache_dtype: Optional[Union[str, torch.dtype]], + model_dtype: Optional[Union[str, torch.dtype]] = None) -> torch.dtype: + if isinstance(cache_dtype, str): + if cache_dtype == "auto": + if isinstance(model_dtype, str): + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype] + elif isinstance(model_dtype, torch.dtype): + torch_dtype = model_dtype + else: + raise ValueError(f"Invalid model dtype: {model_dtype}") + elif cache_dtype in ["half", "bfloat16", "float"]: + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype] + elif cache_dtype == "fp8": + torch_dtype = torch.uint8 + else: + raise ValueError(f"Invalid kv cache dtype: {cache_dtype}") + elif isinstance(cache_dtype, torch.dtype): + torch_dtype = cache_dtype + else: + raise ValueError(f"Invalid kv cache dtype: {cache_dtype}") + return torch_dtype + + +def create_kv_caches_with_random_flash( + num_blocks: int, + block_size: int, + num_layers: int, + num_heads: int, + head_size: int, + cache_dtype: Optional[Union[str, torch.dtype]], + model_dtype: Optional[Union[str, torch.dtype]] = None, + seed: int = 0, + device: Optional[str] = "cuda", +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + seed_everything(seed) + + torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype) + key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size) + scale = head_size**-0.5 + + key_caches: List[torch.Tensor] = [] + value_caches: List[torch.Tensor] = [] + + for _ in range(num_layers): + key_value_cache = torch.empty(size=key_value_cache_shape, + dtype=torch_dtype, + device=device) + if cache_dtype in ["auto", "half", "bfloat16", "float"]: + key_value_cache.uniform_(-scale, scale) + elif cache_dtype == 'fp8': + _generate_random_fp8(key_value_cache, -scale, scale) + else: + raise ValueError( + f"Does not support key cache of type {cache_dtype}") + key_caches.append(key_value_cache[:, 0]) + value_caches.append(key_value_cache[:, 1]) + return key_caches, value_caches + + +def create_kv_caches_with_random( + num_blocks: int, + block_size: int, + num_layers: int, + num_heads: int, + head_size: int, + cache_dtype: Optional[Union[str, torch.dtype]], + model_dtype: Optional[Union[str, torch.dtype]] = None, + seed: int = 0, + device: Optional[str] = "cuda", +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + + if cache_dtype == "fp8" and head_size % 16: + raise ValueError( + f"Does not support key cache of type fp8 with head_size {head_size}" + ) + + seed_everything(seed) + + torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype) + + scale = head_size**-0.5 + x = 16 // torch.tensor([], dtype=torch_dtype).element_size() + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x) + key_caches: List[torch.Tensor] = [] + for _ in range(num_layers): + key_cache = torch.empty(size=key_cache_shape, + dtype=torch_dtype, + device=device) + if cache_dtype in ["auto", "half", "bfloat16", "float"]: + key_cache.uniform_(-scale, scale) + elif cache_dtype == 'fp8': + _generate_random_fp8(key_cache, -scale, scale) + else: + raise ValueError( + f"Does not support key cache of type {cache_dtype}") + key_caches.append(key_cache) + + value_cache_shape = (num_blocks, num_heads, head_size, block_size) + value_caches: List[torch.Tensor] = [] + for _ in range(num_layers): + value_cache = torch.empty(size=value_cache_shape, + dtype=torch_dtype, + device=device) + if cache_dtype in ["auto", "half", "bfloat16", "float"]: + value_cache.uniform_(-scale, scale) + elif cache_dtype == 'fp8': + _generate_random_fp8(value_cache, -scale, scale) + else: + raise ValueError( + f"Does not support value cache of type {cache_dtype}") + value_caches.append(value_cache) + return key_caches, value_caches + + +@lru_cache +def print_warning_once(msg: str) -> None: + # Set the stacklevel to 2 to print the caller's line info + logger.warning(msg, stacklevel=2) + + +@lru_cache(maxsize=None) +def is_pin_memory_available() -> bool: + + if in_wsl(): + # Pinning memory in WSL is not supported. + # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications + print_warning_once("Using 'pin_memory=False' as WSL is detected. " + "This may slow down the performance.") + return False + elif is_xpu(): + print_warning_once("Pin memory is not supported on XPU.") + return False + elif is_neuron(): + print_warning_once("Pin memory is not supported on Neuron.") + return False + elif is_cpu() or is_openvino(): + return False + return True + + +class DeviceMemoryProfiler: + + def __init__(self, device: Optional[torch.types.Device] = None): + self.device = device + + def current_memory_usage(self) -> float: + # Return the memory usage in bytes. + if current_platform.is_cuda_alike(): + torch.cuda.reset_peak_memory_stats(self.device) + mem = torch.cuda.max_memory_allocated(self.device) + elif is_xpu(): + torch.xpu.reset_peak_memory_stats(self.device) # type: ignore + mem = torch.xpu.max_memory_allocated(self.device) # type: ignore + return mem + + def __enter__(self): + self.initial_memory = self.current_memory_usage() + # This allows us to call methods of the context manager if needed + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.final_memory = self.current_memory_usage() + self.consumed_memory = self.final_memory - self.initial_memory + + # Force garbage collection + gc.collect() + + +def make_ndarray_with_pad( + x: List[List[T]], + pad: T, + dtype: npt.DTypeLike, + *, + max_len: Optional[int] = None, +) -> npt.NDArray: + """ + Make a padded array from 2D inputs. + + The padding is applied to the end of each inner list until it reaches + `max_len`. + """ + if max_len is None: + # Unlike for most functions, map is faster than a genexpr over `len` + max_len = max(map(len, x), default=0) + + padded_x = np.full((len(x), max_len), pad, dtype=dtype) + for ind, blocktb in enumerate(x): + assert len(blocktb) <= max_len + padded_x[ind, :len(blocktb)] = blocktb + + return padded_x + + +def make_tensor_with_pad( + x: List[List[T]], + pad: T, + dtype: torch.dtype, + *, + max_len: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + pin_memory: bool = False, +) -> torch.Tensor: + """ + Make a padded tensor from 2D inputs. + + The padding is applied to the end of each inner list until it reaches + `max_len`. + """ + np_dtype = TORCH_DTYPE_TO_NUMPY_DTYPE[dtype] + padded_x = make_ndarray_with_pad(x, pad, np_dtype, max_len=max_len) + + tensor = torch.from_numpy(padded_x).to(device) + if pin_memory: + tensor = tensor.pin_memory() + + return tensor + + +def async_tensor_h2d( + data: list, + dtype: torch.dtype, + target_device: Union[str, torch.device], + pin_memory: bool, +) -> torch.Tensor: + """Asynchronously create a tensor and copy it from host to device.""" + t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu") + return t.to(device=target_device, non_blocking=True) + + +def get_dtype_size(dtype: torch.dtype) -> int: + """Get the size of the data type in bytes.""" + return torch.tensor([], dtype=dtype).element_size() + + +# `collections` helpers +def is_list_of( + value: object, + typ: Type[T], + *, + check: Literal["first", "all"] = "first", +) -> TypeIs[List[T]]: + if not isinstance(value, list): + return False + + if check == "first": + return len(value) == 0 or isinstance(value[0], typ) + elif check == "all": + return all(isinstance(v, typ) for v in value) + + assert_never(check) + + +JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"], + Tuple["JSONTree[T]", ...], T] +"""A nested JSON structure where the leaves need not be JSON-serializable.""" + + +@overload +def json_map_leaves( + func: Callable[[T], U], + value: Dict[str, JSONTree[T]], +) -> Dict[str, JSONTree[U]]: + ... + + +@overload +def json_map_leaves( + func: Callable[[T], U], + value: List[JSONTree[T]], +) -> List[JSONTree[U]]: + ... + + +@overload +def json_map_leaves( + func: Callable[[T], U], + value: Tuple[JSONTree[T], ...], +) -> Tuple[JSONTree[U], ...]: + ... + + +@overload +def json_map_leaves( + func: Callable[[T], U], + value: JSONTree[T], +) -> JSONTree[U]: + ... + + +def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]: + if isinstance(value, dict): + return {k: json_map_leaves(func, v) for k, v in value.items()} + elif isinstance(value, list): + return [json_map_leaves(func, v) for v in value] + elif isinstance(value, tuple): + return tuple(json_map_leaves(func, v) for v in value) + else: + return func(value) + + +def flatten_2d_lists(lists: List[List[T]]) -> List[T]: + """Flatten a list of lists to a single list.""" + return [item for sublist in lists for item in sublist] + +# TODO: This function can be removed if transformer_modules classes are +# serialized by value when communicating between processes +def init_cached_hf_modules() -> None: + """ + Lazy initialization of the Hugging Face modules. + """ + from transformers.dynamic_module_utils import init_hf_modules + init_hf_modules() + + +@lru_cache(maxsize=None) +def find_library(lib_name: str) -> str: + """ + Find the library file in the system. + `lib_name` is full filename, with both prefix and suffix. + This function resolves `lib_name` to the full path of the library. + """ + # Adapted from https://github.com/openai/triton/blob/main/third_party/nvidia/backend/driver.py#L19 # noqa + # According to https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard + # `/sbin/ldconfig` should exist in all Linux systems. + # `/sbin/ldconfig` searches the library in the system + libs = subprocess.check_output(["/sbin/ldconfig", "-p"]).decode() + # each line looks like the following: + # libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1 + locs = [line.split()[-1] for line in libs.splitlines() if lib_name in line] + # `LD_LIBRARY_PATH` searches the library in the user-defined paths + env_ld_library_path = envs.LD_LIBRARY_PATH + if not locs and env_ld_library_path: + locs = [ + os.path.join(dir, lib_name) + for dir in env_ld_library_path.split(":") + if os.path.exists(os.path.join(dir, lib_name)) + ] + if not locs: + raise ValueError(f"Cannot find {lib_name} in the system.") + return locs[0] + + +def find_nccl_library() -> str: + """ + We either use the library file specified by the `VLLM_NCCL_SO_PATH` + environment variable, or we find the library file brought by PyTorch. + After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be + found by `ctypes` automatically. + """ + so_file = envs.VLLM_NCCL_SO_PATH + + # manually load the nccl library + if so_file: + logger.info( + "Found nccl from environment variable VLLM_NCCL_SO_PATH=%s", + so_file) + else: + if torch.version.cuda is not None: + so_file = "libnccl.so.2" + elif torch.version.hip is not None: + so_file = "librccl.so.1" + else: + raise ValueError("NCCL only supports CUDA and ROCm backends.") + logger.info("Found nccl from library %s", so_file) + return so_file + + +def enable_trace_function_call_for_thread() -> None: + """Set up function tracing for the current thread, + if enabled via the VLLM_TRACE_FUNCTION environment variable + """ + + if envs.VLLM_TRACE_FUNCTION: + tmp_dir = tempfile.gettempdir() + filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}" + f"_thread_{threading.get_ident()}_" + f"at_{datetime.datetime.now()}.log").replace(" ", "_") + log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(), + filename) + os.makedirs(os.path.dirname(log_path), exist_ok=True) + enable_trace_function_call(log_path) + + +# `functools` helpers +def identity(value: T) -> T: + return value + + +F = TypeVar('F', bound=Callable[..., Any]) + + +def deprecate_kwargs( + *kws: str, + is_deprecated: Union[bool, Callable[[], bool]] = True, + additional_message: Optional[str] = None) -> Callable[[F], F]: + deprecated_kws = set(kws) + + if not callable(is_deprecated): + is_deprecated = partial(identity, is_deprecated) + + def wrapper(fn: F) -> F: + + @wraps(fn) + def inner(*args, **kwargs): + if is_deprecated(): + deprecated_kwargs = kwargs.keys() & deprecated_kws + if deprecated_kwargs: + msg = ( + f"The keyword arguments {deprecated_kwargs} are " + "deprecated and will be removed in a future update.") + if additional_message is not None: + msg += f" {additional_message}" + + warnings.warn( + DeprecationWarning(msg), + stacklevel=3, # The inner function takes up one level + ) + + return fn(*args, **kwargs) + + return inner # type: ignore + + return wrapper + + +@lru_cache(maxsize=8) +def _cuda_device_count_stateless( + cuda_visible_devices: Optional[str] = None) -> int: + # Note: cuda_visible_devices is not used, but we keep it as an argument for + # LRU Cache purposes. + + # Code below is based on + # https://github.com/pytorch/pytorch/blob/ + # c1cd946818442aca8c7f812b16d187ce1586c3bc/ + # torch/cuda/__init__.py#L831C1-L831C17 + import torch.cuda + import torch.version + + if not torch.cuda._is_compiled(): + return 0 + if is_hip(): + # ROCm uses amdsmi instead of nvml for stateless device count + # This requires a sufficiently modern version of Torch 2.4.0 + raw_count = torch.cuda._device_count_amdsmi() if (hasattr( + torch.cuda, "_device_count_amdsmi")) else -1 + else: + raw_count = torch.cuda._device_count_nvml() + r = torch._C._cuda_getDeviceCount() if raw_count < 0 else raw_count + return r + + +def cuda_device_count_stateless() -> int: + """Get number of CUDA devices, caching based on the value of + CUDA_VISIBLE_DEVICES at the time of call. + + This should be used instead of torch.cuda.device_count() + unless CUDA_VISIBLE_DEVICES has already been set to the desired + value.""" + + # This can be removed and simply replaced with torch.cuda.get_device_count + # after https://github.com/pytorch/pytorch/pull/122815 is released. + return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES) + + +def cuda_is_initialized() -> bool: + """Check if CUDA is initialized.""" + if not torch.cuda._is_compiled(): + return False + return torch.cuda.is_initialized() + + +def weak_bind(bound_method: Callable[..., Any], ) -> Callable[..., None]: + """Make an instance method that weakly references + its associated instance and no-ops once that + instance is collected.""" + ref = weakref.ref(bound_method.__self__) # type: ignore[attr-defined] + unbound = bound_method.__func__ # type: ignore[attr-defined] + + def weak_bound(*args, **kwargs) -> None: + if inst := ref(): + unbound(inst, *args, **kwargs) + + return weak_bound + + +#From: https://stackoverflow.com/a/4104188/2749989 +def run_once(f: Callable[P, None]) -> Callable[P, None]: + + def wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + if not wrapper.has_run: # type: ignore[attr-defined] + wrapper.has_run = True # type: ignore[attr-defined] + return f(*args, **kwargs) + + wrapper.has_run = False # type: ignore[attr-defined] + return wrapper + + +class FlexibleArgumentParser(argparse.ArgumentParser): + """ArgumentParser that allows both underscore and dash in names.""" + + def parse_args(self, args=None, namespace=None): + if args is None: + args = sys.argv[1:] + + if '--config' in args: + args = FlexibleArgumentParser._pull_args_from_config(args) + + # Convert underscores to dashes and vice versa in argument names + processed_args = [] + for arg in args: + if arg.startswith('--'): + if '=' in arg: + key, value = arg.split('=', 1) + key = '--' + key[len('--'):].replace('_', '-') + processed_args.append(f'{key}={value}') + else: + processed_args.append('--' + + arg[len('--'):].replace('_', '-')) + else: + processed_args.append(arg) + + return super().parse_args(processed_args, namespace) + + @staticmethod + def _pull_args_from_config(args: List[str]) -> List[str]: + """Method to pull arguments specified in the config file + into the command-line args variable. + + The arguments in config file will be inserted between + the argument list. + + example: + ```yaml + port: 12323 + tensor-parallel-size: 4 + ``` + ```python + $: vllm {serve,chat,complete} "facebook/opt-12B" \ + --config config.yaml -tp 2 + $: args = [ + "serve,chat,complete", + "facebook/opt-12B", + '--config', 'config.yaml', + '-tp', '2' + ] + $: args = [ + "serve,chat,complete", + "facebook/opt-12B", + '--port', '12323', + '--tensor-parallel-size', '4', + '-tp', '2' + ] + ``` + + Please note how the config args are inserted after the sub command. + this way the order of priorities is maintained when these are args + parsed by super(). + """ + assert args.count( + '--config') <= 1, "More than one config file specified!" + + index = args.index('--config') + if index == len(args) - 1: + raise ValueError("No config file specified! \ + Please check your command-line arguments.") + + file_path = args[index + 1] + + config_args = FlexibleArgumentParser._load_config_file(file_path) + + # 0th index is for {serve,chat,complete} + # followed by model_tag (only for serve) + # followed by config args + # followed by rest of cli args. + # maintaining this order will enforce the precedence + # of cli > config > defaults + if args[0] == "serve": + if index == 1: + raise ValueError( + "No model_tag specified! Please check your command-line" + " arguments.") + args = [args[0]] + [ + args[1] + ] + config_args + args[2:index] + args[index + 2:] + else: + args = [args[0]] + config_args + args[1:index] + args[index + 2:] + + return args + + @staticmethod + def _load_config_file(file_path: str) -> List[str]: + """Loads a yaml file and returns the key value pairs as a + flattened list with argparse like pattern + ```yaml + port: 12323 + tensor-parallel-size: 4 + ``` + returns: + processed_args: list[str] = [ + '--port': '12323', + '--tensor-parallel-size': '4' + ] + + """ + + extension: str = file_path.split('.')[-1] + if extension not in ('yaml', 'yml'): + raise ValueError( + "Config file must be of a yaml/yml type.\ + %s supplied", extension) + + # only expecting a flat dictionary of atomic types + processed_args: List[str] = [] + + config: Dict[str, Union[int, str]] = {} + try: + with open(file_path, 'r') as config_file: + config = yaml.safe_load(config_file) + except Exception as ex: + logger.error( + "Unable to read the config file at %s. \ + Make sure path is correct", file_path) + raise ex + + for key, value in config.items(): + processed_args.append('--' + key) + processed_args.append(str(value)) + + return processed_args + + +async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args, + **kwargs): + """Utility function to run async task in a lock""" + async with lock: + return await task(*args, **kwargs) + + +def supports_kw( + callable: Callable[..., object], + kw_name: str, + requires_kw_only: bool = False, + allow_var_kwargs: bool = True, +) -> bool: + """Check if a keyword is a valid kwarg for a callable; if requires_kw_only + disallows kwargs names that can also be positional arguments. + """ + params = inspect.signature(callable).parameters + if not params: + return False + + param_val = params.get(kw_name) + + # Types where the it may be valid, i.e., explicitly defined & nonvariadic + passable_kw_types = set((inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY)) + + if param_val: + is_sig_param = param_val.kind in passable_kw_types + # We want kwargs only, but this is passable as a positional arg + if (requires_kw_only and is_sig_param + and param_val.kind != inspect.Parameter.KEYWORD_ONLY): + return False + if ((requires_kw_only + and param_val.kind == inspect.Parameter.KEYWORD_ONLY) + or (not requires_kw_only and is_sig_param)): + return True + + # If we're okay with var-kwargs, it's supported as long as + # the kw_name isn't something like *args, **kwargs + if allow_var_kwargs: + # Get the last param; type is ignored here because params is a proxy + # mapping, but it wraps an ordered dict, and they appear in order. + # Ref: https://docs.python.org/3/library/inspect.html#inspect.Signature.parameters + last_param = params[next(reversed(params))] # type: ignore + return (last_param.kind == inspect.Parameter.VAR_KEYWORD + and last_param.name != kw_name) + return False + + +def resolve_mm_processor_kwargs( + init_kwargs: Optional[Dict[str, Any]], + inference_kwargs: Optional[Dict[str, Any]], + callable: Callable[..., object], + allow_var_kwargs: bool = False, +) -> Dict[str, Any]: + """Applies filtering to eliminate invalid mm_processor_kwargs, i.e., + those who are not explicit keywords to the given callable (of one is + given; otherwise no filtering is done), then merges the kwarg dicts, + giving priority to inference_kwargs if there are any collisions. + + In the case that no kwarg overrides are provided, returns an empty + dict so that it can still be kwarg expanded into the callable later on. + + If allow_var_kwargs=True, allows for things that can be expanded into + kwargs as long as they aren't naming collision for var_kwargs or potential + positional arguments. + """ + # Filter inference time multimodal processor kwargs provided + runtime_mm_kwargs = get_allowed_kwarg_only_overrides( + callable, + overrides=inference_kwargs, + allow_var_kwargs=allow_var_kwargs) + + # Filter init time multimodal processor kwargs provided + init_mm_kwargs = get_allowed_kwarg_only_overrides( + callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs) + + # Merge the final processor kwargs, prioritizing inference + # time values over the initialization time values. + mm_processor_kwargs = {**init_mm_kwargs, **runtime_mm_kwargs} + return mm_processor_kwargs + + +def get_allowed_kwarg_only_overrides( + callable: Callable[..., object], + overrides: Optional[Dict[str, Any]], + allow_var_kwargs: bool = False, +) -> Dict[str, Any]: + """ + Given a callable which has one or more keyword only params and a dict + mapping param names to values, drop values that can be not be kwarg + expanded to overwrite one or more keyword-only args. This is used in a + few places to handle custom processor overrides for multimodal models, + e.g., for profiling when processor options provided by the user + may affect the number of mm tokens per instance. + + Args: + callable: Callable which takes 0 or more keyword only arguments. + If None is provided, all overrides names are allowed. + overrides: Potential overrides to be used when invoking the callable. + allow_var_kwargs: Allows overrides that are expandable for var kwargs. + + Returns: + Dictionary containing the kwargs to be leveraged which may be used + to overwrite one or more keyword only arguments when invoking the + callable. + """ + if not overrides: + return {} + + # Drop any mm_processor_kwargs provided by the user that + # are not kwargs, unless it can fit it var_kwargs param + filtered_overrides = { + kwarg_name: val + for kwarg_name, val in overrides.items() + if supports_kw(callable, + kwarg_name, + requires_kw_only=True, + allow_var_kwargs=allow_var_kwargs) + } + + # If anything is dropped, log a warning + dropped_keys = overrides.keys() - filtered_overrides.keys() + if dropped_keys: + logger.warning( + "The following intended overrides are not keyword-only args " + "and and will be dropped: %s", dropped_keys) + + return filtered_overrides + + +# Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0. +# In particular, the FakeScalarType is not supported for earlier versions of +# PyTorch which breaks dynamo for any ops registered using ScalarType. +def supports_dynamo() -> bool: + base_torch_version = Version(Version(torch.__version__).base_version) + return base_torch_version >= Version("2.4.0") + + +# Some backends use pytorch version < 2.4.0 which doesn't +# support `torch.library.custom_op`. +def supports_custom_op() -> bool: + return hasattr(torch.library, "custom_op") + + +class AtomicCounter: + """An atomic, thread-safe counter""" + + def __init__(self, initial=0): + """Initialize a new atomic counter to given initial value""" + self._value = initial + self._lock = threading.Lock() + + def inc(self, num=1): + """Atomically increment the counter by num and return the new value""" + with self._lock: + self._value += num + return self._value + + def dec(self, num=1): + """Atomically decrement the counter by num and return the new value""" + with self._lock: + self._value -= num + return self._value + + @property + def value(self): + return self._value diff --git a/vllm/version.py b/vllm/version.py new file mode 100644 index 00000000..135a818b --- /dev/null +++ b/vllm/version.py @@ -0,0 +1,2 @@ +__version__ = "0.6.3" +__version_tuple__ = (0, 6, 3) diff --git a/vllm/worker/__init__.py b/vllm/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/worker/__pycache__/__init__.cpython-310.pyc b/vllm/worker/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..625607bcd2e5ad5f437f052a9d91eab03dbaa394 GIT binary patch literal 152 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hueo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_etCXTc50D+e0*kJW=VX!UP0w8 V4x8Nkl+v73JCLEpOhAH#0RV%NBQXE~ literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/cache_engine.cpython-310.pyc b/vllm/worker/__pycache__/cache_engine.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3facbcff9363f28d62a7d38e253093c66e5ee625 GIT binary patch literal 3461 zcmZ`5%W~t!5i=NoAo!HT)g#$BjGZ`PvMbqnr)n#9qIDdXV`a-$TnQ>J6oeUxpzxq) zKy8Typ0wBegH)Bv9`XzMi(GKc#X0+CucLGiNQsm;Ag8CNdoa`QM%HS%20U;62};mnqE`mwYZhEy>`;^I!V{-5<@h_+6D2}-!-_y>(2~czo6a*H$O54?)1Ta5DvvV z=}DA|ei#Qz^^Y^zPl7Z6(n&ubivIf__e0#Z`yY_h7?6_vAyWAPAl(y@`$3+k{!tK~ zij)sn+0ZBMXX$ZtQZ^rpvnUkvRpWz38QQqYT zJ@PgT^>+Zm@CewQasqayf^kz=oC%v-qQ-4rJ9f3bnCEa;)Oka=+!hVd9JO|gW1Ba5 z>lyRfyp0sSpYaaw0++4IeLa2;Xlr79O|zjj8}0o781fG!7~?8($9bxB?~5bR+`~>GT4zVP+qnX zbP#j_lr7Akp$K@DKJ{C48#(DD!K4-`I?<1Vb0JmPdnr{H0OXvW=>t{STv@qCaTcEH zYrHmcPR0{o)0DS4j>aN}y5rw?7_+(B2g1+ym5nMmbiS9Co@5<$!3yfFqr@tXI7E~G zG)j5))LVmkaI3(v6ryY{SE+K{hsu+MK`u~meZJS2hZUBUvW{|UdRa$NG&--+BF=+Z@3RBkBkO79S@xp**j(8M}@{t5_$opDJpiAE55BS@eTyJ9m)A05^k zxpiTr(7$Hv#F$w*gzTtxK^_~QbY^y8T{?Lkf*IdxK-y7m{*7G`VtjhPupngF^hdv$ zyF)FNkyIBsH5OrX90}g99em|`2=E%Oedp}&Bhgt9 zOXBNrsK9IZ)++$CIZ?-A1$4Oup@ zY|mpw_L28(0ME_JAP)d9NpwfqvZh~((mwH>fYpRputCcko2nII(7+6ac3lq;FdSwA<^j3F zFEub+C7@YFUISa(!eA)gfL6Bp8Q^yXWVrbV7z!`yk!g3?I>=nfOAJ|bvzF!&Hw{UvcROo?Rhua2@L=(tu1e*wSN_`VC4C|^aqrV4@yG0UH&Z}eH!EU2; zgn6A)abDHd%3$8wSrAXeG6}OW4`O}3=iYww-L+&7`#^}rt~yHh908Xql+m$Won-8~ zZ8Fj%F4RsJs;EPz{bh^4?2thZi*gzy!uLzp_u=bg5+mO9{ZA)BJm=JWpJ$=(OAHr@ zi72s*O8gX&-vv;bQJR;g%B4hS%O3*p>_d^NOzQF=e~jdxBKSFgvZ-?TjtZ;qE%{5N zzq$%u!(|fy_ICAK0FD8*q(&+IFE!~`&@o<6i+)LM_Y3-9L3Nn>qIa)n^(@$fmFIfF zp{)BtpEu~qM{rtcLpOxRx+HKLo7e75cdeRmLldjdw@QDl^^Fm{YD4vDFC949bM-5e vJc`wA*yx%pf2D7pZ$|B2rgrH(svPla0y`9jCo`JFp-s9C4(<`hTsQv*b$4)u literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/cpu_enc_dec_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/cpu_enc_dec_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6a4074f5ed9051e86cb497f28998dd6cc4eaf9a GIT binary patch literal 8135 zcmbtZ%X1sad7l>sgCPKdZ&ISDS?$^pR4Zz=uEdF~tXHd*wGXd$V@d1El($Ahbb|yn zgMoSmdN6}LOu2kWDV0NxK8RKEL5KVcsr(T+O(mzC_U0=2;$2D1@9PHfO=`@a6(6V+7T>7sCYN)wdsJnV- zxJFoVOQGqSVc9K*mTQF-x1#3owrc}k52nJZTa{x*Pz&pBUG_`Cbl7kkvTp`6;jB9w z&bf2)y&TMk3+{sKTft(uLc~4!fLGkP+@hxva9c^u7i((4mcm*xV&u2|Nt6%=&CidmQ&-&di7uk}0 zk%#;w5}5p;-w7Bd-a0(-b8I5<>~uFr2Wi>&JfAVoPm-<|78H3KBJvp#>~93zW5k@M z1dEAItUn5r$R9E31HW~fci9aLnr#v7fKq5Si&;D~94WASU%}?Mk-=_KFX={CAj%JMY*Iem7&zoQ(%gvc;WOEXeO>U4mZA z=0P-V>2*jHyd(lg;%sShv=_x4lBzf}1yA~-+fv7vq@hpBdKpjrE}B%?rbor2;nA7; z8)Kj*MoJ^4T}@RIQ})Z$Pe}!|YfU4oPT(+|zm3io%h+11plMp-IQ3Vl8I6iLyk@lt zVG;8*HuI8XJkc^E=0Q7~DukHC=!qgu0sNa!&n02+T(8qRoA_drCue^c_P4_G;k|P~ zXX9M&L9!Kf&)p4z@Z7yf+~(q3tJn9yFpu$;7n1mULdySo??JZe$A=_#?8G%37ZZwx zW~lUMKC7Ctw{K`WKYGRGjp@?*Q9Fw_i14ap#zrpYVgW-f9nJ7ZeG?D;muMtPq9)p| z@>s|9XnWeOc3pX*;%Y<%AYutkR*L&QXz(f&COn{wv!OX4PQn!Etss_2i0@(56E&-N zUMuk9*z+Wp#hbL_X*6f>#H44Msi{@fc=1OGKk8`leeBen$t=(7`XTqc%=YL~`mhyV zR`t9)eLpB>%AUuf77&s$M4ZShNk}p97=j0PZ7xBa-$u^{5R-aWDHLa={#&cWKUYNS} zTr-xRN!GUjH1Vq!Z}A1Ft5WaKcXW5+t1#>{-BQJG;5o5gtx zn49A^o9Byon_W=~SW6na1-AIG<}QsOm$;6dmiY>xfg><4{PhW!~6%_b58oU%C9X6baf`{0L2fuJ|$fnGrtna_TXZ2!=Q=}F#1HznRp1dIgf|E_xqwQMQocf3uCUJrso($%bfw<8jG0{HFCPM2p3?M}B7Z^4ZTih1KC_Bt$X z5qpL|LX~_3$yZFOm?-d+(C1W3^*w!HBwC`U#;!InAaxY%7<<|;^g(H04$4Vs+f0qL zl$vQ7%r$c6>f!|UIS65zW$`B%$mV*2_k6*}?ar7Be#;xq55u%^p650r&Za2-mtKj- zSs5D!J}irsOFl7Qd`8VRI&f-i1Bp|1e7JRfk$GHLsB5-1IXMc^iuv5bNnw7Ij`$qS z4LmVR;I$it?u20{?6&5S>J#A3hgc;dXbY0WwItNy& zeyDtPX;4Wk+oi|mo-(jyuZ-SQYCN`5drw82TVZNy?`nWOKBxl1POH$bsbg?OgIeBS zR0j35Mx*meT3=EI)2W5^)3pB1k5lt;HJze% z)>p)nO{J@*^Ad-Dx$}?73_}?4P)SQ4DzwXBA(@4+FEBNk+nxt)3(R~}8kCZm)L<$r zKhg$^z+@3Bq7Ie-vy?7vFVj6RYe^ZcsXWpL%XnK!m%zv6?PKXOvzYcP3N-9T26%#d z9IT`(z)6jtexm$RO^!nap?c|3x|jlow)qP)UGY}&oWQe&=Oi8n&l~AV+DKo;`l6u*JlL}kTWxxPa*qqq*OXmZDMD` zr(`VZI=-{nxy!o_43bQ#9ch);of}&`<_@x4nPB-rv2e5-tbo&}J%Ekh-wcu70|Cbm z1ozp4v#lsZ+J@;{S6-7E-WqL1_P{}$(r+P!ERR^ z@BqR1k&}hpnScrX`=kAfO(y2$4SI19C0^+FdcgS=(;Gq5y6wqI127n&e;KK;h^kwbLU{a=`0L1^JMkv1PCE>lYmI?nKR`o&jYLobw}zM*EPuzUEhO z>aaSV$*(%Emc$p1x_E~QVX-pYK(3uW>M9ENFf5)8~Q;@^a zVCCW_@ACUS@m@wd%MAbT4aHLJ{hb>}Mta{Ryji~p)%1h-z4g&Cs34U|>w#XTqaZ2% z66eZH1O>h_<82>x&#x&k6c^D@k%DzX36<#)N~5R5&uNlkvaHgHfd;H=B0i#b<8FuF za~nfxCix>hcgrXc(TQ%2I?|9)=()2m zTia}c7BaC1RX;kFMuzd;+$)S}Hv6 gdBW88?rOMVu6+iCfg%rY551pn%uKt2;}F zPloJnxj@LR957hzxYhg|(6+(F@GWY$W1hGMS$08Dr?^SmuDp!xkUG^9j6fC(;qjuj zH%5U7X_J5;7RIk11!#e*k6L*UKbgCUF91(8#{Yq)+N`R!Vd#cxX_jj2YqpGZpATbS z%d|CHwbeR7VBXg1sD#=2qG}k3j8P06w&uQ7vIqw4EV3qoHc>p3yQ(IlX{)TM*1lFm z)3T`eZPm81AI`*1$h4lDv@6a-&#bCuWf*zm|2NtnWC0TM@(-9A`#YEZ-})-P0w1!< z?YmyfZz1N%=24>xK?GiFGb=ze6Z_K2;jgWo2(p&3LX&N=5N7`>yR!a4b`df8irX^O! zjQOwXKdFNnZJ3MM$er0n%TcEhcVT%bEMl(vJ}(a;D=2yjD6 zAX0T{5Xg?hJextZ*-2utM)M?`&FRek(hvIliV%^=N;1lLWg%TArBy6(E2uHwzJ~}T z#>X*2o-$Ho&H0!-&#f1(3aT(V&~3ncGJ%K}--l$8JTqkxo;U5>97szbaVLC5LZ>zI zz$a%7ecf^A^ZqaqE>Nj$br|Da$O+<;SoE#nu#ow;AoUTXN76L8c=GnHy2zjB%-xss*w(*-$A zw@x2z))A=Y-$*4l+*5LJT=F8Z9{3Nqh}UzG6_P#fbPh;bAeBH!4XMdd_$Qgw<=iCB zySAKwisL&7`E7Sq4&>UfPM?hOGxJYHqCx1(i=QRn6XU?+fF5O(@>7@mw&gk!a75F3 z@v$df_cstp_^m{K_Q@&~tjJuF5)zNEP0ROq?n9{rJRP_s%llZ TkOrQ4wV@jZxu;ome(iq&VtupM literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/cpu_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/cpu_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b897138ece7c16bd74cb17095ba150197ae392b1 GIT binary patch literal 13343 zcmbtbTaX;rS?=5P+-GNJ@4DDyT`Z3+tt?++$5tH6lI1wDWnsyQ)7VUAr+as2H9a#v z-D`QbW^l+lBm@))Pys?hR#geXgiu9Q2vty!5O~M~FFaAzMHQrY@-rk3zG(Trzh`E5 zRq zm$F2&;Ey?De$gpP-foWjC8y+1I1~P)GwDw`Q~tCw?aw$f{;V_W&pC5aFWKDV?{)S{ zI@R3g?|1e~I^Dd*zty=_(wS!2zs>Ab!XiQMT7R-Y&Qf-p)w(+L~xR;?K&F0OW( zUSwTd>v&P(xfV*%nbB%hjA;B+7^~|ld118{hOO#Ct$xXCxp-OjbgjPZ zoo=@l8%t5{jQ4V*KFDRBYS-NU6QrL#)lX-iZo6KypDCQLsamr+%uk$GZNC$qa%-K? zQ~gR~7w`^uwK*ujM`jzXMp$jOmzF$*Z1FDMlKRZKWuQN7#``37e}Z@#;mI+c17ta-YdDsd@a&c3oaUum z^CW=FN$)7Jq$J}d+=SGyC7(r&oaB>|&m&)ud`j|T$QM~6WoaoJA1TYAEbEoL32)LX z@coO%oEBeh&dtAOIaA(r+5j9Bycrk$xMNa#mUY}BzGocwtg9`i2pz8_02Xtu0al;F zj^iekB|qC%r_VpPelb(VkLzC+)-TuFt+3W;1?9n-lv|$XdTx2Kt;*rDS6*OUpZ&1t za!tAA4ltwKZk5rYbayVwHMqsAVH+GCL}PKT(+(QMf7$ZQZXg@gn$=6_ zcPWSpFE>=UioDlaYP7s46?iXIo9r?JhEQv9+E!k;s#=j)S~zL{{K67o%*S)X>^VOJb(_z9`IL}{S*G>KjH6Q|amrOyoE8FJdxZH_pjSRCIL&)n7N& z!??-2)Jdi|O)>c(Qqex)=OLna?oJeV1T_NkYRWb+R%@$pqmkaMhu8IUl|50aT5Z*Q zuUd^VRn9hq0n)i@^(E}F{*z?2>bC3EYP{}Aep@1JtNLR)3|~= zK?axrJirDHv5`??5)VR4(utkeU~EG{#okVDBZA_ZsHaHoC)r*)Sv4;8FX0M`AeI51 zy_U1cyVe#s_tg8DkehPz4(7T!|K5T3gKzZ>9b6mpaXv)qMtrP3gvu2|v5j!DA~WtD z8h7e1;}Sqh*(rU)hXQ6V+XzexTb4)rUwmzk{&#u zFWR5e7Il~Wl6P!!%A(<>-3+)V1)c&w36Fr6+`N~CY|lZii}oO#X1hgq9CgRs5_*MB zf;L5Oe5EAU1j?v&OfKSmi}aot7mGU`mZ)2K83IpDf81L-##&FtQSkHre$ zxyX)%QdEG1-@$>=)FO1a0D5gzQ9}*FpxSVQnDfYKgCEldF5ZKWgM`|Kwy9s$bZrS- zSQXBzn&iF}3;BekY@~3ZEQu5@q*6%XLMn|EE~GL@WnAro7H^eoln7Qko+8|fidl_j zs5dN8UEWqnn=oV4Iebf0>`UHQOMys?#(e5GXiX)CGLj-IDg@02O#tZDEw_C+$}BY7 z^-I;DvF=5-D^^Q1yHksa+xxW+z6Pqr2hfSUQJqKXb=VD6bPnAKGGrUvpT)BkTtNvW zW7@i{XNG?TeZBl1t293xZRz^a);arw-(V$9qd>q2vxwE`>Q{9dtb=(p)CE+CY<@n> zG$a207f`l~{h`UhHWNpA zN5FhrjEmf+$c7Fm6OnEN)s7gJPyss1>(rEYeSz!Mv3JySY?zFv>~5r0m1JAFl=sLh zMx{iQSRm7_qjo?sZ|OM`vI5cqa(oN20usa6vbO-RC^Z1I@@E;k{pR}q_nK|#GuEEP z2+!4FAFqbsTPV}fgzs}4buXad7^~iOV(H+QLn4bymgM)5z%*0uv#uIFh$Hq^a}=-JKN26_39WF)n24|lQ z8uC5Z6fy2(X|H*&z>DFvDn<24zF0u5fMF2Af~t%;GGT>8xwsH*=xIqWFIM}>JUtkL zj|$W*x^X607DL#!<+jWhG)Rb1>JlsF`yU(krJ5*<KJ^sygZIj}6#<7Z>fpz+{J4ufyXnqk2Hj17n#fJ%_&% z^_8uRmC`LUW++k1sG13TSwFS6?0iaJzvsOe?7d$n*f9dX`|$1|H{A)KjCCm77mJ17 z7+Pj5tyfeH@ISe)*v+FXchv$vj^lX}?Z&0unCcF``^9Xp1n4czY3OT=ch-2R-O6;$ z&2(r&^UQY(8^(r+AM}@(ubXi7-X!p$Aa>?g!ZBCx7MP#aU~NtgT44?j^`^R0-N{Ru z`Zh2p6&9ucuXiUj?>dxZ>9_3u`)iXb!TR@-Yi>mXX6$x?GN##8@l=z zV4B$icJ(aG3BIg17f!59b`4PWIqlk4%pP--U9X&sBZt6<9wwiWIS}U* zQvY29p-T0l)x$pelJOFdaho~dvoPlGj`Y61AxtCYcjhG3A@a;4-)A=>zP0s--*dl@ z%D?%O(1dEu;1MJS{k&nKqc$-eRK(<*c+LH}Eixv#j4rkO5B9SUVHvByvHs4IeFZ8TxSdMkf1Bs9FoI zUyA0)5`9pR$-3mN1xm4Wh8W1chXOP1UA@SsHh>oh?G$%{j9?mdAi*^CIi{%cMkYoE zXT$kZXQ~&TdgfyF$usH`QeJDVsmGbJWB(4mrQKSR$%+-vyf}6T4mq`-brTEpC|CBy zh|0C!)-Ix{zUE13H^ksTeV(;A?TCo%aJEV1pehsa0{c#KQJ~02cAQY07AGGoKm92e z*p6#rD%H#EIwV;!Qx?Q*e+2&!aZeV^a4C1DLrGPKzYyL87?T+cz!KoT8SLJCR>vx z(sl;53#gl+P@t6#|49Kt!gw>4MGGTk*wV9oEd^o0ci8XRV#nLpZ0KPI-T`e)1c#*F z%;Zwg)U2)bBkz-)M#Kdz)1kOH^#2nVUyH?s5^({jHAOt!CBnl(K0$#%;n7V4SU)Sm zA~6&e6cRazgj6gNAU4tz8{NE{nAIRQvMaE{;1q1(NfuAW!rW$_rGWkXRkK&zAri(R z65yxaEI^3hE&yC>B_I+8qDNa9LvNZ|@8(;HkfH`9fd0IQr#xHWi^t{bO5KtOhmwH( zM4SfLPeKT2DQ~wQt0bQWL7dA(XiEJW3n&xL4rIa*q10#iY=h*p zB)?Ab8VLclvS0lM(_bJVd^^QhAFMC0wqlE1{U)CgxYTd4!Z?u}#`diUNQDUHOYa7a$6lC6J~B zPN7}nj{s>QSV6$em_-AS3Vzl`qH!od zKdE_1Cxy@@1Eg@NoYc~0+Da$fifemmH|b^I@S z;0}8Lj?Rh2NpWx{oJo&v%ZWuJc48uUIqlxrcUI2ejKi!q?#)T4v*ej@WAdbf_MWF#11;`|W67l6M{AyB0I<5%&%Wa~yUpfbhI}2D7LxfRU2dq zmtL;MRYm`fruqsMj$-g+#GqOcUIJG&UPehAj;vGR82s``aL;or;_7B-q)PvfR@Gsh z(0z-bDwGL*RSU+&)}#}c?l!t|>AbbObl1ZA0i}N@cHLUB>lVHqtqNbxX6`DT!TD|i zejcq_2q~Dg%_2*2m!~VR-H?N$k9CkMZLrUY5Vm5Kw%ODs?K>zG$WtIsZO+Dds0(m6 z)4hp6@7cD@)c*%5jV@Sht;CfSmd8Gj>$*xH9jU&T~wYI3y8gJkcCG?c}sLvuQ?NS z<>6FFUm(SVl-DbmYzzGjitM8da;N%1RLT;LByluz_3Lb zfwyCS%nnUKz=xps8o{67V~^2EO(@^~%Roye_){BBiKb0`+kVT=K<5SsK{%M}Go#^z zp$GfMp*sTTXur-8G;r7F;Ml*h%mby2e$5>MhpWUjOtNFZAkR zxv?lwL~(c) z&uf)q?kd+>zz|xn?5l&fc)@&>+xdobXCsRSF;2a)xMD?AL$C`}yijy;sW!;K>;6|P z5CI@W)}V#5?{N(%Pl|M2!E#yOziUoqs6#j4h~oflwKOi7)H{)gClwP*hNd4KY`j9( z1wH7{N3W2T;ldRbf_Gasm24p45!ppUh7y*6UPt@oebkH+@})R0Ap3Q6tWicFPHKFg zL#CSpn!%30+*|8&yN15qMna<_R+WJch8v2#g*+QVhcvd-6>b`w2GNBR5xMyo$M9hi z(cSK6YS$MYi*58Baig0BHGec%Hw$VDIOE#-$z31+3w0+>Mu{qooZ6t?!All+9tryG zL!F@$;3N2>ha)gB^pZV+Api5w8g_;b9=0O(=(L>~RTv>|i6& zkL6g9%R@l5@)9ITs|?qHoLQ47J^4uDZ}a8zaRT5ILCy{WfR- zuFD84$sr@S@Pp9n1V_u>{L=i<^5A^Hd}i1I0z2Qpq(sTfUhNXROHl$0{K{Ia$5Dkt z1=)<^T{$mR`$AlaKE7)U?4#(As8FBj;xkZk9xwRU!HSnMbULSu^}BYh_{~~Nph^yv z)VXU1KZ<2?zbx=$NJwA?LK#a7-bGB$gcC6lLO5e@Ti|B|TgcC_854|x^=t{aSP;x` zHeaFTXsw6+C3^pB9Ix$B1WoGARl?qIjz<=W$HxZiAgi-uG0s*DCj;jkf*XEYtwou> zann`-K=3JyO@4jVvU-$x;Kc< z$>N@1U0LDmj)ek{%{@BLZUg$&HOIXQA znUB|+tKQ>EwUzoV>mDN^I!X`XU3JnpHrLmD?&cGA>J$bsrJcQsu4K!u_9~eeFSu~= zH}N~vKe2H!-dFKps93_%k%4*r(+BbEocwabbv+3PhF+ll8EqrGEhmk4i=6yO2XArv zahs-6V&`tzh;Jc8%;53Qa%pZ!<4qh2)9wE$jHPHbZ_#augJB|oY-@eru8ld;J*40_ z(K$3{L)5c#>QQ8o!E14R9LN1$W&ZZv*0P9BT0rr^wkMc+nuO4+N+c5`v^munNr9wD zLU>c-Bw}e#A{AK+?RHa9|AeE1%99G$REu@~nuJ?EHnqRUluc45A!@|d_z#$R9K=bs z7#oHoO6+j5u9_qH5y>{kOG;EOnn15|p1*)ZMuVNbRWvxKn8Tl7s;?Px%+7enNEmNJ z1Afa$8vkpjjGx(Q<4rqb{M5*1eq!8H)H5j~%>zb;32QuuV|qwuq;2LMV*%V>ekzQ3mngIImd6w*u}TR%&dG^{XM^%j&NmwAe^wqXooUcLa`Cgj?9Tn zEk#GXEJTagC6PtSDe!3O{Lm_iT@+%$WaUXe(km+^YbMJjE496%TrOEwCvFsi8?b>8{e-Pzg2;+G%*QPl89Bv+5NSd}MQO1Mq0kF&6 zon_A~C~OufTTsf*kMdWQ9FlSsc+|mFIp>gjs#3Y6Du+3x$|qG$u@WbdEF#}~J+nU` zm3V=9(>>kO{kr@0d*6FK;nb9!!|#Qk{rz_G^<3`flo|i!k$DwYWNEpa$hkS83B9GY zbysg2uF*DKvz>SI?Sfm-sLW^;-6G$u^j>P)uFYj;tK6=*6;9_{Q|+o-ZBM(??HPBb zJ?qZ4=iIsWygT1sa2MLg++$p?&{}LCcaOK1+$GKzTg&YgcZJhd>#6n$_XMX)t*6^3 z-IJWQThFu|*Wq-zb*g>ZJrDHsdzRBvt!LZk+;i>o?)mlw_kxxSR)X5T=05kW zoT!TF_j6);Uw6NN+>Dq-ZZoqNz*$QIMkK>NF;n(j49Z|EA z@|s`Y4%WiXW@9U3%c#TdYZuqT6KM=_Y%6I}}l8udC90)Jy z>@;N9X$PGc!zpulE8N-&B(l>HMkoD_2-{v11Okudue}@8@g_5TE4beaB6@M@_MKba z)jQw6dCj}?HvVs|y`i!wJApaA)~SOlc{Sjhdb!u>a4%Jqk3TVz6F0)T-+EhK9WnY{ zUZdCiC`eXrgwkIRi>3EX2=;;~LU5N8 znlhHMYn`n|C+PpGP>+O~t=o$W@XWZgkR22nTc zM2(G>sy~#LMrX^x$~$dpnKHCf-n`3AaP!fsb3JzaRunp28SXR$x^D;Zb|@n5xf@6@ z(56?19EEu3(^iTuI#I8_&8_)hC+zN`;wtBp3b*go(;cMbIn7NEk-_)(`$Pze>ad(VvnU+lTrTO4!;~FCCbsusJPJz>-SW9ReHSD z?RoqxN;nVRvV2*so*>qeJm2IoVvbh8og)^aRzqO9y$21bmIv-^BO)CN6$HufHf0bb zm2sV#%z4mRBOYF#B)ySs_yu^RQf(`YPu4y0-nH2Smk zidNAq-7;<6)+)MgSek|BCjLOLfn-I~a`Ow_cw{m+%oJX#(uU*Z(Q{6KM#z8T*nk5qErcGx40|>8{yFWi5lV|i- zuB0;=jqS<~ZSGZh3SHH7`8;k(K?zu`ELV|L%a+Z(7j63xyBZ4Q1tMqZjXVqm%*HuN z7yND)#xTjpp{#Ew`2hB7mm99-Od1MzUW8i6LW6WT2;W4|CWII@p8w?~xXVW=9J8b&y|x{$I%L6$iyV zYfu;z2iEZ%^3dsI6XaQFaAVKzn6Wv~mUDY{oNpGkaswNiNk1r}U=ZQQ^g!Yose!%6}7f^ChDBVh+WP~+ptuh+eNR9KRQrnBRlskPd*j4&Y z&8a)PW_)nMrz${`5J} z{+S~S@}xFu$4FCl{C41ZiR}>*1wuu-;(7Obek(00dY%aDo+oLqNm@QRMWjUJ%OHu- z=)}^XteUnvJZ-E4BngNlA(YpMkbX=mQ4Cm7SMVb-0B_}+R5;oPoMJ`t6be>wMZW=J zY9G2>-r;Iw;$ac4 zMK|GTjYG_mUNUf*;za;V6L2ywiU6Ayz-B2>5VIWEqJnEGD2r-P34k2LH11{GXK=6L zJ{wGnIiYW!5Cj?X0F=kX;`=#wmLVeav{)7^$j^(fiKoN~fXW5&v^a?;#{fMS0b*@} zotwINMmQ)vE>5Ax)3~VBf;c12;_1@xi7B2H=kR11?VT4FkXi|zLQB(Vp(dVVP!Q`UUYtuKO(d z`I7iD>YPIfcokm}Uq$Y`&|$v5o|GVk3gY`~nBglEz8lc5A3K3^P$K7o?*J^u@P5LM z6K>WR95a}7@W5w48v)h?af*zuVM}BULgB+34C@Vm4&gj##M^iy0%!qF1`kg9BG7a& z_6ph9?FW~GB5PcM25YCwI1{<^)$e>dXE;_+u ztT(ngV0>%}xH+^yO>;jAZLhNedwryMl9?!TEmKglj?PXj1+b z5i%(99qM5f0gTN?i-bJr`0zBvs&hyBbvTR<8m*SI5jYV#6HXldhVyK6r>419FABV^ zexqA6T-&e5jU8;`AaZTmi5|3l{hnKevk+4=o`jRKTXRM-EWbmYlLMff0%q;Il>HtN z0veJz9Wv?8#uB4OQ|%CL28AgSn;e4GUfgI!@^|n`Vj@}+xyyHN+<42w#`NBP>!x?- zja%2QT)pkRc76S7GOcz5q}hW5(EtFBx~)bmkJH=B9t0ZOl>#B2P>~Q7tdV4QlM3%j zPjXzuJ!;e_X79{f}vaJBT+&~*i1v3;k+5vkF5F9wF@rRmPlo5`hX-THIFq^-B zz@v`~AU1GXRbyzMvOiJpbH^YFG|PJ+DKD~xG@R-(cc(mo9U0)2f5xRe8v?4?>U~}_6D-kE z8n~FHSCQc5)b1H_A=VI;F%F>2n>lz7aKZ5`kBE>)PBj!cya6a+n386xY4771M{D$T z>7gAs&vo3?9yP*lhIpIQ7ioqaxkc4U*5xdbc_K6+6yQMkRXI?J zff(1(B~g)E<+T3`-iXL7+Hj76Z?QXO^!~BYyl~&D?!-)t3wA_ymd1UcWn!Ey(I#;O z;}0ajD(guyA|IgAzv5zt4pO-=>VEX*MDOIo4cCDr4)aTXoEEc_iLi&ZlH1qzbogaD zwwb=C5A;1l;0X+3WE`?3fcItk|*=)qXVR6VksgNXM@PaTR11@T682*g9uvTMz zAY4E!0<;2I)&pk;hJ|2fuMN}at8OSl%du5p)^7oLHZ~n}NnLLQ!yd2@il29VK?rs4qq#h~cd~$)of(cH1VLp+{O zRV6Xn{<}4k)ulV5ig}ro2mA$UfZ}#X_sz-S5_4d5C>m+t;^%lJngOwBBOowW{y z_H++&&v)pigFNq03hq&SEacdW`cf_5#yaO-_MD5$5+)eG$ zaMgyft}%hV&0-qU3Bm()XeL64>R4!uWE8=Dh+!r#lX3`I8ul=px6$V6gq`OnnM?N? zWGD;h9TA=WDd#aJMoq9Jtk(*O$?@nV4^fkbph)th^eddm5<*Lbkn;%dLN(JV3ydSj zuZES5%F+zIqK?{Bc;!=CTaq5zs!+T#Zv&WES_NU)k@OsWX(H%G5<;e(rQ~%aI7X#O z3n9wPA%Tq=Xoz2tTo_I0#(Snbg>niLBM&*kGtm4`^8`&q0di98zj(`UU`GcOql`$a zkRF_z6OxEVhbQdloQLaJkD(e(1&Gd@s1=^`R7n!jhiG-`FmmKRJ3knr=Qr=JU0HkM zns@u=m9=X>a2{+29aKzbF!PAj)UWoJ$xC5ta~{V`{<}^T+;>`d<3fMEMvS&7u220# zGZGW1(`fhFG^AAf#~}`A>4*&aR_RX%IEIupg1Ol?L3^9EU!Q4=7)X9NHgu{?q6gOg zpCPS7=h=5DhR29c2_VPR83IjBw3W#o$);PTB8;CpIU<8{fC{rPfe}S$E#TpbS%Koh zr57GS1P}YCN22$6n_X_wWP*msOx zK_YABvB3~T;=wE-!Gp1QR-hIzuiE$!^z0Z_?I4c8<1FHbcrnotQ<-F5{s_}RsHaKR}{1_7?2sGe`P8)wjGir~RfE%RU1tz|)(}FN!gcEZaHRV2$$!?Ra4=jaoNp-)1eV6ciz?aIUiqrstX$r1!5j+F&8FbUqSe?%4F2=EXWlsA85fTJm9 zw~b}AB5+iH4BC=1!WxCw*_mLl4 z=OGjF8z}fj)@4B-_p^rwe-(u*V14Ne4&p8QQJO@L`}`UbI65bp)im&&1}ncw1FfQ-ZIT!OGaj0cOs&5(n&-(ElG!ZJW)6|4`xE0Q9ZVOGzl#K* z07f4b)F%v0001~jtc~~JD&YtV2Uv(GA|*7W3Un5HP!!O?4|SALU29(xd4SFftZnH@ zDSh$+Ke0J7sUs3e;Ig=$9T9&O-SaWiM~C=*L`$Ql)KH%Tka60QVXqs>Kc=z|K;RA2 zEiquV+~dOfJTkGu}z=HXj?^~;I5(+KdLz-YDjxU$i4=Z-vr8zNVzv-Ftx zPpKv9R{jMMW`+fFD3!@VR|YsI2-0;^lJ^F!-f6s2L14&6sEE)%0z{7xEP6C&%ps@# z3ZL~)j#l(ZO-*bZQs6^8cLN#PIEX7?i!*+MKO58H0kI9RY*;cA@dcbe;#fZqbB8m` zgVIA5=~#VxqBSe~2m%XI5BdK2>luJj&e~W!9OC;kd$Dz;G`>jvRkXi=c5UOjO0(6?d4QX>fm%ReCUW02HU zJKFH%+HgNEEyW4EQovGh%r2YyUoB!aD6(fg>Yp1e0-M$`jwrH{ed9K<7-)F37}P_Y zo@etH7CII|MdS+)X>zsx%eRzgP4JRT3E@WePstPZv(up=l4Q{w0q0s#fr>ZL(HPjf zNZAp_c!Khl@1jbFW`S(8Y0$B(`KW((G~&{La>$MA#{S$}2cpb6QJYcz1?h8tY5#Tbg56HK>js-8&XYO<;ZqU!#L=p$b>!m&K3H*&k5@_Y z!v@YvC81=wOoVKLl6wxZGP-BUU_luX220jU=mQ@vyg~{hItyavPS7`A_$s2JUC?F> Px@Uh2nq^dt#qa$uDc-@^ literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/embedding_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/embedding_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..336c63b7d42ecf5a89691d833932f7eba4341e08 GIT binary patch literal 6121 zcmb7ITXP&qm9AS?S65$kvo4nHwtVfeEinTNv0!G$GoBd+7>{8*3*<6eTk1?n)voGl zXH|QUqV%xH5%7R_+2@4_DT0@YfIq+kFZ>2x>Xj$nSr+CdzLV7@$ws>yTOD;eFDFl) zJbBKSCo66?T@BBRzxvJAe_qnG-&5t}Q$ghwyh%sbG_G+y)ENC6p&l8`h)iZi7PF!X zt3)=lqbjRL4s-N^x5jGVHA6S5vwGBEji||*QH!;rHfu*6)`_OrR5Z<|qZu}%##rHO zG{@#txe}g<=GnX|+u=gA$QD((8ZJd=*;!R~!sX~3JEzLE@O*TET~K8=>_!*a#b|}C zM3>m5=rX&k+Uwz!=qkG!U1Qgx>+HI&iAB-d)7c9`8{FUx-h8URsqq$XKhbzwT-!6z zcM~lgK82PkaaG)0)b>nvi|g-dz3D%Yt?u=zxp`-}om+QobyJw0!1iqsp8 zyiFO8N2t#vsBgsb17C8l9}iRUgA~p6U>Kxc7;kP0iE8_P9EZVh^KFs(+)sTpb>1F_ zX#hR_@U7u!oF-^%PS{AdDCt@G%v%sBBf$ed6%WKPiDi;Eh)c+K#wmv7jd#VPu^9Hn zUC7Pn%G#Tx-_dktKKwNNw4bOWt_QOd})$@4V_dIzKQY6K3-u66yIE+)DJTvjU4>fs>I*EMr)>;~k)<(hT zR_aUm!>unx+p^I5SuT{~7Qv;eTG7#dwHYS zJH$BQPN92n{z68=U@=7uu2}!Z0V9)C{aw`$xhP*{g zt3=j_e2&PAL|!8DKS}&h&uboR?;cVe^5hYKv6+v9@nfKeZkhx`q)I`DN5> z#dBjaqN-{yO-4A>kGD!A@Q(xn((g=oYXf(;Lal@PE^#$>t62pcn%v6lK@(ITwA2VT z8POWF395G;%-+cWA(?|(ZD^uCwOa$7-k$-TJ3?SQ zuYtB(2j4tNmp`Y{U_t&oT^uY8&T^YqpX$4fXKKx?vA=wv=~{Y@G~Uzy{XAAw<4w{L z?|Jwq+B1_oKdYn{2HlM0U*xr?#xL}L(|23nfuCCV?U}kZbY9ew-+rL$kf&?k!+WI5 z?*y`l+5Qqv!HZb-9!&n@}melC{#Sdj^$D31xY(pm>}6*GOwJzdz}UZepzU_3p0%^iymp{xtwE31Z);iWUD6Xe zBrBVUIh3acw=!D&DsMbBHYtf}-IHI1qwRdmc*XRu0v zPIduu$WfF5@A_?px`FeSP zp2PvMHN%)bv=4W_c-lf&i^Yfnw^wh6ao-PSBh_r@(onjnYpPz_R)0 zp#Es|nddr6IZG$xJj;^EuR}4`8A-7bgyGS0S(}SK(vqWwy!AMcDbgcP3^#+JV6Ab2 zb8F0fWWD}qi!};Mrtw1nB*1^m9|_h}eUsD%t;$^L0AMZ3nF&Kfe_O~z9SC_v`om2D zvyzk2R${(~$f7qBb*iwXevI_IpH5EalXjPPv5?#%jq=LtkHs)0T<*t`OY&jXF7!Lz zGIA>B1+B{STTkZx)FvNZ$rIvKiuzIpR>~G0<80-kv5TQRp zQc8X$ShYMNOe|5p4({9vQV}tmQc)q5L-eK!pMRNVC4W;6Chw5!#>w4a^%FZG-=_Xf zVbyg%?QeMrDbOkWu!sTjaOR#c?YC&q%(JeFwa65gN0oY(b`(R-&P|**mN>VM?-`{t ze}iPr$1>PFZ4b!H<=2SN*_=0@2})XJ@=FlMp)=Y#w9S^@(jDD8Dx*Zd&sr4+?KXN_ zdIKdxw~5O=2l$v8HX){MmLON6eN<=k8Ks}K_UebKXvP5C}F zRcn;*Q%S}7MJlNnQAEYa%qg>f2)=~ysX@LBKWvfD;@>tY5_Ybix;du^%Ig!jrxo-m z7cC$9Y(@F1@=T7xrnYOQuq9<}@6UZ7_Jq~+tTHh6%>&BgY?LkJi{`$%5o{O-$fh3I z$m@`EVvK{#(%C1jqNkgJ7xxD8baS_s)yUg-zIwkfcK5lfeHR#{F7&Xs`f|}#A5*&J zt-VV=v)VIr_sE};Tx||*f?UT6dQM&wfIT5`?Byo*gTRc`y@aQ*!0n~M{dlj)CPBzTf|9I9iR2b$&CDGbo?4`vJ7I` z9blTRcXZ$kN`_I~SL{zMd*{lj`}(}!^(=W4lHRy?59pzCKL7#cZuffEkFhNlS`k9oUafIg^%okT)>yI$+XNGBfYFNf6M#cEpu#Jz5 zs__@wG5&1VjQ=)V<4<HU9nuXaI#ThVZyg!ca(wdqRQ&@p=h=E zh*0c^+8ebkv?ImNsclf(r*?(*kag5ZP5>LMk8x@gDb1{EF3lb} zf?_GYYOEGY6raW<#g9BkM7c1zdTtkr=d~j&E_|N+;sn$lyB~LXqXKo8^v!ypG6wrn$x;lwRPw4uUq|(2mk;8 literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/enc_dec_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/enc_dec_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f44f9d6c99dfd2180c07b00abf3bd9bfcde4328 GIT binary patch literal 14144 zcmb_j`*Rz|eZLnDzySoohxm{v8J1*8kVWd@S8OY?WJy-#NR%SUd0g$DKuGrCA zwN-p;bVt04 zF3NnWKI9JD!|sSZ;*Q#*?lyayEH~?8?zlbfPS_Lfq&?|wx3{}H>>ciuJ>~ASce=am zUG8J{V-hx9FS(D~kIQ_fzT4en?{W9qd*yew{)D^F-Y4@`eZPCaJ|OeCdf7c_AC&og z{Ym$beaM}*r`;KQMwSiK54%U~BQjs8A9at}$K0pvr`)IQr)Ak-{TcUJ`&sun`#Jet ztbf9N-hSSF!G6Jg(SFfAZXcKBL-kL(C+rjMOZH3hd$@kmJ!PL#6~2$ZysFx#xw3SI zjj++X>N$mNV`Fy|HpX9E)lqjACF5)YB@_I4es)q>)$Lc9dQ~Y;zDs*HTTX}ism5|> zoT*j)(6~_Z{4jH=<=2{xN+`rfh^6CxPC%zBN1Pq9kN=OP9gyo!NYyPgza1x7r~ z1XH+O5zMJJ8$N%N)~|4W_VR1joGWL~oxghR%ICv@^Nm*9zrq)=_F|b*)!p&Te-m<5D>a_ZrA)pr&VE~kKuEdUz0m= zrQK+7@p8rE;kYc2sK3$_0C>4wt1~Xby_+jOi;j{>B&dYj97#%|(pm&{`PD_otF3VC zz!Oe|#Z=|S1nbNRkOxt?q)0}mp)A3?c=eq}fMS>SRy?USlRp-!bd~UdU=}fMt4h4DFaT33#T=n_p4pRhDDHQ~3xlz3yI*5Kio zAo}-wr^Y-?$?^D2%7)wKaeO`S0w)IWWbv>|XVG!#FgiED5*EC2UOY|RKSN1(An`1H zK1azXD0!ZeLrB5_$tM#s6w_2T^Z|@`|z#8>x-IGWN zBE$P?n01_L9jD)MK3$8p$-8g z=%k5L^qG^>0=+FZnVe8K5Kpzr>-A*1L|5V*B_}DNr96N~?!N^J7}kf?F-d>|pKfZXug5r27qjnm)bf?SszPzV1rnR;B|X*&b{NBq!a8n~a%3~;m* z&vI+tWG2tCG|z+Y4S?Sjn8gQKju%;;5Ak8}z5%>P@GkICHi-N-H{ZFo=bn;HWglnHu-#l|d(c06#t3_X&tANr;8SQdgz_gQ74UY8?PL4zq`*WRAK?dx=L1FuK4o?g{g$&&W>f50b_lHwqHY>}%*Z}4>lik}4znY7QudP(4(vI0 zlpVWcpx;B?c7)LcpXSt7VwJq-u@hM5r`XdN?XZ*^N5D}(5l(J#s3Tzz7KvuTL?RZ8 zQv}kTiv&6jkQW-2w%>eZHq1w0P?$)G8*IhO7imybQ1lW-D=8UZQqN01^HtITldoJI za=q_ERFaRvjayCx%6!sqi#5+-qIL^%Dq?=&ww6@%paw`yFKIa1>{k+#(n2&Z5$kbb z&UGOHBBicc=p%B<=kQ48M29a>)|7h+1eCf;!VC;5&;uh#1t9ex6J(io7gmuE5x1(s z0wB?)R}SNRNbyS2_@Rql!lL0&x2xa*GgcS=dcE0QhwG(EgO#ptIaSmN-fh)xv`Ok8 z@M_TdNb-BUG|wxL)!ZY%61Ylfnfs-#;!}e5TY?B7;eQxf5xu+J$!+$^rMb2rcLtHv zc41mXy|vISK1z!mT~kL@Xx-pik#1fh-Lcf|jP#H>=#0~nXKkD|tq}cY%er_2>n(nX z5(f$N&I68HS)SvOsnMshLBK>tPGXpdGiX3OSZe2vn4$Is@k#3Ivy^;}lFw6OQ}TB0_^C4&h)n0F#=7#Q-R@41SxNV=~tD&5c>k%*Ov4f zy4V{49t4SUS(g&W0xhn4dSe~cmyMO>?tVd^2Myi2aN%OfZBf0sN3@PVy(JZs(GbqTL^4XJyJrO4P<(R*OVq~q=3@iKOksm8LrYYkboxjU{(E+ z8cA9$9GPwU=V5elqDIcn3egl|9{?i>FE^^YTj7wX)I6ThiNl5X0(DFyWb~Cc9=v6Lt)%)-}0O!d>E1yJYES&#xzr=nxQ|<(2bO( zTdDy?P`*Df^p%%?{*?+-PMyr?r!ZcXHX!*?p0pvN{2xRbA#4ZGePlf_^)4v%zBUW( zITeWmGml>dnT2#8b2gU=z`fb`d1F7{Bijpg^)R^D(gNYq){S zKr%_;CfAyLg(Tn>lzR|d(E(G%O_bhOqvhL#*oed7Tv^_*ZubE$*UeH_rXPXXgDoG5 zm}sNV1RbC9w7cmRBX(O@4IBNr9mC_27R5S7z!Bm@+>60>s7wbqU^swkl>Qx1=Gj#Cd(m zkU3cYZPk11RYgLlmP`hXu0z&y8f`L@N|3_0*&w?*+aZhE5-*}AhbPaBx3z{^Q#u4Y z&^m=iHG#S9+Um9cSyILm~o6 zWi2w#$xtt+?QEp|!2w`ryWRtPh;MLwP;IxU3nc<{Azq-QL!GJe(85h+!Ub6KJ@Fz{ z5n-0Mht_8*^)` z*`1F=kcEo33bTb(3PXi9Gehhs#lRliIl$km~R5#$&PixI*-f1X{8&J=%>SKHEi zXOF=bQ|Aq*ve0OHeyy5dL#`Bi=VFa9RKoqZ;n2xBdg51U5VI-$s++mNKr{xA44JW*Iq`tBFo39J7)4|t_P``?NHhpOxI)Mk zrb>SXRAjQ?rMWK)#BuuE#1^Ne6St3-Igj#Ryo(;Bnq@$brKMOo_wcFwpL&)jF&rn? z+X8u^yEGqZ0n(F3PP8h`KKT*rB&AQT22mG@C#;4JrUDn@8m)|RPkUYOq&j9NO$u#f zjRkNj-ZdnQ0caIW4~&zb!8DB}VrW@BR*-?kW_G}+#8ZJvjCCp3P?^!O;=ZzAOd3nw zHL&t#g*f+_2YyO zNRx@oqQpa3B1zI`1YVV9Ty?SCxB;s!W+M-BLbAMKHkO!hC5fWf)5)(#+%IzMUZ+0b z4}&SkNV%WmB@SDSQ#XE1aOmmDU{Qlovu6}`K_VuG_T($>!x{uCS!LH>3YWUIz`>;| zuyH*}5oG8Q(+smvQX|7wa<+2bp2D|W>J83_NVoICz1`Y`are`B0+pi(m$Wmv7pBo& zy;9|NK?WOoi0mW;((!0^hG-N#$+InNpjejqh&@`ZS_6dZHpQ~|IvSO85%>EV@{u!G z`hg`gq^XBH66E?P}PP*KUg5*qaW`etY z5E-OKCNWIS53IC_;~o8h`@OHF^WcKodSbqsA3jQX+hXJm%G(Igprfl3Ix9Fc(00_X zf{`(80`@0lJbmG?b~O1mI{JdrF_2HKnMl((u&E%$3}{Mw{LEUGrS3tyz_BzD89=b~ zjPgbm8V#H;+m%4R$CMk&ICPu93^LC_8$vI+r9AZ&WT;mjDy+hpb0g)|UR78&$h@U6D=B|V`8ND+(8Lh(ZYy3p$VYEL=BV5s^ov1yRfSfS z4ze{cD5eBCJXY%tVDon~oq@nu8wv(!MmNiZ;o30L5zM(D{g?x+a3_O#0*fCn-YK9^x+p)aySa)O0N1CMG4`Mg=20X^Dnu-ozlwlVUeswsi}>k~tabmEO2;%&olXkh(8* zMvCV|@yC2pjv+khhO;M$hf}fK8A+cpv~=-FjT}Mw(5N-$n_)^+8u*%{Q70r#%B?0} z2}vDLJSZ~M!SzW?rz-Hu_DCcxewSeWh>|}>Vj~22)5C=snv0vv*y_`oGm*knV_}+TR3aHu zz6@m&J=(jHl4y3D=~etb4b_!64*~i%0a<+$lNx}n2x?t3OuXrdo&@kq!%{bY0U)ErnDfiS=oynOTx>_Y_#fluIL* zgEYtl7D)q?A%M_UI{6?kUJWv9mJdrfutwpJftCHZpGT0vTGi0k4oH#g(m*f(CAP2~ zaiUHk%8x0X!Jt5;qe?IciC9EGnV=}8&85Ixvx1x)D~;XgULvS%25r>cx-K)nZe{ejY=9>e}bi12C_9pvK2j!`Dh=E zG1cF`v;(U+#jU8ej8>%;%)~nJ7ILFogIML73^4gj3|Pc<4U(p?ylaM!dEHS zfgPJ#dOVnd{-J%DVt@I&26h_r>g)`5V)p7cRPTAh-``DhgJjnb) z*@I^JfK8>nKu&rf zP^#1gudwCm*xi7j))r^>JmGM?+ZC}n7)BH!Ll~lc0w@D3>mlk^VS&i43_d*wygQe@lB;zHGd3Fy|R3}w2!}cCo z87ur7U`QNn>SRNGq@#HCCP~2-?2?5os{2$JV9-VXbP{QV@mFf_+Z$G@v~iuGrXf~G zhHba`furS!qTYlmLW?yomj^2?i?8?;#!9?AJ z1FCl)>Y>$L6Rx+`r;OUB5bu=6Ag-30C6tzvxR8!PseC?rQ$9tBX$7HAaPT3WCRcymDycTWIqtNIM7`Y4H zUB~a?+A4X>q_%2wpm_J`s*z&)2dLgkMg33GTPWfRRa(uguUy%J8^wzB$8kGsrD03s zYfHpEjO$Gx`65{jX|ftf_xKEa<2Mlx=tW|_1DIIZwsU<)d((`<>~V6kyx9^bewdgD z+7}E3p3n7_PyfHoEExlV`7|*rsHLHf3!;L8)sbIZ4bzOzS716t<_>OTv6?FZLLGkF z@eCmZ$}DinzVdUcn2sWtpTZ$zeCM_kAO|w)P+he$loCTgpzMj zu@$Wk-6BPJ4Tnm|z|k@R{0=2wL}HI8MnnV+;Tc&T;z{ZwTZ1Rv$3*udpoeeNsiw33_fcQm1Yq@?S)6aPSS8h(&%i9e%? zYinOUsdo`S(=)JxczG66u@V=q(q&j*2=Rn*Ts1U(TzR zf%rF~;AE0$mNY;GW9 zR0x_3B-u9%7)`q6wDpWMo+wCR>4?Iw8;HK6EqcqWBQzn+AZhZvpSLWGg%b1+6X-*e zMg&I4nFST-#U_({|3rI_TJ`&OT;Bg%9OlD$#zN}qhyqlF-nLliXlpfu?zA=2nw$X^0Y)IxiN2el(sAexE(yZ4h2is5) zU!_F41*Fmc=k!V9Ls*m$5s535^r-6`eV(L*yehOD-rpjz;JPWS7c~uAk^C9o*Kqfe zVo;g^QcG#?<5KERwX}xlllGpO)qV^@{E?Z{{>RK~Khy@aA83Wb_cI5zJ@DRs1Y-G6 zGm~7=Eb53$rLfWOQ?D8H3L=rcT4w&I+Viln;rX>BaT^~{I~o0wFxLC#5s=rXlG40E zq&ob8oJ0AXxJ&c-8zi`Lu}msxv^sK~Xw9R&k_#huoGfg)1@@4<=sLrQG1qXbj5E7L zcbR9PhEZIh*_es%yLdB|IRI6vzPvjuSJ>Y9Fxa+V!^K*L^1ik zi#US>oRh%OwZ(|mB-KfRle8vjQWBUXOi6D*GAPMN{dEbhrbFpp(l3#FR;9N$p8QV literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/model_runner.cpython-310.pyc b/vllm/worker/__pycache__/model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b85fb28f12bc42b8edce049d83ae1ab2df53dd6e GIT binary patch literal 46582 zcmeIbd6*p6bsySYUA<0EPtU$!YwTzO7!V-1f#3pyAP5l{LRd(tB&yk%su@h9yJv8# z2P8&4Tm;IZ96^+$*s^RY88~vDX*${%2O%E*keee6;A8+QX+qZ7reL44>bI&<4P>Zzoybfwqotz@iBrO)cCWUXwa-|DXnSOb+oYf#dJ%R|<%+mAsX&++p2O*=_Bv?6LM#_F8)@lh$NqpS7=Yr*)^qO_ldo4p;}o z-&4M;a?m;`{&e~7%01RS;_ofrTe;7=PyCtk{gp%3p~?f+1M=QiK3sXwdQkk?@q2GPnwIxm z*{(R2Q<<@5DznyX<)U@5GH1-_I$+{$QHkZqlidCspt!ic7ny)Na3zf^(<%+UY z#kJf@&8me%PV!S>^(1D%7L4ekwTO@hoh#SE*7NpO>jgYlXG8Y39}C;t?H#X%oF#jw zz3ZCsSm;`KG-Q3)&Rcb7z`n!YZST2mSj$e_-fK@@4<8TN`|RDXhV0!=^;!hEz1Z^Q zPJ2JT>~~(2FE1hP0sAh5+~v$WFOG+cK52#d5iLq3n3(V{_9r&pbKj z*1W`13$?lVYO(A^&N?+OdTw!nG0t6GaJ)F(r;Eyqp06TK-thX5)M`$Z2~Ih+B1`nL zvres0tktT83&rV6PSr+TGL2|wYsH#_kfEdVmCAf|RrsKb)YBNzxw3=bYR!3}hUi&D zM^KR^opi1^<$TiXIa-{)=p3D|&dkkv>BpQabJM|F>g0UU_P@Y?=7{g_IW=!PW&dUH zsSB>7t`slKmFH?#{lLuWqC&ScBb`<#xpu@ZF4P?5N9sF^g6+j}5WosHf}UEeEiBet zgr(UN3k!u=HNUuk_ug4mT)0@6Mplc8ogj^!nODyjmECBp;bl%toj!lAaOT+Y6KBtz zdB4|tqKX{PII|cZbrtD)=BjhGLV136)=_+A3r`gn7Urt6!Rs04@}lFi>&8!>IWIl+ z%)GkfC>^(2WKsjwE>;%GNY@y?;m#0Ai+r-D&dk?}>gut|1;@5^QN0q`QM%2Dlw^}D z>|~}m=TzN!HMi90C&TM?7Z(=hRn0~9RS`W9yjB*=wK=3LGUd3;f`!Gx{LD-lLs_U4 zUnpF-T65gI=?$JbfAZXkQ%^m1irZdb)7&fbC^xuuSk?IA9mb zm@%#wn{x{n=N7zC{9=w2Dw1QNc%_IH#r5CU7zO)kwOEgAR}kcj;KGb6gd3++M6KAU2j);rWZ`Gv|eAN8lf)Zv!|zssY*869BFG z$k)U8`MN>Jgo9nH`NvQO9>wDh!I=%!L(j&R!?jR7JQVr?eR8RbC@K?pv%hDuev^ygGjRJQfnOYzaa4L@&%QR_$}}#n7)yn88Tg_D6j2 z5j^gT=prXXEYLE}12Y8vXIQ2abz-IXc*sfEk;B0ItYk}wDIqC@^f>A9ki?2gSZ`Zc zOu{lw)YdsA7ZvQEFBE-7IHh?A0hdgOf&afakQE$_pUX2eg^D`BGu8$f&IX|!%YzHh|$O_JB3yyqnTW`{9RyZm-%N7Ar2+2hVmXV-Yho@mCg*P(^$5jNuNnn~Cj?2WHRfuH1U zOu2kcJ&loAdOMZF-?@u(ZtenBbx!bu+%)#t;#}3u1#2o-bsXEVbJ&}5wTq69E7OWW z+^%v9KrAG)3yZZ}b-vb2%R~$Ft~-Yno&yA!)A&TrtvL(so}4o|JGmzpEY8WCOfAVt z&sY~F7Hc`YtV*17FU~KP?VP)KL07!ShM>|H=j~i=p7AB4mO^LdoU-jsO7N7I)=RxM z54^Dphp7gzH=`l=&$IkgTqXUu{Fjd1-D}*oI`2^?Kz^={?p}8OG>sgG%1W zf*=lnZGt&8l8$zS2bp*SvIBDSES3wG(8sf`H{yRS%-L?CHeX=<;Pr-Q00&)ccZDE1 zgB2l>m$@>hYKzFjsm{(-9WSGIh-qLoM992RW+Es_6Vwtuc>QjG)fDD64&%j?15~BD z3dHrGAOUd?!BY>F_=NEocp`XAJW)I`JaIe;JV`tJMhbv1RcFkeYepC#5x~%x%BR%( zk&m+Ie2C7obS}_&j!uz|Lx(U%*>Ljdcb`T*#?%EkZyKKII^`L!$KN=)Z!|%!I*T~o z`l8#h2em}5B8EEw$281v=YQ;5>9iSM+S>^=CvTUd7nv@*s=|`fGL?MvSrKMWN8kj1 z^aEP!5y-()2VK1wR!M9!Mm>DZSVj-q;W_m0=fj`WeJqX3ht=bV{$@nY<44L;3-mQM z8^7hUQ8u<}OtY4jzla1bL3<)*u;sThMwgb}E{8Y6Q~5i*M4>zzI&+evA=H7n#WX5sF^di8%W$x*zIK!`GdZ<}C_tUvZ zX9FFMkK!cpdIVHyAf>o))Ok8Coo2s_kJC=zG-J4j;iNFX24Vx@A^d0M(e8Tq(|*W^ zVPp-{d?yw*Zlw(4W-Ma7W0={uS@X6(_TM9TTm`4wer!ju_rz?y_r|doCFGs^Zp=wK zDW}KZi{f?ydr{iy#a@)Llg)27c5J%`dk{D|Gch}D_sSlWkv*u-+t#@vo|&s%4AA6L zKH|5uMWFS07aP!ptFr${GL4M-pj_=l8vwF8u|_XRBv(MMnnRSeLDzfm6@ch(2)b%% z0b{)Z7*i!=m(Tn^0ZQr^vQRJ6xkRTqIb>dNGW{JqZVt}cP=_nhG?tEhFFk!*c+>(O zZV!QWp*}=!uZ013G$iV+!PgBEsmq9*H`HZi=9p7+V}@}+%%j-ED>%1n(wVy zw*x+^hCK6qY7sx`3Z3Wayg=t=I$R~{!*pJOq zsQWScW=Z^!QrJUgA?nUuZD&c7P9MkEH>+wwIEJQo}~d>uvHG4#Y9doOYu{}?t% zYXW%ho!)5YN_bS0s7nXLqw&uc(k92*lj};9Pp%rKyD&tLaG1dFY#Sy$NE{#x&h7qV z0F?l_cKnSYJAuX}otOhs27okS_sFotb%-W6?6m*A7i>okYfQg$Xzjj^_pJXeIWh*~ z^h+2@fPbLHKiJ|QYVi-Z_(#Ow$ju(@cpt;NPp*ML@mqp&CR+UK;O|A)di-Yay8-Ro zh-VX?oD-ipU~jg!yc)xF+iGvaB+lAV4CD5tJ!=f(r0`T81LqO6Dn?DXC`}KS3)>|pxbS4#MuEW$MerQu&~g=Xqp-u{p_TBB5b#!Dy3IEm?}dm> ztk$%{*emIaHhi&$FW&Ga=tHbzBfnI`*VFK&8@^ucvqR_#V@mLRf3NYP(`Bcq1nu#L zD{6kh@%b4IvPKab@_NCDn-v0r1dgCYUYdj-7_6G+nPATp-0~cmdxhucs`mWzg$w2R z=}WFRAOsJBoEI9D%%Ft%@1Swo({1SZ+MdIkVO2(bJGKPy4d=k9bU^Qr z1N6%r7h>O;a3&lJr&j%E!b^wV&4ih3%U(Ko>hb)L7pt<%+?H+qqpY0Zm)}R9pq3w_ z?=?Cfr^5-O>6pJo-zVU#A_slCt)1!L5i6@%lL2M12$oL!&_%Hu6_T>tlFAD@iWzPlc;n>v46W4$`%ds`l4R|P(q#6J6>3TvvR|k<>=v^arZ%gc10T`KjN-fn>NYhvC>nI_do#f2f{?|GnZLuJ^w+!*vHB3cPSl6tuB(s0ZOm-=_Ghx-epOvAExiKbY$gneW_2<;Tra$s#wKI&#Md&%J^@#oUOQ(|GR)iok6))OoP169R&OaIXo!*IwHA03CG+w0unk6 zvN;xs88K`m{=YGRP}4|+2aIgvH)iON;7hk(!-(A)i0CgVi66KzgWpmphrj6v{!vEo zpD;{@VKYla%v*d=Bh)5s#0=ziYw7NHwV4GiZ2(#{7Bj2`ry(p3fEuPEuo>HUHBl;$YEX{h%if)_3{!-SnBha8Xx2z`UOWkX9-@3J#nX8Gh{;6uf- z`w%?AnaY|KRx2p4r7}0#U)d(98zd7#>hJT7V+6$RN9dO|By0K&zP|gFbDhaLmMC)N`D5s?+%h9ig>_wr=v|?&gOB zbp9S4*0`k=(jNAlH`ERB|3!Rp2@(?_Gn_DtF(YDtdjL0rg~z|(JqYD*IQC^M*0*C( zHZhh9qWRG(+v z69D9bSWa_F27ke1C*>im524h#G84f2BU95s#Nd~`ImpOFg=@ftxQu7RuZ4!d9VK7+2x7Dl-03|552-El)*LcBAdYKhs|E0rA zlkct)ZOfNEN^JBgJZ=>ZfF$rx;)rg7I4JNk;|khNKbT!Onp=(_4g`UgdFT-?8Ou@N zozW{{wUgj<1z6|}t$OyP)Gyy04k{7gkz;BcQG$=0kUk6mR;zXa*Pzy$4MHbNU#1BKSq#1k8X2_M`Vwb z?Cq8jdyV}&hD6*i6a0^G*zt>f3wz{kdcAvwZM_~${G%wq-3SLgiN0JhZiImC9-NIxa@@dj{SE-5v;no<&SXt3%8CyJtErQd|77P{8LTR`)<6U!;uEQy%L zIod3VoTLyrARS7n63GG}{O}Q(#{(acbv*D98OH-35pX>4_4BLBEy91-EjH z!=SEhGMqyo1 z-~`8gUSCCIH5vhO9Oq@5C$ZqrP`oYE(VUc~*j>n0c|Fo1KVpwl70DWgdBYto@Z${o zaoWFnW2+h@tR4)8jT!3o*bdnYh52gvDv!<^z?_JXHC!oPa+(LqK7jY)&pX9S7$`4_ z!d|%Q4YwVV2m63GI(>1mdI^jO8Pqb*uGtFP8*XoRz>Zq4n_n!?%~p}BJyJfRw+{gi ztU9D^k_B`(Bo=T$*no91$KK$oe*%$G5!1xk3I1bmn@PAgW6>sGiCiY22{>OPT5d#v zpqmt+NkwB}^JXeCAjo~tAdU$#Y>4& z(%mIKfDyH~4o2lzXgOZRj-^xK7yJ#Ri+z;)TmpH70cH&BWAtUj7l*G;d|?0?#9QgF zCvI>*3zr7;du@<;Tr(sT^#DK!mxd&bSsJzt99_SRkP-NBph64=Z=(Pa;lfC5Y-PNT zSj^!@gt3^8$BA;J7+HGmsAgh`5DR#jIgrb?Bf|tLnY$i-_m-;=od`JPOEoX$=wlr80eercKc!&^gh`RTt^(P6O^)Qpkf|pA zW9SaZ-)h2Nh=>oez#+{g4*0`hB$m;pk20B%8+rJwF{wRy!x}aPvE6)9udd%_{4dk_ z9Xem3^Sf~Lj7Bfu;Je9P_F}s69hJF3g}X(Vl#&YZqeFTb#<&QUxlH2X<1&h0 z3ak4iC8o8BN?Zzo{g{ATt3ocOx{qRvgIMUAdrP?FZktFUx!_lFwoO^gQW|sSlaDU_^wEpX^rhU~ zOb*=ZoZhK(f{o>N%}qL#|C+BrtV|9UazzC>v9fZC_EoH6&63VH7vKrWWlwH#0gKPc z$$a!zvMlPx)T9U)cc&l|iVif^x^gNIal^@mUurXdeV+35TlmvRNRQ^H2^IdoV}58( zmN2q^m9MhGWWhCwa;_}MF$vw$u{1u*G)!%^GxiB7{!f1UFY;{>TzsD$59wIdDUPIm z0kOWrrE&qDSUeF)VS#|ey=g+;BW6+}AZ=t!ED-EWW@HTHuPl;VI3IyT04Oxymom~| zH*wh@Mzk3o_)^AL*!V}22EwF%^k_GljRNE$%gheI`0ZXV9b{~+X#XW$& zh)_{xDXIb7kyNxIUn`ulb(tx4cK2n-zk}jze(R0;aEdF1iEs-Kf9?EMeJP31pR$R`Yk#< zl=5jo)jln)@ip6N^#ur(-z`?cU7VSbtxd3}CJOaWkpEpE-`qRkF=K?%M2xbSZln^; zn?fhP3@#{0Do8$H+8GoWyX&uo)|^~HHFRgSO)U~?4RolA7fgzfi&IxX<4B3q%p~Mi zgBWmQ1qVVUkl})qL>rSYwi2(CiwQ{We@plwkx~*<3T`RIy#W4bHEc(8Jb`iM(jmYl zDpRzK;hqNM(I0qRszH*5ffTDRcIuz!>BGj4 zGkOeysNCAWpD%q4DX0a}H1PU-%pMvd@Lh28v2K9Li)&KI_L2>0*UtT!{It^?UsB}& zGT|7w;@E}513-iEMn3s1aMj*P+*;cG*N$nE^lGDqIhI6q1x&mOPs>;u7)y*E#xm^o zaV*__8Ds3ib_7hBF=502*%eldSvMee8pffYQHo*VNA&0lN{QWi42-c+qLp#^Vw4i< zFk%7`#;>VI7+*2w*u~7HCosagCm~rZQoLQqT28mS4RZrRw2$|{L<_xse~1F{V2rvf za>2m3naF92wYei2@yS5G*T<;jjQPJZ#h=jOX>;3n3wriv{Pd@AtQ528j*R_Ne80fK z-UCl8&cOz%70zI%!R~@12@Es``zDShu|>V;|S*ma&%_bjJzCYvmb;S87P6I2u~Kg>4C zG>b1Cz%&yAU@uVrVp&V8U}F%fWDkYfb3mei+fWi;I&PBseS?(>1s$gymzqDmz<6e^ z0?jVak#it7EoiQ9!8#HEO&MUzu~E)K?Ikr$NwogMKi8Tr+UM zU4Wb~MB8x+idK^75)3Lo?qyq<7eb`{D^}sZ(GfX>pOng=hoKrnii?smUIbK*^7tir zaRzt|c`cd<5_0GbrWVRZ_~n1VSD}suNgYWC@d#9F52!!q=l`3|U(op&6;lhz#X4EZpbuS5yfMp8`2EWWdyz}} z3_P)jFgUO`3wtAykTb{uP*Uctl$nvmjy;=bp*|%+yx^bvx%sWwtysnoC(HTM;us2@VN^ngN)jhTwj(iUAX_1=!K+3BU!gELZx#Q2fGr zUjl1i$TaZ&BarWS^zwuFGJr2}LIH9=DEE8$DVsd|_|mo`Sfj-z13}n0!HNyeI%Kvnf0m5g)<{%{R*bUbt^3_o`#>@(sgD?jja_raXKL?i>kd|yL8*svQz>Ge8QAivX2 z!J(3_H`<~oc(!ovslw4Cr_Y@~bIcoIo&5#?Mh1Gt;NwzRmW?KWZ25x2zcLC_Z_$zY z^IiJhrqhnsgI!MQ!q-%`F=$X+{tXV=03v6>7}crvP4C*md@376);GnK|V`jumkmxm+x|xzjutDVJTaT4sQjUd>NJ;g6d%@1jGjkNl$WYBCJdb*_x8~N#;X3K&apn}EeX=tTFq)}*fBxGw_ z`t-S?eWeIpYoHs>!gGzhW{QF4w%&QPcrmiX| z=%!}HAM?doISuc;O*>X)#dftft7W#JM;ZzIMcF=*7{5b}<;rfd1E^qgl-B|YSQ0ah zjM+i_Yw0}_id^T?=~%M?{=9d8vtKpjk2jA3)PDf{Q@=q+pmR8k-DrOA|^n~*;$00^1nn^oB{g@Omy=WKxQm~w4xBPv=d(!=>b($9s zTo4RHE$(5008C&rPK1~4ToVUm&AVjWcJ{vs1!VBZ2JlJjuCpPkFP3=n-@6QHixP3O zIPt(00(x}H<4{Qm>tf$4F z6u(hQxqlxtYA+sqM=OnG9{HzBy`_xe7o5qJ`VgAA{F0rplg<8EmgD$Zf4v8?D4BZ3 zPF;^IXYr1nTJEoB?Vd{!^{bG`&4P;xSN#V2yEd>gXvgc>8~W^yB@GEB{S|JkwPD~p zBcOF9T->p1=JJ3(4xuD(qrZF6-+rI9=Xv>0k&D?Sm%-{V`>Q=R;wHPrrDU$!!G4qY z>6RLguS|gMMo7K?hPgacTX!wAJdAOIG;V1Z7abv;oRg?QzFy8|{&~P;FCvv_6JBa_Nwry&j>Q&+>@WU>r4=z?aRa!B3zD z{T($JM-2u#a$P4i_-uV$ml_Oq#9d$80_phmmT0Ro@`l>hm2LG6H^R#l#E3xF zdJs~|?4c3JDQ~3wc4>QkqCRd9Urz%=9bG#A_#Dgu0hN?(#3yI4DHD6rrc5-;0tc)O zkT@-S7&fB4w%Fy#X$4|m5g)KJ*Imxj1tbt@wpY;ipu>0)9HL0c(~w`iY&4~R!b0=< zGM}0#=|EwVhaoEFys)4i8U<8wiD()xQ(kT0*4iLTTGt|$7G%)|vH%x_3^Si?tD{ z97LiT1gGC=o+%V!v+B$TwnC^1&y%Q!4x`v_X#CtT61QTvzW1s(@_;Nx`xYgeo{++A zZD9-fxO{SOWc+({TL2ZKgwiqx;_b8GLx_t)DEX_B+%LnbWi+JwV+gJya}lXDUXF!K z@~p#|TWvMqpa#A9jS;+vq%nFE(tYU5Yhi7>CFsouSs+>QDvInoi{yweph(q&k82jA zdV{1G9?9_gZE5Uw;MT4UD`9A_)vK?etj457IjaYe z?USnY1=WSwgHIrE&5?Q?NmTZBBPB|_Qe#-K2tbmOzVho{Ljfp9SdHEKi8JD}>?d-& zVt_)rm)gu&6j;d*(D@*pXX$W$Yik74^x1Ttqf?~A#hc%ugqb5}(f^5|{~V4N0p+K8 zJ0E2jN$ZxpUy&%l>YIEAVIhfrmZEr(mOV|*0m|2)0Szls%2@zm;>9#XSBIJ7Rpw)s z=c}_`s%ctE{aXe{1)!g5>^7emO|z0KY^9QI&S)H)$MK8ur`x zXk~F^>9bw^6AZQuHXiI3z%g}U6pNIbH6&b-4)&}yA)(zhB-uOGU^Dk7Q@1%B4J=zs zLNN9!j9EA<)}Ss~*U-0|VL`5zGJlPX{cvr8LIzoeTOf)+0U6)tP%ph6IDxIvRm@bF zQG@iIo*?azT#aE6S=G-bsK^NSJ1107t@-qsYmTU_gnG zN8sTh{@)phG6(w2cVbZ-D?+fC(sigGY+Qg9Z6t2;jqAmKLkiRt`GFlBV?5m&X;(*F zBu8AtL_A3AK}tgN6kA=*Z;X9&X~S)IW-WCb7N%l;T~|Ga&vUrybl#*ReAj1Cv@wQ>6-i9E`J8j{S)H@^lM8SY?tnF_D-X6i@ zK7lWebz0jP` zKvTZY?$u_%pedhq25=(R=M36e_+j`9n(%|rfX_O^@DDm8_K-6=v)&%IM_^81%syvp z0|(>DSTbgxhYtGy3~cFK*5h{qsYdNsU}(!)2MzZ15@Px$4K_eOeqzS3x7b@@l3*ip zc^bwIHaR(@+>RVJu|~`VCLCh+4tpoc-Rx{x*?J>{(rNSn;%C-2F>nyFw%dE09nMa~ z-|Os>a$+#F$Yl`57WUf*5ON0$j!yU?G&X8lyIVqNYSgs$;QJ1?0yU(e zQPbLMAF>~S#{Hyy7+?0Gqz9cl{gNKCA4ZJ*_9Mhpw-ae4DpLh z(5rV!KMdOTqR;J1c0y_!d=~#)@YfhUbLon&@@?}}7Wy7;!B!AWRRwL>Pko8ehBPk; zdf~PWts0!0f?4PdLCm9R%rq!6qjw$Un-2AQaI6@-O!pzT;IAJZN^2d(hoI6${g9M8PSiCI2eHOP0dX8? z7%1q%L~UJ($_P+hpdRgvf%lDe5D!?dnJAuI#uGs_k5X1hwe*gbMj;qIO4nh-g^v z7VYUh^~}oNx>lZ;MEX4|``~K9Bha>z`<>$MuN|n}g>Po*pbgO^av#DW(sqTr5sIF$ zW6)Utz1ls}tM_Up3Van`KHi`D39nyg=v!7@0K)9`;VcHoDxip7qn=S+@S>1dn0Gyq4QGj3 zkAa7$g7;ccuvP9lfVOC+E#+fi=N>{5tz5CE6MCMG+#XolLWfy*sEmIzINTr9u=%EUa-+@B&~d z&sFAN*O!gacaqKGE0GJz3@Ebe1T#G0(kbc@N~*$blErFP*9j23np`qH2>->+!| zk9l;PM>2(EGI5f0olMO}W;g+)4|LBC&04_u~QRrjd$X zDTBx4y|s)DWM^H7CE-1xK2akh5`r1-Q}1C&q?9f7|0u+_J_r4Du%4;ChOg@V$Yapf zf*h36k3gX$e$7xb+Fu%3HmVf&KzM1CZ#6PxQzaZ8z$FV7I&ET~lAB0`dTekvLbWq- zo!b?+kWI3M;OK!Z!~16S(r4V7kOzCKnyyXS z(pRPIH3L1v{>VzqeWg2V`%4Ed8DKJ-wY#ptu!)L+)jUu>Sh;(dc}K2=7QST<;$Y;S z(!DIHwy$&_*!M%%x#dR!{XG8Who_d-pQT{B94OA`h`;#A(sp-pauQ<3vI=utXU%2T zT$F_COIv7}ojLIMwv1EYx!H>~n0%d|zm(fPd2l9o{86==5!dDHc}E|j`PQkmwCz== zHo5dZ81IG=TFHkZYsd_ObCM-2%w9Qp@>Gt}%>a`;)ul>6VBwnLbWA1!x;Sbq!#>6$ zU4S0c5-Xv zi8FWqj9a=>4ty8pv58EIEoX=crU1^TFO_LCe`zOchytH{x@!epUBm6}(#&}t59b~~ zeP1&!^{3h!Yk5w!ME62gqXMm=Xr)M>&GW5Q%)iA`Pfe8iK}?HfwSo7UzjQh-p#a&ieKN-~Or> z6*-(d>{HJymdmm;dvX6`{(di}?eKf%9L09H1iYwpg*pVk+D+%SYr^US72v|M3(VY$ zWp-;o()(|a#KHXZqBMkGQ+t`+zo)ZEXA%ySEQ^XFDP9~X2evQe#eBL#rYvmSLXFE0 z*}+tMF_X3U5H~!*JW#l1Sb3={uyAof;|#>Jtp3I-@XfTk)_60>7rqESse70tz!pT{ zJJ{z*y{LQW+{-u{e2l_pT?c~DL2TKsA3|(v6b4~&#l(`Bcoj4Zg0IEeMK3{{#?%w_ zsCg$1PxxKodM15d@8gTSYvCE)L5ju{z4RGKh|E=-V@hGXOs9IqwZ@(X%z$v!8CYaL z`qZh@Cr%zKoILjQv6E_$RY*Vf{;4CUo+=zUTR3r6Jt9H0xIVd1UWAclR~<&ini(4R zr;oKoXL;=Gxq_ei@$)B79zS#BG>Xt7T)se-7p`E)HQZ?dIXcsam~%U8&r5MZX+6NL zCid$UR7i8#gc~PV12|@Ht-qB>Y7u?NZj;MKeBoxwgbJMr&R+zWhDdSiRK!^z=K1Yx z3Px(+2geQkIq>y>k(jr#X4)J;2)H`PEB4)340y>cF`I*25Mr3`P@4$$fXUIjMTtF~ zuMtkalX!b+Ll=Cc*$&T?A&tWY$GPey*dN5Sr_sR1AF&h?B7¨S~p3_XXi-AG?|9 z+n=WGQD_HTN0{tfrf)GX>lp5$%jCFG==bCj%5a| zN`qUexsu?rLA9|<&#g}F_3JZM3Pt;?z0JA^kW@L~lD^Be_8oC92-b7|uS3h_I^H|_ zt>q>Uzt@0yNr$PY*;HD;RL{_nUhoyL8AE^Sn7_kP`$l#BsZ8NKIUY%a-OJ`fv0&F) zP47%Rjj|elKwm?Aq^V=@plMpF1cx^;NdP_PdUygz4UCU#O4QJbQ=}3xAf3<$4*STkJd_U4Fie);02nN(8|~?r~sz&I=@4ojuJq?A1B=o9zb~vZgNvX~EqLl6#}olboMiwfKX4~){YNW~x34g7J$r)_Rzua` zv`>t^yPx6GBb7QZn7|=a0@~r-nNhdNqYY3HHZZ>k9=9q~uN{8PC}CF_g9#-l8u1Mj z@UkJl=5k~<^l^%)nRt(?X}B;+1J$BU6o}&6O8iFXbs#vk1Y`KhL>nl%Y#XCgV}%8@ zdhAC-pEDqEh64(=;sh^5e_2dMNL1^7N=*ZYT^%Usqblrdp^NDaXpNlW| zh(Hz0CX&{)5#LTNr-Pb2dk@6Vu-C%2i~Np2j~#8yfB~F=7+D&n53LNd4)qM{b0Y$E zs6L3Djnqc#z1PBa7U}xt0F=S_G}W(x#SGKH!fJ(G>FaR_q9xE1{Q~hm!+ci8;Etn3 z10OSH!Vr*{PFO|!pMC@a^E6(jY_<&prD7)yQ8&jbugq~hls&z#>#dGP7T&aZh)e0 z8<*sMP~7hB){&|0D1l{6xGg*E=+x}b9RKRl(9it&!~Q;G;eLaK!h-&}WD&Km0KhF} z_W_jdEAE3_Re2v#rF_In3VO|}64W1J0{idsF>IL->%P=B*|{SUc_K#JCeZL2cEcJb z-L>!t&ZF9eF@6M*u_L}0KMMCHVw>>>acNQoccbt^6kjvd)z}T0%wo~**SSyw(V+}Q zhYobvCxQ9%PAlG2)QxQ{q}h@Q3H@D5#-7}gncKH7_f|N+boQvPC@*5kgnn8Ud%K$_ zMF=WdXgypbQw1#uQ{FY&+;#9nm>%R<$1XS8wXNr`AbXOd@JaNi6ote4K_jiL_UV5YIqKscXQ z_rj}=Gu{{Jdz?N}uljsIXw7WPbetB0UvD5PZebB$mI%S9%a=w;}S8=!~3~LynTD{?^jfgImVMGn0z_l!>wYN#^EIGK6 z@GADkrU<4lPvSPmN*`j}WL4jxL!m3^770xg&@5LOW)cnXdU%iAMKxaq{g5BiD6p)6 zMxWz|d-cK>c0VT+frH}N4ov;8ie}-qoy#sRZe^ba>c_aZ6|ceaEzmwWsBs8vYB8FG zM(v;&{a;vpQF$EES+|P8h!7*^rPgfLJ?kX{=A5f>jvh_I3HIgVhPgAkZl3cLz)_B4dPQo5%#1+UWLN;T1PIBf+3|25(yG8hDhA-9sdTT zIPgo2IcRO;f}YsTj1W_lxS^yEzOfWg^>P*ggS3>a$%Z$IB+t^u+e2w~bK{SGLJQI@ z-&hZaM3x(`;ncuav4-e;u+xFiS{e$hB~URUuzG=x zKteCu5>eLpX2K0QqC`i-`1{r_4e3myW$P$*)kcpI7bFCdcIDvd)-D{6gEG^*xOxF~ zZ*K?GqYB{p;_SZB4n&Z7^C9G@Mf*B1-)u`FY10nEV`&p61W3(pZR$*;{s`H(k^LMZ zHc6T@0Z;el+<@}j>W1_XX!@KvY#-79me8?%cnQX1rT4Us18i}2JA?}23ba_WuIeRD zi&b5l2}V&HWWa_Zl_KEjtW=L5mG#<>dZ~{f$F5VnCE~lSPYfXuZb=NdEV9O{Xd)!0$;G1OR#{$g_=2!_3sZgpQf z2eAk8*UZjU)qa`P_Ls<~y3}}pYq7nmr&!Yf>R<(TV?M46`%p3UDLNgCd4S8by*lq}RHs`{t&YF;;(3h)t{TFR zGql_K7?S1NUK#gU*5+>^$ z=zI%Kus;3{Kedmy`b&Jjzy(2CHcPu|z?GIZcj;Omzw`}@DAKNNvu+3(^y1-G>`uTX zC@NHXhXcGl1gEvC7D~$S3f&%gr;DdlSwH&XF zTnm*(-3yX(%r|JD<0QzpvZFAb6Dp0DCSdCj(h{ZE$3w!e0=EhLs&&FSp(^D1l?_zg zeEmCU;ReW&{1LcNR6j#XF}?A-;4^KiC&m9;FqeZAr5wcwDX3@nfO|tV$1Um$wXOCL z6zjIxX_TCXne*TmcbxeOc4+6yc1l{*GsMu4Zm0S;wOw|fpThqI?+a2BD|w08$C$M{ z5SNk^yI1zyV7m2?yGT=pgSCKSV-}JT(Ry0!Fh*@$OhrG1QNd&4orh6KbbK@L>v-U0 zMeFD%e7VKnme4TMP(ZLWSx=Sr+5K-M!0$r8FNVQo8l%rItKW{nbX}-)C&G8u`&ahc zlz!=3Il$V@$-L2Z#P|JZ(_NZlRy#-@+v`wlsb@=fv)}6d+SE9>RMCcXj^m-~X7%GWy~m*6PS;5bU2c5<^|rHeYYGYfOU zaw##u6Ua{Jb~l*ekQ(rh!NEbIjaV^`P=@t%hz$!P>A;;3DmX$J2#91jExO)~JQ|=2 zcQ@hefrpn6JkXqL7`)UM-4j+xw)?IW6*cl<1(Qb$ja-;4V8y#IpE-L`OC^hBKn~QA zqHG%F8x{jm7j5$r2Fh?+(KK!=e18@f@aS;WvQ$jU%kA=wO9TfQFgFpT$m0m3Q6F$8 zXicJ#i*yXJxxhF`%X!LK=^F6aaC|`X7Gi`U$aOEPxre5#g7X`w+baj7CH3F0JHA;i zIvz{|eMOruZmlDjilO^8+qa#!aUWlSiF z%AUa8&17lnn6yY(bs?goIJ#gSi(Johvyij&WmOkz;8JPXisxv}OV^SaHJdk?dp`<+ z=@NYw2FdwFECJU3xmFg=a~Rp+=5X*~{Q@yAy3!)YGO6;|8nNfbx!pdA-EfM6p->W@ zUv%4L+EtNJZ0@9Z8h-(fLbj`N7^S&s&+zqPBdozS{XEg4IfO4A?6Pl>X z9r_v-bTz*SotUvXREk$X>Pi~qteNjb{rS`GrBDtzTk>*PPFZl zcf$0jGwn`FXXWqLIFRrKg^&3N+5}t)QZZhdz!L%lG^2vsS=m^!3DsTnjnJ8=L)6GZ z3QFHV0*33!OS-VfUL+XO%*r^{A`UQY=myd-B`s^;r%0L;m8W~&;$8vOumiIICFcE z&gc00B{~P`d=g<^AJ?UHEM$ft$*s-vCxnfQ_EgS3%5bepCurfW_p_N6ogZSQe}v9w=`6yrhTD4+V1#01`s(Yqn5aO9+%uttt+b9$ zCjWY&ZNSyY8;J4cn`Kn zG>P02wbgjrv2a%*iv-T*+a|6HZt^*CLdY$KRx>8Q2{-qqX_7Z<_!^VA!Z)5VaA6R`MAMC>7$!{Wp-#Ny3mT3_?9Wo8 ztR3QzFALpDz6r1PR_tafN;7-Vp@fd)M$`i`+_#KhGHz~s+jx0nDDf1{=q?e=pQ3z` zo`AegWU{T3)X!kF+DMH6KpsWUxtz<*^C&wE;voXW4rJWn?%@$?*8vHO7AY*(*p9WG zWC-d4=yR{b4kJSwAUFZsZ{K|bB!MufSZ!qH8(M>H2)Vd&6;`KrY^Qc{UL=g?X67L6 z!IDAwX?fG$w~iilO{l#O8S6YNH@9Dg-&uNLY?g3Ykswl(4f7jc+pbSX{9w zxZ z2e@UAlTGbSguTt>LAuiH<+`J^@zE}93pvcofWIUsjVYrmtFf5{a-KuZQ!tb_r4?`=>Z*C$gGlN;oe$Hw zMrSjfJL!;hswU|0oI(xLVaKTvI-;CAO5ZY_kI-31XPnL!I(N~bs)u?n2cu5k`{>(4 z-v{WUXqZ;_D+=3meNa6{GCzcJ15Z%m8f#xz)Q{8p^T_)A!s z|7Hv~ry008&FD-0MXG9i0O{XmdWfUKWIa@cqp9!WyX2U9CzCSbPy)vr?Fv$jIJOQ_ zKGhV!ECFch)yZ=j_8xC!3O4yE4#XK-ccMktv~_Ceh?iyPS)Z?Htg z4^!S5i_N8umEmobz+qOSwT?dZI-9Y&3WY#@2sRC@w4f3O>=O&B1RnZJ zh@@OP)IAGM!6bK=ld(AuIR(AF`W_+fcs|VJb?k);3c|OrJ_wr4Kj|(I;;$nhNh6&| z9z)L`lFHGg`&)Wo@?GU882*$N)O1*98_qO2qQM`51=sCSJnls};J08q1$PTZscT{I z%M6u6804xTr4%Kv17A&OOTpkYx`|w}B$a4OsT*-z#e;5x>9MfNb4wlv_YNGRFgR~8 z=>=&h7?_485+s??sb(>WkvOe|1!Omroyy;aF90#E-$tZEq-W0cNu?<7vB&}C_AAQUkmw!R7Mbzg(zQzUDnRWsrgtLgD z2?H({#bxc`Lc&HtFEfDj(xSpukXmS(y2kn>Ma~39KixKqzC;BY%lUCYH`=a)pk$&3VMkw#A*?n- zVG*{+Am%whc0JxA@-{ASyyFqy8G1uAoTKUx-n3xoZa+m_QXKG8B={Xq+)t5QfxT)T z03fVKVjlO?^_0>O9z|%Wmw=)~E7iDy7nTaExIwZHF7Dlu6wmo7peSia(!!|XhFW-OA7&Z5=?#>TYSgyNa>E1-F6BWAMUYXf$@g1 zvKI$0S-!(g9xgY;;XtNzhn)g@J|lM&46aN<+do{|hwXs215hV-&n!;p5q76vvwLuG zcBhE$V?0XxVRI4j<&cAH{h*!;^h-S#>H~ht1+;sYr0ibfOKa5_O#D7vd@)csfL!lF z+g7AZ=_$AF&zpUvgUh@w1`799VK2Re+t6YC0DWA$uReH9 zE{$3qsv7kSVhthI{q=!aNRs4xwO+;2-o~=aJ(Pnn&{= zA=qH8K27L00t8rE8SsT$%%l1jwnHNbI6N6S!b8)ho%Zv!(JNvD!J3$!hsBX;sG6*5 zrDoWXP3J|A;c7b=?Sif;^b%B#)wk%-+Lo1si30y_Sb_T7HPkP#ux(5XGjd?<)p(4e z?&GHge)>oJ6!#A{#zlZ z?jaC9Aa#x_dxe$OChl|;3s+gnIYfhHD4bMnq^vh!BT86)So6OGyObHi$)SFk1*U^s zI5bvpqa|PT{uN=QtnN@PDaolPW5CL`GhlVcS%dBUkM2@6h&Hwj3a{!B)*9PxjZ)OH z+pzs}eDz{r;6ulRTvgp0{~Dp4J7*PPPjcsc8$Fvyg5?DR0JuFKj4UYSMd+tEA^vZs z&5Q^n>QEZsA`1*A2yh*f@ebp}gq}-*!Nt3om`fUaF#kcX@qw}*-fkIh8HvQV*WXI@ zFlNF?Azuh2A_f%%(_p7*F-D~D#SOu96J}dFLQzH$apdjx$R9PBmvaVb0FgM35hRU0AU^wkI`%HAJZpGn9iq-yEcR46FhUZHYnZVREHS2IpOjBWM8T+<*7Q-QKU7{3iczCftp63lZkd}aVz z9D^ti+7^<#E~7oCAwU0qDKjlbx=d>n)bbb%R>4pgwwwv|57@^?;CN|YJ-Z;v5rovj z_isb}qUZ*awnE8G!b38)&}Q;TjP69lHKCzGlSj1h0K_i1ggUN)^slo776X}WIS;yk z`_g#WkAN)6adOcx(6csUj}rvw0W?#9=*9&pz+2GLu=-8bs2fY~r|{vctOr*KxZ>fZ z;m*?KaG+)97GNkR4$VFMN14BzpOED@O|(4t<2vFy&kNo6Af8`O$Q!eswQvo{I&>tn zg49MJIYAv@8mIy~|3kj~l zrSeACoPZ(=lBhmZO9BB+!Dx77r3X4vK;WxH8`G!@uYHQtu@?A+zoNzTULZ?7%aIp* zD3QZ@tU`U~TT$O;eRYfM$QwlC`&P0vzE$eKVf=Wwp2WlZyfH_CwErN&VYVCm21thh zE!Q<%f!0|BnpUC0P@S?{wC^Fld~8bnGxR8+0HGwp-+&X`rPQqV?Wfp3WEFo5jcreGz}}8*jATD8~y5UfBxBoBEmC`iZ%YYOA+45 z`ZG^Eam$mil)zSrI~y+u`-(9$>ZYM0cRsCM^&MJiltLO}(o8YbKhjstK+ItCnT-Wmc9LZ=TO5JP>ZZ%z>xlF1Sx4vdqcbgtgKQWr7*|V z5Mh7>XqVkpN1Doyci@>Y`&RX0KvEY$NXP+GmPLJpj+`icfxRbiQNX4kkAhWl)6-;^ z;GEp#6_u6s@e`>#D|S|dLlhyAG)H}fzGf4|C%}rx(U%FVo<1Ufm0nI?cdJeR40a0opd_&P2;^m%!FW~251B0Ei;k&cKU!uz}{jN zWd}!zjX(Mu*cMOC>VQ5_62vaX^~L)=Rhn12*KwpU3f=25{7&E*!uPzn^b^6AN^%U0 zb6qeGv>=-Z%;_84p>nebJ5bGVUpDPgO4x}&o|fy|zbAKKPwwEJT+7v3g(D|V9G^OM zZ0g*eR1SX!?&$t8cUSihuz)4$#O{(hg%)Z)BxfP!n)FI@Tx3W9T|*VlDyrrsKw@!E z0NO7DW8PByF4~T5)0@4>DK2;`dy+ER&~^qKuZ00q!AL|e!Y>z*nVMy(-_Vmr@z0Q} z)U;J}>rn1g@deR|)q;^c-)|9&qm>BhRo%vxo@u?2>`?ReCkXWDBXn5!$~_PBJv=@4 zw{MI)B*x@_iX~+AqRfl34`y6c|!#v@W>BM=ITtN_Pn>K3@~@#)?yf{AkNO z4f1XkC*@~(hrJ5@y()w9y%gS&ZmkOY^YLq#B0%nBGTqevy_f=4k8XpO1?rfYdk~?{ znMrt)NON9Ni%~H=|1tH8_)))2XNArnoxe+msJXTT!CC6XAbiiQkJDA3KNAVj$I$}Q z1kMq$*MLoEa31F=!agsGxwvpu9bl$}jaI)H=$f=?@_Y_Ig%4oIR1r>zZZMWfuLD^@c7thC)H zF;X?@+km1O9+8^j0=GsbtJY{@mCe;Arh5FZas??#F)gMb5(*-mAVLh|{4haBBox*& z-$6g6awda2$CTQ@FIoB0(p|E>TbnwACn=sM=8Hu@QEg)4A&C_*PlJ}^_(|7QqZMj1 zLPSc!57y?;)K*5+6`<1Cq*lxoNe8Q6vX)?9I9cmxk2S-!#bY2(`dafH%uP2`bJjHS zG_T|oOKgN%yCrq|XTR78+RX}ftwC$F zNzFB-xTA=USQFi9ASqmJ>TQzw>bcsjcz06I!+ng_r!#99MAX7B`|(M*y&}cpm)Nd0 z6b#Nm=CV$5>4*>v6Qxir-EAE)ItJ_>M$}U%kTcMx+g#D<$HllbpB8K}8qK+%X;Sh* zx9SiA)B|)5(|M52Lv$Xd^9US&ed@w}J6&4F0g{d|h<$_+AEm?XQawiJ7@f!I9H;XH zofC8x>4-RN39v(*<}2x3EqD6w7~v#4Dlr{Qf&+Gip1hH ulLJ}Aa5|3Mar*;UuknF!;^07dXjkZ^=KoMQ@Iz7LrOc7gO9Sgh`u-<*Xm4}? literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/model_runner_base.cpython-310.pyc b/vllm/worker/__pycache__/model_runner_base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41fe8a49cffc39ef3b0ff5a687fec5f1853f8601 GIT binary patch literal 9631 zcmb_iON`t`dgiOyY)*I2Q*T+OWRLBZH9eBon++Vru=QA5SZl1cv~kR=)8TYg&u~OG zTSfMa)#=$79sxG6aDpJm-326UE=G`BF1h5ILx3Qs90=sJ2#`G_m%XiJDc@h!&lyQ^ zkZ{22VzF3{|MC5gnxs;3HCz||_8&GsTh_E+(Zl5B;NdOY=@~=Qn8x%-^VP5J>v%RI zBQ||AwtP#cb~7r(1-~HcR%FLTzbNa4$camSDRzA~p7CemvR{rXekHE@)p*vQjpzKi zc;26vef4O;U%*&)v=A@)i?Xd4EyaiYL$dBfhvOsu5m_%qN8@AuFwlKy_^jv((KPBInqvvDK_hh{iosQ4=XVC8ue_>DeU*z+w%4XTzmxjO0=J^bN ziMuW1n8p_PEL+6y64P5YJH!s(v;3FY5q5N{*V6b4^g70lqt`1>?P0MK>?C^Brg}Vw znNE#ndY*acah9FtbL!uR_~s>9e~IcXi@nTV zxmWO4-qBc%&EC`4EI0Q|zs|L-Rd$xW`lXIH=hzv%Im7FFCa7{AEi0^!mO3|inJ*vH z_ALJbT36Y5w4R@6eU0fiwc2aHCh1wL8M*oPrOSCWSWhz%G_#mzn+dDwxx<2tXJO30 z((}TO2s&x4lvi$jbmMB{^1D|r|KQr%JNUGHH@B{Y%`7jz!`obh&D^<`aS^OXJh$Ep z(=2!1?_^=p4x-$;b+^OwB9(s-2ux85(?-VIX+jS(VLQwkQL?eY>9KM%h&vG%?{~9K zH$%f*vnvE|XN^u2WUWNRSo-3%HYSTX3$fN)a-=jrbd%rd@^+KIBa&|CI?n>^B)|ys zZ)cc|7BkT}H)^~TG;i}Z`wCOZxx@CxuwSm<%8y(UNx+&xn$hO2CyYne+G=Tq^`zTo z`Hb32gAqL{L6)@}pr)K%90sv-Tkxza+D&Na=%TuX`!eqI6pFsi^sX){eeFK#_w@(b z109+Go=LtLy+3w6T<`Lz!-aFf4DlS6C}^EuY2q|~etTs#i#w~Gu(Of{VuNQZ-;2AO z@r8J2H44{PJ9n|Z_Ud*N#j87sxXs0COr&dwZX5E_SPxQO@7&Fo8v$dDDVjB=RxYV} z0u!b-3a7ZBTaU#HsODw))w<%IID)1r0#o+_Zb{%>wm8A7WxIS%D4z7L7lALy(hDU zq&f_uu*bcFbf?`jFKK&0dzv!?SwuUVX5bSBnPQ$~jbu#C&SyyQn^BOaJk4#@&z}R2 zMsnF`Mrm$#l1^^5gO~&7gWG)cwq}bLK`_a4e|{Ug7_jK}4p^P$1vzAH;>~zoIk`B4 z@A4NLQm3c6+7d}T(0a+!m(Vj^Kw*^@^s4UYu5NrfXE?g0i&xRy)KLsC6BS~pWajUp z!oY$^i}hXT2r-nI%mM?AzQHUo@Hkim7BaAt6~46kuvK7~28I=TTmK`&_QQZT;@6aD z%5e_#on(ukJ&2-V~ zX8QUTm_)iYf<3eM@srv0?P2ek99okUuj%S%0CRHLk|JZW)FemO;>T&mkxR+?Z=qI5xEUi?oW9C^GE}ZBPGP|J?Xo zYk__A=^ju`8W((wE~9G(wJPrCai?3lruCdJj6S5Huajl^5b$PUz`K8^!_w(m?|!C} zrMpj3q3@c^_`=w-eg-+{n<%Yc6n3qwaNlNyKA=nQ<1>Sy|4-ZS$=o<`tr?QN%}urh=yMTWRg1f zta|5m)I0m3fbDN@c(TgRL)cCK#&(tZb@tC!L$yBEo)Yr=B8`KNf3 zQe0_kwo%rtN4C*>pLOHTU?V%504hj6mo9w$Vw5z4D1Ebz*Kpn@o+&>bU2(-@(6N08&g7=GWTMHNJQ@V3-pknY-@&Y>kE(mzxF$tp||>d#bxle6E<&0 zT=74Bk^sB`_VM8kJG%w z;gTTq%Xn_W$IRU@4cmy85QzzbtlU=Q$Ss;bFT#gxLpyR4)8rO0vNn^Gu)tRxIG85X zDlgFzFb=+PTF@WqsKWJ(c*n1(ttfF#b3M#W3?MFII^R{(H82L`<-2H-nvh$qZo4_5 z4^Cqsy?4+hWhe||4kn?hFF`^K#2ID7)osImG*>92)%dh(IlA%KHVoZ;Y*^ISLO&b& z?;jhF4a0bZcmLM=z5j!r2?}T+?;p~p{t(4i z#(0C!c-bzM-BzlappOikYy*MQ*}*?hqdAsFL0ZZJFup3?e77P{UUd`S+Zb13cSQ!Bv0&d zKUU;3%t4!uq>#f(%)M9G1CID*X2A-V1x1j84CfRI!G^*JC{Oe^wAf7hOt0y^uJnf} z+`jsW%#Q2}#s)sV_x?77EM(k6USbeTd)a1?c><}3P;f6zVm=zH*-f(~_83o*Frm;e zpqzw?zG*IGAQsn>>>4S4jD!VeQlXK-un;`7_M_X$Al(-#Pvii8iOr-EWbNMTmnKsI zV+XGOtO3??6EUKS6<)+(D!?Z5G`y%jB%UA^f^a*d&E7~lLyEvFi$NCdZG_t#wom43 zq^m6BTtuN=q;|L$cx}D|I8@jH*c=V^v<-|ph>vN0Qv~>#So3Lml!cVmleCgC%_tTd zujOyh_cxMaGf$`3vj+G|U#nHbWek(M4RZ6l5g})}(YVtMB88)Zh$c!@kRvZBOb~y9 zBA=1*x=QN^^6~^lMxsK+Iu+x+j5Z{t`T<_b1d;>%I;tClJ(l$i5bUuF%&|@58^c<7 zcne}=EC0BOFQzUXPjBH)$+!*`k?B5MMsi`rYiJJs6TRLeKVWFqU`99$yQk7SFQ)@5 zrV>1L?|N|8YbNb12vv@Q@ofaWD`|%}!&cbzC`VEEu7s@?r}W-n7z7nE8;8;G{bsPu zJvQp1sFO;r!>hvR&_C&EjYTX<*0IE^GVWE`x(_HVm}VY=X4Y-O{;l7Y()iwc*X2mO zy%DyNKuct++RcjecvP^nFEsO#^(_SQ$ePKor8;?P)2UMawA<+4|2dEzVOvC z){uIQRbK+uD0KDKdA7r$nric-grFs*S#Yplv{2;5MuR2IM&pUXNK*3mxK*M~vka?r zK+*_f-=aQs6qAz1EeZ4p3j6}<7QsI&Fk3-CD?$#P{Dj=VB&E7x#JHe{x%c|p-nx1u zRyGAENICEUdX5?Bc~kKmDL<*gW!wWm^UYhA-d5mZhz8pIg;BC*lyjM&#u;=$3K%KP zQMZ3Wlfo6VS39tt@doNsUmdhnCdQ%FQF>trw=Z3y?v{iO$5 zsfxp+SzcyyvS)GYknB5;r^6Uy0b^M5%@KL1n(j1yde}0#~y+w2FA%y+DVW~ZEuW8 zH3<-;@5Aqvr`s*q+4h}riK^QLHtP|=#8C`>qwP@u9J=mh(EDtnH?h* zQqD|yJ|n25@Jsp}a=GO7QBXei0VlCWIWl9#A)U!rRHj~@rKqgKvwV_Y;S=QsmU_QG zu|!&3{eLx*Ol7a-M@G0dg1-iZz+6YOle3Fayg@MI%a`_xp_IZ&Wj8SjZR|4clvJ#*i)x>;oF!!3v@JR@L{677 zk9Tm8v_*$VM(>BCZGRxocBL9qrU%)um4UE%W1#W6bl^u(q5vWT6ov;0V}juPRZzG- zJf)Gwx`zhe9gK&i@25(Y4)w;Q{hcU zZItlztH>@paPVw=<66c;qfq|Zt{ArUHDzeZXtial@wIE0e|xel$E+D^YqjsF zgNq+vXk?q#n}VcBZIX~IFHwFeOAcjsM~(GiSXLi895Npu>KH(sEt2|(D^y5kuTkvHZ`p=gvi-3u5axNpe%t5Jw|vFXQi=?wI&RPP>d2 zOa7L1$0!@pwWQqLBg1eX88eitBe`%*_ZwpgZs%i{vU-Q8HtJO#M$s;PGjHf~ve(>q N-J=Wo;mYyZ{|2)`9h3k7 literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/multi_step_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/multi_step_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b05eb7921051223be7d894fb810fa52447ac2649 GIT binary patch literal 21810 zcmb7sYjhmfec#UP>|l4XSUd=T0Qg!y#T7|Vq+S*+%MvNll1xFeNLlg3?sBm+02bT_ zbZ0;c=pu4VId0N0vCm19G_G4PN!_xi>FN6&pVU1~bDBQVN0Odto3u^8=u5x&ORWdd z-|v5CW_JP5c7QW?@4a*1_kaKIJME#Ndi=f%sct!pflJkI0Y$Z zG=`iZl(!ni=CCs?d6~vYbJQ7?xZN0Q?s4`s$DQ%!gfr2cbS9g7oxRN|XG-1=G^U#~ z&P;Qkv#+_|+21_i9B3YN4$8Z1<508Yl$wW~!;;Q5jx>)tN1Ml-W0KA{jyF#@Cz_8q zk2Fs@C!1xbjQl~j(0H_Y$~k4Eyo~B#Y(tIH%`?s!q({BkJBD+1G36G2)o>njhZha! zoIB#3cSpUP_xN;b!E(pkJ?~l06YjV>@m|V#(w%ho;`gb~q}(ZY`n{ApeaCd3_DrN_ z+k7Z5b_UsC+?*2<{9La+{4H@>>c%WvrzXMZaEtcUH`)C zmn)ZEx_s%gFVB4@Oux`t3$vH1jYf68;f2g`sw5oWHhuHa?n z%@)#SBP=X>L1ks7vZ>D@YI4TlJvQXfG~$s%(eF*Hyc@5?pOBuBi6B9~Q4wo2Z|p zhjd!Kwi>Lg2H}V#>#fCCy`ahw>-N!{u6{szTML%jt@^s>R(xr#qC9`K5%}dyc;FJc zYj_;&HSg_JuT}Ft!;wYp?|Hcuc&h2S^=jZ<_ga2i`Qbn`nsB18N-~6EoUI$dVh+8& z_S*ciR|_sxYfD}@QuWtbwMroERhG`XC^Gh$*WY}#?Rt$jR$DDkU99?EcuX>0ZegZZ z+6o2Uss~HkXI!?zy%$xx>ei}$z>Vlu3XfG}b*n+psx+e&sw`lI9 z90YRE>CU1jbx~PHtop+necuEeVS7jPqG9pXrFLtfzR0{60!*Dtez97+>9yQ)`h)xe zKyB#^N4h?&+O_7HZib^@)>{jnQpeD4xVN^n+PaDTTv6Uaz0s(6EzaBxZRG{4s#OCP zOa2(=a|w@MK+s7kD@a{S1%UYb27&#|t8ZSt{&MBo^~#|SLRmG*3YmpQyBf$E z^aQ8)YRvcEuC(O@SLRpi4cAl5_p=C0!!pJV>w$V4i5(+PBU46Bau%c(G1fcgwUl}U zNv!)getJ6N`b*2<(OPoME9_jb&$c0)L+S4kZ1aYC0zW%C=ey_e`1=ufDH2A<#L}4{ z0BJYvTKKix3>M7}$6k++PL*&j$9@M;vOSp$7cBrd-Z46&+L?Xat$s)>d!y{4t518tGBeZ<4vCy+dvLV@uKG&5`pcmPOykX)$zqP(^=R{`X z(m!}coz!w_)3}GThnT5j-5I6T)95SA02O>wQ8tw|3mL`q z4rs_D^LZBh9Rx8nyotl)q&>^axaPAd&t4u_&LW3<+R4w^H{0*-pbfUD-R=2ClEv&-#?b1mW zmQGJaW%@cVEmeJKAOe5SEzg$T@=B{dxY3**z5Zq?07mmc zVopmlx0mX*rP6J$bi0Z{qh7n!SStY)cSlZsgWZ#&)rzNku3ftCB4+b<2J>Bw3i&ZsWUGz%qySv_4Bra5N8`GR!Ng0P-+gSS*kZ9wx_W+D@~n$Xj83r5N%S_T+8!ZF2wC> zJ>V|&OdGlLdUa&>y6LdkzwO~ne;N#10zh-&sD2p{ol4#HuY~zz^zM2gXsg=N>~&$U z!fe6}?up9?POb>L42zo6iMgY2>{eiyr7oa^`V51Y8N9+ka9x7%!dz0_ z!CPUFY%2>YY;EOD>*Mu_9(({EBx0o>KFJbVkx3T&C%J8c&+RtBQhgyl6kY=dS95cv zxpvE&Wq4*jp8u6O^;u5lRR-H;G~BGN&3oNtA?l9iWf?Zf+5-!Ot#@AAYJ}Oiqgn=) z<3Gm9qsV6d1R|WLYpfdnz$>Y*7#)inyKHQl_prs>%#Nk@N=}+&FtC<0n|8;%XZ-vG zQUd^h)N;0yW-EAdc=C5_BlQ;Av{Du%K_k`5o=f>p-Zl)>H&S1~BcPIVg*U5y8oZet zP9oly7fSK4BP@Hmbeot%a!acT0_sKD$?=R(L|tQ1wkLg7~foJw@NX$ z(ON9UJ1WQPbjeeYVY;o6@0g9xV(BDs20^n1ZL zLXU|+GHx^zl-596WZcoznI{7Q(dp@bww$yzcjbC{P;pDuMFe36QlMIM27Bm4m=TN; zTD4WT8rm8WsH?2<0)xv8t{^C9c0F;g^L;;%O6>;neKM29e31vvB!iO$fb#g+AMf9? z3zo5dycbZrP`&N2mnRhYB=sDFPa_BiDiya~t5nnoQ^S=?wT0D?&GC`@1`A%}tITB* zX1xRy4MJO^PK{rz`F6X}ouV*9p?yt|p_*eOuQ8C-=+04M4a+SlYS*3DVZGDi3zw9}nmAc>H;si=8|f zxG7usvH>sartccUk&z#>T?;&x0e;Mas-!46nnN4*M(~y#TGko$#tJNec8*q#6FjPD2Ypc4$xXyj3g9&Wc_dwGC#yw(aFSw6x3V`oQ2w|Qh-a5 zgs+D~b-&V;jIV?Rq;+)&%HbZQ7U~MEb<2d)^FTl`3krQu4J5zOprpJ5aMG9tQGibg zG5`)=HYwQMH5LswJq|Gpa`5&0hLV$f7-b;pCI?8pg^Zr_oxpyWeYR0;&b!s;{xaJJ zFP$1st?%2(Va&$m!V!`CiDxvs?7zQ-zcAZcZBpR#!$HP~qUxR>PD!@#<62=5=|gbR+db|FO(ZG$w)#6zlN;mP2!@eJU};>o$jU&?Nn9b+?3N(SMM zdP8%roK+RHs%|i-GMHz;l~#2I%M5NJ0Ji-ie!~4#zcSyexaw6({iuzF_Ua0FVGsL3 zUMC#XEN+F0EoZpp-R^6&i=PBF5&!-g9)AWw(Z~Y>0}J>3+y5`02L?X7%Z~lP!M8Y+ zf~egh#f& zKc|;nz^-hm5*knk8609jqEb$W=~}~A9E@r+=%0SLUo?Tq7Ep_$HIiJszlRF`7=o-# ziiB0!GPyci#q_T0@q^mJ!B5UId1y+N$sgm%I#BP@d;9=!0*G$<`P8!n*(5cc+RSte z0w8q?Jf6x!<@X8mWqbSKTtu#bCN+e$wt~i0Q)U@m;1c-xCTeIBZPvWX6lv<0Z-zx8o; zYu7pMtR31e)~Mz|TC7tc-ULzFG2291TS7P7CD*uVs=Vx#dDHR>Wc35{4q(wABO8u4 zmYW75ofaVgsby$!SGK?F-ldqQ2MJ*~FzsfmP=4I+ZrZG6!~KzfF*gcR5FeY`{0 zu}Xw;MbDrpTu>R&Eemyu)+tjXB;p2?djU01!t`V8&@-_W{#{?fKo?e9H7Z$4niBZB zAeo9EVF8n-pqHrDYHbKf(IV=V#S-~wJ5sf<&&%%!zoOME%(x1X<9v7in9=!U3iFUt zk|he-(B3b3YF4WBt_aBZ8PT9)@a=dp;?3-79L+r`0@)`Z=RlP+Zm_oEIYZhWRM9v|b%1WlLz;ggo1_W|x@<1ZEt+rA>aX5iRpehnt3k_a z1_fbC6>f_hDze+H#c*55cdcpr~Mcsn@TV!@=d|Idm=1sZlpWu zW!Pnai4RB~Qoz)Zq);|ZJV7 zMQ(PxDjVe3&zd1TWwTs!e+AwS(2MAJ|2)I2b8zcsMb97 z9tH#sn&Om7@PdU_{W%H&uGPIFT zG;RCb1MWfC|MyD^hrAgXi!ItV%Q@ga{N=60$^&!j`(<__ z;gn95E@>yBGZCU+&v)OhpAc(7;z=XCcH&J_ij+1NO6zB4OV_~PfBx##S9f{>ky)>n zZdDtrUWxjTQfSO-!F?DgeiayAI~$zFl0mZ<2?DADIs{KB@e&fnOqiux58V^O{O8%) zWu@9G9D`R66qvo+A{Zr{+SLRJ&1pEiQdQ7#VjFcI;UpZ7JP1NChsAGvI%5&bYLi8T z>A%4-T0$0VO&*;6$@1*tXygnv{iqXY927iDa+QjpC)gIch=$G<=ZV%ILuAY_Eo%5p zD3PG~he8Q@|FlhW2jVuYa`r$TsHc!CYyqWjtsmG8T7LxhlRP@38i^{E*oCH7VJld3 z7k3kQiu9Oz-{82ZM9HVu5AQybp3+*+b^^uw&JKx&h368UM7u1OKPXS`n$V=ePDuT! zA)o+;oVZydcPi~-4qa4=gBliu`u5;;qV(`gvwAZI7EPXikV@_?&W+0i+Y9)1ti82< zao0KishZU?PPn)0V-<5O6_HMgx9Tc@D-Im?76Fvu!~z@?eR?y+j0(;9ea2-HVIm+| zz-2^$3KHx2y)@8`1|x=ME%5wIO-3nrQ6VJ~K#*Zsps0+*;AOR8Bga~{HwJ)~ ztd8BubOx|3)<(9I)!SleR-m6aL)raH7)iJXY%R`J-?BS-F(6jJwE0e!3&EKBX_Qxg z4uMmk(mv*gWw*;Y&29ZE%lsOHUuPivB4?}_MG*=&nS3+}gqZ(TBrDuc0-rrVb_o9; zoyxY)w&^Q~bf6-m zwR$5>#r86kUoWP<^u!m<4QnIQ$t;`xGo8#P)Y{lip&0jI?Pk7ux?_SFpfF~4;6R1E z?4p5K4i5w;P!7wrZklf9E^OtxmloIvihKTZT7mj@UX;T2+;ynHgLke7Sl+k8fynZt zxYF<=NicG3k;=l3B+j|QusP$2SxL+?x>R{E%z4;Dn3chK3oG~{n$u>`zrxrrv-Rvv zZw(%uK*~c}=^0s~w7yfe`fs3Ic|h;|CM$lM6)m7{cu-4^=q4=aL$5nme~sn(0U(^z zW0u{4#x3f|Unfk=qP%Tc<`_I5VBi3zwP9wa9|Yn85w>A}kj=vMp>47Mcm33^u(5j| zeZo5TKY@YYLjwB_3XS8Z{UhXX2ZAv8=JtyHNZp$d!3aSy-<)S{Ue-m*H5j^Q)Ki4Mm1T55V@WtO^ z@B)Jb4%8ZwF(TOHD*}Tv2LG~XEGCuiXj`)$#@Ao<16foQbxPI0Ia9PyM``K6EGo=~FfC1677xpL_nZ{6Arv9jRg~_0%a1{rmns+ z;L;RoF2g^dG7O|NqAnu`zm-vx9NXLjM%4sXy#w-2&!&V)K!EDIKqz2$m6}eH`t-h#h=FuQ2v*kRwFy zMgDF+DLXufvg8zs>t8A*x~9wTc(qk=Z|ZTAEmdRBS;DrcpS9AsN6D155OYmQhMb-C z+1)U+R<}jGx=+QpK0Tj0HwWssvU(jX@}!|+#dR_eSK7X!-02jW)psf~L-<0duOKVT2zhh{B7ZPvs@|$KR$Z^MxUx#T zH}6N7MZL@7d;OdBmBe=n%M+7jQN)NjxW=xjP6>ygUyBsq*fWjU1#P=5@=sVKC*~)( zNI`=JRxqen8|v@!?awgy^9)2iBLv|sCIbe4k-@JpAj72oF#=~|o2kKtfVHwJgh4&e zQkNNAVbMMPMP-^j49E?X3G;RAjvr8OuO?VisKnoA>{$fmLJ#Bd519K88F1J2w#iy> zj*7kj&X}e(y{@?G@9?dVqkfJ={ZqX7Z4#N^LS&=>k&eVg1ifO0B!&-3O7!Iv{8BJpTL%E<T(I&H8 zbQ%x0*8fEWM9P9fg%65(pH3P`btEDI(2;b~3q-SGMuva_@dT*t6a*Y_MI8(LFv$%# zV;CAvNv+KCz#=#$|9q6ncC1CVa3M-VcY?AnL@DsZNWH|=+XsX{PV2ipSIW4u$wLJ& zg4N6yF(u3=GMumOU=HeQ41N+pn8Uf%WfMJgsz5cLK)0)GZg71LyOd3U7%NGqB}YLM|*-`^d5&OWrsFb$C3 z!Qf`mhivR`Ap%NIx*<+e_h4VaR*4`YOt0oaXmcV9JSE$J6ubir*r1GMD6{;dQSN}0 zc~nw{W@49fvh_B$-B`{GZ_#c0({Vkp$-1s=EK0(fbkl5q4zU72E)7X`V0nm~55`#p z7i5FBS|U2pgD9gxKq!J%T^ZpFEwb2PZLz@?25lHAdC$u)4{M!rmy{$%4GkZ4EQ*$O zMKIz*X3vjShg%Q26oB|%9uX|E_|e5$ zCsPx?Jnq7|n(FQ5!lQl{Ro_EH>PbY_5ABL?x=n=EN;Oy_y}^wW)e5tuid?}yEM%c6 znhpe5M-b6t13Lpo_#O;_+Vl~PaLmW`k_zlnir4u-HsDhi-5>b3sH^@RgMZIp+p_-$ zwlFE7fX+sFN%koR?to1GNQ^+z8*=IzZs6a>`2VMmW3K+nSQ3<6K$hN?%k`mQ_w zpHcW1Fa*8x+C}GDzj}VQr;R^F*CPrJPFNJ^gK@ta&iK3b|6kDmFD3mS-*pkTH}udw zQvY%L{D*3=Pr{O{VL#pvVK=tH(eI<`uO$0nJme_$HMe6iz;S4BCz2+tEN_RC{H=Da zIxmN1*X8&FRQy&vimdsNqu6Z(Gh(N6>J~yIPA~!gy}9zq?Iy*2c%|(D-$5)KuIX!> z*jyM2f;A;Nf&mu#IR;|&dx^1o3^*gTm%&{IIR;exX;Y$boWk0@fjkG7ns6J5m)T|A zwOMk4r3V=h83>D}ew#5OPQgHmXIUL#z|)}!n~DIUxH(}q87fx^%(2$@*E|4Nr1O_O z{T6GSV(@hYXR$v1&mfXbjis#;E|!Arf(-x!+8hK+_Ys&sSOLtB^OpIMnaO`>@8dgr zKlD{w_G4xSWj>^(0L^(r<(pLVEX|pw!q?uBe%ztR+k`h)H zW==CbWyrXMtJN~WaC|K#LLhxKpJ zjT^7TTa13)?Ouz4^9H>A;_HWSs(`Huw;XpXr2Uj|y>FN1;+2US+SX3;mdbrUaUWa7 zO~?8=Jxau-Bx%-7F6~64%G77=@|FX3u%cPVsPeSJmTwRp8B5=6D* zW3Mn8sdh02X)C&dc%uiCo-E%etw6hm%dHDNB6#ethnmqkzW`T>uD73-f-WR8(dp93 z)fR5CcSjq~S;pJFY$ReEnod5PkO=q^c|*Ql(9;{n3yz$h$3Se}FP+2|mLtDDZ(Fvs zfJ@M74}cVg=m>@)SvUUXMTI6lHHIR>=Bs~!7&vi?cybuje`D?BKBnF%*cJN+5)Nwt zG#SD+>Kg7#x?hQ}3?^m2P5e5EZp^*Hq}h2J)(Ws)0ZFd{>E&ghQ(PEu zfe4nqY->QjbE9-8yP1O$jXG#D_{C1KUF!^P40ncABcPiwOj38!u$pIqSVlHRgF-L_ z#n}+QCEQ%EV>SdjT_TxFQaZtAaMcWay zu~*tS)Y;oPfKeW*Omz+-oW}SbbS69dC2yv6FxVID?;Kj5**tJh?^;Px(ra+gg%e$; zR5^s*E>%hr9_~nq&Oux&*tdD4b4aWJf5dojH0teOXAq-!O!w2ucMb-}bX(k8cSwID zBdRB}M(H=C&P$yM?3>Yf(Ef~+yRB2f@yL34x=q+#T>VKfGK;Hm~I3tc^r*(I5!zU%EcBzXSkC5(X zrNr4#IzAbX>=F>)L%+*&qbv>^9yq9ED%a>P7k|Fz#xA&9o6*EHE?t$PVQk0c(aT-u zC~kiEW(hBQU+B_PUACle6h!sm0{z4OPQriDgZh&`{3~mqLqB@egx<@DN_s@et=4|+ zAS~(^x(q^Zg{Z%nyhQ9zZ(+F*M90aX3QIlq@nJ)Z+d$cfrn{?(6=ISn73ca`c7U|7 zR@dc69^`YwX|W@yen z7MlLzc6yY5mMjX#_FOiN-pEedz;`2C_SX8Dhu*5i#XQIh4gWbr;`$Gt!c;tkZ0{6$ z2XCtXi85M|`@b05CItN-)V#844 z|4aU))&Iti`V$0B+IQdCt=EaT*ZnEnu!WOCyH$yAldC^ub@1`2^$SV^Q;&Y z@j0x77fZZ_fB||LDp4I_xB4c-`Ef$B_2fomG8sLYXvLx_Q0ooJic`SdYq`I$+XSx0 z6BtXWGbpQQ`7WQ(Iwa9C-e>MI12R2fy4qUP76x0RN+%=kJfR)5x!LNUv#2m5wCjfH z_4D<0LK}k z{|IlnN@1s<5v~57lE6u5wY+!mU9~z?J)temX1+Slp`B&$7=v>RXh80Me@VWDR6eZv zDBbq1}?9B*MLmSWD>4U@;R0 zufLoVeXj#+@lJ$coPyp*T;Yuu+L>_MJ;yLcC;Dvv501w27XiBeN(QT|M?9@Fp+D!e zZM1QAR5oexeYQ~KqUu%V9y$4F8yMe5G@)tFj*ms*y8zgzh#ETSzk@qG^OPaQcIXUBHuf5KU=*bT=ZNHJ;;&6MZ8h7duHjE zxpUxQWyAwi9bjXmkNR?NiB(Q)n~_dZM~n+*iNaEeHpeMR`-y++Z2Yyj+3Gyj54ZE> zda@|2LOWW#GW$Ep;0p{yCv}>!#~D1q;3)>rGY~x$wLQ+vi;>)UHIiAiOnV(Sp0pZD z5Gon-W^$mm2V4xO+`_pMuH0JBlFQX$AY>D9#S7=Gv}7XvYkON# z?MaAM;;DQB9ukR{KJXRz0DOdf<*6@x0Hh|&_%6w{6!x^U_INxV&-`ZQCWAqjz;o%h z-?uKGAmmS+EIu42@4;&w5JeQtNQ7UO(VRsr527IF5vQ}YFbZKU$VA?XTAs(VcHW6P zejH}qychNSSY-Wt5Domel@0TyXo-?fh-$0OJ)$~#a5spS6}?VI-G^{8xjG6O{$ZN9 zru}8<(n%3#O?aa!Gd-fssdY1nRT5h#oi6OeNClFxu=>&WI;ougVxn|*P2a4vMfS3v ze^!*0`*>n*eDyW(I`H{sVz#w`NpCjt5(yr^`1Ixz?-ETSsz}6sA{$gu#qPDEKn0pB zu0t(!OSg4LcgMXAc9v`e@NyL@agRs+PD^z`a-jKH0@8!0%ljSZlrS3pL^z3-z$LwA z<%=qF>2;^eBeN5_U}OMe4BZmE_8bsL?vgRBX-aDLEB%?G<7*eE{B*GRb0VrF=mC{&dad4xbhbLU=!b*Zu{hCSzj5&&iMApfQAT zo7w(+#-nCX$arG zdB@B9u?j&pECwJwP^6;veSa3rrlA|&C3D?j&FVmj`>278my7`mHfz4Syho;@7P}|v z5XXBEO1DxA0AK4qqonA~_Wi=9Vv%2mHBSM80H5|%gIF0+a~eJmXXB_-RJo6Vg#;MJ zcBe>W5@*?ZoNPZ}&1!1pMrt6%j-%D|feB%EejmUfuqGL;Hp{?PCcUN2JeJY$R%%=o zXHplNX`#)taIisKMkh<7%h>3-Hk&xx z^k?`qRRMaV)kAd)sEy88eN%3lNma^RyBKxLEVV8=wE-O>{1{@@N*XXL_;tG@>fu9BmM2qn5!$^zJ^sdeff){qV{Q7B|Bq#S*PG zCvm$cr_iDoB&5}$i+yIwxKPK|tb&mJ7DzW3Qo#h}@MBaQaK>nxihaS61H4DJz@$5LoHkZ=HOtW-7G4QjeazYC^p8bHFaF*oM{@I?A_nE~7}zlTENg!UrtbD|FK zQWb17a}6{@lY+FnjA`du<2G{YP(d9A8bUROq(hCa1IZ}-s+eo;>~`yLk3u;h3v(WMFCsw{M&~xn zBtH%@V0NL`nU_%HLLP7H#dd=<}m1xQ1sc@34V=)810kEP|4FdQ5vn}^^> zu*+TnB0_)|?E`EOZruX}V3&3o0rFXkE1=q4uFio^~7 z*I^b4?4RZh6dBArTcWc}$^{Rspa^E8!GBQLsqFJW+N94Ihx|D>&AR*SExs=p+wTV) z9I^7q%Z^U?*fzrLlxeZ)O)>sNZ{sD<5-nqYyr`f^nRFfg!6=WDXyDmU&MI9b+FvX> z=Xvv{iT!i2Ao?3er=Jv=Nt-qvb!KsKUXs>ht7m?~%PrjcxCXV~ow@u9_798RXp@t) N&HAhlvB%%Z;2$Zmvxfix literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/multi_step_worker.cpython-310.pyc b/vllm/worker/__pycache__/multi_step_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..614441315574ed921b6dd82989dea1b78ea68898 GIT binary patch literal 4768 zcmb7ITW=&s74ELSOnYv&$Cvd&0!$np~0KH^~m6@hwPylTtmIhJE(WW#zQOE;^tkW-~Adk8@qk8 zT;fq44U$NyzFpe4<3V29AIB;$y-z23JkFw|bnZz_=MJieQYNi}AgWkQy{yLpt0;WW{WGgr)a=hN#M%k~33MHrKLWhmHhsxXutw0>pCYxwBwU$=Mibh0yvC$Hs^ z#J0Znb~-&s-$)_eB$`OxsD)Y^ zse{zz=Hqr@gbY2-X!+FU#=cqD>(CdjaLQM?0RtJGdn*AN!ofHj#`{!}QC$hThezQc z8XSm_=f@LKuEZ+jGCmTrKBaPFBFE_@41|zDiF}TlpQns~RBpmmMG|KsoKL6X6H%^S@JmIYRY5Bk=_DLiF~KQqC6Zxj z!^>6a9zLLNu(}s1K?s7?^~zqrEMFl?g`tj7z>EgV7FpmUqk{TAvYzRhj4{)`^yk(! z@1NeX$L#YaJK1{1v$p4Zx+@h2xu$8@t z5$ytCJ9P_p)PV+f)WrzMoi>UFx2C4LrDtu7mUZnzs&T789o|}fSAhTr{%nqV`1T4* zh0wcJn1#db$4nz-U8=%9+jwG}dWCnkSvXXF0u<%f3Xedp<($Hf0epCW z+N|qBI1jEValaz6Fcmq_nMYv)It(_3K%N4ELRk@*3S+M1tI$-o<{Tg*JeFj3;>TsM;nU_K7<`GHynbq^BGmBa)XcOs>1n!1OzXBX`zHZKk?@xvgNK zqsL+7?+^!W8^87_u8dr{!&za4_Wf5lyK4a3jvoJx9*^2t;|Z9MMxS~4(mw8_+(J}g zBj*Vro@QO>bWW{W+t24#R`x_Xo?wQ)9lZ(!`g7FYsoYE>pIpsXm8${ha5W#U=EK$e zdw1r5_dgmU5WJ!mLl(;J_%g~4Z%|jZ=)bBGIQFANiijU0&}hjE*j>aFJ(V9TfA`aS z3w?JkP4htbnGl>J6Ha*{RnIy%u3Auj5apy$Qz4&CeuXa5=RR-st_cX%T=l0XCOFV0{CZY0a zui=?n=E-|kkoAAXsY~=rVP_7pzo7y;GvpgYvr~js)Nt9&SzJ3ko=AA`03sHM**LQc z^NIN@>(n~6Pn{f>!rXnx)JrvR8uInrD{x&TYJ{exg55N-9gJ&&96`f+j(4?2_lg!x zzlE)5!bMZ=X1)0gpXnK_vsT^)98K;(Kdzh_juy0)au4 zi;2j1obCJL3Ic8R?uWm|{j z)}OP7e1{sk)ur=CC~MBCc;;7pwM)tFCW^Vmq~~)@C-o8&dirJQGE#s^&T>}38FFEpdru=bI%!QhOVf# z**<5Vd$tGqY3eq#n%$;tSCDDjAS3hS`DY<>X_mgZ+waLWvSyvK4a$7V=zkWH@FD4O zr{A5D7z$UXT$WBgmV<-9y{Dh=k~Xp0I6}FkTdll>tnAeH?Sa58q`q_Oi}x!eqHo{V zQQAaObZVZ_W^9Wh!zOD!WpCjo{*+nZVY~Tn_5=H=i+YA$Kcm4xyvJe3P literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/neuron_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/neuron_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f69ffeeae541f4c4415de0f3e2cb859e4c23f30d GIT binary patch literal 9004 zcmb7J%X1q?dY|bT3~*wrH(viBe3;@lNcFL3EP>I0HaE z1IgqLD_&ET-mDC=#vVJhXuLmH(hwWs@`Oy8% z*YEG$y<*YP@Vxqu|MK8Jf2wK!MwRg=gUXwD!=LGz#x<_{noGaT*8}DhE!Q=cexk#MXR1oiYvdkx9Gk7^ToH{yR~#PF|Kzv6Z1y96(#2F zb{Hl0-G0>Wb(?;YSsC zH7!u@^b5e-~(UCVQW3fx4Z4g^Ly*-Lec=s&7kkMyX$vE)TCtqS5!D5 z?+&8=AR?l55qVuPkiD)K&~SAlnY?rV_R6h0cW+$3?Y+Nv^Vagp`@cvg-)crQ;)>{o zJ$b8(=3!E~Gw`D}rrGpUOr0f@x4My#f#B_CBvR@ynO+tT2cp{&H)U_oKbR0pX4|3H zZ)0o`^yH@3eAI0F%~kByRM6ZIUZh6z9=D?hUcbrFeOYbF`-3hv@2zGC!ZC!J1Yv1q z*CVv1?R~4++7Mm70;IJjds~cWk^vJ>a|VX3;?U^y)CRoMlS}GEw_bK?c~?!m zZB^7(Wv|Iw%`j@+PlCaN;ML&qg5O?U=x;_3dfkObz8@?+?#T@y7se&RlZx3I{mtZDGxQDyCpH~1 zD2Lp?L@S{s*=+0TDGO+PU*jY{WOeE(DKw*~>#e4IE1yB1EdyZmP?=t}cQYu&jB5ie zwBFS|*JEv0i}gLIu`hJeVCTT{!Qre0AD=U%-Dvrtyn;?-g8=PU3SOboRRAi4CP2Y* z^wXF2ePhT4G_B7hw&!)5f$+S<@rXa5yHGB8-oruDAAZSt9`Ch0PhLP9No|zb~lojsgzgx931CQegnQ4CGF!rrwuy2@I`C3w&7TzOI z3AH!zhBv_YBL$PH7h!S(qHPL`XGC_*<`&OF1=#%Mp)$xp4LBkX(YHlG6g#DZ*AZnX zl*(EL>ciwkUP3!k8mCgP8H$P&2NVpS5R+m`Oe@u5iy5@9@=1J|%}wQUynfKjv?#+q z&Y{n#c}>k>26LR{b5AnvJf9b5))+s-&!Xj7eh##@D5J0Q{25@*iQ?}0JuOGiv-}dT zi5Y%D^@UZa-Y%l{40?S|wWLv=CH$JhpXVUnRG*B$$X|M5LgiiLIjFo>l8F<_ z>`M*1n-suY!uMLrswURN27FXv*j%OYz>19djmI@ykmk{#s~#)bV$W2~jC3?^ZbD#va=t?8G{?t%7O-3Yq#hssCsa7}lBM8JNZ-s4(3tdYJA2mO9eLbKL--J$tt)K*|h z)GmYA_iNCQC_tVDpg+<}!@an)c>n#oOCJ7jEPin7?L}|-`ki~X;WK$RZY^Jb>-OS} zM8CTA-*>xL(qpKNj0(+w>_$3tdo((kGA`Ddurrr2Cm)8b*Y`FqtDpYmTC>Y*kqG+a zTn(hCHSx9vQo=lpE#_blVh`v7Aww*yrv|L`*4C7@?46j?-F zl7+qQf(2xe$gcw=R(dG%3{{*%S4~b+XOk@{;O2R$yAU+{m~S#S?gET|kw2qOSPvcn zCV`zx&Ux@VhtfAX5s$Y++C`q6_4*Q$*&D#>sy&e8MTU`bKngT|k+v-rcWhpA#=CR< zLvOhI2W+--_n!BjTOiWzAn-!*Fid8K7XDyR#Rg$LBkSn**q*7K7_$##8pxa}f7_!) zg+B+dvX*Y?73S!+VeXr{x$iJrXNHZE2|MBF*1ly~%+wh$%(C#y&;q~v1y&%eUi!WH zKU-%`D7MCEFiD1J>4YamYD%dnvf-&GrBu&kq*MZg9jdBha`u!|R)$1AvN(bu_*>?Z zI~B=hH(cw%ZI|IN9Qd@(4!LG)VL4R;VJdMehxAv{$$nT{g;?-ZD};@DRvMU!oFZ_P z1Cxw8FiHNBe3`&epCfU6xOeHP_VGC+S2Num?5SS(hGl5~w$7d0rNf5FERNY*r)cBA zH-!j=QHwmZ(L+6F@v5!kupJTf`;N}_4KuXKnOfJj8D0agi8q7SiuD`X$LrhKINQ;~ zvqTj!l#N|;PutPAZS-NI54OR=OUfa$xjwIL=ZGfGq2$EQZe~y0&Vw?K7KJ#oYsCd+ zF&IjDRWf$1h(+1Bu!9*X4D1FSYItAuN>+q=E25X;aJ*IZ#Y7)P8gtkxw_S=$@|R!$ zCoW+Yd2T#q+vV5-U72VePDPjlW>SeOYVCimR#k*ssmGP5)G4EFg_}<^+bk-^#tEze znu*vvpv*iqw#}%zI~iMh`t}5@r4gHPCZ1T!z}8qx5?if(Nq`*qAb_0rS|UVXISd)T zFcSmuUXmTE#U#U{P3V?eXz~SatLbo!mzo-xS9*y_s0EvMK$+J}bF;z{!2W?5S zO03j@m$OvKD1BA0rrP8_(O)2N;H;(-#MDD~c6>OcEDoupn2y#5ZaGR0lfl{QNK%KhI$7HcgDj2#^47Ya+!-Z=Al?(|Z z{-Dlm-O)>MiSY}CM>U64^%4|e1$a_;CL`r$ee0O)GaIUoR3Kr1``Uy!kII(nmFR40 zvA=cUq>vw*(Gh_rH~T=Kku&;1fmXtN4?=BqbUClso}oMvW(YZi{f%v#aIvk#IH!cy ziAhwQ^;9%-yIB(GIM3Of27!j~#)aWG67E!_t)T$lQDTak>Tz6vUG?RN06)C=uo~q^ z6gi8WP9CCIfB=&y=13HeVGYm};|yR)QJN4%sudxgqt=8oQ=wclAds0DcY?$TS+PTzlBUrq*wz~-1@8aVms0IX z2~Kf@u-=15OMdG#GN<{j@T#T%76lHQi`WWwtOLs%XXF%8 z8)`E?%4}QP*=>6}w{3PEU=f4u=^q&>7VUD8(Lvidhd-@DF^>hRY;5d7T%KC6z*@%+ z7rDXBC$RW8$ZkV;fz53gauKw!(aL%P=jEb&Y{l8wjvYjFnWtkm4Cc!ja$--0N8GZ} zQ7&UC%7vjA?&HihR?yLY%^o6cr}(q3yUw9siwq}gTy-aps(z)nk8`tgP5Co0daoCu z=~NQU(ik)Mt^KW^o;=2pA|K<5#x>;gntu2?3gjD6UvPnsnySORO1+ROgC3S79}-Ym z!0fQ+RKVW_-YtBkpke1M`-&=WP zd>EN3`1Q)bsqZjoTQB^dHeb#7i1bswi$p?+z-K6cA+?Ua%~XC*56kZ&M;f!=Xn&8i z05W~7!?q1TrOi7g^n3=)W$loII{y(PaZqdzSFyv8UNB%H)^<|L{Dq zeB4mm_&rSKC=c^UNU|*>E>W?pTRzS#BwOy>kfFvbLYn%i+bJ6< z&X^r>#_>LmO)7(kSbXZvVM6M{Vi-eUDypG4CWVq5B3x#r8A5PO>e%c<2u&5)fm;gu zCbM)KF`mWj{i?Bb^(24%fK<>VkFW}97kU+|P==B;<%SmKC_mNFHcY8mD-*H6Bg-G&aDkjNoJn~viq};+}3Mtrw ztD#{UI!tmZD33%xJd$}MVK{K*NC`mqA}*7NuXxJ-5DkuKtOM;2hSm|@8oxE740OHT z$@b^lBtABx7WNPN-|4U*U~hRDPzQ^Xpyc-;B3%~4nh>^AfR+aq8lF*fw@y2kT~bTh z`qLA8P~$yuBtuqHe*BdXE2)W3MQcDeCylz{=28m`gXiW({z<4();N$`SQ(T)WnkJ- znCevMk&64Uz>QAX#w?yiE08KmAbz-GlJQ9t+ zBI?fw{EEQm0B-R?n{(0i!U&QX${pYm3z zfgZuSt$Ja5OPro)IQ;%0`jsIXo|aZPB!lR@2%;tY(nOZ(#!|{4_2=Xt0U`gGz@HEx zNs|S$B`E#VFofhwu%k8AkwBfKGL zIa@Pf@bG-g)Kv%jnq{1?oF6mC0&a@VbKm)fy+*aFfzqo`hA^GDB+262*Mz5*hyhhg zwm{#i(^iI=ieJ8>z6+qR6b1Xru?l}+7pHVd^QgZnZ{sJ)Q$((AaKN0T0+UiZpbRyx z@*0piUnQOpj&CAaxh_pmq&ucIP;I%|F}3ByKgMT|aD-9E zdGv^f1Zh51PEeCc)g)DkDPQ-bIx6?#DPJ0XbCYt>Lu;eX%YErI!n8+9c?sF*staG0 zZga~iHQmF4rs=CxZOTqcWfax-k%3Fug1FFi)CbzbQ00470Oy`p*z{twkq(;Ph{DMV zg(Tx6&r|c7BE;2YdhOmA-d~1|<|=OCnypCPWTtX*lUmgX(Al2Ge2IMxgW;0!b&|C3 kGJuUVU_pN#VHy3-F-n@!-xQLr6rP!=BV@xB-Q3*&0;wrLZvX%Q literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/neuron_worker.cpython-310.pyc b/vllm/worker/__pycache__/neuron_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a00b6b8a20643972f9bbb38eca114afed95a77e GIT binary patch literal 4376 zcma)9TXWmS6~-<=5F{arx?e3rZXB0QEXBP{I+G^j#7;AA6~z;4nhvHjD9Bw(kZ_?F zpk#6AhgR-m{)5vuKlCs3p+5t!eac&BGSf@aMm=Yjph!jP6cmp3a(4IZcdkgLrm7k| z_x|+fgHLBP?XTDwe@f7~4{!QMAWU;LMi>oAL|q!`t{xe#L2yhDOKwU2H|2jLEJv1W zMHROaRo!Y-b8C_9+LE^vPDORM9!yQGfagEMvWp3&qf+`6iFkp5R`D%e?N8-wWa(3;ZzXbC$1T$73Lj2wJ_2Gmpo| z0f-&(IBSx8I^~%s{Fo(?mvYWvXzs&jybY{Um{0lB9#4^TMqO8p_TrcexaRs!()Pno z#KVgNK9g{K;HNxa-ARPMn`BRV-EJb_@Ugj?YCnp*y{wI(jX$%{xesspHy~Vd377rl4M~^A?Mxb8#Iu8h<)+|S zPsF?J3h1+VFe0GeKY%x7Kn7%>dD?&iB?BEO9T-6MK?$faFoBi^WuWH30t&4|?a&gp zWvl@Ie_kQlpmL@WEvvy^6!tLAa0i>=Zc|JHaE)C-;4CocMJb`Y67vo^=tbaN>0Yxg z=72-YBUwPQh-3-LGLjV}t3ceDO!QLlA08zc;=;IFJr2WYtCs~~D%O#)4x~U{NWXC# zLFzFP9CP8FNN;IYb0g(p=l?xe?)*v&#Sb*Fm9h) z#p*T!TRhmN@Jj6>Yy<73_66z&w0uYp^)mty7c}yU>^2Qi0}f#$LF^F?&R!9LvdCL1 z;^16Og*0iIKZJqQ0Ai7b-d~;&YmDuB$xu6JM=uG=hVqD8Fw~^t2K>vx#(k#Zt9T6t zF3BQZhq0f+o8ndLq`x#F!NuYU0d(LUL4#H%4VgSAj6A1kxE%_HO8b9#$TKdYAm-A^ z<6hK)JOi%nANxV*w?gjx@>fm^-0%oIT)H^A`I8gB+YRDi@I!1z{xc7EO8L`a zxItNzuTkkn-~?kluXL&0ypD@5wpINLaV`O#Z?FOrdD;kxq4e*dE=(X+)zB^4e`7)) z|9h=y!aDH-AZ^r8@l0szEf^g@COV?2`3vp$1Va`CA?(*jeF`B9g4iMT42GK2`z-Jc#i=^E2OT32>ISpJu;?wO^ry8?=9G z;`}SwuNC4eDMaCu*Gpj?9jojA2F?^)KrTh7Vm_Z4rp;l=B>2-d%2_d0CC~UBMw4 z^I=GaU<^gkZ-G&ovY{KqI`3~z0D0Al+*Sczt^dx&3e}2j6ya7OM^3sx#|S5(>l#~DubMI5f$IYV$$9L#5MQ> z-Sq;&nbS#xlXjtG4gCyqs>3)WVu*k5I6*cZ1A_=Ea4dT{yYM@#ccDUkbTRj#5Q(@d z*ypQ|T$P=vXyc*Go)kD3Ayew#nLy_HHQzz044X*H!>p2l_dyJi$rAoU<;(QXOOQSd zD0j#ig=A%bcb59UQ^z^I6mZL+4od-$h>P@YEGxig5}*t|6#%EAAMu!BoRbM(R#KyRffeCZ$V=ufOr}_%&=Mtv{83_lL(7y5 zU;EiZ#Tx9h+jOwN&L=uyTW#I^vCE_W5k7uG?jK zLUT>L3;XA0c8ax!ypXz@#NY#%gK8P?w>9>W;MQcdtZEF2r9yQQX}2M{#?~oI!7ICS za!!5Ns3-*9i-n{+C1)TU_YgjboO@G_sc5#P{G&((vNV^~y1O*aHH@h(_@EWju&=

Ym3=DGDV^)c}tdWJMwLz{{=^pf==Xo6M6%41mE0@j()#KB$r8wS~=!^4du@`;pCVw5yFAiP~CmU6qMl*deA# zf}9ViXCR66ppr7>qY@o*Or^+5a_}Klsmk4_9Pet<`yI*&|_xkmFjj&cT6?||1!!Ne~>30<6->5SCX{daJ6#W-~tGKGC_^Pe?%w~SU zF8G?Qsf1^qZW}T$+C}6AujrTTl5EqwvTxd^EbCsyui8~vHoTf&x9k3#J?A&t8ZM*Hy+w=Z{y&(G)y+yxccVxNbE&0p#vMiUqbN+ezyeymE1^=RbQI;#-ihs$z z($!M!&LE0YbItL&yFMCvJT2WD#)B|$yi}KnwXT}7AOLJ^@wn&2anRdv`nx=E zfm^;e@ABAjo!IG?(#lPzzs+xk!RBBqt$xBE4*Ge`yc0U^qy^Nxg#9UoA#Kyj|tq7$3->-E2<2FLEQ7gtt~ES(f1vH z=naCc>AKXkCUIQc8^yy>j80|RnNKr!3u*iAmv`20-@W(A$9H;PeEQk#`|Dr)N!tEH zC#I>^c@Tx-c7SnFIq{s%~imkee&AwE4 zVMpV-tA3D|7)k~&E-SzmP%iPZENe4m9c2^k2J(tqbW2YZs6o{&L(pbwN{-6)rPTo~ zycdU%UX;#dwP6?yNN`bFlfv~U0!&L0f6()&r-?0zy{z2wNhriE$LsAra>Q2DXV^sk zQMrV46Dhg|5G%2|!(25{UG|jCmR)5FzX6WaQv++&)`W`wKUY)3mAcY3#5o|uc>)&* zTqLkU;1WQ$AeND(dK`-WcB*rK81HpiTIhR`xQv#cE8-1g&;RH~><@1Y2gBX<-n=Vv$OfzUM>{#I+r|a$$0_GvM1ah&a?iidq1sVydQE zmA{h8SY4Ivuc;>u_=pDm3qY=(Wwp%Qi*A9G)Fn05x#2=N-6Ai!P)xVXP1ocVUR_q) z3i2BAD)Kt=8lT%-bnEWi6WwmOD{jkeKhf-_JI`CZjrIng-_+a%cM;fy8SEnO>@3N& z%+K-joBFbXejRrS{Vwp%;l(3m+Itza7Tt5pilkTJE9h&u=iLiWG#rRaZWBs&DQ)K_ zON#3YDcV$XMsav6Ert(!{ZR0<3Q=(nWof(Sh7+@sBN|a|cXCWEQ`x*)?UU)tYZVXH zZl)rfywa4X36tS4O(v`FVhwV%8}AKy+89JVR}3C<(aW^6UxiTekMyvJbPp*y1W43G z=_v^VRFeWAOEkbjq62D)0jMV>KqDyw7Ly`iDX9RWzN77!D47YtO2Q7SM+!={#Mr4L zug7!fqaHR0i(Izus5?#6K!N(8);geB+VTA10@aUbMT?0-w-qFyu5~-&yI84so51%7 ze4oH?6TnTah<6D59)aH_@E!p=&*EJGyG7PI@&+!fdhgL7aKlGw{gt}8;f4L(UNqR} zg4ko%XwWu?7zmf@lDBpfyoJ%rZm!;@;E??64JQ_8Y`ZVG6Z2kXFMVe?~wS%)Q z-Fm7;+}r$r_9%CDvQgTZ+$dBLCN(uO)B4J3CUYY1x884|e?%NJ8ixAn&twKORXX4$ zKtt8=L!al&EUDUYN!!10#u-@6NvDS3@kj(~eKN=SM`Z&^+I2F)u~Y#F7^=1dMT1dL zpOUKSQp{|vt7Q_Fa+Uidsl%wxBRi`p)#a({s^TN+L8dzcWXMue9wOP}N2o{S-DO+T z)auLqm9w^VG9>NLZMH?1axcH(dLEpgQPwY_jshLN@C`v7}wo_>Z-d;T#MPEWY3X$AE)yZ<(c~9H^-G&PbxdcVetr;^|*?1 zXNMOiLa8z zVeLp6HvB!Y}l6moq zjArzYKTv+EW^^%fX&x&~+DX%8lT}>L_8k3lQj7i*R6CLr|3Re*t!y2t`1h=*_*bHc za+~O#Q^re4Ct0e%gO?OL0ZFhEsAVN7A1)`ITw;mo8q07}lFotJQ6zRk`I#!}N$r^$ zk=<(%Wy!Nv#?M5<{#vqsD`|qm%ZUNZvcxdV(3Yjb;c~3U=ZL=t>>$$<9sQcJ6}pp@ zX#Qk*GB@TEx3HQj_^s_S%>F6u>%qIp@}$201H~;Kp2v!sm`%$6!MhLMj4vbxbm?4j z4tl3y+$Q>!(9bx6E`rw=cUBSxcnLev-15`Hm})CY;S`Q)D~S$hP5YgfxY8-y1&J%4 z!d;X&^Av6c>%#pzz9icSp3tSz<((_Z`Q$=!Fh;j%Qg2Yk;Ou#GtydXAK449C)5}{X@&;eRv745EMLugKXhW z!I;Q$0@tE&$ckby>cgyZ%kiQRtE6}$nyh&c!v*n(pbJA)v~=diK>VD)6K(?njU@!kxPY}c>RkiPrc{yY8c zQLqbE<$UT(Z`Ru2)`;ZtXb^8_ZI3PTa&U)RLHKC(Yr0<$Kec^Yf7s^{*f)tUzCOmc z|JEcNdC&UE_$}x8A4^iWwq#K9Mrx3U5V&ap6G;C$^Ug10 z0jWlJP8M2`TBO=8-;2^R{1EIK@k`pC?kt-X!ic1QWOpX4oywO7&W6X`voV`tL)ojf%tt%2A(1s1LhnS%YC)nbtu^9^AiLnOfu?&1{BReYKOc-q0I>MXp ze@}c!)E0ebm-kL$u}NSmC^|`Xd4jrEGhg8|pk{(a87KZR;mA|5JF`ocRw9yjzqmtP z?h;s|ey!InfL$aU_#wWe9!6FZKcq4D2~(Z28LZ8E6OWAv(;M;}KvASezdeZUA~c-X zW|vXOyga)y!#8{3v@hF}ZINA`YP4M=)jO#+rrO-gcxf-cuGD$cE6O|>a6VhWMB^ta zJ{ck?wjg1gQun-3#^t2=ulj5Aq^D$|CqcC2`vy8jOE~LV9Zo4D$28B)0wPmH)Fz@* z0}u&0J1)VAr6^lFHZ_f+SDm2^j_fy@)>2Kaq?xi8`-TJ{KbH&#tXB=*hw9j=lj>r@?+0p z@lv9|v4FGjBV)|Qg|Rl)$HurwuP+D?bqWJ_3NF3K=;M;SVl@PX#Z4UnT4C*0SGCRI zAb_Xrhhh(o{8u0;=mjAyPNbTd((;+d?~J#av>8nv;->ERs|vG_>q{0*WttgXfU-ny zV&+nf{qHQVv_e)RX&WzCId9Wm?03)N;hEIaavp6*UG}^jK3rXw7q{S;v5x>?`qfa< zktByb6s8>3PxuS2s^DhInA7WqChuSFu?~@*U7u`!$_B@w0 zB#+aJGlWn3wOPg~Jl1|yuhk2{lU~3^>|Z`h8&1)NcW?ugZz2B#`C8W!-$5?u`BHQN zQWie6wVQYu5~K%WjzEI|MWKRDpQsR!k9G7uBj{l!Ex_wbwT&?J1nG^qMSu}lB0z6* z+0i1+lGmXiRnMOL=(x$3J}EuBJ_t6spr>VNP(_=X2L#?Y4JcX`s{pj}=qrGd!fG&} zztmV}B$>v3!*uon&jim|5yz^;er=T5uMCs@k5OU&&8p@v&9|9h$o}R_<m~K~Ox9s68lY*3o6or(6lopBNL}Zge**^rLNo3lR%&Z27LRvff z${8)Ip0meAw3?j}Aq8P~&+3>_q(=xi@Y9vk^M^f`^(R*fy&O-zG-cA3C$uKpbH_AO z7t%?2y7AXWraL6%v&5g4J^)p`HGhaxqeK?Kl0p|Lo_=w~Uc-f!Jp$uugYX-LR_p%& Dd22kZ literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/openvino_worker.cpython-310.pyc b/vllm/worker/__pycache__/openvino_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87ac044dbde79c49eb675618df5c76efdb9a6e8e GIT binary patch literal 15474 zcmb_@TaX;bb!Am`b#?V)dV1cN!K*+JB$^}|AVraqOo;#i2m%QVVE}^cvK56!XU$AE zy1EBh)dNnuN2UmB0j!tneS{shW!W=!*lWhdYIno2KN}nS?buj<>^l5Ye-3}xittO8 zU!q9D=iaP-Ok>EfBRmszD=RZAGb{7veVm)!o|wpK_Zj1Ix8^DoZspuFdzXo8@~hx#xp|Ti`lob0R3Z zMNX%ilR?QX1yk-+FzrqUGww_<>&^yq?p!eM&Ib$b0@t&e`+`MxF<5e!g8lA(&dW3p z1P9%NoVJ^Xg2V1%PG_4=aKt^r>0I-P;HZ0))A{C;!7=w3rwh%ef~Vc5IX%%l9+cfO zr;E*Jf@j@lgXi4m_<6E<%`ls%t+!saanpU3cE}m&QZ?5~T8yA<~bndq0ZC^UIW;F~e-=m2xr{mO_)@E#; ztJk8~yi^aP*nV?8s<&I!W}Lp>S#SEwWj!vf`jJXREa}|7^Rp zQeTY==lqR&t^bs})UJx81nH}1l63xZTlmf7p>(;^jOxg#_H&9?sGoQmgeNi9^iGIrk=2%H2VUs=0$CH+ssScVzS)V^JJjyX<*#14e(~~~=gwRjZK=rh>aA7! z<{&H^@yz+R{2B(vb9B|e)A6wwbJr-*s`(dWyR&}TkE)^?RjJwjs;cUpR*Ofpbg3@Ht(b?h)h=x$0Fz9~|ETA02i(tbu* zzTL=*jGq&>$gUJcPUPP;-MlFHh0!v9!Y`ucBwZ`UqPD25*kVEyQMbf(xrIGySz=O@ zP-{v|aVgqW?{j~qm=-fAo&JQ>ZQ3UkrU|8T|S*1-JKIep;+8Vc3#c}lIe zc%JU1d)5w(Za;DkAZHfOH0Fas1Iw7pvE3IPaSF2Idp16_dJpwUGuBL3n8E4>@aC1^J&s+abB&G;0L3<5u4X3+vt2QE>aV%PT)1Go4yR=+(?=w zHa*|o;9kPGu-F^$QB`8lCteOZYr*ru z-ILAwt&{7U*j}xZ8_i~LQt8Z-{pI!4b7g%qwmsG_p7#`53||2$W-Xomb4Ecg>P4%d z7j)aOOjPHmgFRRav#0MjRA*{3O+)gVTP8N+ z1Z~Ex+({gJ_X_n}wz@NC5{3x}lCSbTs4{I? z68&C^hyhYA$uHm*Gkf+~$_v-3*p;6}fqaq3Gel;nUK%PL3jPJks8EJ=oox-pTPHZD zx(-`GxHH7@!e6PvmLb^~`ENz;L{0j{$I3!+)7Pk{a$del4U!-nGGsh)TkRkgZu^`4tykEnHamXu~AidqRtDEQXX5i~7q zuFw}w)OOaY8@@AqQ=Z3*^2;FQ3Fbw4fim`BKKT_Y7_gqaNRO;Ki0|a zZxFde@6ImwmCXv$v!YuJcJid!7B-Xa9Axqc>gcLKwv}u76im5RMF}lY;effvt zIL?K#=0$B!gi($5VE-?T#6V!tbVkGuI*XvCEhADw3!)0S5Fm@%Jp;n2@YBW?m;!m} zZ3}jS+0%DY5JA&-(xidk*O%YV3}-G*b?fVDDrf`ogu8hjoD!k$r_~HzL20blld@#H zW+5R|a%|?}3^!44?clW^OoPxCvh@OQA?p!|^X~q=W+-{@Uwk4=OvSE=kUvh}dzHv_ zBBuw0mOi$5Ani`{JCy%5kiFGTzgnChu41@r^t7ZVF5c6`gIGo^A`ClIk_n}6tA*5e z40x|tB3>p#?VYk2r<#5%Sr;-&6KTfsu575q`AM`o{sAUc{;wzqNuU(;S+Hq$-(FKr z71jQtee;;kB>BW+caeae(srO3ao1U2kx2=26q?KG=~1d-Hqwwo`$*orO!c7eW+8EA zHR!So^p-^uWI@Ao7DlGEmFwkt*;z6T@x{l{LpdYMxXI5Dd7cPuK6#SJDG=B$Fan_> zSR(LFcc|0J%Rj?A$*X zFHxzPluoB}KQfN@>rb1x4-XwYlsS~ao1{dcY)4xv{YX+zHC(&?NXDPW6-N5FH%|U| zBi*n>YQ=<4Zo(-_`!+o9#3%RjA`9O(=TC^dUxcS!6chfWDEcLTYEcuDxKHC=!X51( zHRaF1r#4m&F=mu@XT@j5oS287JSWIs*4%k9?JvNaFOna=Vu(etgwlOtKiWQki{7@x zL2(GVi{fySV+uzcLCzA|dqNyVYQK2WKQL?webbw!I3}J#>A_DZeOeqx=^;_x^WA5} zvnW06JKINgw5%as63^kwo}ji-|0r1j;)JNYn}$#SBt~*loI?F$D8<)^=fw+ndMbJP zjCc`GPYVNk_2szOmm~@`>Hfu;U6>hs|7zs;3Z_v=06Wc2djw?LjRKsFt?t=m|I=C!b`u=0Sdg!nXlUL3(RmwwDLZ^$MW*9{d## zsYXG*44Ps9<7HT*23w;99!$t{j5rsA$*oCU>rLL%bS)xXNaLV>xxWT_YoLAjSQEXX zFyEtI08iizTns0q6ASYw@qqgBJ}U&BFjBT9&9QT;0wi#rCznOJ@YYrmX&aW)vVob9 zEh23q>qPE=l&8nmSvIMtPUHz9Y;W@0x9Kq;GHi#MaHk@Pv3LXTL-=lv9Og6HT@aete9 ztz7d_It^>L07nS!56S7|-^7!8w*S_ggxXu^Zx$sK20nQWa(eJbX#*E?;65Y(6v3td z2LP4=pp?1?kC|4NEwr*J$$XUaL>v&pE`0I6u4d;nWiz2qNvE zR;u=qroMB@^PsPCy8D~w{KzLz)o0NPuTOJ!)3)}FlP@Lk?{=-$k<^3ejMVhG zo|=Q8zPc7Uw;;?qn9nx8HtejdV2D5ioeOZI17H|(v*Mg3h+JT%g-;L)=|<;fP<<%eEYN$)r70p^Tm9cA^P6X4rPk5pTJyA$HGG1!_PDpwrQs<5(L&c{o8nX2Bc&BzZ~ z;JF9d{qWE24Hl;!&v;U020SZQKw?Y9e`2%VT4~35H8Va*xHxJWT(0(#`gNB_ak4T zL4OWp7aT+gKnfR2f@LIn*z4HMW~4y^%#xFZ=)mmjEIRM$--eR?PNHU&QwCK@ux1Ks zoOW;uid@^aSeQ}J)9n7<#X;O}mt|COt^vzg=Y31)jbb|r&$L_3&B2Ca9EW0OWY>W` zN4E5r!S8tUIp?nLbi#pf7y-}0pM!=5(=*)qt#<1KvBGc=i6j79_ffR*Enap+8$q8| zJAs%a?t>+Zm`|U>w8pa@9K$-|g+8-^>PYGK!O8%3ADRz{h*GQzmIJZo zaQWnb`^Gt~L@%odt8ykGe3b`jW`T4Y@F5vdv z62JuY+q$%)QQHeD1HLyS4g?wjgE)e6W|D>p20|Wa`aAFl-q8i2$C>UkSF3fbAj`1O zsju_}eX>c2E9xztz>1Qz(K423AommAp#UHWBDD-^iebFVO}so5VV@#*Z#s9^{1$2_ z^Vm1p)lyWtizH}S8$XS>U-c~~^zS$jgwFBqa+wC%qyPpD0;*(GBo1{})Fv57H6o0& zMS?ht;8+XvI@B)5ySK~#%#q{UKSfY~>US;U1Z5JkT}T&j$7NU)Z+X-`I>v>2ZIfNL3)x!( z%<>&PM6esS^aQIUoKnRTIhX<%jNA%}B_e2!6geKYGWp?7Xb7Zh9X))KqMuO#67P$Y z2aIzT7$;CpfoN!BLclQlmhlQArM=Yf>CQ7`uCd8RxK+6rn^4@qn;k$;Zhi!G8eayQqFb~K z^b!TSlufPdYU_iXjqsRZr4S<1OS<`ysdtZzFUey%opClfLD}GP{!8~$U zfr3+`X6=2HF*x*of-F)~V`Gjdcn7PHX1#S6qyGVQP|}A8X=4BHs`!Z`k`Jj9f@6Va z@(382`qz7OT^wl<*lZVl%YTD<-GgJZvb$bfRN{tEc0xP<8ABm_tPivP+dW2#6L*A-&{9TZaJQlicH}MWY_tYA8)px~pGZc6wv(JH<+QMux9uK+ zl;pZ2aB3x~90fGVM&XkScUQ4v!I^o4nMgrRP++xZuj)`j_A-!gj;xbBTcr~|p``VQ zr-n<*hGV@IPmg7RQ&=wn0Z0L*m;WB`@yqf*Q0hN|Kt3!CTJ`|H@Zlk^NoSzQiI+e1 z_&btzk}@KGj#BRt`FSEF7~+Xt>#XqlBpmhcsM0SG*<&d30ljTu&^Q$}w)f;OqU;{= z*=tDHY53^y$`Lwe;F&yj10Xd0TOSrJ3Qeki%a}$0AO|A@c@$u$T8}LAu^qc>%#TU- zfq8KWy^{=4J}H|MkjIhYEI>FEM1K#=8v%;l&cGA31wsVd*&U76JAxkxP)BbR?or6Y z#QLVYPhafob!97#U}B>VSH(3~o9#BZA&2t?O?4zhfieVs#F+%9Q}IWH5RV{tSgyc? zU!s}-{$*_~&X08D=3rO?s1co-@20!HY=`n`dVdPUomHW(;h8z|D%^P-jqu^^`MdLh zTd-v~*8OQ}jma5G5w!6?XgLZ0u~NK%SUd%=SRc+qG1Fq&A9tS}6L2i`M}|InL~qx+ zq4l9zNZmvv`599Lgr{LIjN~np{vVow0uqR|gN2!aKkSyrzW9@U(p}#>fp$oSlD-SyM$2S@ zJj_G}q*V?uBtma6Y#}+DI23CD)(O&Eqy;I*N^&w=;L8WuhW*e2%|4`b?qM1<|Ih|4 zJk+C!helLckfnfbKacdF=OsF-A||5!0Q-w# z@?8_hPYzH(j@!#~8b@Bxb^)U}C}{L~eE*@w;a*`U4auG9WiiJ8898FASAgR%{T<*) zICPNZuwP~$1@>l^yC**N1OT~*2{gb^2n*{P)^G4f2sS}tGq}Q8EGniQn4!@3XaAk1WI1z!fHZT-TFDo$FUJki9!Qz+>rUm$a?i101Y-VD{_> zsht!I)%;dLq*wJ`ZdL42P4$W-DDS*X z2mA0X(;n098Q}2g#w@YIH`3AE_Pnqk;CLMvgC;t=%u;XlyBGebJNw080?8<4`l{L9%CYW|t1=!p&mQ7T{bQ z?{+UDSRix@9GyrS!O?IEPmocA_yoP){rWYW7>jnpVJem*Izd2g9~2y%t+wNng$(Dr z4>NQ^oF8RR{g^t_sbI0FdfSHxy1Q0y`jzs6%ZK_l5KXzI;s>w*D!)N34iEsz@{9%DS5SZ$AAJXZ ztOxIwCQt!?azDN26M3Bo*#oS)mGiJkk6n=Rr2JJ%|2mN!BEJFRmT=-uVFXE_2TG>j zbds-p5f@naozS(}9-?K;(Atr=M_=U@>4|574lsuD*C=yd4SIaFWpMS@CPB}-^C&IiHZK}q(bsKy5dm5hIt9#0cF0}@ZEkVdk*3HbXHuwN)pMo0bt zqdl@A^hxBy_xZ@eX&OLiuv?Jxv58}WI3h@vicwWD^Zp|EZEAn zF&`N?enq)Zo1Yz1A|nH-yuyLF@Jp4)#<=?X|Cj#$?A2-sqpcI}3itqGDv3p|0i|{J zW|=P@TLC5vXPx4bcqym?TCS3WYXZ|wP9RrtO1+Qp5+=+?jT1)YE4&G^3VHt<05$}G2+}>k(VprV$;lpbAM^vBk$({S z32vGR75oYH!fDh?e;a4=papP(;M>}F2%621!=u0E8gIN78<%lic#Y?&PUB!P-%V9s zSouJA$~rqIKz_cCfw}1p?C20{gBt)#;lHJ(>@*71o}8vb@YSd<@5snIM-xcVF4}ne z*lpPN@vPZ>>7TNF9&fH&yH1B{oMUwUp5hf>K(gYT3EBYqiM3TOj{4u})CpaK4uEJ- zJAi(dqSQoRSnn3%UpywbIfeHPnn;>tlePQbSb6ZirkzXsbrksHBR?YuWpXw#Y2s(N zxPPi!WqOc)TNP(~=V9Cltmmuf%i}udCT+e1;Y6jqH|Wjs5&11zpnpYV2E<)zCn1&! z_!n{`s{3I@4K&UTv*fc>`QToa5h=qje9&Q>9VAsWo8`MhT^%7pl3vnTCP`qRyU4x; z1U(0jU>D#x0Xl?j;1>A5&_w<_u2 z1$0X%kiSE9=G13EDT7If4VC&PeHmMdn^--nF#8U&Io2-p#qxKF{3j51lBTrZ zBF_8c$7#%Sqlz3qc~L(vNQ&qe0--xe621Q_O58cl=+{}pF_N`*6DODVUbnxeGVWh8!AnYc_QT8#pbPcyUC~AzC{@R!{zHgLrf0g^Zot{rw zPhvS9VG$lpFXW~U%^gY^IBibH<8mKa7Jeo1F^*$=Y^4ex<(@CjO=l&Iru1o&Tffc8tF>`X@s%xthaA zt@>Wf8QZU*Wt=7b66A$Cz;0>>ATl_<2*~ibKAS3}$O!NMmpEk`#gz85v#0(MJgod` literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/tpu_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/tpu_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ee4b2722688736f3296e5f66ac687ac76fc5031 GIT binary patch literal 17905 zcmb_^TaX;rdERtSPfyQvXJ>C%+&ILI9Fkb@CXym)iXZ_{l*lC{K&I4^G8|0z0y8@| zobJVCtH-7V+H&GRqI0cSmTTdXV#lc@m5N`Im!zDo(6#x@{R~RK)7amN4;Zs_j&KQ@AzkyQ{D;uN($)%{sX9S zpLaiMyzChlQuX?~ya}syBhGovu-WQ1gP@*^OBcU({%qsL&!2tqi!ZOf6sJ%3Z^q7x z&2G1Osq4qqnRY9TtykJX7(1^G!}g%x?8cdk8$-Ulcys8-nXmLwif%K*X4tw?H{;pU zVd(c+_f9Y}w^s@rkl#b)b@|Kgy(*1jAU&-mBct#K~@%An~bZwjvtJinV{%IBL3%b4V6 zE}#oCs36Y1GPpccgG=$kt|sGN zF<~p5EkMvhR&c3cJ&41|d(g;qfq! zHDuLt7?XNyYS3%o&@e-(#q~~3@dbTk?bILt+B4_geG|9nBHr6*vxyb+R!#qr0zHo> z_}>72$~C-{Ykno=r#qIP@r-B3DHAChCFx0tHA!VADI2LQYGv{3_}S$Y(vGAhC;8m7 zDKGEl{5;+jBwdhn5$Td!l=fw$XWWus_GhF`1^HRO;?Me3yvN+u%;l7Rh|9p5Ie$Jk z%`<`eknkzo2t#j$m^J*uLb=0M8D@6*(R`m~#Vl@%&ffTsXj` zZ?wH2E)EAln-@lMrCrAm=i5BZMi&PW8)0k=<3fL<*9hcZqNpF|1OIB{GVfS$F`jD& zjkUH4a7j11VY?B8{xB}05V##mPJzy+;_|h&3OAbFhTp&3?)$58xf$H-w;CRPwkJ9T}a z)_rvfG(yvch7!zN8QzQ!HiJDVH#O2T7$!IfV5KeN+PBN*T~_*OJ=BfW`o7p{ zH2Te+-)O}726rFi9O+`CaTV7)d6NmrS7KWe+<2zZX!iSqut}r~8VyzAY}4(2sIo}O zHH)oFgF$!avZ=@T#v#ZMP;QFTL8w%L+0Qat5Dp3odS&1#E=_TJ)w2Y=QB#2XqfP+u zt_RHJQ+YFQI7VLnIY!Z_;NicDVVd(se%q{?mi3`+nA>^N{K!t5ADUL>1L>uO4tD*` zAoo0;;5$IzG0Dvt-bv5&(_Y%QJd)mwZ+o_%^|HR>Ieu=f?B%@tTNyX+t#~D`43byy zX8fX8!87X%sq9qznNEcy55%tO=e&YfT+4fN-aKm0-nABMSl)uSh#FOhJtcn*Z3`s4 zx;KAb`m{i9Eg_wjyYLh1?e`9#KZsIEe+Mzg={`$-5&fm{*xn)UFzPPl?#UIsQ6gzOhJLVtsm;FOR-1)}xj(aE2%VB?J>z+HQTrqc__po=r zzaML6ABUumI?^?)?g5!i$-ful=e-BLhu$JPIN}vS@*auH-~dw$`&?Ya#r3<5mKJAb zhKZaqE~o@!L1kQ0@#s$oPR=nXRhNpGYh5cP%?fvh(&5eR#O1l}TvZ;;GuLZ96acF@R$`Az|4W~T_4 zTQiX_dB$z_Sq^7dd#iFsn8((v{_RvzYIWuSb#L>Yd5e87TxO4pUJCn`rlgwjLWr2> z>PzbLn5}x5;0pv_BzT2@ltrxq#Q96zL2JDcv^V{@)N9^Is56*pyvQs$sZ>iYj0a-5x>qyX4(M>;(L@DreXgmpW9s8eOp(?eX7UN-|8+2 zfh>iDPp;F?A;H6@e9$mLb1Qu(1qs0NjL7iJ+vazdjZ`=-E3o#Cya2uPc6MaG4h_`8 zzguY|)ps6A1rJ^~4CswU>T7r+bE|+}&9Jytx`TXJj?7yeZ$_?+wYvFba_AL}U;Yih zwEgI#*KrMtT0Cl8Yj!vMdR8Be zTs*mlaxLXD@@~37NYzXbvR;v^8rvF!WAlb`%a^-@OAy8e3Zpl>ib9s!M<7Qo{qF8E zwtNhY#=UsvdJ`&3LJtN*+*=_niTV_0WC7sSyh3jN288-ygR4+Y4zRS&FN~z})&Zh?^i= zL7W3JKmqCv!xowA_>WAC+*NMVIY^(8r9KZK#aM!UVn3dFo;>fD5t$$}i4=-^lOAP0 zq1@h8PH7eF5cNrRGLTxOulbZ`MOlxwkIZd*RWvG zmokr}A7H|s73J?bi^5$?DRE;p9@sqhW6wx-@=R#Um$t{mG=zjZq0J^7hk`Qolz z(iUr~fL+;<6Xl{}REo-AWR{n`ouf?$jKo=emsjiE3cx$25NAaKDXP&sc3dQ17%G3Q z-R%Z$bt=~otu1h8S?2jIaP6chE(YBJv@X~zKtR;%=q@gTpNIYpt-ZZVP3j#(t>LFW ztDeV?x=8S|1RP#i%oC4mJ^*&L?l;ETxN8miexSb0TG=&V7FkiH@#^WXHdep#YU9G` zSI@t4_JW&jcsC*P4qUJt)hf z`k~Z1rJ?O5A`Ne&*SkqA=+{v(_#FT%TY+Fp;$eT37j0+f7ox0i8zC}={l4R{p7T-B z`lJ>X_>}bZp(6yz{>VxCWM4bI+3znpix8o$T-hueRim6PgH%=IA5*re$J~MW_dZ6* zc2?5@4VwH>Rp2Ak2?kGK!r8Sb&pea*=EEak*vQI}yQO&(!Tl^?qJ?_a zRz|oMTiSeg2V9NdhbDJmj zw$bRkw@JMa+u9r$TkZbZAhyDG&sQP(P=AG>K_G1UOH2vB-w`g!_o(;M*b}I)paz5m z5$1?*aRz2M6{-mqx_J^7T{}+0D7u$F7q2%}Z)2zh*&m~e;5mSNhGLtAi(kcW*(l;# zQ-dTl|IB>TDOL=PAk#JW8Rn0i+`O@QU=Qb>9wT1jZi{(oLX8?-7=L%+!U872Z53Sj zJ;4PS5P%KqR^X63UIsScN#n8bWFiBY@vV^^*&QQT7XE1>o!+wUq;46bEKCqtv^T+B z@8?@^V`Dir!tDkQIY{NA9C2p^t{COfq7YeIc2pqtK_&#ZlT>;u8=9dL6>ecRQet;< zG;#zNq*vsmx1z3tIqe^nglE8FwI3=jxLygkkY7xVX23^~0xz9eN{zr_FkXdyUi}8F zS{Pw88_heZZw(*WNd%QW*)@zT+JWUW~jQt#NS z!ute2U_csfs5VYhbpYyfaT?;9u!h*;$;S?_gc$Lyb^m6d6`&n^rcJwj--~UqKK^DP zZNi)`+~YiYjF%deuZOTr12qy<6_K#mIDh&~e7H;~Ah@%WWmDYMs}=zRy8${GsC6z+cW7k z9^xmW@=>5!Kw4Ru$m9+zNMvuYBAJdUwj>iw52Zzv^Ig$!98w5jf~Y}759|`wq)+V{fF_VOv>}maJBB)t(1z@UHaL-k_g@7C zKv|Rnj^?Ah(1HRmA2i^y#`OGF7WAPQ6+Q4nV09YrO36FoZc*q#w&Msrcui9o=%4Jd zEVvG8rn2WMf8ino58c%+04zKcN$oJotWywM_CBuy-d1D$*%|J`^D#kLd$T(o-c& zLxKk!(-5Xhq(GBax*u?RFBMgP5oW!qck`_6nVMQnl`mkGmCmeClq#f^dCZJ@(xT9m z#b|*vMNxmOSJj*N(Q@T4Fhz_~w*cH(5&>~bj7f)ji{)7<2gTVT7>m;cV@DcLzs$~_ zC-|!bB1!L|5lcHXLj5&X`W1qSL>%XIcWv*6`s;jio8UVHg8T4N!o?aP;~4T8T(Ang2aF(pfQfT>T%Xf1nOAx3l99gMaFi_64l;poKbA~6{_y@(S2 z5}VCGb>^LkU~8M7gw49Vl3;srg7rT=mg}j;V>-5T8oPW8d8~q}ya)tey*uLX76yh| z7y#O*w@jjTf}R$HfV4UhW`Yx;?ZJf(VZf3aR#GGDS*YgSgkG?=EKu*xlRF2Gh%;sS^2jkalUNd!FVo@oGGHYol zv!yB4y?No{V0dy^6Yn6U?siaXZ79mw-J6iP1NwYDk&r2mgOJw}=X=fD@N40q1~Vw+ z=Ux+TYcvivAd|nljX&M<`d@C(oB0>~Agm46YR}a+k8`qc%G835;c%cr%=6MsQ5&YF zxw82rgmEz7OToDM&iX#R;d5iQ{TFNhKX94x_rE@;ejYy}9>J3LCercDc)P`t*b6in zgGaH4-E6{(>br124I9I_)Hu6(`h{1{HZHz)zHvUDTOEWiGg_dB1MR(+zlp?%H2 zOX9SOc9o1a0aAFNSV7WZZ5Pw|B$bB-4!wQ4M6_&cJD*tu(b@;~ge0p9(IjttvN)lK z)p98<*YUy{Y}oV`pqX+N(z9pA{7jBItAfhv!Ne z%}$wSaogVBd}5D)Hx@fZeTP2*@dzu=!m-lq2G1cuU638YOYtVU(Y*d|GxhfXpyE=s zS6eLLeHMQuP)C$titggeS9h4A`N_3qgkXwW3-1&zdbbOGk}XdW?76{ncbWKq<7#Qt zHf=gqX^8?mh4&*b7O4Do(S%LNggwcE)o6S3$WPu2@qh1j(l!kH zG~Hm3^l)&Dd@w|r!Rc{E+D}aFCNb7)Cw^gS+xM>Uq&5Iub86bIY6%w0(16u-E_f zLTMLv20=vV|A*1f73c;wMFGs&mU)YWylzngdD~EJFnL-tZYHBtI#tjNqchi;7m8Ab z<}-se=CCs88(7&dU}bm@x&e`6DMi_QYXR%3go{xHdq8bN>lsF83DWyaG)pZ7wf130 zma!XEwBLW%H*#b^mN4}&^qJ?D0Av=8>2LbQ*2A>!HWXy)p#poW}2_7Am5_F%zhh(apL32n`V z_jt6tEX(bvp+0Ou*t@5{VS--okM@ra;LaS-mZZ0h;3?Vt1G3`>(dH0#+vwa&8<4b{ zA05V>K`uIs8*oo_XsZw&i4I2h;NNoFxFtgA+s4(iyieiL=#ZDXWpchKtB1#^-^mTY ziR?awNiNy>XueY&m&s1gcaBfxkK_G`sr(7d^1wI`Jqvq0$#Ea&Cbwerj+K_M$p|@ggOkv(9kUl=T zH@bJLj=MCs^}rokxgXpQ`}61sP8w(du3vM{?tLLGh@%m)VV5tCuz3U8dN+io^-uq!bk z&F(~5B;~O~FFga}BoN4*M^;DIO}XdQ_dv|Fsrq9~oLoSdIGwq|#2o}x8CxmPDC16Q znuVJpSi;^lAV;m6>Nn9OuIQO)-IL=X!RrL{WyQITegJXON6-gZQfyytLOD{fbq?UjJI{I( zJ42kgj#qyDxE5c?{Iuij9j1NYDBW-o9L_TNWO%Y8Zu z$Ci-5Ehpmdc%y#E0YpBU9jNwY1ei>D8(SPWC%y#SksGvjiu01@MK=o@EK8^SdOHHu z&i4I2XHjJ23&WqOV#T?MiFC&-5*JvQOg%O_acS4YwPi&%cVZNA^K|cP`{thO8<%$7 z%oHy&BJWd0N5`P#VaTBIlK({OK3?y^jqdG?<|G_sTi+NQ0@C z4&kA!q6fNL2ssXCj9#|sn{+TwdK~pCY>de$8x2`bL#))7-bI5-#wo&6SrH?THsYXX z_)8tM;R?8Fl)uTDaBqqdx1(3JdGOPazHXcYO?e!Lyoz6JF@IM@hN~Y{KZCrYCrN#d zV2*%-tD<8^%@R5E4WLhX_gp6TLufF-yHeoh1+n z`0q1i6IcZQ0Km=m2XH?%J#~OZg1i5QsV8>>Mz5%s064MWHvycKnYZ4=Iv%6F5bKc5-U_OR3 z`GD_Ol9yicKQ<5XM&Vv<+criutRHh6XvQDf$vC!Md0*yB+}-uZ&F5dhts94KdWMcj z|8{B(8m5bpEWt;1~~g}r_gKW=$v^RWKFT_8Ja$@e$-3}pNPj{5hkTNxLCZy~zP^VPp&PnKqm z+^N_?o_p@wsvrq+t27ko=~Il`#b~EmU#oFX=vtU;~4BmgdaNx-Qp)OGZ6!P zbSM+++7Cc*&uQU{j#jOei_$=NnhQ`(`E!gXD zReL#z=VC>UIBLNeMl`u`=D-T#g4s9 z#e3aU|2A6e?8DgT4I?j4(#umLFGq_Q844Ul_vRnrew>xqO+OHma6$rVLwvDD`!TKA zBHPxQeNP*ylbV!HVAw|Z#Y#S3Oa4ydGCn=Yv2Y@IJzcwqzQmZK3u`Z*ks-v4Rr~sm zzhe@I^39c9U5#Bsl-8$v8<+pNJKXg_knZt330qK`1Q6H8hQ8fX(KaCnfyqRgEq?Qp ztPaJO!z*`R+nyzbdepIgKS4!#6?6vU#c9ZHG61nzmv?(jPlL zU0W5ux-`>8bXj<+dA&xvSItK>X$`__I}B>q{1yX!cb&{6;8f2CI*D8tiQ%oaBouXp z$A1%pLH5<&%8smXXIe9P7Bwl~FTxHADAQPBt0 z0fD5s(p|)tN0{X}JP4Lme2oW^yT1jcIXE} zp^B(A!@DUtJA8p+m-up_X)y*%?{FL2U)D36Z29c?+E3!2TnP}xZDP=Wpkuc5X9S`y z#zsT-$1TZ60dlPDxOAnBh?hP>`*FTtipfq?^iE9ae!2djFN#>`W{56`H`Ui52_zFc zz#nn}yBVsmU(!km3(4=Z;2aiB2Ucy!zoKD45BXr;S+ptbh!#c7<-;V>a+{h5w4+H> zr1e43MyN=-XjIb`(R8Y!FEM1qK`H-HCxI1(@$ubLRiE^3^{RVK+8zXpw=U-T04bN=qU#97KYj#jP6(Hs=cv z3y_0oK%35U5X|tIiUcoe)yhEZJ3lBES_^yse?{x5Y0Heijk1I2KcHB`c-D%c%^fY| zC$c8euztdEf#dx_QUN&C3?7+@hlIE|uq_n*Z>KwOa-akVEys$otD3|mgbd%Q`9TZj4Uk?TaOyu|Cj5|Z zt)7$XgrF<9l4Dy)Jcv$fqZe%S)PI(0O#O4D7%d&FgVr+MyAfw3z8doIWaVxtAqniv z_uPv9g;>lmapGr@pOvhwk!JWeCOK)fcccFkMm)uhE+SEdKXe|_?jmqY@r@F+CVSYm z-FXyHh?VW`AQ58wVE2}oN_ZR*Qw)pm2}TKzY}mEoaYTNBzL5xmDPyI!@C6P8L;6I_ zu_YVU)HDJRtg8?o@jeX!EwN``&4&mwxJAn;uF*fjh&us-^^77Ch?`p-W@=(qgZFu| z0QE2V>JJG1F2NrW+$Q)}1m7X}*93cr*gr+tQ{13qNa*lx9qg@RT(y}6&SVfG1c`4u zrecfw7t~Gs#D&TKWPo^i;tDLPHzgz=T_$z~UR-Tv$TB5H2=1=_X0w2lTiO5{2bB)- zq7FH|2cv-+=vN> z8rB}$PEr*&75rW;vhk5m`Cv-3Ath!Qw|2LRB87v=H=zy6Crs|~yH)!bPgTntyCijC zVU+(kLEkH}`^be8GZHVHdr0c(?-`SCyYwg5_|GH)U){rn)(Gw;I0BH&cCwi(iQOSs zX4m?3za<0tROHidLqQDVM(3y&F!a)Oz@ zsCV72`s%y4F8bAKVBzz^FMqxL{KjGC38V;&;l$OXB0&z8ri zUM{g*X1T(0m1SMpn-#exkALh!rx&IDw$)t!ly>BL)6Ly$?KI2%HwRg}-;29>;pTAA zRe9l^9`a2uFJF#Z+v;+^x7psxD_7L*c59LguJvO%t|`CKm#RC?)NaH&?slj7xwkQd z9Cnp94Vof9qk8F3t08Xw{^dls9x5z z^D@_W`&(N|BRiL>EYfjL_LC@8N}_1t>W8X@o~+wj>ef)D8RlMj`{rBGm7DL~xEkGj z6aQ~re$8a_+7`x;+3r9^DRjUPwVOBIG2Mr~9_DfK^=tiB+1OWreo1LLqJCNZ>^}svv;` zSx`mkDPI;{A=N-H^pMBpI&uqY5&1ImC*(;+(lIVsQp@;T>8ui(dGu5A zG}o`mGwL|{tZ%yV3HdFIaYCL|>pYt)pOoiNa#B`+#`Adz%WPIBuNX+Q4C3ad&=?og z(+QTMaYj)$}xyy3-FuWK~iORP7GOxGONb2@&rK3B% zySb-THq^cAtui)f@}YDI&t*L6KS7M_kri1Z0cnr?k=3yyF>;V|MlPf~DnJ%S9;7!a zLKdkN+LR#uQ2>cl$L$2t-Vy23uUkLGh@C>GEX9t{-$B~yRM4v2D`r(wvuaU3t6^@M zi{|h~fpqTC_;b?23cJ@sCmRfNHy&pFrmfFF)n;9<;7hNPI7MQO#BmbqBuJyg7i%n-FFPo+21Ctl{OV3l&1&GN;YIAVPxtpr)=AkLSaW>ZFG>t=3eq!nK`1<_2 z7qet=anK$-m&F?A^tnGwhTF*t$(@Vc_Qu7*F1D?A@piYHTr?YY5$`+Ry9e-C zqKKwVVV5kg>e;^SJ@Qu<6}w^!f%1pK5%%G~%J23XQ=nd$4A_Dhn|@rRig*|h|AK_H zGk`#fOI9Wh9O>M*ZAt+Q*i?^m*rtx(p>4zp1zUs<|GV-Sb}fC#T2mT$cN*3Ib3)pvh*`ReyW zcdOg)Hw$@yW$#MnO@pQ>a`NIv+}cq+`RPxvAi2l}VSx`J_|?}cmi=CLw|OQP{WLF* zIX*9Becjp)>%{6?8fRGb;*GA7;XIW{wHXh)W3D!sdJ{ZX{v{=(DaKL zOiy3WYz{GfxCCISZo8+BmgnNbBn&?6#+Z>TU+$T2b4uoDJir=SRoVLVwxTqPGZhvi zx!a4Ae(o8P3hU;m&|FcsowPHItiu}CZUc z1RkEIh6F%p9oYBb#}TMLv9C9UK8Nxg?2K(@_W#5-??VTFrndf5WP5m08n9;XEzYK& z^*u5>?U0AZp5V3Y>Ds<<8$*o3_NE{n*b6Bm1s>#kyhFYloy zZ9sUIaDxXx;ln_X^uOA>G}~&aGNltbs=Z+n#kb>jmzW{ifOX$VFI-|HpT3O5WZ+K; zgg2XR7;MB@YnwSLET-yK)K$H_v~$~>qBJbQ>JKz@$v&k83d_)Jc zY>309B>s?_v(j*Xvpz^P)nvQExjQjA$Hw#N4^mC*rq7Zf+nooz8s<&sXx>C6eZ31C zxY_3o$(?pD3zxQFB*#_5tPRIIrh7o{U~X{lop=zXgSdsorFU=6@v+Q<*^B0-+51%v zznlI6saZD8^hUO;Qhl9D>NCa6pt+rd^I1Ow4UQFILT(IG-qpipKK5p_O-7(W#(lDH zW}L7_(^I(tLeBZtG;c8X3UtuSfhT`Ij&7l0dIh4!bQgG@>pk!aAVqcwya#?U5Izy6 zEglMhv;tyu$(eZQiy9n@itWRF@E%ssYVZ927rsVo+7jyBKn$qh(n3em{5O% z1i73YayQ|0;+tj1xSVh#_mS!p4&ZQhil7_zI}XS(r5v15)RjPh9-aRvV96dxr+PpR zB{+ac?7q8CUP^^aX@*IupUsVMt-NA=@Ej0H;Z;;nNBTZ_}jp6ZWK zz$?WVGsl2^+t;ar9o%b_`ZMZSB22K^%u56pP8t(biLq`*7z?1$lgkK<=?g6$0 zb`QQU<-xmxWCeKu-1fc-ocaX*NH+lSy|o#zA9YHO*c&2DLNTM3JTmMuEKN$g4Zj%D zB~*}0K#zVIO_~Mf*08FGv_L{C23Z=mtQ=lk9isIi!!mo#4^Z)IJSm+R-)Z>nXM4*t zm5*%E5q*iBmVG{L2Yd#ARz7X)&BL=dXRYJ0p@-4OX>)PL_)4~HA@61E z>%&MFaZcb8Oy0C)d|1M#W~4G^8NCc}qRtt`H+=&p#!QK5kIjiDG=+0JYS#e3P)Im` zJMIo~A?9l_vkE!a;M*{mg*qO=N4`ebKh%XSi6Pp5#VlzZg51TLU5AOT04`v;v$K`` z=I%$9nP4&X4#_+hos4k9ndOGrO%m-Az?58IK&GS1>?L&J@1#cB3$ zXwIHuMeLoLY4+H*UsH+{W2_PoNU1G^%?v8-Z-iU!PuL=#%r?<$P$c*d`-Pde<3!ulz9Hd;yp@F;3zRP*y_=2>NFL0i)#te@VF9X+( zLjyvJdj%{nn3Rx>yBnKI@`tu`e>h0?7Ig={}OvX!dagn}^^|X&l_ao-%VdCMp0PanTMT@1d>V zqtyFizd)CXgW`R}6Lwtv9(L1M-RIdZmqxfkpq0OGq6j7z!0HAFoLheDo(m|-(46Zpse+QiQHX$Hn9&qd6fmkHzTo=hfA zc0hYlvTmOo=PC5%KXkuz#oo#c;711s^FD#`uQyLo{0spN6-9Xv5o;luk8~x9ZsEY2 z8eJmI0}^EtgnUhgSNkO1gvcG7@7&$!_q+OOD*P@9y4P!$#JeQOH_>FdbqgXVuOFUQ zwyPzTkyX}2mgC#zO%CU1MI~gFm@0ISQg=wq_+Uy3dKne8{ptGEH#8-&+eJNb= zKorCm!V{m1qWGQg#b=@<{znAjzeQR6m#7544VwNlOOAMU)+cxnoT*hsxo!*Rq>cVO zTJYu4BF~f;6ZnMf-c}wE0>+V~fB-O#Ceb1vaqN2D$DuEnFBq%&L z%eiz>2g%bFUPu>CHDBF9`V{ChB`z5Ojt3Nu#sg?oX$Qy`O&1|J9ze`PK8|hf`?Ixd zI+|3|8Hp<-X3I>ptFJ=fhcoLxX{Q#gy|a(9{NwKFN38Jg*JvDpm`yv;h?QZr9;4A| zXrzA%jR0(lN^tN3J;|Zh--8GXxD|ir+weR;qN1OW_*)X1L`ve3VH1=sW7u>J3Af@sX(eb-Qja+L}t6FX`v;g&@%`od|P~X{*io zr}s41s}+v_M$qU&tFP4siotShFn1{*mZk+JAc7DA;>Fw^5tOW*AP757=s&`!VTpbl zxIjTw{U?+!bKZn!4NEa!k;ab@Sy<<$hkh#JNHPbT$HSOkAvM|iaDiK!U$G3G_O+%! zWLV`II6_0!Ybj={FyJzZy`<*1lLob1=TgJ)7wEQUhBLpV@XssZ^5G_94`c&BL25d^ z`WOjvn)HVdxi?mjh>wZb!2ho)`!^&Ao_UeNaEPz)P954)hCF*y{v|{GFJp79lwt*_Qq zaz40!xPbZ8D_wP9(bfjtS+&(y7V~S&bilT|>cM5b4=w8xS-!pT2`u`9MLQdGch#re zOJ5yZytl!3R(mu6d}i_84Zd%q{A$sDjZzOvx_6xiA*_+sOjQLnDKQ{eSSa}}qkuq3 zph_x&vQ&GRS2YWjEn1ifZHP!%sMwOT#Aq9hgt{yWX#?tn38FGNv<&)sYg^*kAw`ji3-oxa_v!hI zlLeliA1}tE7(MzQo7w17)qC_{D<8+ovfGDl9>e6p^r3IIJI~I~;sqWZjpD^*Hn@ZO z2}x3h6DgM21v-HGz2yuSP~z-2Tyb!j^}VSyCwakH&IB}WqLft1t5V$8JfmQOuQEa9 z6}qb-8Z9|fO~?mHT8c|X>zJJ5`W1BDxnKs0r%>N^xv}r1%SPShQjz9xryz=O&Ugdd zFS3I1OhC2`qqA#I3`BQ9aoH{XQ~z z5kc5PLlk6p_+4J6`P2L=;@R7%cyCfEqIaC<(Utu(M5W2N9v1J{Pl%mqxLfl{`YyqYzA7liNh{l4o#E>({E GH~#@x+m@C9 literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/worker.cpython-310.pyc b/vllm/worker/__pycache__/worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b65ca57ecbd150663cef27dfdeeffe6500cf5288 GIT binary patch literal 15189 zcma)D`EwiBecxRy79aqU;3-n)>aqk&A|>1LAL~fII>NIJlnWl*wYSN~4nzU(}rfu3uf9T}5`~jVv{?M8J{90rLDAQScaFeGG5lodPCNbm$P!JY?HV0qD>)s4|^llh$z!*qh8S} zdSlj@H*Sr46V`+`X-#@l)|5AGO?%s{ZQhJEX9@OD}|yPXRx{u#ZSx*gVLsM)Yn|7z! zHuq^Z;~MViS%u9ay`5<*IktoCd^K%7!*;p(>cWb~cC$UN>ek}W_92b!XC~W=uQN=$ ztdtJ4Cr+EsU3}5J+2Ge)ZkB6~@6WwSTf0=sg!*Eo9EAG0iXVj83(cU?s5`YVb=eKV z^p#e#=7#!}^`;x9U#z33lne6IwMwuaWsY2QxKpcjD_!IbuNj6TkH zC#c+rP$;Fsy>8uaao6@}OuHHNYgg)(pyJdjZI^{R>BDB2GOsMR0+-ot{YHg1>YiH< z!YSVk>~hNw8lDZDY>qL@Zb=REqFJr6y6STJq#0gvueV%3z~?A6<4&ESneVy`S;Lnd z4=czow1Q@f>J+a4xdsOk(VQr++~OSL+D%MkrNKSqP4-Pc9zzc&&b)#N2d-GpC_9`u zTgMuCE~_|!D+lnysmp{|z3e{68?EMfH*lB}IN|nz*^6#1a4@Xg+B5alN}c9AdB$6I z8LQM+<#?@nT}*uTOudZ7<%_O(^%iR3IF4FR&NGhhhI0Qe$@W^f^IW6s)GqMF?go5R zoK0C-rq9-~WMw*~y}!NDhyQ&|QCtPgMzr9d$BC*I6vx~sFan`VYDBeZ4$i|Q!bH`^De2*hNf%gOm&t#Kq3WPVw zK$x#8))bpzvv^N~EO+3&jqPN+@Sb5;Ba)mI#Hd&xOpvC-_F?4x=rNBT58$1Pdd{+g z>^}6ogB@~rim`Ndn3X`ryWHIyd+uO-*K~pZiZ-i&;O}QgUe&=m_Ohdx&oMlNpTXwX zJlgDI$Jq(su^)Imz#c@-0rwz!D5B*<>_ejEeIlpe9>U-5>S3Cz==ovxh$t^b<38%% zkCsPPboLlKDN2q;C6BvDB1})QCq&6H_9S|nbLZXTgeT_q6uxP$6L`;~e}g^EP6MA4 zSdC>yeLsU$ctF%3Tpkoy7@t#CRJOntUmdm{VrSeB3EXw|EPD=ZA7;?6oI#V#fNTxM60^F&Y1i|i%jJlWIF zN7=`a^ORUO>;rq5Sz^{tCpjNypAb2x(TZw(l6?v-p81@jE7k&-aXB>P@pk4RMNjwf zWHVTE0@IZg?wf}l6U-|B*J{+w#!3mruIUj|Gm+sCOP|Z-O#IG(A;&#cd=u=wQ6^r% z%$t>94OM-RaLpAh%q!?6k_4#BUBB6=`;}!BgZGH;Jf~6*@Su?x^TRKhWn%dzcw2;C zT$G?gxDi6;**YrwE)W)d_3&7;QeSbw=*sR<)7^q`Wwj1``={KU@to*v>DjO#MzqUP z`iv&rJ<2Hhv4n`L3^O61;?IJZN&J27EIk5?~I_0 z%GWw0x5n$hVPpD^!W+Rhd;zqckn z57jTwitOkBC#E2^vqErJu)Dg4|A^qNiNM_0%d(w`JDREl`>Ok^2LwD=y)SwnTGf~# z(uX?}%wR*WX)N~|t?GWJT$cTcR4Kp2u?b51(M7SvF_rmijaH4B%dSa>$~+FzFax0m z5Pb`!$6O^LJqM9HEmRC~NSVF1LP`wu9lw-*U05Y7^Nfg@oXPRfPrYIQzTDQhg?(AF@3 z*9crt40EmAjato{YXy~>&mW_XXo}W2w|?mCT9~sSVjoszXuvei|^}v)(c9lC&NUtu|Y7JJ5^oUbn>=APyrmwWvsk zV(MGlUOIQ~ynW@urG@A1i2fO* zrL3adTbG}fyR>el7B0RR>I*L}p0;M?qAoWa&Z50q1FJ^|rO8nD-P%geo^9GLak|8w ziOuY8+=H0DJY3?KJxrgp2Vys@VtjzX3vGw5`d0BgNj9k>E|7&{&4}69@nj;*ScSi# z@U!UY?Z@T=uQ}hWG>--7!VQi+;=38)}bAJz*}!v`oAu{!tnVuweafnMe>b zCU(n*0@&j*d*0CQ2-bI5c|%)j&%}qogy63?plU>9UP|$2(MMRo@&^^qJ;OOM77#EG z-cb4TC=)9#!3$J%SQN&a^Bc%|%K*q>4Mx=VWQ2QPrF%|*pQ8cJ14M}MC8Tlkf<}{H zq%zv>p1})L5|8}_D!T;Go=8URmTno8R*nSbTUh`h!c?q|x>VA{A-86T)CUsu08IFk z=;X`P8*N5bZJY6?-K&>-;=|!`QTR&$F(%^!n9d*^o&Y3k&p6y38+APc|MY&LjcO3l$EIdIY} z%?cqE8ZtN3DLd2}ewc}T2Vts-^LmDMLNLUSA!X&rDG)W;X+t?JU$ZMKHu3&w7-)n- z(vVmT(;^*etK~3LZonS`*G%4pBV;|22L;ddZ16g;FEIR3p)F8n$Zqg;NjUpO?oDA) z-^s>aG;-u$8&f}}7>XY;vQFED*)FcSL0QQwr$5l-$fSu)1V2`T) zeo=3aN2gI>B%5zJg)txVN6;tsSlLixt`*b6Q-mG#b13UVHV?v)S5WDPczo&*OlT|` zyW1?(n@+G6W&oj-*Tbymyke6EQP$BU{!u!ijA?ujPE}PZoJ@T!6`engc*Zr9^BMrs z0?TpIk!*hrvJ`G5Fu=eq>5_$3sczC>6I0_JO8*v*PhMEVkn$q;UVAp0#@2qrF)6so zn-u9TWtw~ZqjU<7An|b|;)!4qF%e-BDzBp~#(zqTlXM&{I7)>SpqiTzZy@g{czhB7 zhOVi(yX~F5NN?_}?*wG15gvi$KOqrdt)<)ox~UCaa1F{!Cn@8O@(JyhdJBe5HPcaN zlv}Az3VciVkC1UfxnEOQssoO~^w-l;8g69oD7W;EF8JG{Oud!HW7H>uAu^b%xoUn= z>7-_qj=o*FjZ)-OZXxK{LI?VT6^5gP6* z@X(P#(2JwA#iRb}N6iYlaB9L0>q_LES1Ml1Gn)-xIu`0J2qI2%BR4d85ukiV$S2bg z&X2NF-#Qb~DU|!jC9%R?_(24}>zUi!oEcP_Cr>8*5rQu_T4Hf(6%SaFk0M!R!bEn` z*J63ybXXPEMC{?X+U2e`nG=;Zyv>dJRbjh9S$BDNJ){ho7wQwd9r3Cq%Szr6;e8{j~;FZOv&mVK16D*W9|8L>G7SrXy83i~)yRg<)x<)= zqvHY*F=nM5Pw57X3FLw|!51@E8kaJQ#1$_SAUoP4@p>b*7V>Ngbi@hAxoBQ!>A4Tu~iVh6`XXC1GkD~ z)RBz0CJ-_J*PuTrgEg_Lo#Zb@jeU*Ba1Jn z{uh!yIpmQnl~4O%#7BSmh}@?&2>{9|fcQu0M39lD8VC*Y{ zRP}0_lq>Qa!CPbqR;a=wq!7DAzOpeyZw4i~q_w|zI$|+>gq-*Xg}0HYjI6uLN(INT zOZsvxdni_%ZKvipaEQ6*(}|_FLckxzVXwfcNt}(ttma@DFje6vahhvjX++%gTS7c) z2X`v)LqMyN8%qKm>d_6vdYCwYVXk{R!!c;pB>rLOz%xNdN9g_GFzEu)@UOY`?uv%v z(o&RtOYRwt*o3m(YRVQcTZD8>wqf&$DJC=s_XN=kmQ?XcB8uJ9Q*CdH z_Iz`jKCww=4DByHKG{#77fK3w&;0BQ%4b#KKSzM})oh@l3`+cMbrFK?^Y8$Qvedvb zL#WpJQr}>tNRXVKpuzS5tm0sNa`=`S(6H#TDu%#YzpZ~syAEk5R-lyTdr@0Dc1XXK zTIW%uN?NGR9S>EzN0k$XW$siBGN+8--;)!xNCi|j6fKo|9~Sam<8J#{v^-n4W$u*l z;QCJ@5%;lW9;Drgd632{=8?TcGXM$9VlKVdYhmOqmb=cMMhhrHx9|!z_;9<%7-7yo z4KPC}lVu5%wH(j|lM-$%IvQab4nQ!q`zbvQzl95Do7?Zp;-5jG7^$awF_0LejnXk1 zgLLI0_(U$1eJ_{JX@&O_MRDuG^eqBfceTR%@Bzxl#Qi+p+J@i>!i*y$PpKNQxg}Nc zj5LymU05F+d>g(;mrH>$T!yPViMz@v&>Hhq0^PL|w!E0kub==Sd#N~*-=LUkPsO|2 zM_^kfKqqnmk2oH30;K;DlUjO{W{7>FW#*3)7%+G^#Ts2-+cJk~fv`nz0BwaWU(gaA zc3{OV7}1~tBT{AwBhbK0NJ71S8n1ykWMhZyVOj;6lc6PkpSUa>iy}f1cjE5#J}sY4 z8$Dw1&maL$y8_i%t*WB@J80NK zC8*TH@l9D!X2X$gUbz7u-UuT0K?{ODV#-4#jQzlE`lVt1MH=}_0AUL3JlvKH*WGmD z5kmE`3~8b8*HGU|LveZdp-{h3ac}Z3P&dC#KuqNjQl+e9_MfNRWElQi^hq7`v=K)= zIq%_Qle_TvyHV^1bczV~QQG>GXlJAhlKapC$acpM9lJLnN7hi1|}IU))(LuZIgF(NHKgF~I_Af~&aQ$FzsywjrQF-j#h zk0mt?%8zPhM9oFyWRjZCCN;B^Pc>ysh2koLn1xDR8j=ew_XpeT)&!rlGSPVvYi?!5 z7mkY9F*t{CN@CIc6--ZdvT0LBV(OC1^a5>%iOPnS)1bNuwXHujjqTCZgj4mm=U+zK zaB>C0+s9oM#3aP^5WB+sKHZGOX`-9`4n9`Nh=qjyp|+a z=1-Air&u1`5$RJ(eY+^pZ^L&f^RPSVj!x@XFC?vw_qXb#Hpp;A53>r@ysmdNjBDJI zxyBR9^`|>2tiI}yIJ+bQkEUZ1y^sx8CIyC>@kJ6-Tre@?8zv?KEU@)jOIJ2qIB*tn46RCh=RuxM9usZk_osf^j{CtYNuFu$>#w3h7c#(lu$0 zbioB?**ZCVOwt0$enMEgRY%Z+Lgu7v3j0VXn_?d3GJ@3%gC#vV<|={`b@S?=J-pgg zJ#lp^*WF!yn)W?cL8M*UReTRJxCvn8VZYVZ;RVDjLPHpHEHYliapB}q3wPArn}gy& zwCuPI7l(x!Tz15NUv!-y30-&7$=)segmSkAbZpT{CmE?1C~lEcN1%L4Q^mMDVrUx7 z6*5dxD7~B2K?z!7j{NeT!%48vpJ1L4jM}PHo~!wACq{D`SDw%_kv2{4DEo|#uJX5j9G^c5;31B&u38V2(&G@-|!? zFyx;j@9Xix=6W2>Ni%5o{r5lqzwPd>bUf>E+NS{hfa^*_YTKJh0jaKppHZ)L^y8r?#i{5Y+aILX^S?%`H)!zhA(1F&gPLy{9<78rwj~Gr| zi3R~NG#HwAq^T(sjrPgsTS*1)@PqOY+1l(*1(E1`@EJ#vTc}3gq$)y&A4Y0F z9{=9}vI=f&jCTzx{bw4CFD<3L2Se*!4YxZqLwj4xX#b0A8vlb!y#Ec;;J=K#_6wt+ z{oEMV-qJ?2|ICeQ|6vrhpJ`*cf6smj&b)Ls1?S8zv{m(YsYe4nB0Pp3bMN3Pi5MgI zo{^bRjR#57qQF^pELYGnxNnl$t)hP99xmK9c1^0d3gK;$w$bT$q@$ly@Vl}f~-l1uk<_*Vozg#oNt)DYhE zEjREvF%VoXYw#K(s}zcuK!$Yj+(rs<);`^8xEd-Epd*5nxGr%lfai=sPxI%I(h+D>@kYRs{4h{#amX2z z+4@jN-7qK&iQuG8*B#)Y$uQVo2vnv!DL_MpD`P9Tlv#T7B1UTOV-4KHqf0w$h%cKC zqIMpXR2SA*Gx|@8_O>+ygSO_I?T`Q(KvX&*cnO!OAX2Bk&#cIfV&Fmn+~_ zkJrJPBAzWT8}w$6_GA+awnoqzZd{)N0pPD;?j@(zl9!)g8vG-Tkn8C-O93U(Fdwax zXqE5jVzUZvqmQ4`S=)z%4DFDUA2-D`gnmHHWG@g+;~v>Bbd^*6KKk2wQ~49ei}M;@f@B*nco{}^4Dg(GJkWzk`yL&!3lj$Agn z#(HI`5l1k)S6I6-X4);Xx+61MMxrSbXCVo&9Fu`gy4rWt#DChD?J4QSGvNp82CV1W z4Gfk9(&AwU4MBHO*KuUZxK2vf+Rb|gXhpa*38KWoPw6b2YtNEvK-hwZD1iP7Tn*Q8 z%F%^Cg?;b%FE&QO6h|mjrGGd z(qwLZO59Z$0lwFi;A#E|&JT{Rj~Bm5;1vR-AzFFrioBo{-P*(wZ=k}z&@m+uTL42O zQB;88pG_5zM&L!pE$?!w2|Y(jkRQ>b z%8Jki1Wk_#c01Q27px(1d6#a!@@LS@%8PG#J6CE(IqFTQM?xPG8kA6dgc?Pfk<_I~ zQL>7ndCve+ZjdyQB1iTwuhQ~qYBDC8kd-?ZnQlTku_i>mbRkIFzQOqnwb(bHWUl6{ zgTKuA^1r}@rehr$R8jUwjw{?JkPZ`)Z$d+}MnsGFzr%dV`*z68J~K1HmG;pZ7B#Tt z;@`leP9_wtL1{(n-hc_InzblE}BzRyy^Zk-3!2fgK^?)mf716Zcl>ymg8`R zthS%3^$jPd9Q7h}*Mn4NTW?o#Xq$X={63UQo$~J}MM79=Go&E%e;`2On_neBj8q)f zo}-?q0AcnN28T-iB+k6ze+)p!<;|>m0DL)%%WTKhNxguh-~F3GE`1g8m(_|X<P!0Eb~=4%@zk02C9h2$I*x7jJLdws zSc;> z+tZzf-QcoXd&ZtY9iu(lnX~6OuhgFJ9J7yg7VL%2qP@s@X8U+&$zJ07a{EN*q|{p`yHZ_=B3r{GPA(w(BcjOS^uf#-%;#`9TG*jn*syxDg(yqWXn z@n-&xj+*E2e9T+G^8%NA4$q6;aXcTtqubAWOQInj6E*Q^F|+lVb$zKomv>?;$M8;i zr`|E_&wAR8Li6;`XjfL7dQ!ab(#2%TU5jGrw&IS6H$$&kMRu^8R4%&hw!79AiE+tq z#ffp%kK&~AYB%=7z-=eR8zN3hH+x+wadWpTlG5t|UNyC3?$xzO$ZdDcZ~O7?#V}a+ zH=4!d*qRJoujNLu6N?}UrQ=b<^$ig_-LA7C!(LZC#px5;n(zZZcG}^_hLH4lHGJ*D zYvQe*h+;fW-f%k@fPA$VcYBo4X!WEN0UB((@p>pbnC0B%?}%0}7S}>gw9~rD+?5~} zvLig-jm1qKPK2^gh=Ato(1HkOy%)4-x=ssg;h;Gu-ju@il2g5|hkA|(wtX1}9qMu0 zl|I#tFxADEvZ+!IcYGzl2$N|wh+Yr~`H~xnWImU{MJTsL8cXfNy+=T(lVS>_q~mv5G?oe=ErW@+guZ6DFZ4r1%k^fFGbiRyejerKy|8iWTcZI|{9gH6p7O2C)p=7U(Jvc+TqDWD_kWprHFenSZzTv4nK5H*F{`EH;}}I?`ch~zjXyH;k#|WFDy@N z`a{j4H?rE`Ge6kCq6n))RBwec?Kce0syf^;O}-;Kq1@%QQ}am&Mepu| znODR3O15soyDViWk!?hxy?(zSAIG@f|I)d*(>>SqyJurpf}PKPuG8D>Jm1+l*Y?-W zb$20Gf^*yLcITX$cKYM2f!0^LyD5{WbD`PlNQ_c{VYKBdEC*DPhpv)*1ofJl`>$BG zOM>{@$7azg1ij80Ry|y|+-=uqxn`}k!`5vk0xt6M_1CShyPFhyS-L+&du(~o_hZMwK40m8t^dv;7--?JDylj}m)@Zm9?JWk& zgTa0bU%tk45?QXrlJ^^!m$T9BIchG;G*72BeE!z5wF6?~>F0_KTLYnItDyJ?&u4HF z_}k))6$ZR4>8@ZwFq*(x!`gJaBJhNV;VbebN*V7Mgb#+|37+BQ6W*DuCqLkI%*al) z8-y{M9u4fUF65)wUHKR#M0G0p6z-BohO6k%IEGuCF3VFW6ER6m_M179I=WUeHNLp& zd9=8#BAU%Ubf58RN`?I|=lGO1bx2%0e!R(RyMcuc)MjH<)EjEzN_tN1I^QW z1v%;IHwtfQO{4$Q^h8aCOFIm^5XBZr*>)S!MsazLY`-4%n0lfZC21|V%G$0a_-H2n za4k<_IWoy;@gjlXM5i0?S|K0sqqVsqgmsHrb0owUPGZ%ileUK=n|ioK5*~vDS!1uk z_c_saa}`o-MwJ&1OG&i#Ruf9aSY}h_4MSUSXojYL z*ncv&Cr4D3%c$0(eaSv_KSp8qwN<8&`)Yt(K)yo7%(6C-rtHUD{^SACF#M`5ZM)A}3qm&rrqz?yI<- z2U3&nn|P^?ExkXN8%*{-DRbAp_y6tnbBFXgK}5eTxZ`iod*b=A(=u1!lXfUSe@IW2 zu1~ClMH_aLpI!F$XaX87<)PBM*2ZzjokXo+FUN9gIC>9eJdGl&++y=YQgNKX?Fh$7 zY7X&m530M9)E(!op4(2}lpV(lTaF`()c6yW5KksWD36kk2Kfvn^GNI|KVU@=x<5$- zd5+$Y@-2^1Ql?~%5@Ia*X-a01B*t18w&e@-{Hv6Ffs!gEzedT3X!wrKEJJnIF z@39C`)+sa}2y;K!4sQ!jNe{_!-q(}b8*aNNSl>;I7zVEVELzF!C2u3`d55|n`8=!jAAk62tYor4 z5>G#Lu#!eBFHv%tl0(>%iL6U6zlx;)!gWBbnz4W6 z>>nyCvSA#Y_+x1I;Bpf??NB~X{D~tbJ`x(M&ANP%>d<*CiQVLvDWO@*3zRV9o~OG{ z%Dpe4@FuRPj)b^()|f5M78eXjZ*D@1FxO?+7eNb;V;_m}?`tm>#I!dd8XnAe zuP$c9te9h@XwoYH4w&+$*#w{WCi&el)Iz@*o&g~)n#p&BS-;w-5G{<`0?$R=K~U_V7l_h0u#+l#DdGwiiJ zvNeW~5!+vfThFUXGmpeu3PXyL8Kp`)ap;is;rKx^dmsziTrxGvYlYoiNm@@b1w<#t z=>cqK1J>iuII~KoKopB^loVU-D4ECsSbQQqz|AE5iIF2Zng@k}b{AWG52`eN_4K!x z{u=b_m6lW~^Pc9F-!-w!mHzFa_H-SJbGkYptWM5+$PI3YBabbho)H8G}ykT@n*<2^Wo8!~8c;Rx8$aTReH zxJsV(o=GMS)?4qN8!pl1?=U8MfB@AFj6IVorX{})B4P#cLZyjG< z*Rk39ehoT~PCCxayf(GZby_Gs_M!gIabNA?Qr}lK3?2vZ#nEglEZv zsD%D90lmh6Fk>y}js|qGp5|77asuz%nkJl+@kiEsxJ(1}Cd3Tt&O&F(aMC&CPKjAy ztVW8960$juQWbGbEI`06isRHG3(;cQDx;;qG*x?tmP2dg05I$mRl_@lk`*xnp?y@l z2D=DK_Q?loOo~$}a!c52$v&Nyn1SPDhPfGEofbY;^zQV=>`#W zbQ})xj(~!=qkvSUEz%J)8Y;I(di;uYqqoLX3{?-hXT0f05R4gioE|s5C__tu+yS4* zQAUcX9{@u0V{VK^?=|)#3?ckwmbz&hA>qw|U~p`t^rnM63&0WQB%B$3OUaNcfcScV zF**nEWiDc2kkpiF4B|*lJj0Tg2~zZFGV1K4iM9!J#Lx+BA(b{AyLI@uX~IgllGA}t z3OF~^$dZO1ISP?U!HuNOJvgd|)nq=*dwCoj@o@8~!;sF*>_hi-ks!|S*=I7sA>@#8 z0zg3o3q6D|h!TKowQkccqa1cW^RHl$cxM=cd~mTc06x`CcpbQ-F{~-mD~2s)LDz%AwWAj?u$*Z!5ohFAGxn2+;%I5pUhz zR)7#If?XUOw0TrOEyGh$3K}XhTCpxeR~$}NDEACVanvTbz#X;V|=GaKJ@F zup`iP=seO&b+=J_tixGPU%w6)2|Rb5W{OK4O>ex<`7sY09AmPf<+vV_qpS&dmXt6!;0KF)wsZ0m# z45ZqK#Yo7p*G~z_yzh+xY4hL52`#edf$A<~)tMgqo>kQoGiU0F(ttKnF0QS%x z>EflkCf^t1@&G#7o-rs6^ntNXeqW>ic?&WoCqckcSUbfy#G}KbfJlHH`OpDc@ruRv z+blCT4a}Ib`cf+LUBM9i-e#V%~`4m#c;S)av3m`LP0?qI=A zxzTRW;%JXGw{^QYt;nTC<%mdp@l>w@M2qH2~y`oKlxayjz1E2nLf;3{?SFT7<1O+e+3PfY~)SYp!z*6G;@PxU3=vkFJ zmswv|oRvEVR&$h4Qv1#VgJlKboU%>HVF$%zMisomGmzpD)iNuDVbo#P*R+b>f9A+l z8*5^0jV9655XrcT#^b_W4IBh+yQ?d?IVkdaP4ns$zggTf_DXx^ptM!oE2ES#C@CHT z+M(ap<>MPzGf&&Ac*VVHY~Cqsm7`bU3g1=Z+8tQV?~)v@VErf1vL;`lJMiKwgUZ(A z-6^jWPv312j&)DJJA<)Q@qTt|&MOZHQ<>*FHr-(i$Ce5K_H$4fls)r3V_^PR`=Pd1 zQ?nT8#|wKC-19>8F7?87-=;hC_crx2m>5(SAteU2tx2-ux%47ErDJ8f*kF1{>;E{d z|KmZ8+r#gr!newM^+9zou~-iPfD*d&~Dm)B_YiuE6h}2j_jWCBpYZToq z5+e21lk*UGTC{BS0z`X}?_fxKkOb-_yD1R-vF*crTT!&NfsN!(Q?nZsP`2UX{E}(l z+pH2P*-;z&9nmx-v3F9<)V3&@XMHjaTT5Z2hyglg{pJ!}6{wli$#8~E)+6;(#%M(q z5_I=9N~V$6lQ8nueF^s)tjpVSf$|zsw6^mIl_XeB-lBIWTF`VwD^C68`N7&{b@&1y z#mObVPQ^`?l^A|3IxInz<7|x@BxupDV*o&xRR4A(LSV^Q*F6+!PL70LQd3TInxt(q zzeT)AN*b~tB)C$NlfYgKrN42=Smg>8p|%q(JRmc9gijFmw%|-FJjbyp6gsBqVsYBX4GU5&s=NXLN**t;1?5px2LYfO36L$o&``NZ zOyCZOZV_U_fS7oU#6%3$45FY!rMJMq%-|Jfsuz$~L1_SN%;36LdapPr5qpEN%Ke`V z*&^kzl&4_a-B6JO&}uiN+p(?z4Du10XI-Kgxu^)SNm7f*pW;JjJh4~=_WVr!{W4ZJ`J`| zXaJZA6!LU-*BBO3p#>w=0`WV1L~mm^c*7K**5pvBkO934DH(Ams|RN(Z^$cSXFrkm zK?UHz**|y4ggl~aBK}$pw7GA9R@IqLN@L9+`UW*Z6UZpo_&_h2zwAGE1!Hhl} zi*RWRisnS))H@H3L<@pHcub`GtNtkhgx!n4+&G$GhoY4rMs6JyzkN#~x@er618hXT z%{0i4y(b70rT|64kgZ$vDmBz@r6F%X3DvM19*46Y&{CvcPvId-Dn@)8#sLp!DwHUi z(!#-+n-llW4X9Om6(f>6h>~C^DVw;OY1Ue}Mt)XU!T`9uUCbXb%57Tw4#j3roL!1! z_XwQKMyVE$Luqo#aXNTjL>|jmR+HIuUbz~{%-CG1crp`9z}i`C9%8*jGCfvmJy1w7 z8I{?Gcvi=C5tnkC;=FPSfC37RM(2)$e}^E20VtOUP@w;Z?3KuPyaR~YBc+y%fud5k zV9zb^-n1wx(sMS~(+HsmlJvLHhx{HT?;uG`HMpckLCPz=7?FbVckm*UxufEDvO4@e zCpy^PLOc?8>HHZOguQxZ*@UV<%0e9~g7IPh#G41XX(g+a82CRyGzPEGdJNa(89YF3 zhqH1}*oFo-&|;j*bVt7f?TdvNYjp=_IIn}gYR_MM{n7>JjVm{6jP1Oigwz- zMJixK0y-}64r;NsrIQQKE8?F!#g4I8Bz=V5LDwkBC1@!nm>Wh=kIlOA7Tjt)Hr(1Q<}AuR;7GJ*UFuhi3`&k_*jl|Ub6|BslQEquO8(b13!ey|0#)`~)C z69gi{`=op@YXbf!dY|$?am`5Vj1-6qa-&E8EPxe(;)HvrZkv&aQk)7u37~;q3x3y8 z?L>Rdo>{$k@v3w2^=p@}IhR*2V8hrV1sTs%JU=XXICc=1&~5h+m=IA^%OaLe{yrst zfW)pd^``IT4=JCiyQ;V!vG~cPx=lHO*&r!H?6@(Uxy%pWN41pf6N8Z{R;sBrJ3}vd zCP@)O-~g?pk^G9A{+4KF9`(!;BG;h!l9F84>skXe)Ia?o{p2L+m<3F9M@Ku8%NlO_ zi`Z5WIWPj)r~%%*z?KHU3QLZrXQv0oX6jeSqi>i!EEdhKs)#yC*OkLCfiZ{z<2bR^_l zkXdc$v48|=ZRH;$NvfB#IJd-9e{M>u!_YTim1n4ssXCN@Le*#&5>x%ZBB`iTBb13I zun*D3!F?D4y{d_S%(g<;=7B~ufa=ssWQA1T*8jIZe$0|DaERxt>P`JCtIfut${g!h z7ifUAOOoWdiV-+PcRxUq)Cm&xTj^wo_gSRKKc%}LQNn81C?QcQ-$wES8a7Eij*o2U zy74|KOga=M-OxWUOQ5NJ9VS-oeZ6rCDq4Mk>m>$*t*23QOzw~*E#kU}D>{$FFYG~v z4iFi5N0)OT43eue14vm4#__ZlkrLoUQVB4~BXoz*k>Q(WvHveP?5Vdb#wj8rwhH$GyrP5uGl}Uzw4EZo;E?;>0>gAPW;qVEpHjSj_ zLv0`xt%X1R(V*pBtfD;t>Hi5LHt5u;(&j0yz~mTDgmylBE8VPLfR zf_#fw_(MuqzI_LGNzq+vN!mj;4ZsY$h$-AqF(^!UOj~R$FfnofE(s1rk#)LbTw;O+ zW$Lje1>0_LPL6RE5<5**2>!5D6~(G@hH_@ON*lqiD=D(e5ry^{&WCIih#_NW36j3; z?m}T*fwfA1SL7ICdx{HYe^iWA1o9+jW;l`xJ7!DecWEdr@aep^PjfjXXjfEc%A?AE zs!~P}n^z~P-iws{IVH@NFX1k!4gqFmMv=wC+@tI!wzO!w4&%0IHrroOFTg~haA*QY-EKXiix;{X5v literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/xpu_model_runner.cpython-310.pyc b/vllm/worker/__pycache__/xpu_model_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee9897b6c218b4605f8450c1a73c84666af4de77 GIT binary patch literal 14474 zcmb7rdyE`MdSAb$XQt=5v$Ic<((om5L~<$Ve5X4fCGjbWC%!DM?tGm)w`Qk$moqy( zGpz0y4@yaTT}}HF%8dR}$ehC! zW(-Xen$SC%tM0n1bKYMSFn69bQGqYm_DK{0^t{tS^bdYf~LDtO%j_U+DHy7mH zd{A%;TvzWD-6Ho>3d(Mo%dE~sP;o1qPIV@Os#^`F+^JyNoepN)nPAqP4d&drVBVb% z4!8$`gYH4DYj+L>&$!QUI^B6TsJXS^uzQ&Ena+{msC$&t+0L=xxO<$_PUpGcdG~ov z=Q<~Xx?AUTzVkwG(mffRa!&=P-P6H>yTIjz&Y9pv_eD+@J1+$nds@Qkta2A7*7uSZ}F3BsCz^l zMafbBRsXFyZOe48qwJVCjq%BLqDHgR2*Y|dDJ*{d z)@ASFo0l)Xb@j%T#5~_!Pcj!9olaw^<0saoRx?VhYppO!GB^8CtJiIGlGNf_pB^r* z_x&VI^s5btj>{{4n$~!tA1CY|J=n!^NxS9*IjO{B>7AJy;gIWbFTFoV((`B>c=g7ucNV?3FJHNOd-3hBCxxrs{#x|5zk>0~bu`Gex~<6T z^j21U8sOCJM$qrHx+~ZHs6n&HgTsRYKk((vwFn)cAWLrwXdHDbnY{kawZ*H~Z(cfo zZSP%^7a9?L;G*9Rdy-!qCgtmEov4NJH#&oo%3jIHcNK>pLK6{$7|P_;E`}cXqSc6e z)n|zB_3y6v-KKwq20Z*|)UK+9xb6o%xsJ(OZef~|YS38qy@)@^yWfiLc>RVz?TPad z3Tg3&T_ok<2MkN<>7>N#_V!x0>&pv`&`+u(1w0%sJ@8oRTB{=lO^$mFArub1G+2qR)iPY_4exmq|_4e!&c9`kq&nJstgErVZ%1LY1+s9KNrxW6XWw!AwX7ZjU+ zxP#w^8HiWi{7}d3LjbS#p#{>U?C?jWRCp0rSOrhWs6(UU)W>mBecvH3^|zCX3pTkDENq-;#!23Qb{ z$9|;3&fyCGJBY85VsedlpsZX|7_Q}~e7l{V)BKDu&q7tX*^v^9OB_EXQe4C4d=548 zoKJJUfP9hj8P1oGFH;GZWw~r(tjyuE3d$zAP7e9JU-hT_>1AV1TQ)>N6yLYp8Gklw zh$&I>=luCOO_cZQ2=pi_D9iiv%Na2#s_&-&;s*p)wmvOi#b7s1I5qs<34KvpTCYWS z{MrqFO+vV69+mhkaFrWLzD0oKMLnQtn3Pnm-wRtLjG=rE-6e%F)=nw`K?p5zqvNgK z2UCPe@m@rj^t@IkJtbg>bxcbi@l$V&K~*wT;IWcM)^Ht|+w z;b)MDwKiS4(0|JSBO9@foVjJ_U}-S3aiea?bEqfJgVe3Pv^+_r7eGER5-apO%Smnk zJtUEPEH1xHTcE@M%3HX0sE@YxJf(&*wmc z$ulTw(ujvYO79AB4Vf6W)S51{5Mqpe3$qY|jawN1@9N)HbHIbC>+&?Je_+ZRxbeH> zO-k(w^I)V5IlG|ei&3Duf(AU&jBV(aZtWDGG}2G&;RF3f{a}*uylx}#Juh)Q;uWY# zr1PG47i=BEPGT>zjz~(L*XVY8QG?kLHEEG~O(ijiBnFX} zsG-^FM)ERc(?s4NLSvC+tV+V0#Dvb3G_&$LWpiwy0I~1%grv2VXNkN`WN*eeMGV5C z^&%4CDG&#oP%$fdng6PK758bw$QqXQiLDzuj$u5qP2&^8Du3KW{jnd7^c=2m3FJx8 zBuv1AEeM8Ee%jCY*#SJ*A`N(O{2bt+AToQ;1mH_JfQ6LMmTi#}d4`2N!$L7Rwx3tO z2Iae7!)_dT3^%yh7-Goh@y*G^V|q-!iSnmQ@(XzSbV-gZmg>|{s^wKwuiHD9LY7YFCuxH!1s6!TC8nrT)Wt5E7iS2~lxKR7S-_QoYAyC5uA(h2H}i%1Txj z6=4WdSib2)m6OT^=j+S%x3pzlkdzc$o8$v3H7gv*Oa_t!DPqZh1c?ILLH(mN=jYo6 zzKSR-LGsGW=yBN+MNvY_i6_@xhK!<4Sxlf#XiGMU-WWEYhAc?0^64Wgt7 zt7b&9lBs1_kReQUPm(zuMxoadp^_-l2E!kv4P0~&NtHy}mbR@w)O0rVJl2h#N&2yE z#hRDml#LWFl%A7>cDR{YrSNKbU8aTduA|MbFzUi zNXi3SPgxGoaY-p4GXt&3upp&JIw&cIotA)O1GB#?diN7&snct&dSPqBPi(X^p3K=ar|i? zJJwUYSZo(P)@u-<)$hZ z+9veHCUJ)aXfk6fw&PTs62=1?ylmc(I{F_2;tMEHx|fwL>DhWpO+>a8dVThjp||_e z?>D6Xc!BVn;179+8m85h-K9Ry5!qAk+&c|%|6Hb)NSuBNPgoJ>b)Y>WTO$}GbGNDlw(k2VfHZML~j9>o|V9Kc(M$9SqMSTcp^LSMDQ%vZEu(pR)i1DA=* z!j-~h$NDAho4>x9j?-;DJV3Qzw5HqUw)GHUnavFP$ZYAtST({I=xLmx#oNqMoj8k> z6Fb|fhuUTicN_OSTIA5O5T~}&aexSpVII)Ol+ai7&FOd=Gm)bHqtbRco_?rr&Y=EeJR@{rJuuo3 z2fQKmK;N82YN|agY_NGMo{gud#R}#{kbjIm<0?{kBYDm33Q{v7{lFBN2f59;Xm)!p zw$b}M3(6dNm|spoNU};u*oB3okrENYRdN+v1LxEft>6Qlb8xQtF7MH!oFt}C@?ock zFb)s|(8%U1yH4<&yt|1LG~OfBKrSf}dXPgku+`y(;h~ZxMi#1+@2&+TcnG!thbtqw zp@TO;WrJ1&pD(X~m}4JdeLgYKllnM|P+qxm6pBi$RewEn3lw(HDSI?N+kX#Ult@yt(M5#oFY%bYTjwu%x3Aq?^sZi#Eh;iG7}-Wj5}qZt^1KqO z)5EOLRJwL^EeO|w#DIUm`lP_2q+z^8{sn6MEHzFoQP`_~KtT&3^a!-9QBYnNT;2ppuNx)qC=b)bU`p zF=0zkvV&Z2c{%hE1tn&1d5OkeB!8Y-nh3+V^f|OX8z>oZgZ5d95>NjYuJGF+(7A0 zKd*3DG7ba#4949fjLIhR79lO6a-0FuH5rAiT@)tt%>irE1FUtnGla)6+^Sg(cx>+C zvDStW1;o7>=ew!M0UDzOIGSh7&NEV{+c|~2Ku$(lM$VE-137`QXq5+YmVulTjGPnP zQuVIOU!`8D&OG64F|Oobj{GtrN1Mgi#jjP&F!UmlUGL)p(L|6?hNLVhbB6 zI>p#IwQK@6(<8Vk*RZ&CGcgekW_)C!e`A1>a#v~<-;+4Vn8+p(GN#%sp6R@N&~!4rv=;RIAIKE1>#SYEEb+ ze~rkm6CtEhNOYW1gj4bX2tMa6Nd4>3vm`T& z(8<3_RsR~1zfRxl zGvLO+uq02jUfu7>vj57Fp>N>1^oblL(kDVuQyCEvaCaYJEF2j!jN+qWzGxaVK+H!KU@zwK6XRXV zmCed0+_s5kW4}Ylox>I00Rj$P;deCOBBVF-l z#H2sFd{9)y)cYxSPMi@lVis=jynjGU`v>7>&x!f>EySe{`Oj=W`%ps-Y(sKMO&szM z`;-0=3TdIX2`Bm)@hn;&&8EbQqUIm-kK@TeDtkh_Eb7?kd;xtv(>{SdPon=i?x)axn&0&Ty=&PKr^NyX&rS-m z?>;A&Fz0dw1mR0|6nsg3ZDKXnqTU;bp$F{nHq{nn32qN>9l|2wJ;`!7+$J2B6qLU^ z%;uZ4gE`FQI@pXF$;ieqQ(V1=*f~{Y8#5`x-x9KQ51*s9UXrPS-GHq_+NT^gtg1M> zcc&!;EK8Uqksl`2-sr~Xux?TL?Q*b1ndE{1Q3h8oNx=vTI;3LMX#qC*g?2(>Botf=frr;!R%sy!* zYpt-xPC@+{`CUwp{B0tCo5(hiheUpd$oGi+9+BTC@^?X!R9{j+`|nTztCinH%B_$_ z8b^0lPD&+FTu_^Z0%R&QDQ*hi+0KOpjbYBkxEKA2(|`Q zmpD+d-$Z6l1yANUGz8A*w1O6*Rj561^xbLf6#Ce~hUY(&1O5<-{UMa}*oG=IbiJe$ z_8a8V02e6w(tWL2SipxM) z+7z*)XbeT^5EghqLX-U(*v-Pr* z}<^KTeiI(#-eQK zjbp96X0&*iS&p^tMgM5zZjnUtUa#3$Vu8aR9GiOM)>Y!{)%b1tP6JIHsx4)wyJYs;R;}mG-yXnnzl-+g&!c)vos4px8v2rnZ3@+1) zaG4M}oYw&AxKC@V8k`pgFzvje!L5R`vP}TYaNkZnG{0eN<`B{91rV|tJ;!Z>-Zr=>`tRwdugF|+m9j=z*a<0Y3CkYwt}^wcCXP{ z^J~3j)LO(*7-z6kfXD)SIqWVVN;Je*0^cBH$HL8bdN3je4czi5s#9Etov$%|BknYD zktrQkK=>F*ilWQy!66bBL@@h5#8ZgpE4X2Q)xWz>K>2jXe;smsMcY`|Cw(wj=x7if z6);*d(Z@I_X596RZ zN2v}w89l1V_(1F4X{`G)R93*Q-{__p;6C><4=0%%M$Hv4vMDAh_~90y!k1eTJ}M$< ztMU)1>{$+h`8b!h4#mB9IBO6h3|aXn1^+$2iNKuNr*Taj&vA2f4&x3^guq0oPb$Rv zNA%tQgvdXo&T(MkE`ohLKRC(6F+F!;bQZmy#ECo0f3Gj)A5xQy+9U!;l9Q?&*l9IS zkMZ0;rUuU5yd>G7aU1fA!|vSMMM|nq9Ns>R&5iKV_lV_w5$~v&6y>rkI}KH^lTi^Y zPO+N6MnPsrZ4+3JEYpJTO0g`i2@jY2Q|#mK7%61~DBl#;z>o9UqFJ#GSOV6gqKb#1 z9~@{>wt{YdQnb*IzA^uFhUdNHu{akAo`s8no;DsLfIM%F8!!gwgb$9p=xn@Uv+UiV`t;+zVHA4>QF^In&@%t zgi8tqbn!6BI|lY@kxsi-FvN|Aq?1Uuq&F>u*sv?bVJ&DJ+Csr3=o|m#O}_)9v!+gO z5LON(M`0$7x5m;_Tf#X(QR{VUjoJ!arEU#;%_kQ5tV0by=GukYEwy14QlEVEvexx| zfwmBD?v;8PSVkxUB#2rcJCn7gwWx+8D!cvdwx@;&)PyaZDy(5YkIz&!I<+wJ`{AjY zzp%1!sy0OI1!uPd?5BMPlLFZB8>@)slQYnJZ(W5dC`y3}d+{Wi-6TEmOC?#C`Ikp+39Zn_S3Xa;s`;tZm4JsBpcCVAvgfgXm*Gf{WszeQRh z|6WaXa3($AQ*~5{rbfAUe-Ei7N9u&PyWyUv!H=Cr(;see3XAg*?&ibUpy}N z@}J>>4KxXlAYnO#eTeQeo{>q+F*Tr=ou!ohXyeGfH6MKC=-xitJ?gdpl3x3-K$0w5 zG{nzM66oQNJW=8)UjIKsBDN@2FR}M2;MHT#lQyeqrPaw8+L;3{$l1su5l2ip1v69! zPv~%maI695qbbbaN-;;k;pxAC<8N?vXorsu!NEw)!@Mhmuf*xw^hEW0Y0&SbLBC}5 zA##9Ngx)TJ7b)&g4u56^pn?*ZiHJbBh%HRekqWAd*msudV*k*9Q9JDSKBe%z_hZ^; z)p5_nHg*JZh*mh>WbCZafp|o?U&h!sK0iLWHPQ1!LW~+tTJ(D|s@dj`8F!fs%g~N( zI;8W-vS8bSEe75CN!})zvp>pYEs88~jSSfdJ2qnLwEKqBfsjP|k%QecYCq`DwvCO0 z;|#MfX2Ah!-oyvoQ0H|FMqr$QRlHU|wLkF9M#2gWhZMUy!u2ktNX5t*B8P}f5FtlX zP7x^*p)EH_aXv}HEXg2Y%Qmr=dcBUUQl8ucIZLEQ#3%CiL9j<5dr;BQx{9B(iB5K} ziwNVgFX^8K7+&NbQxlsAsSx?EiI4(OmNY>r!~tWid^$iyDURk9nc9qI?M`vT5=qECnWbUrpF@$6$Np|>~| zy~X*7eE{zQchK82D7U%X_=#heKjJWR{igi)G_L{bLeSx@V`y?holqKS)IrW zL{1VpMdUP*1(3nmcUN;^aH2e1Xe`m`JREU9Lv5cvX;Z-KZoBmV<(+grTp zT|9ql@twCX%d=GFRU(WY?^24S>Ph03WX@uMINkjkL{7z6GYG+vEqO}MKu+?;DLrqN okyn4SC{d@TkLgyLEKWjB+~JAktqjgoXO2|#>LKkHX21D=0Ycx8j{pDw literal 0 HcmV?d00001 diff --git a/vllm/worker/__pycache__/xpu_worker.cpython-310.pyc b/vllm/worker/__pycache__/xpu_worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4cc9e5fe8405052f400833db9e8c675cbebf376 GIT binary patch literal 6076 zcmaJ_TXP)66`tvxon6gdb(dvh4`9FxW(B#CI6!cOAb^lvHn((DY8dUbq*={n(=#hu zTkAYTyz&dERHVvN9+IDs%42@RRNj&o-U3NT0s-ecJ-e&5G1;v-)Av4o`t*0cp14-C zH2j|Z-5>ToeMHm#LYe7b5t*0pB&M!up2js#_gJ8FUA-A+co%{~XmBGe@?vOmG*opZ zUQ+KeFRS+qpHXj1y^Wv}R(Un7@mg5t^{~MkVUst*Sw0)i@wsrG&xZ?qAzb8(YP@2w z6fX1SaD}g^ycwJcSNW<+mx8n5Iet#1%fUn8d467{XMzi%&25#of``LL_#-M^2_6kE z@{1~64IT?0=Z~v&EqEfl#4qWZI44?1I)Cz2&8vHj4>YeK9v2sn4E~hIZfdROVE(H8 z-u1WbTd~|1((VLqlC(c2vu(8snX&07X;!}0OZ_-l~vZ=&YoD0qaA-Y ztG+A_{LVOMZN{!QszLh3)ls_gX6%Vzl$pP_od|j0Zu^0s9*zp@*Inrb!D*xGG7fv` zRnP6ELXH|WZen7uA502nZ}vo|AGoQ1FjAq#vWG>K^rdh@n$zix#&!J2PklG=2g1wF zQO5DGKlz`Krj1MddjP0{pbk)B0*&SN|0y?JlpgCd{v&U=gA(gy~g7YoOWC#V%m z0weCQGujUGmc5k^4EWhH+MV%M(QXB)v)(zR&L~g2kj-Fq`EuQta00L1<8yYp=ccx} zjjc>YVqbD?Y+8!liX%JTX`x!!YBOyVxb~~p-)>treoAb>8ONjfi47NyJH89++PD04 z56u%l+6{zqbWize+kOiJj+`D{)G$&cy*NtzZ8Y79r5(C{l;S~WHUAs$*d20v8@oPI zbzC)3jr3H5VX;1jVP;uZF`26BFbs-+Cph z=I-6ek6~lt=A(jUGCnHKsHB#!ew?kU<&1Nc00(!Rv4Xhc2IS!TnW5zJ)C$x=azc^r z#a@Pf(jE7kOHNPYxNNbf&ava>2YFoF#BuQ?POWtEjmQ?_X;c`uGo?uTGU_zo^y43; zt9aJ&B!7e$8eL%ZK1)ikYroV}{YXoylp5+o&0#|ZSs0dwTDRaB!vb=}p#fE`O2_{r#xic#**eS%>2shnpZf|F?$oU zGR$Jo=v|t%f~VA-r7^m7&-jqWof~SpMvtcfiHloV0pKBVGPJ?>UsiS*J7}^O_XE$~ z7B=mK{S5h23WS4?j`!SFFx74qC4q!|6oS|H51f35z4Wjr zmBd*R50f}Y;vo{}NvKh6N+Go7^ciPF)EcfBLzwzZX6m-dYR%?X}(^ z{5D!U2!e1e_q;V&e&mGh-eFdDatC*4ammXN^;t_dnW>j`^MU`e-qe|H++_uZcm6kY z18q^Nmp?P=WqmL=0l@Znf(~u>_(y32k8$=AP*eO! z`xQGO_tsORYaYXSyZVW)YbsYhfn%u@9JyOO(Ywl(qnWho!To7m>QC-E)Q)SaHC(&f z=r)fqhH^<`OSYiJz<_nG49*~I*-0L^?A&{7@}4VpYgRc5H}lfWm^f8dOk>&ElQxD? zV{wbZ=BQmoAi5vsE}AV;xE(nq@B;3d`Xb4yX`BLwLJ`LDFl(q7yo=2m6;u%^Q$G|N zQpPfSR8{9&`c08t&XX?W?BsaD`>if7QwjM^F4|O=NBdl#pP?WjuUya<^tujDz@D=&t0I(s}f(JR;59F}&7M;6^b!+XKk zcKiV0n?hAjxchd#O?E$VcU2Ue#4tOJ;Y6TyAt;i)7vDm(jSf|)LsYjBYPS)A_7fBz z67Yc+A|Q=JdfZR#YuDaP+Sa7UTbNQP!iAt81on_`Egqnv$eZSA(5`n@YxCgYKUKaKm!IDsEROET=l%A`p?$UQx_ z4{g`$_VZO!hxboUlG9H?#Rp)~arC~5Dv0hvp4uZ1Ol#_(fxwKdHZd!a9YwCE*I*Bp zF58{H1ldXNEf>VVK)m+cUJsGXz6EouMV#tx-*OW>0_fy$5FFxmB3vRz{T>Vnh9;oM zh|rIT*um!Jn(@}YY+{KwHsL-LCw7*|NUWc=)=DB<3`3}%D z&)<)M=O+i$p^ol>>jy5#@u+zSQ>fr*H7o=F2`sX^Wkz`leF%xn{0b0hG5{w6BVd!= zCCDWBw3vmEO3&cZDSZC_Y41T)3YqjMM5WlN0++s(FXaa)xPMV~=awr&r{Bx(hQCIW zgh3c=uyU%xy^ZdfvV_L;s44%71b`C|4XFc|wJy6;NDEzqAY4ACQUu`u=%L;%?HA<5 z6dw_W2A~-r-YpSO=MWuzYgkC0N@qM=q(CCAyB0^h8Y=`3ALDM3hM9jAwmV1SbGF1rgP|b4Lte2XC*kZ^&Lcf218R zq=n-}uX3UvFP&(_CA{iQ^|We(Cr0NRuC5O1)O7&BJGnWx2ce4j?df{Dlm$|_-r?zq z#W}{cTMeZ;5ZMLErK4nfF@gvk{m@NuwHuS%(T$+3TB<{5Um6tcgQRT_SgSp_IW_0> z6~zt#dW5|oAGQzfOr+#~1!=kmD9Pv7*Eb!$;krvI0JNWTnxb z<=EDwPIRe~PeSk|P-hV!GT%q$XQ|QxI64yFU^;>q!SLfm;cKgu6RFcH1TBD23&x&O zkS}j+z2iK{s0&nazdi3ezw$W@`WrmS4G6P9bevE`VT;a~@s*+Q1{hN$8gAS*OG>gb z($rEBbwVN{ak}{hD-lsQzOszfg7M|x=~ECOE01mk-^3eH&p^*PZ(og;db*SGuVGwk$u5T_BkuD|FAOqH@-goi&^Y5v%>yqR@pzy8v8q| zvro+i`-C;Ezgf?j%@y5xwr;37T9@S}&G!O{4T#LdjR6r{5)kK$C7t-bdL&ZB!ydF3Ki&rMh7t>h9_EPuVi*~WJ%xlFa^rfcaUoa>U` z&(-~1zJ_+0IntWk None: + self.cache_config = cache_config + self.model_config = model_config + self.parallel_config = parallel_config + self.device_config = device_config + + self.head_size = model_config.get_head_size() + # Models like Jamba, have mixed typed layers, E.g Mamba + self.num_attention_layers = model_config.get_num_attention_layers( + parallel_config) + self.num_kv_heads = model_config.get_num_kv_heads(parallel_config) + + self.block_size = cache_config.block_size + self.num_gpu_blocks = cache_config.num_gpu_blocks + if self.num_gpu_blocks: + self.num_gpu_blocks //= parallel_config.pipeline_parallel_size + self.num_cpu_blocks = cache_config.num_cpu_blocks + if self.num_cpu_blocks: + self.num_cpu_blocks //= parallel_config.pipeline_parallel_size + + if cache_config.cache_dtype == "auto": + self.dtype = model_config.dtype + else: + self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] + + # Get attention backend. + self.attn_backend = get_attn_backend(self.head_size, + model_config.get_sliding_window(), + model_config.dtype, + cache_config.cache_dtype, + self.block_size, + model_config.is_attention_free) + + # Initialize the cache. + self.gpu_cache = self._allocate_kv_cache( + self.num_gpu_blocks, self.device_config.device_type) + self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks, "cpu") + + def _allocate_kv_cache( + self, + num_blocks: int, + device: str, + ) -> List[torch.Tensor]: + """Allocates KV cache on the specified device.""" + kv_cache_shape = self.attn_backend.get_kv_cache_shape( + num_blocks, self.block_size, self.num_kv_heads, self.head_size) + pin_memory = is_pin_memory_available() if device == "cpu" else False + kv_cache: List[torch.Tensor] = [] + for _ in range(self.num_attention_layers): + # null block in CpuGpuBlockAllocator requires at least that + # block to be zeroed-out. + # We zero-out everything for simplicity. + kv_cache.append( + torch.zeros(kv_cache_shape, + dtype=self.dtype, + pin_memory=pin_memory, + device=device)) + return kv_cache + + def swap_in(self, src_to_dst: torch.Tensor) -> None: + for i in range(self.num_attention_layers): + self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i], + src_to_dst) + + def swap_out(self, src_to_dst: torch.Tensor) -> None: + for i in range(self.num_attention_layers): + self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i], + src_to_dst) + + def copy(self, src_to_dsts: torch.Tensor) -> None: + self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts) + + @staticmethod + def get_cache_block_size( + cache_config: CacheConfig, + model_config: ModelConfig, + parallel_config: ParallelConfig, + ) -> int: + head_size = model_config.get_head_size() + num_heads = model_config.get_num_kv_heads(parallel_config) + num_attention_layers = model_config.get_num_attention_layers( + parallel_config) + + key_cache_block = cache_config.block_size * num_heads * head_size + value_cache_block = key_cache_block + total = num_attention_layers * (key_cache_block + value_cache_block) + if cache_config.cache_dtype == "auto": + dtype = model_config.dtype + else: + dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] + dtype_size = get_dtype_size(dtype) + return dtype_size * total diff --git a/vllm/worker/cpu_enc_dec_model_runner.py b/vllm/worker/cpu_enc_dec_model_runner.py new file mode 100644 index 00000000..8ebbf6db --- /dev/null +++ b/vllm/worker/cpu_enc_dec_model_runner.py @@ -0,0 +1,311 @@ +import dataclasses +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast + +import torch + +from vllm.attention import AttentionMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.multimodal import MultiModalInputs +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata +from vllm.utils import make_tensor_with_pad +from vllm.worker.cpu_model_runner import (CPUModelRunner, + ModelInputForCPUBuilder, + ModelInputForCPUWithSamplingMetadata) +from vllm.worker.model_runner_base import ( + _add_attn_metadata_broadcastable_dict, + _add_sampling_metadata_broadcastable_dict) + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + + +@dataclasses.dataclass(frozen=True) +class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata): + """ + Used by the EncoderDecoderModelRunner. + """ + encoder_input_tokens: Optional[torch.Tensor] = None + encoder_input_positions: Optional[torch.Tensor] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + "encoder_input_tokens": self.encoder_input_tokens, + "encoder_input_positions": self.encoder_input_positions, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + _add_sampling_metadata_broadcastable_dict(tensor_dict, + self.sampling_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "EncoderDecoderModelInputForCPU": + return cast( + EncoderDecoderModelInputForCPU, + super().from_broadcasted_tensor_dict(tensor_dict, attn_backend)) + + +class CPUEncoderDecoderModelRunner(CPUModelRunner): + _model_input_cls: Type[EncoderDecoderModelInputForCPU] = ( + EncoderDecoderModelInputForCPU) + _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder + + def _list_to_int32_tensor( + self, + _list: List[int], + ) -> torch.Tensor: + return torch.tensor(_list, dtype=torch.int32, device=self.device) + + def _list_to_long_tensor( + self, + _list: List[int], + ) -> torch.Tensor: + return torch.tensor(_list, dtype=torch.long, device=self.device) + + def _empty_int32_tensor(self) -> torch.Tensor: + return self._list_to_int32_tensor([]) + + def _empty_long_tensor(self) -> torch.Tensor: + return self._list_to_long_tensor([]) + + def make_model_input_from_broadcasted_tensor_dict( + self, tensor_dict: Dict[str, + Any]) -> EncoderDecoderModelInputForCPU: + return EncoderDecoderModelInputForCPU.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + ) + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> EncoderDecoderModelInputForCPU: + model_input = super().prepare_model_input(seq_group_metadata_list, + virtual_engine, + finished_requests_ids) + model_input = cast(EncoderDecoderModelInputForCPU, model_input) + ( + attn_metadata, + encoder_input_tokens_tensor, + encoder_input_positions_tensor, + ) = self._prepare_encoder_model_input_tensors(seq_group_metadata_list, + model_input) + return dataclasses.replace( + model_input, + attn_metadata=attn_metadata, + encoder_input_tokens=encoder_input_tokens_tensor, + encoder_input_positions=encoder_input_positions_tensor, + ) + + def _prepare_encoder_model_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + model_input: EncoderDecoderModelInputForCPU, + ) -> Tuple[AttentionMetadata, Optional[torch.Tensor], + Optional[torch.Tensor]]: + """Helper method to prepare the encoder- and cross-attn-related + model inputs based on a given sequence group. These additional inputs + are used to augment an already-computed `EncoderDecoderModelInput` + data structure which already has decoder-related model inputs + populated. + + Sets the following attn_metadata fields: + * `num_encoder_tokens` + * `encoder_seq_lens` + * `encoder_seq_lens_tensor` + * `max_encoder_seq_len` + * `cross_slot_mapping` + * `cross_block_tables` + + Constructs a new model inputs data structure, based on + (1) the existing fields in the `model_inputs` argument, + and (2) the following additional fields which are + computed (or in the case of `attn_metadata`, updated) + by this function: + * attn_metadata + * encoder_input_tokens + * encoder_input_positions + + Arguments: + + * seq_group_metadata_list: list of sequence groups for which to + compute inputs + * model_inputs: model inputs data structure with decoder-oriented + fields already computed. + + Return: + + * Updated model inputs data structure + """ + + if len(seq_group_metadata_list) == 0: + return (model_input.attn_metadata, None, None) + + # Since we are not supporting chunked prefill either the entire + # batch is prefill or it is decode + is_prompt = seq_group_metadata_list[0].is_prompt + + # Build encoder inputs + encoder_seq_lens: List[int] = [] + if is_prompt: + # Prefill phase. + cross_block_tables = self._empty_int32_tensor().view( + len(seq_group_metadata_list), -1) + + # Extract input tokens/positions, cross-attention slot-mapping, + # & seq len from each sequence group metadata + ( + encoder_input_tokens, + encoder_input_positions, + cross_slot_mapping, + ) = ( + [], + [], + [], + ) + for seq_group_metadata in seq_group_metadata_list: + # Build seq lens + seq_len = seq_group_metadata.encoder_seq_data.get_len() + token_ids = seq_group_metadata.encoder_seq_data.get_token_ids() + encoder_seq_lens.append(seq_len) + + # Build slot mapping + for i in range(0, seq_len): + block_number = seq_group_metadata.cross_block_table[ + i // self.block_size] + block_offset = i % self.block_size + slot = block_number * self.block_size + block_offset + cross_slot_mapping.append(slot) + + # Build encoder input tokens + encoder_input_tokens.extend(token_ids) + encoder_input_positions.extend(list(range(0, seq_len))) + + # Convert tokens/positions & cross-attention + # slot-mapping to encoder input tensors + encoder_input_tokens_tensor = self._list_to_long_tensor( + encoder_input_tokens) + encoder_input_positions_tensor = self._list_to_long_tensor( + encoder_input_positions) + cross_slot_mapping_tensor = self._list_to_long_tensor( + cross_slot_mapping) + + else: + # Decode phase. + encoder_input_tokens_tensor = self._empty_long_tensor() + encoder_input_positions_tensor = self._empty_long_tensor() + cross_slot_mapping_tensor = self._empty_long_tensor() + # Extract cross-attention block tables & + # seq len from each sequence group metadata. + # Cross-attention block tables are empty + # during vLLM memory profiling. + cross_block_tables = [] + for seq_group_metadata in seq_group_metadata_list: + for _ in range(len(seq_group_metadata.seq_data)): + encoder_seq_lens.append( + seq_group_metadata.encoder_seq_data.get_len()) + cross_block_table = seq_group_metadata.cross_block_table + cross_block_tables.append([] if ( + cross_block_table is None) else cross_block_table) + + max_len_of_block_table = max( + len(block_table) for block_table in cross_block_tables) + + cross_block_tables = make_tensor_with_pad( + cross_block_tables, + max_len=max_len_of_block_table, + pad=0, + dtype=torch.int32, + device=self.device, + ) + + # Compute encoder sequence lengths & encoder + # sequence starting offset tensors + max_encoder_seq_len = max(encoder_seq_lens, default=0) + encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens) + encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] + + 1, + dtype=torch.int32, + device=self.device) + torch.cumsum(encoder_seq_lens_tensor, + dim=0, + dtype=encoder_seq_start_loc.dtype, + out=encoder_seq_start_loc[1:]) + + # Update attention metadata with encoder-oriented attributes + attn_metadata = model_input.attn_metadata + assert attn_metadata is not None + ( + attn_metadata.num_encoder_tokens, + attn_metadata.encoder_seq_lens, + attn_metadata.encoder_seq_lens_tensor, + attn_metadata.max_encoder_seq_len, + attn_metadata.cross_slot_mapping, + attn_metadata.cross_block_tables, + ) = ( + sum(encoder_seq_lens), + encoder_seq_lens, + encoder_seq_lens_tensor, + max_encoder_seq_len, + cross_slot_mapping_tensor, + cross_block_tables, + ) + + return (attn_metadata, encoder_input_tokens_tensor, + encoder_input_positions_tensor) + + @torch.no_grad() + def execute_model( + self, + model_input: EncoderDecoderModelInputForCPU, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + if num_steps > 1: + raise ValueError( + "CPU worker does not support multi-step execution.") + + model_executable = self.model + execute_model_kwargs = { + "input_ids": + model_input.input_tokens, + "positions": + model_input.input_positions, + "encoder_input_ids": + model_input.encoder_input_tokens, + "encoder_positions": + model_input.encoder_input_positions, + "kv_caches": + kv_caches, + "attn_metadata": + model_input.attn_metadata, + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {}, + device=self.device), + "intermediate_tensors": + intermediate_tensors, + } + + hidden_states = model_executable(**execute_model_kwargs) + + # Compute the logits. + logits = self.model.compute_logits(hidden_states, + model_input.sampling_metadata) + + # Only perform sampling in the driver worker. + if not self.is_driver_worker: + return [] + + # Sample the next token. + output = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + return [output] diff --git a/vllm/worker/cpu_model_runner.py b/vllm/worker/cpu_model_runner.py new file mode 100644 index 00000000..795511ae --- /dev/null +++ b/vllm/worker/cpu_model_runner.py @@ -0,0 +1,549 @@ +import dataclasses +import weakref +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union + +import torch +from torch import nn + +from vllm.attention import AttentionMetadata, get_attn_backend +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ParallelConfig, PromptAdapterConfig, + SchedulerConfig) +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader import get_model +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs, + MultiModalInputs) +from vllm.sequence import (IntermediateTensors, SequenceData, + SequenceGroupMetadata) +from vllm.utils import make_tensor_with_pad +from vllm.worker.model_runner_base import ( + ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase, + _add_attn_metadata_broadcastable_dict, + _add_sampling_metadata_broadcastable_dict, + _init_attn_metadata_from_tensor_dict, + _init_sampling_metadata_from_tensor_dict) + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + +_PAD_SLOT_ID = -1 + + +@dataclass(frozen=True) +class ModelInputForCPU(ModelRunnerInputBase): + """ + Base class contains metadata needed for the base model forward pass on CPU + """ + input_tokens: Optional[torch.Tensor] = None + input_positions: Optional[torch.Tensor] = None + attn_metadata: Optional["AttentionMetadata"] = None + multi_modal_kwargs: Optional[BatchedTensorInputs] = None + virtual_engine: Optional[int] = None + seq_lens: Optional[List[int]] = None + query_lens: Optional[List[int]] = None + + def as_broadcastable_tensor_dict( + self) -> Dict[str, Union[int, torch.Tensor]]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + "multi_modal_kwargs": self.multi_modal_kwargs, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls: Type["ModelInputForCPU"], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None + ) -> "ModelInputForCPU": + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +@dataclass(frozen=True) +class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU): + """ + Used by the ModelRunner. + """ + sampling_metadata: Optional["SamplingMetadata"] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + _add_sampling_metadata_broadcastable_dict(tensor_dict, + self.sampling_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "ModelInputForCPUWithSamplingMetadata": + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict) + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]): + + def __init__(self, + runner: "CPUModelRunner", + finished_requests_ids: Optional[List[str]] = None) -> None: + super().__init__() + self.seq_group_metadata_list: List[SequenceGroupMetadata] = [] + self.runner = runner + self.model_input_cls = self.runner._model_input_cls + self.attn_backend = self.runner.attn_backend + self.sliding_window = self.runner.sliding_window + self.block_size = self.runner.block_size + self.device = self.runner.device + self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper + + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata): + self.seq_group_metadata_list.append(seq_group_metadata) + + def build(self) -> ModelInputForCPU: + multi_modal_kwargs = None + # NOTE: We assume that all sequences in the group are all prompts or + # all decodes. + is_prompt = self.seq_group_metadata_list[0].is_prompt + # Prepare input tensors. + if is_prompt: + (input_tokens, input_positions, attn_metadata, seq_lens, + multi_modal_kwargs) = self._prepare_prompt( + self.seq_group_metadata_list) + else: + (input_tokens, input_positions, + attn_metadata) = self._prepare_decode( + self.seq_group_metadata_list) + seq_lens = None + + return self.model_input_cls( + input_tokens=input_tokens, + input_positions=input_positions, + attn_metadata=attn_metadata, + multi_modal_kwargs=multi_modal_kwargs, + # query_lens is not needed if chunked prefill is not + # supported. Since CPU worker doesn't support chunked prefill + # just use seq_lens instead. + seq_lens=seq_lens, + query_lens=seq_lens, + ) + + def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data, + computed_len: int, + mm_processor_kwargs: Dict[str, Any]): + mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs) + + # special processing for mrope position deltas. + mrope_positions = None + if self.runner.model_is_mrope: + image_grid_thw = mm_kwargs.get("image_grid_thw", None) + video_grid_thw = mm_kwargs.get("video_grid_thw", None) + assert image_grid_thw is not None or video_grid_thw is not None, ( + "mrope embedding type requires multi-modal input mapper " + "returns 'image_grid_thw' or 'video_grid_thw'.") + + hf_config = self.runner.model_config.hf_config + token_ids = seq_data.get_token_ids() + + mrope_positions, mrope_position_delta = \ + MRotaryEmbedding.get_input_positions( + token_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_token_id=hf_config.image_token_id, + video_token_id=hf_config.video_token_id, + vision_start_token_id=hf_config.vision_start_token_id, + vision_end_token_id=hf_config.vision_end_token_id, + spatial_merge_size=hf_config.vision_config. + spatial_merge_size, + context_len=computed_len, + ) + seq_data.mrope_position_delta = mrope_position_delta + return mm_kwargs, mrope_positions + + def _prepare_prompt( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int], + BatchedTensorInputs]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[int] = [] + input_positions: List[int] = [] + input_mrope_positions: List[List[int]] = [[] for _ in range(3)] + + slot_mapping: List[int] = [] + seq_lens: List[int] = [] + multi_modal_inputs_list: List[MultiModalInputs] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert seq_group_metadata.is_prompt + seq_ids = list(seq_group_metadata.seq_data.keys()) + assert len(seq_ids) == 1 + seq_id = seq_ids[0] + + seq_data = seq_group_metadata.seq_data[seq_id] + prompt_tokens = seq_data.get_token_ids() + computed_len = seq_data.get_num_computed_tokens() + seq_len = len(prompt_tokens) + + seq_lens.append(seq_len) # Prompt token num + input_tokens.extend(prompt_tokens) # Token ids + + mrope_positions = None + if (mm_data := seq_group_metadata.multi_modal_data): + mm_kwargs, mrope_positions = self._compute_multi_modal_input( + seq_data, mm_data, computed_len, + seq_group_metadata.mm_processor_kwargs) + multi_modal_inputs_list.append(mm_kwargs) + + # Token position ids + # NOTE(woosuk): Here we assume that the first token in the prompt + # is always the first token in the sequence. + if mrope_positions: + for idx in range(3): + input_mrope_positions[idx].extend(mrope_positions[idx]) + else: + input_positions.extend(list(range(computed_len, seq_len))) + + # Compute the slot mapping. + block_table = seq_group_metadata.block_tables[seq_id] + # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID, + # where start_idx is max(0, seq_len - sliding_window). + # For example, if the prompt len is 10, sliding window is 8, and + # block size is 4, the first two tokens are masked and the slot + # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1]. + start_idx = 0 + if self.sliding_window is not None: + start_idx = max(0, seq_len - self.sliding_window) + + for i in range(computed_len, seq_len): + if i < start_idx: + slot_mapping.append(_PAD_SLOT_ID) + continue + + block_number = block_table[i // + self.block_size] # type: ignore + block_offset = i % self.block_size # type: ignore + slot = block_number * self.block_size + block_offset + slot_mapping.append(slot) + + if any(input_mrope_positions): + input_positions = None # type: ignore + else: + input_mrope_positions = None # type: ignore + + num_prompt_tokens = len(input_tokens) + + input_tokens = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) # type: ignore + input_positions = torch.tensor(input_positions + or input_mrope_positions, + dtype=torch.long, + device=self.device) # type: ignore + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.long, + device=self.device) # type: ignore + + attn_metadata = self.attn_backend.make_metadata( + is_prompt=True, + seq_lens=seq_lens, + seq_lens_tensor=torch.tensor([]), + max_decode_seq_len=0, + num_prefills=len(seq_lens), + num_prefill_tokens=num_prompt_tokens, + num_decode_tokens=0, + block_tables=torch.tensor([]), + slot_mapping=slot_mapping, + ) + + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list) + + return (input_tokens, input_positions, attn_metadata, seq_lens, + multi_modal_kwargs) + + def _prepare_decode( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[int] = [] + input_positions: List[int] = [] + input_mrope_positions: List[List[int]] = [[] for _ in range(3)] + slot_mapping: List[int] = [] + seq_lens: List[int] = [] + block_tables: List[List[int]] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert not seq_group_metadata.is_prompt + assert seq_group_metadata.token_chunk_size == 1 + + seq_ids = list(seq_group_metadata.seq_data.keys()) + + for seq_id in seq_ids: + seq_data = seq_group_metadata.seq_data[seq_id] + generation_token = seq_data.get_last_token_id() + input_tokens.append(generation_token) + + seq_len = seq_data.get_len() + position = seq_len - 1 + if seq_data.mrope_position_delta is not None: + context_len = seq_data.get_num_computed_tokens() + next_pos = MRotaryEmbedding.get_next_input_positions( + seq_data.mrope_position_delta, + context_len, + seq_len, + ) + for idx in range(3): + input_mrope_positions[idx].extend(next_pos[idx]) + else: + input_positions.append(position) + + seq_len = seq_len if self.sliding_window is None else min( + seq_len, self.sliding_window) + seq_lens.append(seq_len) + + block_table = seq_group_metadata.block_tables[seq_id] + block_number = block_table[position // self.block_size] + block_offset = position % self.block_size + slot = block_number * self.block_size + block_offset + slot_mapping.append(slot) + + if self.sliding_window is not None: + sliding_window_blocks = (self.sliding_window // + self.block_size) + block_table = block_table[-sliding_window_blocks:] + block_tables.append(block_table) + + if any(input_mrope_positions): + input_positions = None # type: ignore + else: + input_mrope_positions = None # type: ignore + + max_decode_seq_len = max(seq_lens) + + input_tokens = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) + input_positions = torch.tensor(input_positions + or input_mrope_positions, + dtype=torch.long, + device=self.device) + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.long, + device=self.device) + seq_lens_tensor = torch.tensor(seq_lens, + dtype=torch.int, + device=self.device) + + block_tables = make_tensor_with_pad( + block_tables, + pad=0, + dtype=torch.int, + device=self.device, + ) + + attn_metadata = self.attn_backend.make_metadata( + is_prompt=False, + slot_mapping=slot_mapping, + seq_lens=seq_lens, + seq_lens_tensor=seq_lens_tensor, + max_decode_seq_len=max_decode_seq_len, + num_prefill_tokens=0, + num_decode_tokens=len(input_tokens), + num_prefills=0, + block_tables=block_tables, + ) + return ( + input_tokens, + input_positions, + attn_metadata, + ) + + +class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]): + _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = ( + ModelInputForCPUWithSamplingMetadata) + _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + is_driver_worker: bool = False, + *args, + **kwargs, + ): + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + # Currently, CPU worker doesn't support chunked prefill. + assert self.scheduler_config.chunked_prefill_enabled is False + self.device_config = device_config + self.cache_config = cache_config + self.lora_config = lora_config + self.prompt_adapter_config = prompt_adapter_config + self.load_config = load_config + self.is_driver_worker = is_driver_worker + + self.device = self.device_config.device + + self.kv_cache_dtype = kv_cache_dtype + self.sliding_window = model_config.get_sliding_window() + self.block_size = cache_config.block_size + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.kv_cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) + + # Multi-modal data support + self.mm_registry = MULTIMODAL_REGISTRY + self.multi_modal_input_mapper = self.mm_registry \ + .create_input_mapper(self.model_config) + self.mm_registry.init_mm_limits_per_prompt(self.model_config) + + # Lazy initialization. + self.model: nn.Module # Set after init_Model + + @property + def model_is_mrope(self) -> bool: + """Detect if the model has "mrope" rope_scaling type. + mrope requires keep "rope_deltas" between prompt and decoding phases.""" + rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {}) + if rope_scaling is None: + return False + return rope_scaling.get("type", None) == "mrope" + + def load_model(self) -> None: + self.model = get_model(model_config=self.model_config, + load_config=self.load_config, + device_config=self.device_config, + lora_config=self.lora_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + cache_config=self.cache_config) + + def make_model_input_from_broadcasted_tensor_dict( + self, + tensor_dict: Dict[str, Any], + ) -> ModelInputForCPUWithSamplingMetadata: + return ModelInputForCPUWithSamplingMetadata.from_broadcasted_tensor_dict( # noqa: E501 + tensor_dict, + attn_backend=self.attn_backend, + ) + + def _prepare_model_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForCPUWithSamplingMetadata: + """Helper method to prepare the model input based on a given sequence + group. Prepares metadata needed for the base model forward pass but not + metadata for possible additional steps, e.g., sampling. + + """ + builder = self._builder_cls(weakref.proxy(self), finished_requests_ids) + for seq_group_metadata in seq_group_metadata_list: + builder.add_seq_group(seq_group_metadata) + + return builder.build() # type: ignore + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForCPUWithSamplingMetadata: + """Prepare the model input based on a given sequence group, including + metadata for the sampling step. + + """ + model_input = self._prepare_model_input_tensors( + seq_group_metadata_list, finished_requests_ids) + # Sampling metadata is only required for the final pp group + generators = self.get_generators(finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, + model_input.seq_lens, + model_input.query_lens, + self.device, + pin_memory=False, + generators=generators) + + return dataclasses.replace(model_input, + sampling_metadata=sampling_metadata, + virtual_engine=virtual_engine) + + @torch.no_grad() + def execute_model( + self, + model_input: ModelInputForCPUWithSamplingMetadata, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + if num_steps > 1: + raise ValueError( + "CPU worker does not support multi-step execution.") + + model_executable = self.model + execute_model_kwargs = { + "input_ids": + model_input.input_tokens, + "positions": + model_input.input_positions, + "kv_caches": + kv_caches, + "attn_metadata": + model_input.attn_metadata, + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {}, + device=self.device), + "intermediate_tensors": + intermediate_tensors, + } + + hidden_states = model_executable(**execute_model_kwargs) + + # Compute the logits. + logits = self.model.compute_logits(hidden_states, + model_input.sampling_metadata) + + # Only perform sampling in the driver worker. + if not self.is_driver_worker: + return [] + + # Sample the next token. + output = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + return [output] diff --git a/vllm/worker/cpu_worker.py b/vllm/worker/cpu_worker.py new file mode 100644 index 00000000..b8456285 --- /dev/null +++ b/vllm/worker/cpu_worker.py @@ -0,0 +1,371 @@ +"""A CPU worker class.""" +from typing import Dict, List, Optional, Tuple, Type + +import torch +import torch.distributed + +import vllm.envs as envs +from vllm.attention import get_attn_backend +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ParallelConfig, PromptAdapterConfig, + SchedulerConfig) +from vllm.distributed import (ensure_model_parallel_initialized, + init_distributed_environment) +from vllm.logger import init_logger +from vllm.model_executor import set_random_seed +from vllm.sequence import ExecuteModelRequest +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE +from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner +from vllm.worker.cpu_model_runner import CPUModelRunner +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, + LoraNotSupportedWorkerBase, WorkerInput) + +logger = init_logger(__name__) + + +class CPUCacheEngine: + """Manages the KV cache for CPU backend. + + This class is responsible for initializing and managing CPU KV + caches. It also provides methods for performing KV cache operations, such + as copying. + """ + + def __init__(self, cache_config: CacheConfig, model_config: ModelConfig, + parallel_config: ParallelConfig, + device_config: DeviceConfig) -> None: + assert device_config.device_type == "cpu" + self.cache_config = cache_config + self.model_config = model_config + self.parallel_config = parallel_config + + self.head_size = model_config.get_head_size() + self.num_layers = model_config.get_num_layers(parallel_config) + self.num_heads = model_config.get_num_kv_heads(parallel_config) + + self.block_size = cache_config.block_size + # Note: In CacheConfig, num_gpu_blocks actual is num_cpu_blocks + # for CPU backend, because we want to reuse KV cache management + # in the scheduler. + self.num_cpu_blocks = cache_config.num_gpu_blocks + + if cache_config.cache_dtype == "auto": + self.dtype = model_config.dtype + else: + self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] + + # Get attention backend. + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + cache_config.cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) + + # Initialize the cache. + self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks) + + def _allocate_kv_cache( + self, + num_blocks: int, + ) -> List[torch.Tensor]: + """Allocates KV cache on CPU.""" + kv_cache_shape = self.attn_backend.get_kv_cache_shape( + num_blocks, self.block_size, self.num_heads, self.head_size) + kv_cache: List[torch.Tensor] = [] + for _ in range(self.num_layers): + kv_cache.append( + torch.empty(kv_cache_shape, dtype=self.dtype, device="cpu")) + return kv_cache + + def swap_in(self, src_to_dst: Dict[int, int]) -> None: + raise NotImplementedError("Swap is not supported in CPUCacheEngine.") + + def swap_out(self, src_to_dst: Dict[int, int]) -> None: + raise NotImplementedError("Swap is not supported in CPUCacheEngine.") + + def copy(self, src_to_dsts: Dict[int, List[int]]) -> None: + self.attn_backend.copy_blocks(self.cpu_cache, src_to_dsts) + + @staticmethod + def get_cache_block_size( + block_size: int, + cache_dtype: str, + model_config: ModelConfig, + parallel_config: ParallelConfig, + ) -> int: + head_size = model_config.get_head_size() + num_heads = model_config.get_num_kv_heads(parallel_config) + num_layers = model_config.get_num_layers(parallel_config) + + key_cache_block = block_size * num_heads * head_size + value_cache_block = key_cache_block + total = num_layers * (key_cache_block + value_cache_block) + if cache_dtype == "auto": + dtype = model_config.dtype + else: + dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype] + dtype_size = torch.tensor([], dtype=dtype).element_size() + return dtype_size * total + + +class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase): + """A worker class that executes (a partition of) the model on a CPU socket. + + Each worker is associated with a single CPU socket. The worker is + responsible for maintaining the KV cache and executing the model on the + CPU. In case of distributed inference, each worker is assigned a partition + of the model. + """ + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + lora_config: Optional[LoRAConfig] = None, + kv_cache_dtype: Optional[str] = "auto", + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + is_driver_worker: bool = False, + ) -> None: + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.load_config = load_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + self.lora_config = lora_config + self.prompt_adapter_config = prompt_adapter_config + self.is_driver_worker = is_driver_worker + if self.is_driver_worker: + assert self.rank == 0, "The driver worker must have rank 0." + + if self.model_config.trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils import init_cached_hf_modules + init_cached_hf_modules() + + # Setup OpenMP threads affinity. + omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND + if omp_cpuids == "all": + self.local_omp_cpuid = "all" + else: + self.local_omp_cpuid = omp_cpuids.split("|")[rank] + + ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner + if self._is_encoder_decoder_model(): + ModelRunnerClass = CPUEncoderDecoderModelRunner + self.model_runner: CPUModelRunner = ModelRunnerClass( + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config=self.load_config, + lora_config=self.lora_config, + kv_cache_dtype=kv_cache_dtype, + prompt_adapter_config=self.prompt_adapter_config, + is_driver_worker=is_driver_worker) + # Uninitialized cache engine. Will be initialized by + # initialize_cache. + self.cache_engine: List[CPUCacheEngine] + self.cpu_cache: List[List[torch.Tensor]] + + # Torch profiler. Enabled and configured through env vars: + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace + if envs.VLLM_TORCH_PROFILER_DIR: + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR + logger.info("Profiling enabled. Traces will be saved to: %s", + torch_profiler_trace_dir) + self.profiler = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + ], + with_stack=True, + on_trace_ready=torch.profiler.tensorboard_trace_handler( + torch_profiler_trace_dir, use_gzip=True)) + else: + self.profiler = None + + def start_profile(self): + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.start() + + def stop_profile(self): + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() + + def _is_encoder_decoder_model(self): + return self.model_config.is_encoder_decoder_model + + def init_device(self) -> None: + if self.local_omp_cpuid != "all": + ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid) + if ret: + logger.info(ret) + + self.init_distributed_environment() + # Set random seed. + set_random_seed(self.model_config.seed) + + def load_model(self): + self.model_runner.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of blocks available for the KV cache. + + This determines how many KV blocks can fit into the configured CPU + KV cache space. + + Note that since vLLM assumes a block resides on GPU if it can be + modified, we return num_gpu_blocks=num_cpu_blocks and num_cpu_blocks=0. + This allows us to reuse the scheduler of vLLM without generalizing it + to different devices. + """ + # For CPU device, the block number will be calculated based on the + # cpu_kvcache_space. + cache_block_size = self.get_cache_block_size_bytes() + num_cpu_blocks = int(self.cache_config.cpu_kvcache_space_bytes // + cache_block_size) + num_cpu_blocks = max(num_cpu_blocks, 0) + + # Note: To reuse the cache management procedure, + # use cpu cache as 'gpu cache'. + num_gpu_blocks = num_cpu_blocks + num_cpu_blocks = 0 + return num_gpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache. Currently, swappable CPU memory is not + supported. + + Since this worker does not support GPUs, we use the num_gpu_blocks to + determine how many non-swappable CPU blocks to allocate. + """ + assert (num_cpu_blocks == 0 + ), f"{type(self)} does not support swappable cache" + + # Note: To reuse the cache management procedure, + # use cpu cache as 'gpu cache'. + num_cpu_blocks = num_gpu_blocks + + self._validate_num_cpu_blocks(num_cpu_blocks) + self.cache_config.num_gpu_blocks = num_cpu_blocks + self.cache_config.num_cpu_blocks = 0 + + # Initialize the cache. + self._init_cache_engine() + + def _validate_num_cpu_blocks(self, num_cpu_blocks: int) -> None: + """Raise errors if the num_cpu_blocks is invalid. + """ + if num_cpu_blocks <= 0: + raise ValueError("No available memory for the cache blocks. " + "Try increasing `VLLM_CPU_KVCACHE_SPACE` when " + "initializing the engine.") + + max_seq_len = self.cache_config.block_size * num_cpu_blocks + if self.model_config.max_model_len > max_seq_len: + raise ValueError( + f"The model's max seq len ({self.model_config.max_model_len}) " + "is larger than the maximum number of tokens that can be " + f"stored in KV cache ({max_seq_len}). Try increasing " + "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when " + "initializing the engine.") + + def _init_cache_engine(self) -> None: + self.cache_engine = [ + CPUCacheEngine(self.cache_config, self.model_config, + self.parallel_config, self.device_config) + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + self.cpu_cache = [ + self.cache_engine[ve].cpu_cache + for ve in range(self.parallel_config.pipeline_parallel_size) + ] + self.model_runner.block_size = self.cache_engine[0].block_size + + assert all( + self.cpu_cache[ve] is not None + for ve in range(self.parallel_config.pipeline_parallel_size)) + + # Populate the cache to warmup the memory + for ve in range(self.parallel_config.pipeline_parallel_size): + for layer_cache in self.cpu_cache[ve]: + layer_cache.fill_(0) + + @property + def do_metadata_broadcast(self) -> bool: + return self.parallel_config.tensor_parallel_size > 1 + + @property + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: + return self.cpu_cache + + def execute_worker( + self, + worker_input: WorkerInput, + ) -> None: + if (worker_input.blocks_to_copy is not None + and worker_input.blocks_to_copy.numel() > 0): + self.cache_engine[worker_input.virtual_engine].copy( + worker_input.blocks_to_copy) + + @torch.inference_mode() + def prepare_worker_input( + self, execute_model_req: ExecuteModelRequest) -> WorkerInput: + assert execute_model_req is not None + virtual_engine = execute_model_req.virtual_engine + num_seq_groups: int = len(execute_model_req.seq_group_metadata_list) + blocks_to_copy = execute_model_req.blocks_to_copy + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy, + device="cpu", + dtype=torch.int64).view(-1, 2) + assert len(execute_model_req.blocks_to_swap_in) == 0 + assert len(execute_model_req.blocks_to_swap_out) == 0 + return WorkerInput( + num_seq_groups=num_seq_groups, + blocks_to_copy=blocks_to_copy, + virtual_engine=virtual_engine, + ) + + def init_distributed_environment(self) -> None: + """Initialize the distributed environment.""" + + parallel_config = self.parallel_config + rank = self.rank + distributed_init_method = self.distributed_init_method + init_distributed_environment( + world_size=parallel_config.world_size, + rank=rank, + distributed_init_method=distributed_init_method, + backend="gloo", + ) + + # A small all_reduce for warmup. + torch.distributed.all_reduce(torch.zeros(1).cpu()) + + ensure_model_parallel_initialized( + parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size) + + def get_cache_block_size_bytes(self) -> int: + """Return the size in bytes of a single KV cache block. + """ + return CPUCacheEngine.get_cache_block_size( + self.cache_config.block_size, self.cache_config.cache_dtype, + self.model_config, self.parallel_config) diff --git a/vllm/worker/embedding_model_runner.py b/vllm/worker/embedding_model_runner.py new file mode 100644 index 00000000..a7f5b2d4 --- /dev/null +++ b/vllm/worker/embedding_model_runner.py @@ -0,0 +1,208 @@ +import dataclasses +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +import torch + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.distributed import get_pp_group +from vllm.forward_context import set_forward_context +from vllm.logger import init_logger +from vllm.model_executor.pooling_metadata import PoolingMetadata +from vllm.multimodal import MultiModalInputs +from vllm.pooling_params import PoolingParams +from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData, + SequenceGroupMetadata) +from vllm.worker.model_runner import (GPUModelRunnerBase, ModelInputForGPU, + ModelInputForGPUBuilder) + +logger = init_logger(__name__) + + +@dataclasses.dataclass(frozen=True) +class ModelInputForGPUWithPoolingMetadata(ModelInputForGPU): + """ + Used by the EmbeddingModelRunner. + """ + pooling_metadata: Optional["PoolingMetadata"] = None + + +class EmbeddingModelRunner( + GPUModelRunnerBase[ModelInputForGPUWithPoolingMetadata]): + _model_input_cls: Type[ModelInputForGPUWithPoolingMetadata] = ( + ModelInputForGPUWithPoolingMetadata) + _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + observability_config: Optional[ObservabilityConfig] = None, + ): + super().__init__(model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config, + lora_config=lora_config, + kv_cache_dtype=kv_cache_dtype, + is_driver_worker=is_driver_worker, + prompt_adapter_config=prompt_adapter_config, + observability_config=observability_config) + + @torch.inference_mode() + def execute_model( + self, + model_input: ModelInputForGPUWithPoolingMetadata, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[Union[List[PoolerOutput], IntermediateTensors]]: + if num_steps > 1: + raise ValueError( + "EmbeddingModelRunner does not support multi-step execution.") + + if self.lora_config: + assert model_input.lora_requests is not None + assert model_input.lora_mapping is not None + self.set_active_loras(model_input.lora_requests, + model_input.lora_mapping) + + if self.prompt_adapter_config: + assert model_input.prompt_adapter_requests is not None + assert model_input.prompt_adapter_mapping is not None + self.set_active_prompt_adapters( + model_input.prompt_adapter_requests, + model_input.prompt_adapter_mapping) + + # Currently cuda graph is only supported by the decode phase. + assert model_input.attn_metadata is not None + prefill_meta = model_input.attn_metadata.prefill_metadata + decode_meta = model_input.attn_metadata.decode_metadata + virtual_engine = model_input.virtual_engine + if prefill_meta is None and decode_meta.use_cuda_graph: + assert model_input.input_tokens is not None + graph_batch_size = model_input.input_tokens.shape[0] + model_executable = self.graph_runners[virtual_engine][ + graph_batch_size] + else: + model_executable = self.model + + num_layers = self.model_config.get_num_layers(self.parallel_config) + # use an empty tensor instead of `None`` to force Dynamo to pass + # it by reference, rather by specializing on the value ``None``. + # the `dtype` argument does not matter, and we use `float32` as + # a placeholder (it has wide hardware support). + kv_caches = [ + torch.tensor([], dtype=torch.float32, device=self.device) + for _ in range(num_layers) + ] + + multi_modal_kwargs = model_input.multi_modal_kwargs or {} + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_start = torch.cuda.Event(enable_timing=True) + model_forward_end = torch.cuda.Event(enable_timing=True) + model_forward_start.record() + + with set_forward_context(model_input.attn_metadata): + hidden_or_intermediate_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(multi_modal_kwargs, + device=self.device)) + + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_end.record() + + # Only perform pooling in the last pipeline stage. + if not get_pp_group().is_last_rank: + if (self.is_driver_worker + and hidden_or_intermediate_states is not None + and isinstance(hidden_or_intermediate_states, + IntermediateTensors) + and self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_end.synchronize() + model_forward_time = model_forward_start.elapsed_time( + model_forward_end) + orig_model_forward_time = 0.0 + if intermediate_tensors is not None: + orig_model_forward_time = intermediate_tensors.tensors.get( + "model_forward_time", torch.tensor(0.0)).item() + hidden_or_intermediate_states.tensors["model_forward_time"] = ( + torch.tensor(model_forward_time + orig_model_forward_time)) + return hidden_or_intermediate_states + + # Only perform pooling in the driver worker. + if not self.is_driver_worker: + return [] + + return [ + self.model.pooler(hidden_states=hidden_or_intermediate_states, + pooling_metadata=model_input.pooling_metadata) + ] + + def make_model_input_from_broadcasted_tensor_dict( + self, + tensor_dict: Dict[str, + Any]) -> ModelInputForGPUWithPoolingMetadata: + return ModelInputForGPUWithPoolingMetadata.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + ) + + def prepare_model_input( + self, + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForGPUWithPoolingMetadata: + assert seq_group_metadata_list is not None + model_input = self._prepare_model_input_tensors( + seq_group_metadata_list, finished_requests_ids) + # Prepare PoolingMetadata. + assert model_input.seq_lens is not None + pooling_metadata = self._prepare_pooling(seq_group_metadata_list, + model_input.seq_lens) + + return dataclasses.replace(model_input, + pooling_metadata=pooling_metadata) + + def _prepare_pooling( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + prompt_lens: List[int], + ) -> PoolingMetadata: + """Prepare PoolingMetadata for the sequence group metadata list.""" + seq_groups: List[Tuple[List[int], PoolingParams]] = [] + for i, seq_group_metadata in enumerate(seq_group_metadata_list): + seq_ids = list(seq_group_metadata.seq_data.keys()) + pooling_params = seq_group_metadata.pooling_params + seq_groups.append((seq_ids, pooling_params)) + + seq_data: Dict[int, SequenceData] = {} + for seq_group_metadata in seq_group_metadata_list: + seq_data.update(seq_group_metadata.seq_data) + + pooling_metadata = PoolingMetadata( + seq_groups=seq_groups, + seq_data=seq_data, + prompt_lens=prompt_lens, + ) + + return pooling_metadata diff --git a/vllm/worker/enc_dec_model_runner.py b/vllm/worker/enc_dec_model_runner.py new file mode 100644 index 00000000..d58a7a17 --- /dev/null +++ b/vllm/worker/enc_dec_model_runner.py @@ -0,0 +1,546 @@ +import dataclasses +import itertools +from typing import Any, Dict, List, Optional, Tuple, Type, cast + +import torch +import torch.distributed + +from vllm.attention.backends.abstract import (AttentionBackend, + AttentionMetadata) +from vllm.attention.backends.utils import PAD_SLOT_ID +from vllm.attention.selector import (_Backend, get_env_variable_attn_backend, + get_global_forced_attn_backend, + global_force_attn_backend) +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.forward_context import set_forward_context +from vllm.inputs import INPUT_REGISTRY, InputRegistry +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs, + MultiModalRegistry) +from vllm.sampling_params import SamplingParams +from vllm.sequence import (IntermediateTensors, PoolerOutput, + SequenceGroupMetadata) +from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad +from vllm.worker.model_runner import (GPUModelRunnerBase, + ModelInputForGPUBuilder, + ModelInputForGPUWithSamplingMetadata, + _get_graph_batch_size) +from vllm.worker.model_runner_base import ( + _add_attn_metadata_broadcastable_dict, + _add_sampling_metadata_broadcastable_dict) +from vllm.worker.utils import assert_enc_dec_mr_supported_scenario + +logger = init_logger(__name__) + + +@dataclasses.dataclass(frozen=True) +class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata): + """ + Used by the EncoderDecoderModelRunner. + """ + encoder_input_tokens: Optional[torch.Tensor] = None + encoder_input_positions: Optional[torch.Tensor] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + "encoder_input_tokens": self.encoder_input_tokens, + "encoder_input_positions": self.encoder_input_positions, + "virtual_engine": self.virtual_engine, + "request_ids_to_seq_ids": self.request_ids_to_seq_ids, + "finished_requests_ids": self.finished_requests_ids, + "multi_modal_kwargs": self.multi_modal_kwargs, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + _add_sampling_metadata_broadcastable_dict(tensor_dict, + self.sampling_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "EncoderDecoderModelInput": + return cast( + EncoderDecoderModelInput, + super().from_broadcasted_tensor_dict(tensor_dict, attn_backend)) + + +class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]): + _model_input_cls: Type[EncoderDecoderModelInput] = ( + EncoderDecoderModelInput) + _builder_cls: Type[ModelInputForGPUBuilder] = (ModelInputForGPUBuilder) + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + observability_config: Optional[ObservabilityConfig] = None, + input_registry: InputRegistry = INPUT_REGISTRY, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, + ): + ''' + EncoderDecoderModelRunner constructor. + + `lora_config` and `prompt_adapter_config` are + unused (since these features are not yet supported for encoder/decoder + models) but these arguments are present here for compatibility with + the base-class constructor. + ''' + + self._maybe_force_supported_attention_backend() + + super().__init__( + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config, + lora_config=None, + kv_cache_dtype=kv_cache_dtype, + is_driver_worker=is_driver_worker, + ) + + # Crash for unsupported encoder/scenarios + assert_enc_dec_mr_supported_scenario(self) + + def _maybe_force_supported_attention_backend(self): + ''' + Force vLLM to use the XFormers attention backend, + which is currently the only supported option. + ''' + + def raise_backend_err(): + # The user has specified an attention backend override + # which is invalid for encoder/decoder models + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_BACKEND) + + maybe_env_var_forced_backend = get_env_variable_attn_backend() + maybe_global_forced_backend = get_global_forced_attn_backend() + is_forced_by_global = maybe_global_forced_backend is not None + is_forced_by_env_var = maybe_env_var_forced_backend is not None + + if not (is_forced_by_global or is_forced_by_env_var): + # The user has not already specified an attention backend + # override + pass + # logger.info("EncoderDecoderModelRunner requires " + # "XFormers backend; overriding backend " + # "auto-selection and forcing XFormers.") + # global_force_attn_backend(_Backend.XFORMERS) + elif is_forced_by_global: + # Backend override enforced by global variable takes + # precedence over vLLM backend environment variable. + if maybe_global_forced_backend != _Backend.XFORMERS: + raise_backend_err() + elif is_forced_by_env_var: + # Backend override enforced by vLLM backend + # environment variable + if maybe_env_var_forced_backend != _Backend.XFORMERS: + raise_backend_err() + + def _list_to_int32_tensor( + self, + _list: List[int], + ) -> torch.Tensor: + return torch.tensor(_list, dtype=torch.int32, device=self.device) + + def _list_to_long_tensor( + self, + _list: List[int], + ) -> torch.Tensor: + return torch.tensor(_list, dtype=torch.long, device=self.device) + + def _empty_int32_tensor(self) -> torch.Tensor: + return self._list_to_int32_tensor([]) + + def _empty_long_tensor(self) -> torch.Tensor: + return self._list_to_long_tensor([]) + + @torch.inference_mode() + def execute_model( + self, + model_input: EncoderDecoderModelInput, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[PoolerOutput]]: + if num_steps > 1: + raise ValueError("num_steps > 1 is not supported in " + "EncoderDecoderModelRunner") + + if (model_input.attn_metadata is not None + and model_input.attn_metadata.prefill_metadata is None + and model_input.attn_metadata.decode_metadata.use_cuda_graph): + assert model_input.input_tokens is not None + graph_batch_size = model_input.input_tokens.shape[0] + model_executable = self.graph_runners[ + model_input.virtual_engine][graph_batch_size] + else: + model_executable = self.model + + seqlen_agnostic_kwargs = { + "finished_requests_ids": model_input.finished_requests_ids, + "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids, + } if self.has_inner_state else {} + + multi_modal_kwargs = model_input.multi_modal_kwargs or {} + with set_forward_context(model_input.attn_metadata): + hidden_or_intermediate_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + encoder_input_ids=model_input.encoder_input_tokens, + encoder_positions=model_input.encoder_input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(multi_modal_kwargs, + device=self.device), + **seqlen_agnostic_kwargs) + + logits = self.model.compute_logits(hidden_or_intermediate_states, + model_input.sampling_metadata) + + if not self.is_driver_worker: + return [] + + if model_input.async_callback is not None: + model_input.async_callback() + + # Sample the next token. + output: SamplerOutput = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + + return [output] + + def make_model_input_from_broadcasted_tensor_dict( + self, tensor_dict: Dict[str, Any]) -> EncoderDecoderModelInput: + return EncoderDecoderModelInput.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + ) + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> EncoderDecoderModelInput: + """Prepare the model input based on a given sequence group, including + metadata for the sampling step. + + Since chunked prefill is not supported for encoder/decoder models, + `input_tokens` is assumed to be either entirely prefill tokens or + entirely decode tokens. + + """ + model_input = self._prepare_model_input_tensors( + seq_group_metadata_list, finished_requests_ids) + ( + attn_metadata, + encoder_input_tokens_tensor, + encoder_input_positions_tensor, + ) = (self._prepare_encoder_model_input_tensors(seq_group_metadata_list, + model_input)) + # Inject attn_metadata encoder/cross-attention fields & + # encoder input tokens/positions into model_input. + # Frozen dataclass fields cannot be modified, so use + # dataclasses.replace to construct a new model input + # instance. + model_input = dataclasses.replace( + model_input, + attn_metadata=attn_metadata, + encoder_input_tokens=encoder_input_tokens_tensor, + encoder_input_positions=encoder_input_positions_tensor, + ) + + generators = self.get_generators(finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, + model_input.seq_lens, + model_input.query_lens, + self.device, + self.pin_memory, + generators=generators) + is_prompt = (seq_group_metadata_list[0].is_prompt + if seq_group_metadata_list else None) + return dataclasses.replace(model_input, + sampling_metadata=sampling_metadata, + is_prompt=is_prompt, + virtual_engine=virtual_engine) + + @torch.inference_mode() + def profile_run(self) -> None: + # Enable top-k sampling to reflect the accurate memory usage. + sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1) + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens + max_num_seqs = self.scheduler_config.max_num_seqs + + # Profile memory usage with max_num_sequences sequences and the total + # number of tokens equal to max_num_batched_tokens. + seqs: List[SequenceGroupMetadata] = [] + + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens( + self.model_config) + if max_mm_tokens > 0: + logger.info("Starting profile run for multi-modal models.") + + batch_size = 0 + for group_id in range(max_num_seqs): + seq_len = (max_num_batched_tokens // max_num_seqs + + (group_id < max_num_batched_tokens % max_num_seqs)) + batch_size += seq_len + + decoder_seq_data, decoder_dummy_multi_modal_data \ + = self.input_registry.dummy_data_for_profiling( + self.model_config, + seq_len, + self.mm_registry, + is_encoder_data=False) + encoder_seq_data, encoder_dummy_multi_modal_data \ + = self.input_registry.dummy_data_for_profiling( + self.model_config, + seq_len, + self.mm_registry, + is_encoder_data=True) + + # Having more tokens is over-conservative but otherwise fine + assert len(decoder_seq_data.prompt_token_ids) >= seq_len, ( + f"Expected at least {seq_len} dummy tokens for profiling, " + f"but got: {len(decoder_seq_data.prompt_token_ids)}") + + assert decoder_dummy_multi_modal_data is None or \ + encoder_dummy_multi_modal_data is None, ( + "Multi-modal data can't be provided in both encoder and decoder" + ) + + seq = SequenceGroupMetadata( + request_id=str(group_id), + is_prompt=True, + seq_data={group_id: decoder_seq_data}, + sampling_params=sampling_params, + block_tables=None, + encoder_seq_data=encoder_seq_data, + cross_block_table=None, + multi_modal_data=decoder_dummy_multi_modal_data + or encoder_dummy_multi_modal_data, + ) + seqs.append(seq) + + # Run the model with the dummy inputs. + num_layers = self.model_config.get_num_layers(self.parallel_config) + # use an empty tensor instead of `None`` to force Dynamo to pass + # it by reference, rather by specializing on the value ``None``. + # the `dtype` argument does not matter, and we use `float32` as + # a placeholder (it has wide hardware support). + kv_caches = [ + torch.tensor([], dtype=torch.float32, device=self.device) + for _ in range(num_layers) + ] + finished_requests_ids = [seq.request_id for seq in seqs] + model_input = self.prepare_model_input( + seqs, finished_requests_ids=finished_requests_ids) + intermediate_tensors = None + self.execute_model(model_input, kv_caches, intermediate_tensors) + torch.cuda.synchronize() + return + + def _prepare_encoder_model_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + model_input: EncoderDecoderModelInput, + ) -> Tuple[AttentionMetadata, Optional[torch.Tensor], + Optional[torch.Tensor]]: + """Helper method to prepare the encoder- and cross-attn-related + model inputs based on a given sequence group. These additional inputs + are used to augment an already-computed `EncoderDecoderModelInput` + data structure which already has decoder-related model inputs + populated. + + Sets the following attn_metadata fields: + * `num_encoder_tokens` + * `encoder_seq_lens` + * `encoder_seq_lens_tensor` + * `max_encoder_seq_len` + * `cross_slot_mapping` + * `cross_block_tables` + + Constructs a new model inputs data structure, based on + (1) the existing fields in the `model_inputs` argument, + and (2) the following additional fields which are + computed (or in the case of `attn_metadata`, updated) + by this function: + * attn_metadata + * encoder_input_tokens + * encoder_input_positions + + Arguments: + + * seq_group_metadata_list: list of sequence groups for which to + compute inputs + * model_inputs: model inputs data structure with decoder-oriented + fields already computed. + + Return: + + * Updated model inputs data structure + """ + + if len(seq_group_metadata_list) == 0: + return (model_input.attn_metadata, None, None) + + # Since we are not supporting chunked prefill either the entire + # batch is prefill or it is decode + is_prompt = seq_group_metadata_list[0].is_prompt + + # Build encoder inputs + encoder_seq_lens: List[int] = [] + if is_prompt: + # Prefill phase. + cross_block_tables = self._empty_int32_tensor().view( + len(seq_group_metadata_list), -1) + + # Extract input tokens/positions, cross-attention slot-mapping, + # & seq len from each sequence group metadata + ( + encoder_input_tokens, + encoder_input_positions, + cross_slot_mapping, + ) = ( + [], + [], + [], + ) + for seq_group_metadata in seq_group_metadata_list: + # Build seq lens + seq_len = seq_group_metadata.encoder_seq_data.get_len() + token_ids = seq_group_metadata.encoder_seq_data.get_token_ids() + encoder_seq_lens.append(seq_len) + + # Build slot mapping + is_profile_run = (seq_group_metadata.block_tables is None) + if is_profile_run: + # During memory profiling, the block tables are not + # initialized yet. In this case, we just use a dummy + # slot mapping. + # In embeddings, the block tables are {seq_id: None}. + cross_slot_mapping.extend([PAD_SLOT_ID] * seq_len) + else: + for i in range(0, seq_len): + block_number = seq_group_metadata.cross_block_table[ + i // self.block_size] + block_offset = i % self.block_size + slot = block_number * self.block_size + block_offset + cross_slot_mapping.append(slot) + + # Build encoder input tokens + encoder_input_tokens.extend(token_ids) + encoder_input_positions.extend(list(range(0, seq_len))) + + # Convert tokens/positions & cross-attention + # slot-mapping to encoder input tensors + encoder_input_tokens_tensor = self._list_to_long_tensor( + encoder_input_tokens) + encoder_input_positions_tensor = self._list_to_long_tensor( + encoder_input_positions) + cross_slot_mapping_tensor = self._list_to_long_tensor( + cross_slot_mapping) + + else: + # Decode phase. + encoder_input_tokens_tensor = self._empty_long_tensor() + encoder_input_positions_tensor = self._empty_long_tensor() + cross_slot_mapping_tensor = self._empty_long_tensor() + # Extract cross-attention block tables & + # seq len from each sequence group metadata. + # Cross-attention block tables are empty + # during vLLM memory profiling. + cross_block_tables = [] + for seq_group_metadata in seq_group_metadata_list: + for _ in range(len(seq_group_metadata.seq_data)): + encoder_seq_lens.append( + seq_group_metadata.encoder_seq_data.get_len()) + cross_block_table = seq_group_metadata.cross_block_table + cross_block_tables.append([] if ( + cross_block_table is None) else cross_block_table) + + if (model_input.attn_metadata is not None + and model_input.attn_metadata.use_cuda_graph): + # We will be using CUDA graph replay for this decode. + max_len_of_block_table = self.get_max_block_per_batch() + batch_size = len(encoder_seq_lens) + graph_batch_size = _get_graph_batch_size(batch_size) + assert graph_batch_size >= batch_size + cuda_graph_pad_size = graph_batch_size - batch_size + # extend the cross_block_tables and encoder_seq_lens to match + # the graph_batch_size. + cross_block_tables.extend([[] + for _ in range(cuda_graph_pad_size) + ]) + encoder_seq_lens.extend( + itertools.repeat(1, cuda_graph_pad_size)) + + else: + max_len_of_block_table = max( + len(block_table) for block_table in cross_block_tables) + + cross_block_tables = make_tensor_with_pad( + cross_block_tables, + max_len=max_len_of_block_table, + pad=0, + dtype=torch.int32, + device=self.device, + ) + + # Compute encoder sequence lengths & encoder + # sequence starting offset tensors + max_encoder_seq_len = max(encoder_seq_lens, default=0) + encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens) + encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] + + 1, + dtype=torch.int32, + device=self.device) + torch.cumsum(encoder_seq_lens_tensor, + dim=0, + dtype=encoder_seq_start_loc.dtype, + out=encoder_seq_start_loc[1:]) + + # Update attention metadata with encoder-oriented attributes + attn_metadata = model_input.attn_metadata + assert attn_metadata is not None + ( + attn_metadata.num_encoder_tokens, + attn_metadata.encoder_seq_lens, + attn_metadata.encoder_seq_lens_tensor, + attn_metadata.max_encoder_seq_len, + attn_metadata.cross_slot_mapping, + attn_metadata.cross_block_tables, + attn_metadata.encoder_seq_start_loc, + ) = ( + sum(encoder_seq_lens), + encoder_seq_lens, + encoder_seq_lens_tensor, + max_encoder_seq_len, + cross_slot_mapping_tensor, + cross_block_tables, + encoder_seq_start_loc, + ) + + return (attn_metadata, encoder_input_tokens_tensor, + encoder_input_positions_tensor) diff --git a/vllm/worker/model_runner.py b/vllm/worker/model_runner.py new file mode 100644 index 00000000..ee4442f7 --- /dev/null +++ b/vllm/worker/model_runner.py @@ -0,0 +1,1932 @@ +import dataclasses +import gc +import inspect +import itertools +import time +import warnings +import weakref +from dataclasses import dataclass +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, + Tuple, Type, TypeVar, Union) + +import numpy as np +import torch +import torch.distributed +import torch.nn as nn + +import vllm.envs as envs +from vllm.attention import AttentionMetadata, get_attn_backend +from vllm.attention.backends.abstract import AttentionState +from vllm.attention.backends.utils import CommonAttentionState +from vllm.compilation.compile_context import set_compile_context +from vllm.compilation.levels import CompilationLevel +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.core.scheduler import SchedulerOutputs +from vllm.distributed import get_pp_group +from vllm.distributed.parallel_state import graph_capture +from vllm.forward_context import set_forward_context +from vllm.inputs import INPUT_REGISTRY, InputRegistry +from vllm.logger import init_logger +from vllm.lora.layers import LoRAMapping +from vllm.lora.request import LoRARequest +from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager +from vllm.model_executor import SamplingMetadata, SamplingMetadataCache +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader import get_model +from vllm.model_executor.model_loader.tensorizer import TensorizerConfig +from vllm.model_executor.models import supports_lora, supports_multimodal +from vllm.model_executor.models.utils import set_cpu_offload_max_bytes +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs, + MultiModalInputs, MultiModalRegistry) +from vllm.prompt_adapter.layers import PromptAdapterMapping +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.prompt_adapter.worker_manager import ( + LRUCacheWorkerPromptAdapterManager) +from vllm.sampling_params import SamplingParams +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata +from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d, + flatten_2d_lists, is_hip, is_pin_memory_available, + supports_dynamo) +from vllm.worker.model_runner_base import ( + ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase, + _add_attn_metadata_broadcastable_dict, + _add_sampling_metadata_broadcastable_dict, + _init_attn_metadata_from_tensor_dict, + _init_sampling_metadata_from_tensor_dict, dump_input_when_exception) + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + +LORA_WARMUP_RANK = 8 +_BATCH_SIZE_ALIGNMENT = 8 +# all the token sizes that **can** be captured by cudagraph. +# they can be arbitrarily large. +# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192. +# the actual sizes to capture will be determined by the model, +# depending on the model's max_num_seqs. +# NOTE: _get_graph_batch_size needs to be updated if this list is changed. +_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [ + _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025) +] +_NUM_WARMUP_ITERS = 2 + +TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU") + +# For now, bump up cache limits for recompilations during CUDA graph warmups. +# torch._dynamo.config.cache_size_limit = 128 +# torch._dynamo.config.accumulated_cache_size_limit = 128 + + +@dataclass(frozen=True) +class ModelInputForGPU(ModelRunnerInputBase): + """ + This base class contains metadata needed for the base model forward pass + but not metadata for possible additional steps, e.g., sampling. Model + runners that run additional steps should subclass this method to add + additional fields. + """ + input_tokens: Optional[torch.Tensor] = None + input_positions: Optional[torch.Tensor] = None + seq_lens: Optional[List[int]] = None + query_lens: Optional[List[int]] = None + lora_mapping: Optional["LoRAMapping"] = None + lora_requests: Optional[Set[LoRARequest]] = None + attn_metadata: Optional["AttentionMetadata"] = None + prompt_adapter_mapping: Optional[PromptAdapterMapping] = None + prompt_adapter_requests: Optional[Set[PromptAdapterRequest]] = None + multi_modal_kwargs: Optional[BatchedTensorInputs] = None + request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None + finished_requests_ids: Optional[List[str]] = None + virtual_engine: int = 0 + async_callback: Optional[Callable] = None + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None + scheduler_outputs: Optional[SchedulerOutputs] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + "lora_requests": self.lora_requests, + "lora_mapping": self.lora_mapping, + "multi_modal_kwargs": self.multi_modal_kwargs, + "prompt_adapter_mapping": self.prompt_adapter_mapping, + "prompt_adapter_requests": self.prompt_adapter_requests, + "virtual_engine": self.virtual_engine, + "request_ids_to_seq_ids": self.request_ids_to_seq_ids, + "finished_requests_ids": self.finished_requests_ids, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls: Type[TModelInputForGPU], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> TModelInputForGPU: + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +@dataclass(frozen=True) +class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU): + """ + Used by the ModelRunner. + """ + sampling_metadata: Optional["SamplingMetadata"] = None + # Used for speculative decoding. We do not broadcast it because it is only + # used by the driver worker. + is_prompt: Optional[bool] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + "lora_requests": self.lora_requests, + "lora_mapping": self.lora_mapping, + "multi_modal_kwargs": self.multi_modal_kwargs, + "prompt_adapter_mapping": self.prompt_adapter_mapping, + "prompt_adapter_requests": self.prompt_adapter_requests, + "virtual_engine": self.virtual_engine, + "request_ids_to_seq_ids": self.request_ids_to_seq_ids, + "finished_requests_ids": self.finished_requests_ids, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + _add_sampling_metadata_broadcastable_dict(tensor_dict, + self.sampling_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "ModelInputForGPUWithSamplingMetadata": + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict) + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]): + """Build ModelInputForGPU from SequenceGroupMetadata.""" + + # Note: ideally we would be using a dataclass(kw_only=True) + # here, so that this can be subclassed easily, + # but kw_only is not supported in python<3.10. + class InterDataForSeqGroup: + """Intermediate data for the current sequence group.""" + + def simple_reinit(self): + self.input_tokens[0].clear() # type: ignore + self.input_positions[0].clear() # type: ignore + self.mrope_input_positions = None # type: ignore + self.seq_lens[0] = 0 # type: ignore + self.orig_seq_lens[0] = 0 # type: ignore + self.query_lens[0] = 0 # type: ignore + self.context_lens[0] = 0 # type: ignore + self.curr_sliding_window_blocks[0] = 0 # type: ignore + self.lora_index_mapping.clear() # type: ignore + self.lora_prompt_mapping.clear() # type: ignore + self.lora_requests.clear() # type: ignore + self.prompt_adapter_index_mapping.clear() # type: ignore + self.prompt_adapter_prompt_mapping.clear() # type: ignore + + def __init__( + self, + *, + # From sequence group metadata. + request_id: str, + seq_ids: List[int], + is_prompt: bool, + block_tables: Optional[Dict[int, List[int]]], + computed_block_nums: List[int], + n_seqs: int = 0, + + # Input tokens and positions. + input_tokens: Optional[List[List[int]]] = None, + input_positions: Optional[List[List[int]]] = None, + mrope_input_positions: Optional[List[List[List[int]]]] = None, + + # The sequence length (may be capped to the sliding window). + seq_lens: Optional[List[int]] = None, + # The original sequence length (before applying sliding window). + # This is used to compute slot mapping. + orig_seq_lens: Optional[List[int]] = None, + # The query length. + query_lens: Optional[List[int]] = None, + # The number of tokens that are already computed. + context_lens: Optional[List[int]] = None, + # The current sliding window block. + curr_sliding_window_blocks: Optional[List[int]] = None, + + # LoRA inputs. + lora_index_mapping: Optional[List[List[int]]] = None, + lora_prompt_mapping: Optional[List[List[int]]] = None, + lora_requests: Optional[Set[LoRARequest]] = None, + + # Prompt adapter inputs. + prompt_adapter_index_mapping: Optional[List[int]] = None, + prompt_adapter_prompt_mapping: Optional[List[int]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + + # Multi-modal inputs. + multi_modal_inputs: Optional[MultiModalInputs] = None, + + # Whether the prefix cache is hit (prefill only). + prefix_cache_hit: bool = False, + reinit: bool = False, + reinit_use_defaults: bool = False, + encoder_seq_len: int = 0, + ): + if reinit: + assert len(self.seq_ids) == len(seq_ids) # type: ignore + for i, seq_id in enumerate(seq_ids): + self.seq_ids[i] = seq_id # type: ignore + else: + self.seq_ids = seq_ids + + self.request_id = request_id + self.is_prompt = is_prompt + self.block_tables = block_tables + self.computed_block_nums = computed_block_nums + self.n_seqs = n_seqs + self.encoder_seq_len = encoder_seq_len + + if reinit: + if len(self.seq_ids) == 1 and reinit_use_defaults: + self.simple_reinit() + else: + if input_tokens: + self.input_tokens = input_tokens + else: + for seq_id in range(len(self.seq_ids)): + self.input_tokens[seq_id].clear() + + if input_positions: + self.input_positions = input_positions + else: + for seq_id in range(len(self.seq_ids)): + self.input_positions[seq_id].clear() + + self.mrope_input_positions = None + + if seq_lens: + self.seq_lens = seq_lens + else: + for seq_id in range(len(self.seq_ids)): + self.seq_lens[seq_id] = 0 + + if orig_seq_lens: + self.orig_seq_lens = orig_seq_lens + else: + for seq_id in range(len(self.seq_ids)): + self.orig_seq_lens[seq_id] = 0 + + if query_lens: + self.query_lens = query_lens + else: + for seq_id in range(len(self.seq_ids)): + self.query_lens[seq_id] = 0 + + if context_lens: + self.context_lens = context_lens + else: + for seq_id in range(len(self.seq_ids)): + self.context_lens[seq_id] = 0 + + if curr_sliding_window_blocks: + self.curr_sliding_window_blocks = \ + curr_sliding_window_blocks + else: + for seq_id in range(len(self.seq_ids)): + self.curr_sliding_window_blocks[seq_id] = 0 + + if lora_index_mapping: + self.lora_index_mapping = lora_index_mapping + else: + self.lora_index_mapping.clear() + + if lora_prompt_mapping: + self.lora_prompt_mapping = lora_prompt_mapping + else: + self.lora_prompt_mapping.clear() + + if lora_requests: + self.lora_requests = lora_requests + else: + self.lora_requests.clear() + + if prompt_adapter_index_mapping: + self.prompt_adapter_index_mapping = \ + prompt_adapter_index_mapping + else: + self.prompt_adapter_index_mapping.clear() + + if prompt_adapter_prompt_mapping: + self.prompt_adapter_prompt_mapping = \ + prompt_adapter_prompt_mapping + else: + self.prompt_adapter_prompt_mapping.clear() + + else: + self.input_tokens = input_tokens or [] + self.input_positions = input_positions or [] + self.mrope_input_positions = mrope_input_positions or None + self.seq_lens = seq_lens or [] + self.orig_seq_lens = orig_seq_lens or [] + self.query_lens = query_lens or [] + self.context_lens = context_lens or [] + self.curr_sliding_window_blocks = \ + curr_sliding_window_blocks or [] + + self.lora_index_mapping = lora_index_mapping or [] + self.lora_prompt_mapping = lora_prompt_mapping or [] + self.lora_requests = lora_requests or set() + + self.prompt_adapter_index_mapping = ( + prompt_adapter_index_mapping or []) + self.prompt_adapter_prompt_mapping = ( + prompt_adapter_prompt_mapping or []) + + self.prompt_adapter_request = prompt_adapter_request + self.multi_modal_inputs = multi_modal_inputs + self.prefix_cache_hit = prefix_cache_hit + + self.n_seqs = len(self.seq_ids) + + if not reinit: + self.__post_init__() + + def __post_init__(self): + self.n_seqs = len(self.seq_ids) + + self.input_tokens = [[] for _ in range(self.n_seqs)] + self.input_positions = [[] for _ in range(self.n_seqs)] + self.mrope_input_positions = None + self.seq_lens = [0] * self.n_seqs + self.orig_seq_lens = [0] * self.n_seqs + self.query_lens = [0] * self.n_seqs + self.context_lens = [0] * self.n_seqs + self.curr_sliding_window_blocks = [0] * self.n_seqs + + self.lora_index_mapping = [] + self.lora_prompt_mapping = [] + + def gen_inter_data_builder(self, num_seqs: int): + return lambda: ModelInputForGPUBuilder.InterDataForSeqGroup( + request_id="", + seq_ids=[0] * num_seqs, + is_prompt=True, + block_tables=None, + computed_block_nums=[]) + + def init_cached_inter_data(self, *args, **kwargs): + assert len(args) == 0 + assert "seq_ids" in kwargs + seq_ids = kwargs["seq_ids"] + num_seqs = len(seq_ids) + + # The inter-data cache is per model_runner + inter_data_cache = self.runner.inter_data_cache + if num_seqs not in inter_data_cache: + inter_data_cache[num_seqs] = PyObjectCache( + self.gen_inter_data_builder(num_seqs)) + + obj = inter_data_cache[num_seqs].get_object() + obj.__init__(*args, **kwargs) + return obj + + def reset_cached_inter_data(self): + for cache in self.runner.inter_data_cache.values(): + cache.reset() + + def __init__(self, + runner: "GPUModelRunnerBase", + finished_requests_ids: Optional[List[str]] = None): + super().__init__() + # Compute functions for each sequence in a sequence group. + # WARNING: The order of the functions matters! + self.per_seq_compute_fns = [ + self._compute_lens, + self._compute_for_prefix_cache_hit, + self._compute_for_sliding_window, + self._compute_lora_input, + ] + # Compute functions for each sequence group. + # WARNING: The order of the functions matters! + self.per_seq_group_compute_fns = [ + self._compute_prompt_adapter_input, + self._compute_multi_modal_input, + ] + + self.runner = runner + self.model_input_cls = self.runner._model_input_cls + self.attn_backend = self.runner.attn_backend + self.scheduler_config = self.runner.scheduler_config + self.sliding_window = self.runner.sliding_window + self.block_size = self.runner.block_size + self.enable_lora = self.runner.lora_config is not None + self.enable_prompt_adapter = (self.runner.prompt_adapter_config + is not None) + self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper + self.finished_requests_ids = finished_requests_ids + self.decode_only = True + + # Intermediate data (data in CPU before going to GPU) for + # the current sequence group. + self.inter_data_list: List[ + ModelInputForGPUBuilder.InterDataForSeqGroup] = [] + + # Attention metadata inputs. + self.attn_metadata_builder = self.attn_backend.make_metadata_builder( + weakref.proxy(self)) + + # Engine/Model configurations. + self.chunked_prefill_enabled = ( + self.scheduler_config is not None + and self.scheduler_config.chunked_prefill_enabled) + if self.sliding_window is not None: + self.sliding_window_blocks = ( + self.sliding_window + self.block_size - 1) // self.block_size + self.block_aligned_sliding_window = \ + self.sliding_window_blocks * self.block_size + + def _compute_lens(self, inter_data: InterDataForSeqGroup, seq_idx: int, + seq_group_metadata: SequenceGroupMetadata): + """Compute context length, sequence length and tokens + for the given sequence data. + """ + seq_data = seq_group_metadata.seq_data[inter_data.seq_ids[seq_idx]] + token_chunk_size = seq_group_metadata.token_chunk_size + + # Compute context length (the number of tokens that are + # already computed) and sequence length (total number of tokens). + + seq_len = seq_data.get_len() + if inter_data.is_prompt: + context_len = seq_data.get_num_computed_tokens() + seq_len = min(seq_len, context_len + token_chunk_size) + elif self.runner.scheduler_config.is_multi_step or \ + self.runner.model_config.is_encoder_decoder_model: + context_len = seq_len - 1 + else: + context_len = seq_data.get_num_computed_tokens() + + # Compute tokens. + tokens = seq_data.get_token_ids()[context_len:seq_len] + + inter_data.seq_lens[seq_idx] = seq_len + inter_data.orig_seq_lens[seq_idx] = seq_len + inter_data.context_lens[seq_idx] = context_len + inter_data.input_tokens[seq_idx].extend(tokens) + inter_data.input_positions[seq_idx].extend(range(context_len, seq_len)) + inter_data.query_lens[seq_idx] = seq_len - context_len + + if seq_data.mrope_position_delta is not None: + if inter_data.mrope_input_positions is None: + inter_data.mrope_input_positions = [None] * inter_data.n_seqs + + inter_data.mrope_input_positions[ + seq_idx] = MRotaryEmbedding.get_next_input_positions( + seq_data.mrope_position_delta, + context_len, + seq_len, + ) + + def _compute_for_prefix_cache_hit( + self, inter_data: InterDataForSeqGroup, seq_idx: int, + seq_group_metadata: SequenceGroupMetadata): + """Check if hit prefix cache (i.e., some blocks are already computed). + If hit, update input tokens and positions to only compute the + remaining blocks. + """ + computed_block_nums = inter_data.computed_block_nums + + # Note that prefix caching does not support sliding window. + prefix_cache_hit = (computed_block_nums is not None + and len(computed_block_nums) > 0 + and self.sliding_window is None + and inter_data.is_prompt) + inter_data.prefix_cache_hit = prefix_cache_hit + + if not prefix_cache_hit: + return + + assert computed_block_nums is not None + # The cache hit prompt tokens in this sequence. Note that + # this may be larger than the sequence length if chunked + # prefill is enabled. + prefix_cache_len = len(computed_block_nums) * self.block_size + # The number of so far computed prompt tokens in this sequence. + context_len = inter_data.context_lens[seq_idx] + # The total number of prompt tokens in this sequence. + # When chunked prefill is enabled, this is the token number of + # computed chunks + current chunk. + seq_len = inter_data.seq_lens[seq_idx] + if prefix_cache_len <= context_len: + # We already passed the cache hit region, + # so do normal computation. + pass + elif context_len < prefix_cache_len < seq_len: + # Partial hit. Compute the missing part. + uncomputed_start = prefix_cache_len - context_len + inter_data.input_tokens[seq_idx] = inter_data.input_tokens[ + seq_idx][uncomputed_start:] + inter_data.input_positions[seq_idx] = inter_data.input_positions[ + seq_idx][uncomputed_start:] + context_len = prefix_cache_len + + inter_data.context_lens[seq_idx] = context_len + inter_data.query_lens[ + seq_idx] = inter_data.seq_lens[seq_idx] - context_len + elif seq_len <= prefix_cache_len: + # Full hit. Only compute the last token to avoid + # erroneous behavior. FIXME: Ideally we should directly + # mark all tokens as computed in the scheduler and do not + # schedule this sequence, so this case should not happen. + inter_data.input_tokens[seq_idx] = inter_data.input_tokens[ + seq_idx][-1:] + inter_data.input_positions[seq_idx] = inter_data.input_positions[ + seq_idx][-1:] + inter_data.query_lens[seq_idx] = 1 + inter_data.context_lens[seq_idx] = inter_data.seq_lens[seq_idx] - 1 + + def _compute_for_sliding_window(self, inter_data: InterDataForSeqGroup, + seq_idx: int, + seq_group_metadata: SequenceGroupMetadata): + """Update seq_len and curr_sliding_window_block for the given + sequence data (only required by decoding) if sliding window is enabled. + """ + curr_sliding_window_block = 0 + sliding_seq_len = inter_data.seq_lens[seq_idx] + if not inter_data.is_prompt and self.sliding_window is not None: + # TODO(sang): This is a hack to make sliding window work with + # paged attn. We can remove it if we make paged attn kernel + # to properly handle slinding window attn. + curr_sliding_window_block = self.sliding_window_blocks + if self.scheduler_config.use_v2_block_manager: + # number of elements in last block + suff_len = inter_data.seq_lens[seq_idx] % self.block_size + sliding_seq_len = min( + inter_data.seq_lens[seq_idx], + self.block_aligned_sliding_window + suff_len) + if suff_len > 0: + curr_sliding_window_block += 1 + else: + sliding_seq_len = min(inter_data.seq_lens[seq_idx], + self.sliding_window) + + inter_data.curr_sliding_window_blocks[ + seq_idx] = curr_sliding_window_block + inter_data.seq_lens[seq_idx] = sliding_seq_len + + def _compute_lora_input(self, inter_data: InterDataForSeqGroup, + seq_idx: int, + seq_group_metadata: SequenceGroupMetadata): + """If LoRA is enabled, compute LoRA index and prompt mapping.""" + if not self.enable_lora: + return + + lora_id = seq_group_metadata.lora_int_id + if lora_id > 0: + inter_data.lora_requests.add(seq_group_metadata.lora_request) + query_len = inter_data.query_lens[seq_idx] + inter_data.lora_index_mapping.append([lora_id] * query_len) + inter_data.lora_prompt_mapping.append( + [lora_id] * + (query_len if seq_group_metadata.sampling_params + and seq_group_metadata.sampling_params.prompt_logprobs is not None + else 1)) + + def _compute_prompt_adapter_input( + self, inter_data: InterDataForSeqGroup, + seq_group_metadata: SequenceGroupMetadata): + """If prompt adapter is enabled, compute index and prompt mapping. + """ + # Note that when is_prompt=True, we expect only one sequence + # in the group. + if not self.enable_prompt_adapter: + return + + prompt_adapter_id = seq_group_metadata.prompt_adapter_id + if prompt_adapter_id <= 0 or not inter_data.is_prompt: + return + + # We expect only one sequence in the group when is_prompt=True. + assert inter_data.n_seqs == 1 + query_len = inter_data.query_lens[0] + inter_data.prompt_adapter_request = ( + seq_group_metadata.prompt_adapter_request) + + num_tokens = seq_group_metadata.prompt_adapter_num_virtual_tokens + inter_data.prompt_adapter_index_mapping = [ + prompt_adapter_id + ] * num_tokens + [0] * (query_len - num_tokens) + inter_data.prompt_adapter_prompt_mapping = [prompt_adapter_id] * ( + query_len if seq_group_metadata.sampling_params + and seq_group_metadata.sampling_params.prompt_logprobs else 1) + + def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup, + seq_group_metadata: SequenceGroupMetadata): + """If multi-modal data is given, add it to the input.""" + mm_data = seq_group_metadata.multi_modal_data + if not mm_data: + return + + mm_kwargs = self.multi_modal_input_mapper( + mm_data, + mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs) + inter_data.multi_modal_inputs = mm_kwargs + + # special processing for mrope position deltas. + if self.runner.model_is_mrope: + image_grid_thw = mm_kwargs.get("image_grid_thw", None) + video_grid_thw = mm_kwargs.get("video_grid_thw", None) + assert image_grid_thw is not None or video_grid_thw is not None, ( + "mrope embedding type requires multi-modal input mapper " + "returns 'image_grid_thw' or 'video_grid_thw'.") + + hf_config = self.runner.model_config.hf_config + + inter_data.mrope_input_positions = [None] * inter_data.n_seqs + for seq_idx in range(inter_data.n_seqs): + seq_data = seq_group_metadata.seq_data[ + inter_data.seq_ids[seq_idx]] + token_ids = seq_data.get_token_ids() + + mrope_input_positions, mrope_position_delta = \ + MRotaryEmbedding.get_input_positions( + token_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_token_id=hf_config.image_token_id, + video_token_id=hf_config.video_token_id, + vision_start_token_id=hf_config.vision_start_token_id, + vision_end_token_id=hf_config.vision_end_token_id, + spatial_merge_size=hf_config.vision_config. + spatial_merge_size, + context_len=inter_data.context_lens[seq_idx], + ) + + seq_data.mrope_position_delta = mrope_position_delta + inter_data.mrope_input_positions[ + seq_idx] = mrope_input_positions + + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata): + """Add a sequence group to the builder.""" + seq_ids = seq_group_metadata.seq_data.keys() + n_seqs = len(seq_ids) + is_prompt = seq_group_metadata.is_prompt + + if is_prompt: + assert n_seqs == 1 + self.decode_only = False + + encoder_seq_len = 0 + + if self.runner.model_config.is_encoder_decoder_model: + encoder_seq_len = seq_group_metadata.encoder_seq_data.get_len() + + inter_data = self.init_cached_inter_data( + request_id=seq_group_metadata.request_id, + seq_ids=seq_ids, + is_prompt=is_prompt, + block_tables=seq_group_metadata.block_tables, + computed_block_nums=seq_group_metadata.computed_block_nums, + reinit=True, + reinit_use_defaults=True, + encoder_seq_len=encoder_seq_len) + + self.inter_data_list.append(inter_data) + + for seq_idx in range(n_seqs): + for per_seq_fn in self.per_seq_compute_fns: + per_seq_fn(inter_data, seq_idx, seq_group_metadata) + for per_seq_group_fn in self.per_seq_group_compute_fns: + per_seq_group_fn(inter_data, seq_group_metadata) + + def _use_captured_graph(self, + batch_size: int, + decode_only: bool, + max_decode_seq_len: int, + max_encoder_seq_len: int = 0) -> bool: + return (decode_only and not self.runner.model_config.enforce_eager + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1] + and max_decode_seq_len <= self.runner.max_seq_len_to_capture + and max_encoder_seq_len <= self.runner.max_seq_len_to_capture + and batch_size <= self.runner.max_batchsize_to_capture) + + def _get_cuda_graph_pad_size(self, + num_seqs: int, + max_decode_seq_len: int, + max_encoder_seq_len: int = 0) -> int: + """ + Determine the number of padding sequences required for running in + CUDA graph mode. Returns -1 if CUDA graphs cannot be used. + + In the multi-step + chunked-prefill case, only the first step + has Prefills (if any). The rest of the steps are guaranteed to be all + decodes. In this case, we set up the padding as if all the sequences + are decodes so we may run all steps except the first step in CUDA graph + mode. The padding is accounted for in the multi-step `advance_step` + family of functions. + + Args: + num_seqs (int): Number of sequences scheduled to run. + max_decode_seq_len (int): Greatest of all the decode sequence + lengths. Used only in checking the viablility of using + CUDA graphs. + max_encoder_seq_len (int, optional): Greatest of all the encode + sequence lengths. Defaults to 0. Used only in checking the + viability of using CUDA graphs. + Returns: + int: Returns the determined number of padding sequences. If + CUDA graphs is not viable, returns -1. + """ + is_mscp: bool = self.runner.scheduler_config.is_multi_step and \ + self.runner.scheduler_config.chunked_prefill_enabled + decode_only = self.decode_only or is_mscp + if not decode_only: + # Early exit so we can treat num_seqs as the batch_size below. + return -1 + + # batch_size out of this function refers to the number of input + # tokens being scheduled. This conflation of num_seqs as batch_size + # is valid as this is a decode-only case. + batch_size = num_seqs + if not self._use_captured_graph(batch_size, decode_only, + max_decode_seq_len, + max_encoder_seq_len): + return -1 + + graph_batch_size = _get_graph_batch_size(batch_size) + assert graph_batch_size >= batch_size + return graph_batch_size - batch_size + + def build(self) -> ModelInputForGPU: + """Finalize the builder intermediate data and + create on-device tensors. + """ + # Combine and flatten intermediate data. + input_tokens = [] + for inter_data in self.inter_data_list: + for cur_input_tokens in inter_data.input_tokens: + input_tokens.extend(cur_input_tokens) + + if not input_tokens: + # This may happen when all prefill requests hit + # prefix caching and there is no decode request. + return self.model_input_cls() + + mrope_input_positions: Optional[List[List[int]]] = None + if any(inter_data.mrope_input_positions is not None + for inter_data in self.inter_data_list): + mrope_input_positions = [[] for _ in range(3)] + for idx in range(3): + for inter_data in self.inter_data_list: + msections = inter_data.mrope_input_positions + if msections is None: + for _seq_input_positions in inter_data.input_positions: + mrope_input_positions[idx].extend( + _seq_input_positions) + else: + for _seq_mrope_input_positions in msections: + mrope_input_positions[idx].extend( + _seq_mrope_input_positions[idx]) + input_positions = None + else: + input_positions = [] + for inter_data in self.inter_data_list: + for cur_input_positions in inter_data.input_positions: + input_positions.extend(cur_input_positions) + + seq_lens = [] + query_lens = [] + max_decode_seq_len = 0 + max_encoder_seq_len = 0 + for inter_data in self.inter_data_list: + seq_lens.extend(inter_data.seq_lens) + query_lens.extend(inter_data.query_lens) + if not inter_data.is_prompt: + max_decode_seq_len = max(max_decode_seq_len, + max(inter_data.seq_lens)) + if self.runner.model_config.is_encoder_decoder_model: + max_encoder_seq_len = max(max_encoder_seq_len, + inter_data.encoder_seq_len) + + # Mapping from request IDs to sequence IDs. Used for Jamba models + # that manages the cache by itself. + request_ids_to_seq_ids = { + data.request_id: data.seq_ids + for data in self.inter_data_list + } + + cuda_graph_pad_size = self._get_cuda_graph_pad_size( + num_seqs=len(seq_lens), + max_decode_seq_len=max_encoder_seq_len, + max_encoder_seq_len=max_encoder_seq_len) + + batch_size = len(input_tokens) + if cuda_graph_pad_size != -1: + # If cuda graph can be used, pad tensors accordingly. + # See `capture_model` API for more details. + # vLLM uses cuda graph only for decoding requests. + batch_size += cuda_graph_pad_size + + # Tokens and positions. + if cuda_graph_pad_size: + input_tokens.extend(itertools.repeat(0, cuda_graph_pad_size)) + assert self.runner.device is not None + input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long, + self.runner.device, + self.runner.pin_memory) + if mrope_input_positions is not None: + for idx in range(3): + mrope_input_positions[idx].extend( + itertools.repeat(0, cuda_graph_pad_size)) + input_positions_tensor = async_tensor_h2d(mrope_input_positions, + torch.long, + self.runner.device, + self.runner.pin_memory) + else: + input_positions.extend(itertools.repeat(0, cuda_graph_pad_size)) + input_positions_tensor = async_tensor_h2d(input_positions, + torch.long, + self.runner.device, + self.runner.pin_memory) + # Sequence and query lengths. + if cuda_graph_pad_size: + seq_lens.extend(itertools.repeat(1, cuda_graph_pad_size)) + + # Attention metadata. + attn_metadata = self.attn_metadata_builder.build( + seq_lens, query_lens, cuda_graph_pad_size, batch_size) + + # LoRA data. + lora_requests = set() + lora_mapping = None + if self.enable_lora: + lora_requests = set(r for data in self.inter_data_list + for r in data.lora_requests) + lora_index_mapping = flatten_2d_lists([ + flatten_2d_lists(inter_data.lora_index_mapping) + for inter_data in self.inter_data_list + ]) + if cuda_graph_pad_size: + lora_index_mapping.extend( + itertools.repeat(0, cuda_graph_pad_size)) + lora_prompt_mapping = flatten_2d_lists([ + flatten_2d_lists(inter_data.lora_prompt_mapping) + for inter_data in self.inter_data_list + ]) + + lora_mapping = LoRAMapping( + **dict(index_mapping=lora_index_mapping, + prompt_mapping=lora_prompt_mapping, + is_prefill=not self.decode_only)) + + # Prompt adapter data. + prompt_adapter_requests: Set[PromptAdapterRequest] = set() + prompt_adapter_mapping = None + if self.enable_prompt_adapter: + prompt_adapter_requests = set( + data.prompt_adapter_request for data in self.inter_data_list + if data.prompt_adapter_request is not None) + prompt_adapter_index_mapping = flatten_2d_lists([ + inter_data.prompt_adapter_index_mapping + for inter_data in self.inter_data_list + ]) + if cuda_graph_pad_size: + prompt_adapter_index_mapping.extend( + itertools.repeat(0, cuda_graph_pad_size)) + prompt_adapter_prompt_mapping = flatten_2d_lists([ + inter_data.prompt_adapter_prompt_mapping + for inter_data in self.inter_data_list + ]) + prompt_adapter_mapping = PromptAdapterMapping( + prompt_adapter_index_mapping, + prompt_adapter_prompt_mapping, + ) + + # Multi-modal data. + multi_modal_inputs_list = [ + data.multi_modal_inputs for data in self.inter_data_list + if data.multi_modal_inputs is not None + ] + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list) + + return self.model_input_cls( + input_tokens=input_tokens_tensor, + input_positions=input_positions_tensor, + attn_metadata=attn_metadata, + seq_lens=seq_lens, + query_lens=query_lens, + lora_mapping=lora_mapping, + lora_requests=lora_requests, + multi_modal_kwargs=multi_modal_kwargs, + request_ids_to_seq_ids=request_ids_to_seq_ids, + finished_requests_ids=self.finished_requests_ids, + prompt_adapter_mapping=prompt_adapter_mapping, + prompt_adapter_requests=prompt_adapter_requests) + + +class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]): + """ + Helper class for shared methods between GPU model runners. + """ + _model_input_cls: Type[TModelInputForGPU] + _builder_cls: Type[ModelInputForGPUBuilder] + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + return_hidden_states: bool = False, + observability_config: Optional[ObservabilityConfig] = None, + input_registry: InputRegistry = INPUT_REGISTRY, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, + ): + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.lora_config = lora_config + self.load_config = load_config + self.is_driver_worker = is_driver_worker + self.prompt_adapter_config = prompt_adapter_config + self.return_hidden_states = return_hidden_states + self.observability_config = observability_config + + self.device = self.device_config.device + self.pin_memory = is_pin_memory_available() + + self.kv_cache_dtype = kv_cache_dtype + self.sliding_window = model_config.get_sliding_window() + self.block_size = cache_config.block_size + self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture + self.max_batchsize_to_capture = _get_max_graph_batch_size( + self.scheduler_config.max_num_seqs) + + self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [ + {} for _ in range(self.parallel_config.pipeline_parallel_size) + ] + self.graph_memory_pool: Optional[Tuple[ + int, int]] = None # Set during graph capture. + + self.has_inner_state = model_config.has_inner_state + + # When using CUDA graph, the input block tables must be padded to + # max_seq_len_to_capture. However, creating the block table in + # Python can be expensive. To optimize this, we cache the block table + # in numpy and only copy the actual input content at every iteration. + # The shape of the cached block table will be + # (max batch size to capture, max context len to capture / block size). + self.graph_block_tables = np.zeros( + (self.max_batchsize_to_capture, self.get_max_block_per_batch()), + dtype=np.int32) + + # Attention-free but stateful models like Mamba need a placeholder attn + # backend, as the attention metadata is needed to manage internal state. + # However we must bypass attention selection altogether for some models + # used for speculative decoding to avoid a divide-by-zero in + # model_config.get_head_size() + num_attn_heads = self.model_config.get_num_attention_heads( + self.parallel_config) + needs_attn_backend = (num_attn_heads != 0 + or self.model_config.is_attention_free) + + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.kv_cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) if needs_attn_backend else None + if self.attn_backend: + self.attn_state = self.attn_backend.get_state_cls()( + weakref.proxy(self)) + else: + self.attn_state = CommonAttentionState(weakref.proxy(self)) + + # Multi-modal data support + self.input_registry = input_registry + self.mm_registry = mm_registry + self.multi_modal_input_mapper = mm_registry \ + .create_input_mapper(model_config) + self.mm_registry.init_mm_limits_per_prompt(self.model_config) + + # Lazy initialization + self.model: nn.Module # Set after load_model + # Set after load_model. + self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None + self.prompt_adapter_manager: LRUCacheWorkerPromptAdapterManager = None + + set_cpu_offload_max_bytes( + int(self.cache_config.cpu_offload_gb * 1024**3)) + + # Used to cache python objects + self.inter_data_cache: Dict[int, PyObjectCache] = {} + + # Using the PythonizationCache in Pipeline-Parallel clobbers the + # SequenceGroupToSample object. In Pipeline-Parallel, we have + # more than 1 Scheduler, resulting in a potential back-to-back + # prepare_model_inputs() call. This clobbers the cached + # SequenceGroupToSample objects, as we reset the cache during + # every prepare_model_inputs() call. + self.sampling_metadata_cache: SamplingMetadataCache = \ + SamplingMetadataCache() \ + if self.parallel_config.pipeline_parallel_size == 1 else None + + def load_model(self) -> None: + logger.info("Starting to load model %s...", self.model_config.model) + with DeviceMemoryProfiler() as m: + self.model = get_model(model_config=self.model_config, + device_config=self.device_config, + load_config=self.load_config, + lora_config=self.lora_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + cache_config=self.cache_config) + + self.model_memory_usage = m.consumed_memory + logger.info("Loading model weights took %.4f GB", + self.model_memory_usage / float(2**30)) + + if self.lora_config: + assert supports_lora( + self.model + ), f"{self.model.__class__.__name__} does not support LoRA yet." + + if supports_multimodal(self.model): + logger.warning("Regarding multimodal models, vLLM currently " + "only supports adding LoRA to language model.") + # It's necessary to distinguish between the max_position_embeddings + # of VLMs and LLMs. + if hasattr(self.model.config, "max_position_embeddings"): + max_pos_embeddings = self.model.config.max_position_embeddings + else: + max_pos_embeddings = ( + self.model.config.text_config.max_position_embeddings) + + self.lora_manager = LRUCacheWorkerLoRAManager( + self.scheduler_config.max_num_seqs, + self.scheduler_config.max_num_batched_tokens, + self.vocab_size, + self.lora_config, + self.device, + self.model.embedding_modules, + self.model.embedding_padding_modules, + max_position_embeddings=max_pos_embeddings, + ) + self.model = self.lora_manager.create_lora_manager(self.model) + + if self.prompt_adapter_config: + self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager( + self.scheduler_config.max_num_seqs, + self.scheduler_config.max_num_batched_tokens, self.device, + self.prompt_adapter_config) + self.model = ( + self.prompt_adapter_manager.create_prompt_adapter_manager( + self.model)) + + if self.kv_cache_dtype == "fp8" and is_hip(): + # Currently only ROCm accepts kv-cache scaling factors + # via quantization_param_path and this will be deprecated + # in the future. + if self.model_config.quantization_param_path is not None: + if callable(getattr(self.model, "load_kv_cache_scales", None)): + warnings.warn( + "Loading kv cache scaling factor from JSON is " + "deprecated and will be removed. Please include " + "kv cache scaling factors in the model checkpoint.", + FutureWarning, + stacklevel=2) + self.model.load_kv_cache_scales( + self.model_config.quantization_param_path) + logger.info("Loaded KV cache scaling factors from %s", + self.model_config.quantization_param_path) + else: + raise RuntimeError( + "Using FP8 KV cache and scaling factors provided but " + "model %s does not support loading scaling factors.", + self.model.__class__) + else: + logger.warning( + "Using FP8 KV cache but no scaling factors " + "provided. Defaulting to scaling factors of 1.0. " + "This may lead to less accurate results!") + + if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \ + and supports_dynamo(): + from vllm.plugins import get_torch_compile_backend + backend = get_torch_compile_backend() or "eager" + self.model = torch.compile( + self.model, + fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE, + backend=backend) + + def save_sharded_state( + self, + path: str, + pattern: Optional[str] = None, + max_size: Optional[int] = None, + ) -> None: + from vllm.model_executor.model_loader.loader import ShardedStateLoader + ShardedStateLoader.save_model( + self.model, + path, + pattern=pattern, + max_size=max_size, + ) + + def save_tensorized_model( + self, + tensorizer_config: TensorizerConfig, + ) -> None: + from vllm.model_executor.model_loader.loader import TensorizerLoader + TensorizerLoader.save_model( + self.model, + tensorizer_config=tensorizer_config, + ) + + def get_max_block_per_batch(self) -> int: + block_size = self.block_size + return (self.max_seq_len_to_capture + block_size - 1) // block_size + + def _prepare_model_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + finished_requests_ids: Optional[List[str]] = None + ) -> TModelInputForGPU: + """Helper method to prepare the model input based on a given sequence + group. Prepares metadata needed for the base model forward pass but not + metadata for possible additional steps, e.g., sampling. + + The API assumes seq_group_metadata_list is sorted by prefill -> decode. + + The result tensors and data structure also batches input in prefill + -> decode order. For example, + + - input_tokens[:num_prefill_tokens] contains prefill tokens. + - input_tokens[num_prefill_tokens:] contains decode tokens. + + If cuda graph is required, this API automatically pads inputs. + """ + builder = self._builder_cls(weakref.proxy(self), finished_requests_ids) + for seq_group_metadata in seq_group_metadata_list: + builder.add_seq_group(seq_group_metadata) + + builder.reset_cached_inter_data() + + return builder.build() # type: ignore + + @torch.inference_mode() + def profile_run(self) -> None: + # Enable top-k sampling to reflect the accurate memory usage. + sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1) + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens + max_num_seqs = self.scheduler_config.max_num_seqs + # This represents the maximum number of different requests + # that will have unique loras, an therefore the max amount of memory + # consumption create dummy lora request copies from the lora request + # passed in, which contains a lora from the lora warmup path. + dummy_lora_requests: List[LoRARequest] = [] + dummy_lora_requests_per_seq: List[LoRARequest] = [] + if self.lora_config: + assert self.lora_manager is not None + with self.lora_manager.dummy_lora_cache(): + for idx in range(self.lora_config.max_loras): + lora_id = idx + 1 + dummy_lora_request = LoRARequest( + lora_name=f"warmup_{lora_id}", + lora_int_id=lora_id, + lora_path="/not/a/real/path", + ) + self.lora_manager.add_dummy_lora(dummy_lora_request, + rank=LORA_WARMUP_RANK) + dummy_lora_requests.append(dummy_lora_request) + dummy_lora_requests_per_seq = [ + dummy_lora_requests[idx % len(dummy_lora_requests)] + for idx in range(max_num_seqs) + ] + + # Profile memory usage with max_num_sequences sequences and the total + # number of tokens equal to max_num_batched_tokens. + seqs: List[SequenceGroupMetadata] = [] + # Additional GPU memory may be needed for multi-modal encoding, which + # needs to be accounted for when calculating the GPU blocks for + # vLLM blocker manager. + # To exercise the worst scenario for GPU memory consumption, + # the number of seqs (batch_size) is chosen to maximize the number + # of images processed. + + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens( + self.model_config) + if max_mm_tokens > 0: + max_num_seqs_orig = max_num_seqs + max_num_seqs = min(max_num_seqs, + max_num_batched_tokens // max_mm_tokens) + if max_num_seqs < 1: + expr = (f"min({max_num_seqs_orig}, " + f"{max_num_batched_tokens} // {max_mm_tokens})") + logger.warning( + "Computed max_num_seqs (%s) to be less than 1. " + "Setting it to the minimum value of 1.", expr) + max_num_seqs = 1 + + batch_size = 0 + for group_id in range(max_num_seqs): + seq_len = (max_num_batched_tokens // max_num_seqs + + (group_id < max_num_batched_tokens % max_num_seqs)) + batch_size += seq_len + + seq_data, dummy_multi_modal_data = self.input_registry \ + .dummy_data_for_profiling(self.model_config, + seq_len, + self.mm_registry) + + seq = SequenceGroupMetadata( + request_id=str(group_id), + is_prompt=True, + seq_data={group_id: seq_data}, + sampling_params=sampling_params, + block_tables=None, + lora_request=dummy_lora_requests_per_seq[group_id] + if dummy_lora_requests_per_seq else None, + multi_modal_data=dummy_multi_modal_data, + ) + seqs.append(seq) + + # Run the model with the dummy inputs. + num_layers = self.model_config.get_num_layers(self.parallel_config) + # use an empty tensor instead of `None`` to force Dynamo to pass + # it by reference, rather by specializing on the value ``None``. + # the `dtype` argument does not matter, and we use `float32` as + # a placeholder (it has wide hardware support). + # it is important to create tensors inside the loop, rather than + # multiplying the list, to avoid Dynamo from treating them as + # tensor aliasing. + kv_caches = [ + torch.tensor([], dtype=torch.float32, device=self.device) + for _ in range(num_layers) + ] + finished_requests_ids = [seq.request_id for seq in seqs] + model_input = self.prepare_model_input( + seqs, finished_requests_ids=finished_requests_ids) + intermediate_tensors = None + if not get_pp_group().is_first_rank: + intermediate_tensors = self.model.make_empty_intermediate_tensors( + batch_size=batch_size, + dtype=self.model_config.dtype, + device=self.device) + + graph_batch_size = self.max_batchsize_to_capture + batch_size_capture_list = [ + bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size + ] + if self.model_config.enforce_eager: + batch_size_capture_list = [] + with set_compile_context(batch_size_capture_list): + self.execute_model(model_input, kv_caches, intermediate_tensors) + torch.cuda.synchronize() + return + + def remove_all_loras(self): + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + self.lora_manager.remove_all_adapters() + + def set_active_loras(self, lora_requests: Set[LoRARequest], + lora_mapping: LoRAMapping) -> None: + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + self.lora_manager.set_active_adapters(lora_requests, lora_mapping) + + def add_lora(self, lora_request: LoRARequest) -> bool: + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + return self.lora_manager.add_adapter(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + return self.lora_manager.remove_adapter(lora_id) + + def pin_lora(self, lora_id: int) -> bool: + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + return self.lora_manager.pin_adapter(lora_id) + + def list_loras(self) -> Set[int]: + if not self.lora_manager: + raise RuntimeError("LoRA is not enabled.") + return self.lora_manager.list_adapters() + + def remove_all_prompt_adapters(self): + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + self.prompt_adapter_manager.remove_all_adapters() + + def set_active_prompt_adapters( + self, prompt_adapter_requests: Set[PromptAdapterRequest], + prompt_adapter_mapping: PromptAdapterMapping) -> None: + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + self.prompt_adapter_manager.set_active_adapters( + prompt_adapter_requests, prompt_adapter_mapping) + + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + return self.prompt_adapter_manager.add_adapter(prompt_adapter_request) + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + return self.prompt_adapter_manager.remove_adapter(prompt_adapter_id) + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + return self.prompt_adapter_manager.pin_adapter(prompt_adapter_id) + + def list_prompt_adapters(self) -> Set[int]: + if not self.prompt_adapter_manager: + raise RuntimeError("PromptAdapter is not enabled.") + return self.prompt_adapter_manager.list_adapters() + + @property + def model_is_mrope(self) -> bool: + """Detect if the model has "mrope" rope_scaling type. + mrope requires keep "rope_deltas" between prompt and decoding phases.""" + rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {}) + if rope_scaling is None: + return False + return rope_scaling.get("type", None) == "mrope" + + @torch.inference_mode() + def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None: + """Cuda graph capture a model. + + Note that CUDA graph's performance gain is negligible if number + of batched tokens are larger than 200. And since CUDA graph + requires fixed sized tensors, supporting large/variable batch + size requires high GPU memory overhead. Thus, vLLM only captures + decoding requests. Mixed batch (chunked prefill + decoding) or + prefill requests are not captured. + + Since it is used for decoding-only, it assumes there's only 1 token + per sequence in the batch. + """ + assert not self.model_config.enforce_eager + logger.info("Capturing the model for CUDA graphs. This may lead to " + "unexpected consequences if the model is not static. To " + "run the model in eager mode, set 'enforce_eager=True' or " + "use '--enforce-eager' in the CLI.") + logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. " + "If you are running out of memory, consider decreasing " + "`gpu_memory_utilization` or enforcing eager mode. " + "You can also reduce the `max_num_seqs` as needed " + "to decrease memory usage.") + start_time = time.perf_counter() + + # Prepare dummy inputs. These will be reused for all batch sizes. + max_batch_size = self.max_batchsize_to_capture + input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda() + input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda() + if self.model_is_mrope: + input_positions = torch.tile(input_positions, (3, 1)) + # Prepare dummy previous_hidden_states only if needed by the model. + # This is used by draft models such as EAGLE. + previous_hidden_states = None + if "previous_hidden_states" in inspect.signature( + self.model.forward).parameters: + previous_hidden_states = torch.empty( + [max_batch_size, + self.model_config.get_hidden_size()], + dtype=self.model_config.dtype, + device=self.device) + + intermediate_inputs = None + if not get_pp_group().is_first_rank: + intermediate_inputs = self.model.make_empty_intermediate_tensors( + batch_size=max_batch_size, + dtype=self.model_config.dtype, + device=self.device) + + # Prepare buffer for outputs. These will be reused for all batch sizes. + # It will be filled after the first graph capture. + hidden_or_intermediate_states: List[Optional[torch.Tensor]] = [ + None + ] * self.parallel_config.pipeline_parallel_size + + graph_batch_size = self.max_batchsize_to_capture + batch_size_capture_list = [ + bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size + ] + + with self.attn_state.graph_capture( + max_batch_size), graph_capture() as graph_capture_context: + # NOTE: Capturing the largest batch size first may help reduce the + # memory usage of CUDA graph. + for virtual_engine in range( + self.parallel_config.pipeline_parallel_size): + for batch_size in reversed(batch_size_capture_list): + attn_metadata = ( + self.attn_state.graph_capture_get_metadata_for_batch( + batch_size, + is_encoder_decoder_model=self.model_config. + is_encoder_decoder_model)) + + if self.lora_config: + lora_mapping = LoRAMapping( + **dict(index_mapping=[0] * batch_size, + prompt_mapping=[0] * batch_size, + is_prefill=False)) + self.set_active_loras(set(), lora_mapping) + + if self.prompt_adapter_config: + prompt_adapter_mapping = PromptAdapterMapping( + [-1] * batch_size, + [-1] * batch_size, + ) + self.set_active_prompt_adapters( + set(), prompt_adapter_mapping) + graph_runner = CUDAGraphRunner( + self.model, self.attn_backend.get_name(), + self.attn_state.graph_clone(batch_size), + self.model_config.is_encoder_decoder_model) + + capture_inputs = { + "input_ids": + input_tokens[:batch_size], + "positions": + input_positions[..., :batch_size], + "hidden_or_intermediate_states": + hidden_or_intermediate_states[ + virtual_engine] # type: ignore + [:batch_size] + if hidden_or_intermediate_states[virtual_engine] + is not None else None, + "intermediate_inputs": + intermediate_inputs[:batch_size] + if intermediate_inputs is not None else None, + "kv_caches": + kv_caches[virtual_engine], + "attn_metadata": + attn_metadata, + "memory_pool": + self.graph_memory_pool, + "stream": + graph_capture_context.stream + } + if previous_hidden_states is not None: + capture_inputs[ + "previous_hidden_states"] = previous_hidden_states[: + batch_size] + + if self.has_inner_state: + # Only used by Mamba-based models CUDA graph atm (Jamba) + capture_inputs.update({ + "seqlen_agnostic_capture_inputs": + self.model.get_seqlen_agnostic_capture_inputs( + batch_size) + }) + if self.model_config.is_encoder_decoder_model: + # add the additional inputs to capture for + # encoder-decoder models. + self._update_inputs_to_capture_for_enc_dec_model( + capture_inputs) + + with set_forward_context(attn_metadata): + graph_runner.capture(**capture_inputs) + self.graph_memory_pool = graph_runner.graph.pool() + self.graph_runners[virtual_engine][batch_size] = ( + graph_runner) + + end_time = time.perf_counter() + elapsed_time = end_time - start_time + # This usually takes < 10 seconds. + logger.info("Graph capturing finished in %.0f secs.", elapsed_time) + + def _update_inputs_to_capture_for_enc_dec_model(self, + capture_inputs: Dict[str, + Any]): + """ + Updates the set of input tensors needed for CUDA graph capture in an + encoder-decoder model. + + This method modifies the provided `capture_inputs` dictionary by + adding tensors specific to encoder-decoder specific models that + need to be captured for CUDA Graph replay. + """ + # During the decode phase encoder_input_ids and encoder_positions are + # unset. Do the same thing for graph capture. + capture_inputs["encoder_input_ids"] = torch.tensor( + [], dtype=torch.long).cuda() + capture_inputs["encoder_positions"] = torch.tensor( + [], dtype=torch.long).cuda() + + @property + def vocab_size(self) -> int: + return self.model_config.get_vocab_size() + + +class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): + """ + GPU model runner with sampling step. + """ + _model_input_cls: Type[ModelInputForGPUWithSamplingMetadata] = ( + ModelInputForGPUWithSamplingMetadata) + _builder_cls: Type[ModelInputForGPUBuilder] = ModelInputForGPUBuilder + + def make_model_input_from_broadcasted_tensor_dict( + self, + tensor_dict: Dict[str, Any], + ) -> ModelInputForGPUWithSamplingMetadata: + model_input = \ + ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + ) + return model_input + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None, + ) -> ModelInputForGPUWithSamplingMetadata: + """Prepare the model input based on a given sequence group, including + metadata for the sampling step. + + The API assumes seq_group_metadata_list is sorted by prefill -> decode. + + The result tensors and data structure also batches input in prefill + -> decode order. For example, + + - input_tokens[:num_prefill_tokens] contains prefill tokens. + - input_tokens[num_prefill_tokens:] contains decode tokens. + + If cuda graph is required, this API automatically pads inputs. + """ + model_input = self._prepare_model_input_tensors( + seq_group_metadata_list, finished_requests_ids) + if get_pp_group().is_last_rank: + # Sampling metadata is only required for the final pp group + generators = self.get_generators(finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, model_input.seq_lens, + model_input.query_lens, self.device, self.pin_memory, + generators, self.sampling_metadata_cache) + else: + sampling_metadata = None + is_prompt = (seq_group_metadata_list[0].is_prompt + if seq_group_metadata_list else None) + return dataclasses.replace(model_input, + sampling_metadata=sampling_metadata, + is_prompt=is_prompt, + virtual_engine=virtual_engine) + + @torch.inference_mode() + # @dump_input_when_exception(exclude_args=[0], exclude_kwargs=["self"]) + def execute_model( + self, + model_input: ModelInputForGPUWithSamplingMetadata, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]: + if num_steps > 1: + raise ValueError("num_steps > 1 is not supported in ModelRunner") + + if self.lora_config: + assert model_input.lora_requests is not None + assert model_input.lora_mapping is not None + self.set_active_loras(model_input.lora_requests, + model_input.lora_mapping) + + if self.prompt_adapter_config: + assert model_input.prompt_adapter_requests is not None + assert model_input.prompt_adapter_mapping is not None + self.set_active_prompt_adapters( + model_input.prompt_adapter_requests, + model_input.prompt_adapter_mapping) + + self.attn_state.begin_forward(model_input) + + # Currently cuda graph is only supported by the decode phase. + assert model_input.attn_metadata is not None + prefill_meta = model_input.attn_metadata.prefill_metadata + decode_meta = model_input.attn_metadata.decode_metadata + # TODO(andoorve): We can remove this once all + # virtual engines share the same kv cache. + virtual_engine = model_input.virtual_engine + if prefill_meta is None and decode_meta.use_cuda_graph: + assert model_input.input_tokens is not None + graph_batch_size = model_input.input_tokens.shape[0] + model_executable = self.graph_runners[virtual_engine][ + graph_batch_size] + else: + model_executable = self.model + + multi_modal_kwargs = model_input.multi_modal_kwargs or {} + seqlen_agnostic_kwargs = { + "finished_requests_ids": model_input.finished_requests_ids, + "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids, + } if self.has_inner_state else {} + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_start = torch.cuda.Event(enable_timing=True) + model_forward_end = torch.cuda.Event(enable_timing=True) + model_forward_start.record() + + with set_forward_context(model_input.attn_metadata): + hidden_or_intermediate_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(multi_modal_kwargs, + device=self.device), + **seqlen_agnostic_kwargs) + + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_end.record() + + # Compute the logits in the last pipeline stage. + if not get_pp_group().is_last_rank: + if (self.is_driver_worker + and hidden_or_intermediate_states is not None + and isinstance(hidden_or_intermediate_states, + IntermediateTensors) + and self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_end.synchronize() + model_forward_time = model_forward_start.elapsed_time( + model_forward_end) + orig_model_forward_time = 0.0 + if intermediate_tensors is not None: + orig_model_forward_time = intermediate_tensors.tensors.get( + "model_forward_time", torch.tensor(0.0)).item() + hidden_or_intermediate_states.tensors["model_forward_time"] = ( + torch.tensor(model_forward_time + orig_model_forward_time)) + return hidden_or_intermediate_states + + logits = self.model.compute_logits(hidden_or_intermediate_states, + model_input.sampling_metadata) + + if not self.is_driver_worker: + return [] + + if model_input.async_callback is not None: + model_input.async_callback() + + # Sample the next token. + output: SamplerOutput = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time + and output is not None): + model_forward_end.synchronize() + model_forward_time = model_forward_start.elapsed_time( + model_forward_end) + orig_model_forward_time = 0.0 + if intermediate_tensors is not None: + orig_model_forward_time = intermediate_tensors.tensors.get( + "model_forward_time", torch.tensor(0.0)).item() + # If there are multiple workers, we are still tracking the latency + # from the start time of the driver worker to the end time of the + # driver worker. The model forward time will then end up covering + # the communication time as well. + output.model_forward_time = (orig_model_forward_time + + model_forward_time) + + if self.return_hidden_states: + # we only need to pass hidden states of most recent token + assert model_input.sampling_metadata is not None + indices = model_input.sampling_metadata.selected_token_indices + if model_input.is_prompt: + hidden_states = hidden_or_intermediate_states.index_select( + 0, indices) + output.prefill_hidden_states = hidden_or_intermediate_states + elif decode_meta.use_cuda_graph: + hidden_states = hidden_or_intermediate_states[:len(indices)] + else: + hidden_states = hidden_or_intermediate_states + + output.hidden_states = hidden_states + + return [output] + + +class CUDAGraphRunner: + + def __init__(self, model: nn.Module, backend_name: str, + attn_state: AttentionState, is_encoder_decoder_model: bool): + self.model = model + self.backend_name = backend_name + self.attn_state = attn_state + + self.input_buffers: Dict[str, torch.Tensor] = {} + self.output_buffers: Dict[str, torch.Tensor] = {} + + self._graph: Optional[torch.cuda.CUDAGraph] = None + self._is_encoder_decoder_model = is_encoder_decoder_model + + @property + def graph(self): + assert self._graph is not None + return self._graph + + def capture( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_or_intermediate_states: Optional[Union[IntermediateTensors, + torch.Tensor]], + intermediate_inputs: Optional[IntermediateTensors], + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + memory_pool: Optional[Tuple[int, int]], + stream: torch.cuda.Stream, + **kwargs, + ) -> Union[torch.Tensor, IntermediateTensors]: + assert self._graph is None + # Run the model a few times without capturing the graph. + # This is to make sure that the captured graph does not include the + # kernel launches for initial benchmarking (e.g., Triton autotune). + # Note one iteration is not enough for torch.jit.script + for _ in range(_NUM_WARMUP_ITERS): + self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_inputs, + **kwargs, + ) + # Wait for the warm up operations to finish before proceeding with + # Graph Capture. + torch.cuda.synchronize() + # Capture the graph. + self._graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream): + output_hidden_or_intermediate_states = self.model( + input_ids=input_ids, + positions=positions, + kv_caches=kv_caches, + attn_metadata=attn_metadata, + intermediate_tensors=intermediate_inputs, + **kwargs, + ) + if hidden_or_intermediate_states is not None: + if get_pp_group().is_last_rank: + hidden_or_intermediate_states.copy_( + output_hidden_or_intermediate_states) + else: + for key in hidden_or_intermediate_states.tensors: + hidden_or_intermediate_states[key].copy_( + output_hidden_or_intermediate_states[key]) + else: + hidden_or_intermediate_states = ( + output_hidden_or_intermediate_states) + + del output_hidden_or_intermediate_states + # make sure `output_hidden_states` is deleted + # in the graph's memory pool + gc.collect() + torch.cuda.synchronize() + + # Save the input and output buffers. + self.input_buffers = { + "input_ids": + input_ids, + "positions": + positions, + "kv_caches": + kv_caches, + **self.attn_state.get_graph_input_buffers( + attn_metadata, self._is_encoder_decoder_model), + **kwargs, + } + if intermediate_inputs is not None: + self.input_buffers.update(intermediate_inputs.tensors) + if get_pp_group().is_last_rank: + self.output_buffers = { + "hidden_states": hidden_or_intermediate_states + } + else: + self.output_buffers = hidden_or_intermediate_states + return hidden_or_intermediate_states + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[torch.Tensor], + attn_metadata: AttentionMetadata, + intermediate_tensors: Optional[IntermediateTensors], + **kwargs, + ) -> torch.Tensor: + # KV caches are fixed tensors, so we don't need to copy them. + del kv_caches + + # Copy the input tensors to the input buffers. + self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True) + self.input_buffers["positions"].copy_(positions, non_blocking=True) + + if self.backend_name != "placeholder-attn": + self.input_buffers["slot_mapping"].copy_( + attn_metadata.slot_mapping, non_blocking=True) + + self.attn_state.prepare_graph_input_buffers( + self.input_buffers, attn_metadata, self._is_encoder_decoder_model) + + if "seqlen_agnostic_capture_inputs" in self.input_buffers: + self.model.copy_inputs_before_cuda_graphs(self.input_buffers, + **kwargs) + + if "previous_hidden_states" in self.input_buffers: + self.input_buffers["previous_hidden_states"].copy_( + kwargs["previous_hidden_states"], non_blocking=True) + + if intermediate_tensors is not None: + for key in intermediate_tensors.tensors: + if key != "model_execute_time" and key != "model_forward_time": + self.input_buffers[key].copy_(intermediate_tensors[key], + non_blocking=True) + if self._is_encoder_decoder_model: + self.input_buffers["encoder_input_ids"].copy_( + kwargs['encoder_input_ids'], non_blocking=True) + self.input_buffers["encoder_positions"].copy_( + kwargs['encoder_positions'], non_blocking=True) + + # Run the graph. + self.graph.replay() + # Return the output tensor. + if get_pp_group().is_last_rank: + return self.output_buffers["hidden_states"] + + return self.output_buffers + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + +def _get_graph_batch_size(batch_size: int) -> int: + """Returns the padded batch size given actual batch size. + + Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT, + 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT... + """ + if batch_size <= 2: + return batch_size + elif batch_size <= 4: + return 4 + else: + return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) // + _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT) + + +def _get_max_graph_batch_size(max_num_seqs: int) -> int: + """ + max_num_seqs: Maximum number of sequences in a batch. + _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture. + + pad the max_num_seqs if necessary by calling _get_graph_batch_size, + which will deal with some edge cases like 1, 2, 4. + + if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded size. + if not, it means the padded size is larger than the largest size in + _BATCH_SIZES_TO_CAPTURE, return the largest size in _BATCH_SIZES_TO_CAPTURE. + """ + padded_size = _get_graph_batch_size(max_num_seqs) + if padded_size in _BATCH_SIZES_TO_CAPTURE: + return padded_size + assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1] + return _BATCH_SIZES_TO_CAPTURE[-1] diff --git a/vllm/worker/model_runner_base.py b/vllm/worker/model_runner_base.py new file mode 100644 index 00000000..86883cf1 --- /dev/null +++ b/vllm/worker/model_runner_base.py @@ -0,0 +1,275 @@ +import dataclasses +import pickle +from abc import ABC, abstractmethod +from datetime import datetime +from functools import wraps +from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, + Optional, Type, TypeVar) + +import torch +from torch import is_tensor + +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.platforms import current_platform +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata + +if TYPE_CHECKING: + from vllm.attention import AttentionMetadata + from vllm.attention.backends.abstract import AttentionBackend + from vllm.model_executor import SamplingMetadata + +logger = init_logger(__name__) + +T = TypeVar('T', bound="BroadcastableModelInput") + + +def _add_attn_metadata_broadcastable_dict( + tensor_dict: Dict[str, Any], + attn_metadata: Optional["AttentionMetadata"]) -> None: + """ + Helper method to update tensor_dict with broadcastable + AttentionMetadata fields. + """ + if attn_metadata is not None: + tensor_dict.update(attn_metadata.asdict_zerocopy()) + + +def _init_attn_metadata_from_tensor_dict( + attn_backend: "AttentionBackend", + tensor_dict: Dict[str, Any], +) -> Dict[str, Any]: + """ + Helper method to initialize AttentionMetadata based on an + AttentionBackend and broadcastable AttentionMetadata fields. + """ + # Extract the fields used to create AttentionMetadata. + valid_attn_kwargs = {} + for field in dataclasses.fields(attn_backend.get_metadata_cls()): + val = tensor_dict.pop(field.name, None) + if val is not None: + valid_attn_kwargs[field.name] = val + + attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs) + tensor_dict["attn_metadata"] = attn_metadata + return tensor_dict + + +def _init_sampling_metadata_from_tensor_dict( # type: ignore + tensor_dict: Dict[str, Any]) -> Dict[str, Any]: + """ + Helper method to initialize SamplingMetadata based on broadcastable + SamplingMetadata fields. + """ + from vllm.model_executor import SamplingMetadata + + selected_token_indices = tensor_dict.pop("selected_token_indices", None) + # An empty SamplingMetadata to signal that the worker should skip + # sampling. + if selected_token_indices is not None: + tensor_dict["sampling_metadata"] = SamplingMetadata( + seq_groups=None, + selected_token_indices=selected_token_indices, + categorized_sample_indices=None, + num_prompts=0, + ) + return tensor_dict + + +def _add_sampling_metadata_broadcastable_dict( + tensor_dict: Dict[str, Any], + sampling_metadata: Optional["SamplingMetadata"]) -> None: + """ + Helper method to update tensor_dict with broadcastable + SamplingMetadata fields. + """ + if sampling_metadata is not None: + tensor_dict["selected_token_indices"] = ( + sampling_metadata.selected_token_indices) + + +def _init_frozen_model_input_from_tensor_dict( + frozen_model_input_cls: Type["ModelRunnerInputBase"], + tensor_dict: Dict[str, Any]) -> Dict[str, Any]: + """ + Helper method to initialize a frozen ModelInput based on broadcastable + """ + valid_tensor_kwargs = {} + for field in dataclasses.fields(frozen_model_input_cls): + val = tensor_dict.pop(field.name, None) + if val is not None: + valid_tensor_kwargs[field.name] = val + + frozen_model_input = frozen_model_input_cls(**valid_tensor_kwargs) + tensor_dict["frozen_model_input"] = frozen_model_input + return tensor_dict + + +def dump_input_when_exception(exclude_args: Optional[List[int]] = None, + exclude_kwargs: Optional[List[str]] = None): + + def _inner(func): + + @wraps(func) + def _wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as err: + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + filename = f"/tmp/err_{func.__name__}_input_{timestamp}.pkl" + logger.info("Writing input of failed execution to %s...", + filename) + with open(filename, "wb") as filep: + dumped_inputs = { + k: v + for k, v in kwargs.items() + if k not in (exclude_kwargs or []) + } + for i, arg in enumerate(args): + if i not in (exclude_args or []): + dumped_inputs[f"arg_{i}"] = arg + + # Only persist dtype and shape for kvcache tensors + # (can be way to big otherwise) + if (kv_caches := dumped_inputs.get("kv_caches")) \ + and isinstance(kv_caches, Iterable): + dumped_inputs["kv_caches"] = [(t.dtype, t.shape) + for t in kv_caches + if is_tensor(t)] + + try: + pickle.dump(dumped_inputs, filep) + except Exception as pickle_err: + logger.warning( + "Failed to pickle inputs of failed execution: %s", + str(pickle_err)) + raise type(err)(f"Error in model execution: " + f"{str(err)}") from err + + logger.info( + "Completed writing input of failed execution to %s.", + filename) + raise type(err)( + f"Error in model execution (input dumped to {filename}): " + f"{str(err)}") from err + + return _wrapper + + return _inner + + +class BroadcastableModelInput(ABC): + + @abstractmethod + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + """ + Extract broadcastable fields. Override for fields that require some + custom deserialization. + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def from_broadcasted_tensor_dict( + cls: Type[T], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> T: + """ + Pop fields from the given tensor_dict and populate a new instance of + BroadcastableModelInput. + """ + raise NotImplementedError + + +@dataclasses.dataclass(frozen=True) +class ModelRunnerInputBase(BroadcastableModelInput): + """Local inputs to each worker's model runner. May contain + device-specific data. Different worker backends may have different methods + of converting from the global ExecuteModelRequest produced by the LLM + engine to the worker-local ModelRunnerInputBase objects. + + Model runners that support multi-GPU execution should define a + ModelRunnerInputBase subclass, add their required fields, and specify how to + serialize/deserialize a ModelInput for broadcast between workers. + """ + pass + + +class ModelRunnerInputBuilderBase(ABC, Generic[T]): + """A builder to create ModelRunnerInputBase objects. + """ + + @abstractmethod + def add_seq_group(self, seq_group_metadata): + """TBA""" + raise NotImplementedError + + @abstractmethod + def build(self, *args, **kwargs) -> T: + """Build metadata with on-device tensors.""" + raise NotImplementedError + + +class ModelRunnerBase(ABC, Generic[T]): + """ + Model runner interface that abstracts a particular hardware and/or type of + model. Model execution may communicate data with model runners in other + processes, but it should not include control plane metadata communication. + + Each ModelRunnerBase subclass should define a corresponding + ModelRunnerInputBase subclass. + """ + + # Map of request_id -> generator used for seeded random sampling + generators: Dict[str, torch.Generator] = {} + + @abstractmethod + def make_model_input_from_broadcasted_tensor_dict( + self, + tensor_dict: Dict[str, Any], + ) -> T: + """ + Make an instance of a ModelRunnerInputBase from the broadcasted tensor + dict. + """ + raise NotImplementedError + + @abstractmethod + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None, + ) -> T: + """ + Prepare the inputs to ModelRunnerBase.execute_model from an execution + request. This method may move data to the worker's local device. It is + not allowed to communicate with other workers or devices. + """ + raise NotImplementedError + + @current_platform.inference_mode() + def execute_model( + self, + model_input: T, + kv_caches: Optional[List[torch.Tensor]], + intermediate_tensors: Optional[IntermediateTensors], + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + """ + Execute the model on the given input. + """ + raise NotImplementedError + + def get_generators(self, finished_request_ids: Optional[List[str]] = None): + """ + Return dict of per-request generators used for random sampling. + """ + + # Clean up generators from completed requests + if finished_request_ids: + for request_id in finished_request_ids: + self.generators.pop(request_id, None) + + return self.generators diff --git a/vllm/worker/multi_step_model_runner.py b/vllm/worker/multi_step_model_runner.py new file mode 100644 index 00000000..0cd0047b --- /dev/null +++ b/vllm/worker/multi_step_model_runner.py @@ -0,0 +1,901 @@ +import dataclasses +import functools +from dataclasses import dataclass, field +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, + Union) + +import torch + +from vllm.distributed import get_pp_group +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs, + SamplerOutput, + SamplingMetadata, get_logprobs, + get_pythonized_sample_results) +from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors, + Logprob, SequenceGroupMetadata, SequenceOutput) +from vllm.utils import PyObjectCache, async_tensor_h2d +from vllm.worker.model_runner import (GPUModelRunnerBase, + ModelInputForGPUWithSamplingMetadata) +from vllm.worker.model_runner_base import ( + BroadcastableModelInput, _init_attn_metadata_from_tensor_dict, + _init_frozen_model_input_from_tensor_dict, + _init_sampling_metadata_from_tensor_dict) + +from ..model_executor.model_loader.tensorizer import TensorizerConfig + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + +MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "rocm-flash-attn", "flashinfer"] +MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["flash-attn"] + +def _get_supported_attention_backends(chunked_prefill_enabled: bool) \ + -> List[str]: + if chunked_prefill_enabled: + return MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS + else: + return MULTI_STEP_ATTENTION_BACKENDS + + +def seq_output_builder(): + return SequenceOutput( + 0, 0, + {0: Logprob(logprob=float('inf'), rank=None, decoded_token=None)}) + + +def completion_seq_group_output_builder(): + return CompletionSequenceGroupOutput([], None) + + +# Used by pythonization to reduce python object allocations +class PythonizationCache: + + def __init__(self): + self.cached_seq_output = PyObjectCache(seq_output_builder) + self.cached_completion_seq_group_output = PyObjectCache( + completion_seq_group_output_builder) + + def reset(self): + self.cached_seq_output.reset() + self.cached_completion_seq_group_output.reset() + + +@dataclass +class ModelOutput: + """The output of a single model forward pass. + + The sampler_output_ready_event is set when the tensors in + sampler_output are ready (the model+sampler forward pass has + completed). We use the event to synchronize the GPU->CPU transfer, + which we want to only run when the data has been written to the + GPU tensors. Until the event is ready, the tensors in sampler_output + will have garbage data. + + There are two scenarios: + 1. The output tensors are ready and we can pythonize them immediately. + 2. The output tensors are not ready and we need to wait for the event to be + ready. + """ + sampler_output: SamplerOutput + sampler_output_ready_event: torch.cuda.Event + sampled_token_ids: Optional[torch.Tensor] = None + pythonized: bool = False + # On-device tensor containing the logprobs of each token. + logprobs: Optional["torch.Tensor"] = None + pythonization_cache: Optional[PythonizationCache] = None + + def pythonize(self, input_metadata: "StatefulModelInput", + copy_stream: torch.cuda.Stream, + pinned_sampled_token_buffer: torch.Tensor) -> None: + """Pythonize the output. Blocking.""" + if not self.pythonized: + self._pythonize_sampler_output(input_metadata, copy_stream, + pinned_sampled_token_buffer, True) + self.pythonized = True + + def maybe_pythonize(self, input_metadata: "StatefulModelInput", + copy_stream: torch.cuda.Stream, + pinned_sampled_token_buffer: torch.Tensor) -> None: + """Pythonize the output if ready, else return None. Non-blocking.""" + if not self.pythonized: + self.pythonized = self._pythonize_sampler_output( + input_metadata, copy_stream, pinned_sampled_token_buffer, + False) + + def _pythonize_sampler_output(self, input_metadata: "StatefulModelInput", + copy_stream: torch.cuda.Stream, + pinned_sampled_token_buffer: torch.Tensor, + blocking: bool) -> bool: + """ + If blocking is set, will block until the forward pass for the output is + ready and pythonize the output. Upon completing Pythonization, erases + self.logprobs (note that a non-blocking call that is performed when + the sampler output is not yet ready, will not erase self.logprobs.) + """ + assert self.sampled_token_ids is not None + if not blocking and not self.sampler_output_ready_event.query(): + return False + + if blocking: + self.sampler_output_ready_event.synchronize() + with torch.cuda.stream(copy_stream): + _pythonize_sampler_output(input_metadata, self.sampler_output, + pinned_sampled_token_buffer, + self.sampled_token_ids, self.logprobs, + self.pythonization_cache) + + # Erase the logprobs GPU-side tensor. + # Note that although _pythonize_sampler_output() runs in its + # own CUDA stream, nonetheless _pythonize_sampler_output() + # cannot return until Pythonization is complete; therefore + # we know that by the time the CPU reaches this point, + # `self.logprobs` is no longer needed. + self.logprobs = None + return True + + +@dataclass(frozen=False) +class StatefulModelInput(BroadcastableModelInput): + # actual frozen model input dataclass passed to _base_model_runner + frozen_model_input: Optional[ModelInputForGPUWithSamplingMetadata] = None + + # list of model outputs for each step, may not be all pythonized + cached_outputs: List[ModelOutput] = field(default_factory=list) + + # used to pass sampled token ids from the last step to the current step for + # TP workers. Used to append to end of outputs and used by advance_step + last_sampled_token_ids: Optional[torch.Tensor] = None + current_step: int = 0 + is_multi_step: bool = True + is_last_step: bool = False + is_first_multi_step: bool = False + base_output_proc_callback: Optional[Callable] = None + # ping-pong data structures for multi-step to wait on the previous step + step_cuda_events: List[torch.cuda.Event] = field( + default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2) + num_seqs: int = -1 + num_queries: int = -1 + num_single_step_prefills: int = 0 + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + assert self.frozen_model_input is not None + tensor_dict = self.frozen_model_input.as_broadcastable_tensor_dict() + new_tensor_dict = { + 'last_sampled_token_ids': self.last_sampled_token_ids, + 'current_step': self.current_step, + 'is_multi_step': self.is_multi_step, + 'is_last_step': self.is_last_step, + 'is_first_multi_step': self.is_first_multi_step, + 'num_seqs': self.num_seqs, + 'num_queries': self.num_queries, + 'num_single_step_prefills': self.num_single_step_prefills, + } + tensor_dict.update(new_tensor_dict) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "StatefulModelInput": + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict) + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + tensor_dict = _init_frozen_model_input_from_tensor_dict( + ModelInputForGPUWithSamplingMetadata, tensor_dict) + + return cls(**tensor_dict) + + def record_step_event(self, current_stream: torch.cuda.Stream): + # record the event for the current step so that the next step can sync + # on it. We modulo by 2 to keep the events in a circular buffer and + # support any attn backends that may be supported in the future. ie + # Flashinfer would want two DecodeWrappers to overlap the CPU and GPU. + self.step_cuda_events[self.current_step & 1] = \ + torch.cuda.Event(blocking=True) + self.step_cuda_events[self.current_step & 1].record(current_stream) + + def wait_previous_step(self): + # These cuda events are an explicit synchronization to ensure that + # advance_step() (for other attn backends that may be supported in the + # future) do not clobber any data structures that is also used by any + # enqueued forwards steps. For distributed case, only a single event is + # needed, but for single GPU case, since we can let the CPU run much + # further ahead, two events allow us to overlap the advance_step with + # the previous forward (ie using two DecodeWrappers for flashinfer + # backend) + self.step_cuda_events[(self.current_step + 1) & 1].wait() + + def add_sampler_output(self, + sampler_output: SamplerOutput, + sampled_token_ids: Optional[torch.Tensor] = None): + self.cached_outputs.append( + ModelOutput(sampler_output=sampler_output, + sampler_output_ready_event=None, + sampled_token_ids=sampled_token_ids, + pythonized=False)) + + def maybe_advance_sampling_metadata(self, device: str, pin_memory: bool): + """ + sampling_metadata.selected_token_indices is constructed for the + first-step in Multi-Step. However, when chunked-prefill is enabled with + multi-step, the scheduled prompts are fully processed in the + first-step and are processed as decodes in the rest of the steps. + This function updates the sampling_metadata.selected_token_indices + to account for this conversion. + + Example: + Let 2 prompts and 2 decodes be scheduled together. Let the + num-tokens to process for the 2 prompts be 5 and 8 respectively. + + In that case, sampling_metadata.sampled_token_indices will be, + [4, 12, 13, 14] as it is constructed for the first-step in + multi-step. + However, the prompts turns to decodes after the first-step + and the num-tokens for the previously-prompt sequences will + be 1 and 1 as they are decodes now. The self.sampled_token_indices + must be updated to [0,1,2,3]. + """ + assert self.current_step == 1 and self.num_single_step_prefills > 0 + if not get_pp_group().is_last_rank: + return + + assert self.frozen_model_input is not None + assert self.frozen_model_input.sampling_metadata is not None + self.frozen_model_input.sampling_metadata.selected_token_indices = \ + async_tensor_h2d(list(range(self.num_queries)), + dtype=torch.long, + target_device=device, + pin_memory=pin_memory) + + def maybe_advance_frozen_model_input(self, device: str, pin_memory: bool): + """ + Advancing the datastructures of StatefulModelInput::frozen_model_input + is only required when prefills are scheduled with decodes to run in + multi-step. This advancement/correction is required to account for + the conversion of Prefills to Decodes after the first multi-step. + """ + if self.current_step != 1 or self.num_single_step_prefills == 0: + return + + assert self.frozen_model_input is not None + fmi = self.frozen_model_input + + # Truncate input_tokens + assert fmi.input_tokens is not None + assert fmi.input_tokens.shape[0] >= self.num_seqs + fmi_new_input_tokens: torch.Tensor = fmi.input_tokens[:self.num_seqs] + + # Update frozen_model_input::input_positons. + assert fmi.input_positions is not None + assert fmi.input_positions.shape[0] >= self.num_seqs + fmi_new_input_positions: torch.Tensor = fmi.input_positions[:self. + num_seqs] + + # Assert unsupported + assert fmi.lora_mapping is None + assert fmi.lora_requests is not None + assert len(fmi.lora_requests) == 0 + assert fmi.attn_metadata is not None + assert fmi.prompt_adapter_mapping is None + assert fmi.prompt_adapter_requests is not None + assert len(fmi.prompt_adapter_requests) == 0 + assert fmi.multi_modal_kwargs is not None + assert len(fmi.multi_modal_kwargs) == 0 + + self.frozen_model_input = dataclasses.replace( + self.frozen_model_input, + input_tokens=fmi_new_input_tokens, + input_positions=fmi_new_input_positions) + + self.maybe_advance_sampling_metadata(device, pin_memory) + + +# MutableModelInputForGPUWithMultiStepMetadata is not subclass of +# ModelInputForGPU but it wraps the actual input dataclass and adds multi-step +# metadata +# mypy: disable-error-code=type-var +class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]): + # mypy: enable-error-code=type-var + + def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Check attention backend support. + supported_attention_backends: List[str] = \ + _get_supported_attention_backends( + self.scheduler_config.chunked_prefill_enabled) + if self.attn_backend.get_name() not in supported_attention_backends: + ms_config_str: str = "Multi-Step + Chunked-Prefill" \ + if self.scheduler_config.chunked_prefill_enabled \ + else "Multi-Step" + raise ValueError( + f"{ms_config_str} not supported for attention backend: " + f"{self.attn_backend.get_name()}. Set VLLM_ATTENTION_BACKEND " + f"to a value from {supported_attention_backends}.") + + # uses the base model runner to execute the model and wraps it with + # multi-step logic + self._base_model_runner: GPUModelRunnerBase = base_model_runner + + self.is_multi_step = self.scheduler_config.is_multi_step + self.pinned_sampled_token_ids: Optional[torch.Tensor] = None + + # Using the PythonizationCache in Pipeline-Parallel clobbers the + # SequenceOutput and CompletionSequenceGroupOutput object. + # When cache-reset happens at the last step of a multi-step + # execution, there may be other on-going single-step/multi-step + # executions. The current caching implementation does not check + # for this. + self.pythonization_cache = PythonizationCache() \ + if self.parallel_config.pipeline_parallel_size == 1 else None + + @functools.cached_property + def _copy_stream(self): + # used to copy tensors from GPU to CPU asynchronously + return torch.cuda.Stream() + + def make_model_input_from_broadcasted_tensor_dict( + self, tensor_dict: Dict[str, Any]) -> StatefulModelInput: + model_input = (StatefulModelInput.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + )) + return model_input + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> StatefulModelInput: + frozen_model_input: ModelInputForGPUWithSamplingMetadata = \ + self._base_model_runner.prepare_model_input( + seq_group_metadata_list, + virtual_engine, + finished_requests_ids) + + assert frozen_model_input.query_lens is not None + assert frozen_model_input.seq_lens is not None + assert frozen_model_input.attn_metadata is not None + num_queries = len(frozen_model_input.query_lens) + num_seqs = len(frozen_model_input.seq_lens) + num_single_step_prefills = frozen_model_input.attn_metadata.num_prefills + + model_input = StatefulModelInput( + frozen_model_input=frozen_model_input, + num_seqs=num_seqs, + num_queries=num_queries, + num_single_step_prefills=num_single_step_prefills) + + return model_input + + def _async_process_outputs(self, model_input: StatefulModelInput, + output_proc_callback: Callable): + # Proceed with pythonization and output_proc in order. + # Stop on the first one that fails to pythonize + output_proc_callback() + + cont = True + for step_num, model_output in enumerate(model_input.cached_outputs): + if not model_output.pythonized: + model_output.maybe_pythonize(model_input, self._copy_stream, + self.pinned_sampled_token_ids) + if model_output.pythonized: + ctx = output_proc_callback.keywords["ctx"] + ctx.append_output( + outputs=[model_output.sampler_output], + seq_group_metadata_list=ctx.seq_group_metadata_list, + scheduler_outputs=ctx.scheduler_outputs, + is_async=False, + is_last_step=False, + is_first_step_output=step_num == 0) + + output_proc_callback() + else: + cont = False + + if not cont: + break + + def _final_process_outputs(self, model_input: StatefulModelInput, + output_proc_callback: Optional[Callable]): + assert model_input.frozen_model_input is not None + + has_async_callback = output_proc_callback is not None + + outputs = [] + for step_num, output in enumerate(model_input.cached_outputs): + is_last_step = step_num == len(model_input.cached_outputs) - 1 + + # For non-async case: + # -- We simply add the outputs + # For async case: + # -- Invoke callback, pythonize, add to callback queue and repeat + # -- For last output, just add to callback queue + if has_async_callback: + assert output_proc_callback is not None + + # Invoke callback before pythonize (to overlap with GPU) + output_proc_callback() + + # Pythonize + if not output.pythonized: + output.pythonize(model_input, self._copy_stream, + self.pinned_sampled_token_ids) + + # For non last step, add to callback queue to chain + # callbacks=>pythonize pairs (for GPU overlap) + if not is_last_step: + ctx = output_proc_callback.keywords[ # type: ignore + "ctx"] # type: ignore + ctx.append_output( + outputs=[output.sampler_output], + seq_group_metadata_list=ctx. + seq_group_metadata_list, + scheduler_outputs=ctx.scheduler_outputs, + is_async=False, + is_last_step=False, + is_first_step_output=step_num == 0) + else: + outputs.append(output.sampler_output) + else: + output.pythonize(model_input, self._copy_stream, + self.pinned_sampled_token_ids) + outputs.append(output.sampler_output) + + return outputs + + @torch.inference_mode() + def execute_model( + self, + model_input: StatefulModelInput, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]: + """ + Execute the model for a single step and update multi-step + metadata + """ + assert num_steps == 1, "MultiStepModelRunner only supports num_steps=1" + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + + # path for warm up runs + if not model_input.is_multi_step: + return self._base_model_runner.execute_model( + frozen_model_input, kv_caches, intermediate_tensors, num_steps) + + # make sure we skip the sampler on the lask rank and only pythonize + # if CPU is ahead. + if self.is_driver_worker and get_pp_group().is_last_rank: + if self.pinned_sampled_token_ids is None: + self.pinned_sampled_token_ids = torch.zeros( + (self.scheduler_config.max_num_seqs, 1), + dtype=torch.long, + device="cpu", + pin_memory=True) + + self._base_model_runner.model.sampler.include_gpu_probs_tensor = ( + True) + if frozen_model_input.sampling_metadata: + frozen_model_input.sampling_metadata.skip_sampler_cpu_output = ( + True) + + # some pre-execute model logic for multi-step: + # - if it's the first step, we need to reset the sampling tensors + # - if it's not the first step, we need to advance the step using the + # appended sampler output from last iteration + # - also maybe pythonize if CPU is ahead of GPU + + current_stream = torch.cuda.current_stream() + if not model_input.is_first_multi_step: + # Explicitly block on the previous step's forward to make sure we + # don't clobber any GPU tensors still in use. + # This is not needed for flashattn backend, but for other attn + # backends such as flashinfer that performs extra CPU operations on + # input metadata we may need to synchronize any CPU operations that + # might clobber enqueued forwards. (prevents CPU from running too + # far ahead if needed) + model_input.wait_previous_step() + model_input = self._advance_step( + model_input, model_input.cached_outputs[-1].sampler_output) + + # frozen_model_input may have been updated + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + + if model_input.base_output_proc_callback is None: + assert frozen_model_input is not None + model_input.base_output_proc_callback = \ + frozen_model_input.async_callback + + if frozen_model_input.async_callback is not None: + assert model_input.base_output_proc_callback is not None + async_callback = functools.partial( + self._async_process_outputs, + model_input=model_input, + output_proc_callback=model_input.base_output_proc_callback) + + model_input.frozen_model_input = dataclasses.replace( # type: ignore + model_input.frozen_model_input, + async_callback=async_callback) + # Update the local instance + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + + # Execute the model + output = self._base_model_runner.execute_model(frozen_model_input, + kv_caches, + intermediate_tensors, + num_steps=1) + + # record the event for the current step so that the next step can sync + model_input.record_step_event(current_stream) + + if get_pp_group().is_last_rank and self.is_driver_worker: + assert len( + output + ) == 1, "MultiStepModelRunner requires single-step base_models" + + # event for the pythonization so that we only pythonize if the + # tensors are ready. May be able to be combined with the step event + output_ready_event = torch.cuda.Event() + output_ready_event.record(current_stream) + if self.parallel_config.pipeline_parallel_size > 1: + output[0].sampled_token_ids_cpu = output[ + 0].sampled_token_ids.cpu() + model_input.cached_outputs.append( + ModelOutput(output[0], output_ready_event, + output[0].sampled_token_ids, False, + output[0].logprobs, self.pythonization_cache)) + + # These GPU tensors are not required by multi-step; + # erase them to ensure they are not pythonized or + # transferred to CPU + output[0].sampled_token_ids = None + output[0].sampled_token_probs = None + output[0].logprobs = None + + # Pythonize the output if CPU is ahead and the previous step is + # ready. + if frozen_model_input.async_callback is None: + for model_output in model_input.cached_outputs: + model_output.maybe_pythonize(model_input, + self._copy_stream, + self.pinned_sampled_token_ids) + + model_input.current_step += 1 + + if not get_pp_group().is_last_rank: + # Should be IntermediateTensors + assert isinstance(output, IntermediateTensors) + return output + if not self.is_driver_worker: + return [] + + # Pythonize the output and block if needed since it is the last step + if model_input.is_last_step: + outputs = self._final_process_outputs( + model_input, model_input.base_output_proc_callback) + if self.pythonization_cache: + self.pythonization_cache.reset() + return outputs + + # should be [SamplerOutput] + return output + + def _update_sampling_metadata(self, sampling_metadata, num_seqs, + num_queries): + + assert sampling_metadata.num_prompts == 0 + assert len(sampling_metadata.seq_groups) == num_queries + assert sampling_metadata.selected_token_indices.shape == ( + num_queries, ) + # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501 + + # Verify that all sequences are decodes + for i in range(num_queries): + seq_group = sampling_metadata.seq_groups[i] + + assert seq_group.is_prompt is False # No prompt + assert seq_group.prompt_logprob_indices == [] # No prompt + assert seq_group.sample_indices == [i] # Simple + assert seq_group.seq_len is None # Decode + assert seq_group.query_len is None # Decode + + def _advance_step(self, model_input: StatefulModelInput, + out: SamplerOutput) -> StatefulModelInput: + + model_input.maybe_advance_frozen_model_input(self.device, + self.pin_memory) + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + assert frozen_model_input.input_tokens is not None + assert frozen_model_input.input_tokens.shape[0] == model_input.num_seqs + assert frozen_model_input.attn_metadata is not None + + sampled_token_ids = model_input.cached_outputs[-1].sampled_token_ids + num_seqs = model_input.num_seqs + num_queries = model_input.num_queries + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + attn_metadata = frozen_model_input.attn_metadata + assert attn_metadata is not None + + turn_prefills_into_decodes: bool = model_input.current_step == 1 and \ + model_input.num_single_step_prefills != 0 + attn_metadata.advance_step( + frozen_model_input, + sampled_token_ids, + self.block_size, + num_seqs, + num_queries, + turn_prefills_into_decodes=turn_prefills_into_decodes) + + return model_input + + def load_model(self) -> None: + return self._base_model_runner.load_model() + + def save_sharded_state( + self, + path: str, + pattern: Optional[str] = None, + max_size: Optional[int] = None, + ) -> None: + return self._base_model_runner.save_sharded_state( + path, pattern, max_size) + + def save_tensorized_model(self, + tensorizer_config: TensorizerConfig) -> None: + return self._base_model_runner.save_tensorized_model(tensorizer_config) + + def profile_run(self) -> None: + return self._base_model_runner.profile_run() + + def remove_all_loras(self): + return self._base_model_runner.remove_all_loras() + + def capture_model(self, kv_caches: List[List]) -> None: + return self._base_model_runner.capture_model(kv_caches) + + @property + def vocab_size(self) -> int: + return self._base_model_runner.vocab_size + + +DeferredLogprobsReturnType = Tuple[Optional[List[Optional[PromptLogprobs]]], + Optional[List[SampleLogprobs]]] + + +def deferred_pythonize_logprobs( + output: SamplerOutput, + sampling_metadata: SamplingMetadata, + logprobs_tensor: Optional[torch.Tensor], +) -> DeferredLogprobsReturnType: + """Perform deferred logprob Pythonization. + + 1. Pythonize GPU-side sampler result tensors into CPU-side sampler result. + 2. Pythonize GPU-side logprobs tensor into CPU-side logprobs lists, + utilizing the Pythonized sampler result computed in step 1. + + These deferred computations are not required for single-step scheduling + or the `profile_run()` phase of multi-step scheduling. + + Args: + output: sampler output (under deferred Pythonization) + sampling_metadata + + Returns: + prompt_logprobs (CPU), sample_logprobs (CPU) + """ + + # - Deferred pythonization of sample result + sampler_result = get_pythonized_sample_results( + output.deferred_sample_results_args) + + # - Erase the GPU-side deferred sample_result + # computation args to ensure it is never + # pythonized or transferred to CPU + output.deferred_sample_results_args = None + + # - Deferred pythonization of logprobs + ( + prompt_logprobs, + sample_logprobs, + ) = get_logprobs(logprobs_tensor, sampling_metadata, sampler_result) + assert len(prompt_logprobs) == len(sampling_metadata.seq_groups) + assert len(sample_logprobs) == len(sampling_metadata.seq_groups) + + return prompt_logprobs, sample_logprobs + + +def _pythonize_sampler_output( + model_input: StatefulModelInput, + output: SamplerOutput, + pinned_sampled_token_buffer: torch.Tensor, + sampled_token_ids: torch.Tensor, + logprobs_tensor: Optional[torch.Tensor], + cache: Optional[PythonizationCache], +) -> None: + """ This function is only called when the output tensors are ready. + See :class:`ModelOutput`. + + Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place, + adding a Pythonized output data structure + (:class:`CompletionSequenceGroupOutput`) for each :class:`SequenceGroup`. + + Args: + model_input + output: sampler output + pinned_sampled_token_token_buffer: CPU-side pinned memory + (receives copy of + GPU-side token buffer.) + sampled_token_ids: GPU-side token buffer + logprobs_tensor: GPU-side tensor containing + logprobs computed during sampling + """ + + assert model_input.frozen_model_input is not None + + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input.sampling_metadata is not None + sampling_metadata = frozen_model_input.sampling_metadata + # samples generation should have been skipped + assert not output.outputs + + pinned_buffer = pinned_sampled_token_buffer[:model_input.num_queries] + + # We guarantee output tensors are ready, so it is safe to + # pythonize the sampler output & obtain CPU-side logprobs. + # + # However we should check whether logprobs pythonization may + # be skipped entirely, i.e. because no logprobs were requested + # or pythonization was not deferred. To that end, + # + # * `prompt_logprobs_are_requested_for_prefill` signals that + # there are *any* prefill-phase requests which specify that + # prompt logprobs should be returned. + # + # * `any_logprobs_are_requested` signals that there are any + # requests which (1) specify that sample logprobs should be + # returned, or (2) are in the prefill phase AND specify that + # prompt logprobs should be returned. + # + # Later on, these flags cause adjustments to the pythonization + # process to accommodate logprobs. + + seq_groups = sampling_metadata.seq_groups + prompt_logprobs_are_requested_for_prefill = any([ + sg.sampling_params.prompt_logprobs is not None and sg.is_prompt + for sg in seq_groups + ]) + any_logprobs_are_requested = ( + prompt_logprobs_are_requested_for_prefill + or any([sg.sampling_params.logprobs is not None for sg in seq_groups])) + + if prompt_logprobs_are_requested_for_prefill: + # CPU GPU sync, after gathering *only* sampled tokens (since + # requesting prompt logprobs leads `sampled_token_ids` to + # include prompt token ids in addition to sampled token ids.) + sample_idx_tensor = torch.tensor( + [sdx for sg in seq_groups for sdx in sg.sample_indices]) + pinned_buffer = pinned_buffer.copy_( + sampled_token_ids[sample_idx_tensor, :], non_blocking=False) + else: + # CPU GPU sync + pinned_buffer = pinned_buffer.copy_(sampled_token_ids, + non_blocking=False) + + # this will not block as the tensors are already on CPU + samples_list = pinned_buffer.tolist() + + skip_sampler_cpu_output = ( + frozen_model_input.sampling_metadata.skip_sampler_cpu_output) + + # *Don't* skip logprobs pythonization *if*: + # * Any requests require logprobs to be returned in this + # iteration AND + # * These requests are being scheduled in a fashion which + # defers pythonization (i.e. multi-step scheduling.) + do_pythonize_logprobs = (skip_sampler_cpu_output + and any_logprobs_are_requested) + ( + prompt_logprobs, + sample_logprobs, + ) = (deferred_pythonize_logprobs(output, sampling_metadata, + logprobs_tensor) + if do_pythonize_logprobs else (None, None)) + + for sgdx, (seq_group, + sample_result) in enumerate(zip(seq_groups, samples_list)): + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + # (Check for Guided Decoding) + if seq_group.sampling_params.logits_processors: + assert len(seq_group.sampling_params.logits_processors) == 0, ( + "Logits Processors are not supported in multi-step decoding") + + if do_pythonize_logprobs: + assert prompt_logprobs is not None + assert sample_logprobs is not None + + ( + group_prompt_logprobs, + group_sample_logprobs, + ) = ( # Utilize deferred pythonization results + prompt_logprobs[sgdx], + sample_logprobs[sgdx], + ) + elif any_logprobs_are_requested: + ( + group_prompt_logprobs, + group_sample_logprobs, + ) = ( + # profile_run: use already-computed logprobs + output.outputs[sgdx].prompt_logprobs, + [sample.logprobs for sample in output.outputs[sgdx].samples]) + + seq_ids = seq_group.seq_ids + next_token_ids = sample_result + parent_ids = [0] + + if cache is not None: + completion_seq_group_output: CompletionSequenceGroupOutput = \ + cache.cached_completion_seq_group_output.get_object() + completion_seq_group_output.samples.clear() + seq_outputs: List[ + SequenceOutput] = completion_seq_group_output.samples + else: + seq_outputs = [] + + for tdx, (parent_id, + next_token_id) in enumerate(zip(parent_ids, next_token_ids)): + if cache is not None: + seq_output: SequenceOutput = cache.cached_seq_output.get_object( + ) + seq_output.parent_seq_id = seq_ids[parent_id] + seq_output.output_token = next_token_id + + if any_logprobs_are_requested: + seq_output.logprobs = group_sample_logprobs[tdx] + else: + logprobs = next(iter(seq_output.logprobs.values())) + seq_output.logprobs.clear() + + logprobs.logprob = float('inf') + logprobs.rank = None + logprobs.decoded_token = None + + seq_output.logprobs[next_token_id] = logprobs + + seq_outputs.append(seq_output) + + else: + seq_outputs.append( + SequenceOutput(seq_ids[parent_id], next_token_id, + (group_sample_logprobs[tdx] + if any_logprobs_are_requested else { + next_token_id: + Logprob(logprob=float('inf'), + rank=None, + decoded_token=None) + }))) + if cache is not None: + completion_seq_group_output.prompt_logprobs = \ + group_prompt_logprobs if any_logprobs_are_requested else None + output.outputs.append(completion_seq_group_output) + else: + output.outputs.append( + CompletionSequenceGroupOutput( + seq_outputs, (group_prompt_logprobs + if any_logprobs_are_requested else None))) + + assert len(output.outputs) > 0 diff --git a/vllm/worker/multi_step_tpu_worker.py b/vllm/worker/multi_step_tpu_worker.py new file mode 100644 index 00000000..e654f717 --- /dev/null +++ b/vllm/worker/multi_step_tpu_worker.py @@ -0,0 +1,105 @@ +import dataclasses +from typing import Dict, Optional, Tuple + +import torch + +from vllm.distributed import broadcast_tensor_dict +from vllm.sequence import ExecuteModelRequest +from vllm.worker.tpu_model_runner import ModelInputForTPU +from vllm.worker.tpu_worker import TPUWorker +from vllm.worker.worker_base import WorkerInput + + +class MultiStepTPUWorker(TPUWorker): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cached_model_input: Optional[ModelInputForTPU] = None + + def _get_driver_input_and_broadcast( + self, execute_model_req: ExecuteModelRequest + ) -> Tuple[ModelInputForTPU, WorkerInput, Dict[str, torch.Tensor]]: + assert self.is_driver_worker + assert execute_model_req.virtual_engine == 0 + + is_first_multi_step = execute_model_req.is_first_multi_step + is_last_step = execute_model_req.is_last_step + if is_first_multi_step: + worker_input: WorkerInput = self.prepare_worker_input( + execute_model_req=execute_model_req) + worker_input = dataclasses.replace( + worker_input, + num_steps=execute_model_req.num_lookahead_slots + 1) + model_input: ModelInputForTPU = ( + self.model_runner.prepare_model_input( + execute_model_req.seq_group_metadata_list, + execute_model_req.virtual_engine, + execute_model_req.finished_requests_ids)) + + if execute_model_req.async_callback: + model_input = dataclasses.replace( + model_input, + async_callback=execute_model_req.async_callback) + else: + assert self.cached_model_input is not None + model_input = self.cached_model_input + worker_input = WorkerInput() + model_input = dataclasses.replace( + model_input, + is_first_multi_step=is_first_multi_step, + is_last_step=is_last_step) + + if self.do_metadata_broadcast: + if is_first_multi_step: + broadcast_data = worker_input.as_broadcastable_tensor_dict() + broadcast_data.update( + model_input.as_broadcastable_tensor_dict()) + broadcast_tensor_dict(broadcast_data, src=0) + else: + broadcast_data = { + "is_first_multi_step": is_first_multi_step, + "is_last_step": is_last_step, + } + broadcast_tensor_dict(broadcast_data, src=0) + + # Retuning empty dict here to keep this compatible with + # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast` + return model_input, worker_input, {} + + def prepare_input( + self, + execute_model_req: Optional[ExecuteModelRequest] = None, + ) -> Optional[Tuple[ModelInputForTPU, WorkerInput, Dict[str, + torch.Tensor]]]: + if self.is_driver_worker: + if execute_model_req is None: + if self.do_metadata_broadcast: + broadcast_tensor_dict({}, src=0) + return None + + model_input, worker_input, _ = self._get_driver_input_and_broadcast( + execute_model_req) + if model_input.is_first_multi_step: + self.cached_model_input = model_input + return model_input, worker_input, {} + else: + broadcast_data = broadcast_tensor_dict(src=0) + if not broadcast_data: + return None + + if len(broadcast_data) == 2: + assert self.cached_model_input is not None + self.cached_model_input = dataclasses.replace( + self.cached_model_input, + is_first_multi_step=broadcast_data["is_first_multi_step"], + is_last_step=broadcast_data["is_last_step"]) + empty_worker_input = WorkerInput() + return self.cached_model_input, empty_worker_input, {} + + worker_input = WorkerInput.from_broadcasted_tensor_dict( + broadcast_data) + model_input = ( + self.model_runner. + make_model_input_from_broadcasted_tensor_dict(broadcast_data)) + self.cached_model_input = model_input + return model_input, worker_input, {} diff --git a/vllm/worker/multi_step_worker.py b/vllm/worker/multi_step_worker.py new file mode 100644 index 00000000..bf66f32d --- /dev/null +++ b/vllm/worker/multi_step_worker.py @@ -0,0 +1,202 @@ +import dataclasses +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch + +from vllm.distributed import broadcast_tensor_dict, get_pp_group +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import ExecuteModelRequest +from vllm.worker.model_runner_base import BroadcastableModelInput +from vllm.worker.multi_step_model_runner import (MultiStepModelRunner, + StatefulModelInput) +from vllm.worker.worker import Worker, WorkerInput + + +@dataclass +class MultiStepState: + worker_input: WorkerInput + model_input: StatefulModelInput + + +class MultiStepWorker(Worker): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + base_model_runner = self.model_runner + # for multi-step model, wrap the model runner with MultiStepModelRunner + self.model_runner = MultiStepModelRunner( + base_model_runner, + base_model_runner.model_config, + base_model_runner.parallel_config, + base_model_runner.scheduler_config, + base_model_runner.device_config, + base_model_runner.cache_config, + load_config=base_model_runner.load_config, + lora_config=self.lora_config, + kv_cache_dtype=self.cache_config.cache_dtype, + is_driver_worker=base_model_runner.is_driver_worker, + prompt_adapter_config=base_model_runner.prompt_adapter_config, + observability_config=base_model_runner.observability_config, + ) + + pipeline_parallel_size = self.parallel_config.pipeline_parallel_size + self.multi_step_states: List[ + Optional[MultiStepState]] = [None] * pipeline_parallel_size + self.temp_output = None + + def _get_driver_input_and_broadcast( + self, execute_model_req: ExecuteModelRequest + ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]: + """ + Get the driver input and broadcast it to other workers. + """ + assert self.is_driver_worker + virtual_engine = execute_model_req.virtual_engine + is_first_multi_step = execute_model_req.is_first_multi_step + if is_first_multi_step: + # on first step we prepare the worker input and model input normally + worker_input: WorkerInput = self.prepare_worker_input( + execute_model_req=execute_model_req) + model_input: StatefulModelInput = ( + self.model_runner.prepare_model_input( + execute_model_req.seq_group_metadata_list, + execute_model_req.virtual_engine, + execute_model_req.finished_requests_ids)) + + if execute_model_req.async_callback: + model_input.frozen_model_input = dataclasses.replace( # type: ignore + model_input.frozen_model_input, + async_callback=execute_model_req.async_callback) + else: + # on subsequent steps we reuse the worker input and model input + multi_step_state = self.multi_step_states[virtual_engine] + worker_input = multi_step_state.worker_input + model_input = multi_step_state.model_input + frozen_model_input = model_input.frozen_model_input + assert frozen_model_input is not None + assert frozen_model_input.attn_metadata is not None + # clear the cached metadata so that it can be recomputed on + # the workers. + frozen_model_input.attn_metadata._cached_prefill_metadata = None + frozen_model_input.attn_metadata._cached_decode_metadata = None + + model_input.is_first_multi_step = is_first_multi_step + model_input.is_last_step = execute_model_req.is_last_step + + if not is_first_multi_step: + # we broadcast the last sampled token ids to all TP workers so they + # can update their model input metadata in-place. + self._prepare_last_sampled_token_ids_for_tp_workers( + execute_model_req=execute_model_req, model_input=model_input) + + if self.do_metadata_broadcast: + broadcast_data = worker_input.as_broadcastable_tensor_dict() + broadcast_data.update(model_input.as_broadcastable_tensor_dict()) + broadcast_tensor_dict(broadcast_data, src=0) + + # Retuning empty dict here to keep this compatible with + # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast` + return model_input, worker_input, {} + + def _prepare_last_sampled_token_ids_for_tp_workers( + self, + execute_model_req: ExecuteModelRequest, + model_input: StatefulModelInput, + ) -> None: + """ + Prepare the last sampled token ids for TP workers. If it's the last + PP rank, then the last sampled token ids are already in the model_input. + If it is NOT the last PP rank, then we need to get the last sampled + token that is cached in the execute_model_req. + """ + if get_pp_group().is_last_rank: + assert model_input.cached_outputs[ + -1].sampler_output.sampled_token_ids is None + assert model_input.cached_outputs[-1].sampled_token_ids is not None + model_input.last_sampled_token_ids = model_input.cached_outputs[ + -1].sampled_token_ids + # free sampled token ids from the previous step if it has been + # pythonized. Cannot free the last sampled token ids because + # we need it for GPU advance_step. + for output in model_input.cached_outputs[:-1]: + if output.pythonized: + output.sampled_token_ids = None + else: + # otherwise we need to get the cached sampled token ids from the + # execute_model_req + assert execute_model_req.last_sampled_token_ids is not None + model_input.last_sampled_token_ids = ( + execute_model_req.last_sampled_token_ids.cuda()) + model_input.add_sampler_output( + SamplerOutput(outputs=[], sampled_token_ids=None), + model_input.last_sampled_token_ids) + + # free sampled token ids from the previous step. + # TODO(will) we could reuse the sampled token ids tensor from + # the previous step instead. + for output in model_input.cached_outputs[:-1]: + output.sampled_token_ids = None + assert model_input.cached_outputs[-1].sampled_token_ids is not None + + def prepare_input( + self, + execute_model_req: Optional[ExecuteModelRequest] = None, + ) -> Optional[Tuple[StatefulModelInput, WorkerInput, Dict[str, + torch.Tensor]]]: + """ + Depending on the current state of the request and multi step worker, + this method may skip the normal _prepare_model_input and + _prepare_worker_input methods and instead used cached values. + """ + if self.is_driver_worker: + if execute_model_req is None: + if self.do_metadata_broadcast: + # This signals that there's no more requests to process for + # now. All workers are running infinite loop with + # broadcast_tensor_dict, and it stops the loop when the + # driver broadcasts an empty input. Send an empty input to + # notify all other workers to stop their execution loop. + broadcast_tensor_dict({}, src=0) + return None + + virtual_engine = execute_model_req.virtual_engine + (model_input, worker_input, + kwargs) = self._get_driver_input_and_broadcast(execute_model_req) + assert isinstance(model_input, StatefulModelInput) + if execute_model_req.is_first_multi_step: + # cache the worker input and model input for the next steps + self.multi_step_states[virtual_engine] = MultiStepState( + worker_input=worker_input, model_input=model_input) + # if TP workers + else: + broadcast_data = self._get_worker_input_from_broadcast() + # if the driver has sent an empty input, we should stop the worker + # loop + if broadcast_data is None: + return None + model_input, worker_input, kwargs = broadcast_data + assert isinstance(model_input, StatefulModelInput) + virtual_engine = worker_input.virtual_engine + if model_input.is_first_multi_step: + pass + # TODO(will) Can cache the worker input and model input for the + # next steps. See below for details + else: + # TODO(will) possible to also cache and reuse the cached worker + # input and model input. The idea is essentially the delta + # optimization for model_inputs. Where the TP workers can cache + # the model input states and we only broadcast the delta need + # for the next step (sampled_token_ids from the previous step) + + assert isinstance(model_input, StatefulModelInput) + # we need to update the last sampled token ids in the model + # input for the workers so that they can run inplace + # advance_step + model_input.add_sampler_output( + SamplerOutput(outputs=[], sampled_token_ids=None), + model_input.last_sampled_token_ids) + + assert model_input is not None + assert worker_input is not None + return model_input, worker_input, kwargs diff --git a/vllm/worker/neuron_model_runner.py b/vllm/worker/neuron_model_runner.py new file mode 100644 index 00000000..b8c760c4 --- /dev/null +++ b/vllm/worker/neuron_model_runner.py @@ -0,0 +1,346 @@ +import os +from dataclasses import dataclass +from importlib.util import find_spec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn +from transformers_neuronx.config import GenerationConfig + +from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig, + SchedulerConfig) +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader.neuron import get_neuron_model +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs, + MultiModalInputs) +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata +from vllm.utils import is_pin_memory_available, make_tensor_with_pad +from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + + +@dataclass(frozen=True) +class ModelInputForNeuron(ModelRunnerInputBase): + """ + Used by the NeuronModelRunner. + """ + input_tokens: Optional[torch.Tensor] = None + input_positions: Optional[torch.Tensor] = None + input_block_ids: Optional[torch.Tensor] = None + sampling_metadata: Optional["SamplingMetadata"] = None + multi_modal_kwargs: Optional[BatchedTensorInputs] = None + + def as_broadcastable_tensor_dict( + self) -> Dict[str, Union[int, torch.Tensor]]: + raise NotImplementedError("ModelInputForNeuron cannot be broadcast.") + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "ModelInputForNeuron": + assert attn_backend is None + return cls.from_broadcasted_tensor_dict(tensor_dict) + + +class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]): + + # NEURON has an upper limit on the top_k + _MAX_NEURON_SAMPLING_TOP_K = 256 + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + ): + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + + if model_config is not None and model_config.get_sliding_window(): + logger.warning("Sliding window is not supported on Neuron. " + "The model will run without sliding window.") + self.device_config = (device_config + if device_config is not None else DeviceConfig()) + self.device = self.device_config.device + self.pin_memory = is_pin_memory_available() + + # Multi-modal data support + self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \ + .create_input_mapper(self.model_config) + + # Lazy initialization. + self.model: nn.Module # initialize after load_model. + + # Once NEURON_ON_DEVICE_SAMPLING_DISABLED is set to a non-zero value, + # turn off on-device sampling. + self._on_device_sampling_disabled = int( + os.getenv("NEURON_ON_DEVICE_SAMPLING_DISABLED", "0")) + + # NEURON needs to update sampling parameters when request IDs change + # across batches. This variable stores the previous batch's request IDs + # to determine if an update is needed. + self._previous_batch_request_ids: List[str] = [] + + if not self._on_device_sampling_disabled: + logger.warning( + "On-device sampling is turned on in Neuron by default, only " + "top_k, top_p, and temperature are current supported sampling " + "parameters. To turn off the on-device sampling, please set " + "the environment variable NEURON_ON_DEVICE_SAMPLING_DISABLED=1." + ) + self.model_config.neuron_sampling_params = GenerationConfig( + max_length=self.scheduler_config.max_model_len, + do_sample=True, + per_batch_line=True, + top_k=[self._MAX_NEURON_SAMPLING_TOP_K] \ + * self.scheduler_config.max_num_seqs, + top_p=[1.0] * self.scheduler_config.max_num_seqs, + temperature=[1.0] * self.scheduler_config.max_num_seqs, + dynamic=True, + global_top_k=self._MAX_NEURON_SAMPLING_TOP_K) + + def load_model(self) -> None: + if find_spec("transformers_neuronx") is not None: + self.model = get_neuron_model( + self.model_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config) + else: + raise NotImplementedError( + "Supports only Transformer-NeuronX based models.") + + def _prepare_prompt( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int], + BatchedTensorInputs]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[List[int]] = [] + input_positions: List[List[int]] = [] + input_block_ids: List[int] = [] + + seq_lens: List[int] = [] + multi_modal_inputs_list: List[MultiModalInputs] = [] + for seq_group_metadata in seq_group_metadata_list: + assert seq_group_metadata.is_prompt + seq_ids = list(seq_group_metadata.seq_data.keys()) + assert len(seq_ids) == 1 + seq_id = seq_ids[0] + + seq_data = seq_group_metadata.seq_data[seq_id] + prompt_tokens = seq_data.get_token_ids() + seq_len = len(prompt_tokens) + seq_lens.append(seq_len) + + input_tokens.append(prompt_tokens) + input_positions.append(list(range(seq_len))) + + assert seq_group_metadata.block_tables is not None + block_table = seq_group_metadata.block_tables[seq_id] + assert len(block_table) == 1 + input_block_ids.append(block_table[0]) + + mm_data = seq_group_metadata.multi_modal_data + if mm_data: + # Process multi-modal data + mm_kwargs = self.multi_modal_input_mapper( + mm_data, + mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs, + ) + multi_modal_inputs_list.append(mm_kwargs) + + max_seq_len = max(seq_lens) + assert max_seq_len > 0 + input_tokens = make_tensor_with_pad(input_tokens, + pad=0, + max_len=max_seq_len, + dtype=torch.long, + device=self.device) + input_positions = make_tensor_with_pad(input_positions, + pad=0, + max_len=max_seq_len, + dtype=torch.long, + device=self.device) + input_block_ids = torch.tensor(input_block_ids, + dtype=torch.long, + device=self.device) + + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list) + + return (input_tokens, input_positions, input_block_ids, seq_lens, + multi_modal_kwargs) + + def _prepare_decode( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[List[int]] = [] + input_positions: List[List[int]] = [] + input_block_ids: List[int] = [] + context_lens: List[int] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert not seq_group_metadata.is_prompt + + seq_ids = list(seq_group_metadata.seq_data.keys()) + + for seq_id in seq_ids: + seq_data = seq_group_metadata.seq_data[seq_id] + generation_token = seq_data.get_last_token_id() + input_tokens.append([generation_token]) + + seq_len = seq_data.get_len() + position = seq_len - 1 + input_positions.append([position]) + context_lens.append(seq_len) + + assert seq_group_metadata.block_tables is not None + block_table = seq_group_metadata.block_tables[seq_id] + assert len(block_table) == 1 + input_block_ids.append(block_table[0]) + + input_tokens = make_tensor_with_pad(input_tokens, + pad=0, + max_len=1, + dtype=torch.long, + device=self.device) + input_positions = make_tensor_with_pad(input_positions, + pad=0, + max_len=1, + dtype=torch.long, + device=self.device) + context_lens = torch.tensor(context_lens, + dtype=torch.int, + device=self.device) + input_block_ids = torch.tensor(input_block_ids, + dtype=torch.long, + device=self.device) + + return input_tokens, input_positions, input_block_ids + + def make_model_input_from_broadcasted_tensor_dict( + self, tensor_dict: Dict[str, Any]) -> ModelInputForNeuron: + return ModelInputForNeuron.from_broadcasted_tensor_dict(tensor_dict) + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForNeuron: + multi_modal_kwargs = None + # NOTE: We assume that all sequences in the group are all prompts or + # all decodes. + is_prompt = seq_group_metadata_list[0].is_prompt + # Prepare input tensors. + if is_prompt: + (input_tokens, input_positions, input_block_ids, seq_lens, + multi_modal_kwargs + ) = self._prepare_prompt(seq_group_metadata_list) + else: + (input_tokens, input_positions, + input_block_ids) = self._prepare_decode(seq_group_metadata_list) + seq_lens = None + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, + seq_lens, + # query_lens is not needed if chunked prefill is not + # supported. Since neuron worker doesn't support chunked prefill + # just use seq_lens instead. + seq_lens, + self.device, + self.pin_memory, + generators=self.get_generators(finished_requests_ids)) + + if not self._on_device_sampling_disabled: + # Once the request IDs are changed in current iteration, we will + # update the on-device sampling parameters. + current_batch_request_ids = [ + seq_group_meta_data.request_id + for seq_group_meta_data in seq_group_metadata_list + ] + if current_batch_request_ids != self._previous_batch_request_ids: + self._update_neuron_sampling_params(sampling_metadata) + self._previous_batch_request_ids = current_batch_request_ids + + return ModelInputForNeuron(input_tokens=input_tokens, + input_positions=input_positions, + input_block_ids=input_block_ids, + sampling_metadata=sampling_metadata, + multi_modal_kwargs=multi_modal_kwargs) + + def _update_neuron_sampling_params(self, + sampling_metadata: SamplingMetadata): + # Update Neuron sampling parameters (GenerationConfig in Neuron) + current_sampling_params = self.model_config.neuron_sampling_params + assert current_sampling_params is not None, ( + f"Failed to update sampling_params, " + f"current sampling params is {current_sampling_params}") + + top_k = current_sampling_params.top_k + top_p = current_sampling_params.top_p + temperature = current_sampling_params.temperature + for index, sequence_group_to_sample in enumerate( + sampling_metadata.seq_groups): + top_k[index] = self._convert_to_neuron_top_k( + sequence_group_to_sample.sampling_params.top_k) + top_p[index] = sequence_group_to_sample.sampling_params.top_p + temperature[index] = \ + sequence_group_to_sample.sampling_params.temperature + + self.model.model.update_generation_config(current_sampling_params) + + def _convert_to_neuron_top_k(self, top_k: int) -> int: + if top_k < 0 or top_k > self._MAX_NEURON_SAMPLING_TOP_K: + return self._MAX_NEURON_SAMPLING_TOP_K + return top_k + + @torch.inference_mode() + def execute_model( + self, + model_input: ModelInputForNeuron, + kv_caches: Optional[List[torch.Tensor]] = None, + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + if num_steps > 1: + raise ValueError( + "NeuronModelRunner does not support multi-step execution.") + + hidden_states = self.model( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + input_block_ids=model_input.input_block_ids, + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {}, + device=self.device), + ) + + # Compute the logits only if the on-device sampling is turned off as + # on-device sampling outputs the token ids. + if self._on_device_sampling_disabled: + logits = self.model.compute_logits(hidden_states, + model_input.sampling_metadata) + else: + logits = hidden_states + + # Sample the next token. + output = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + return [output] + + @property + def vocab_size(self) -> int: + return self.model_config.get_vocab_size() diff --git a/vllm/worker/neuron_worker.py b/vllm/worker/neuron_worker.py new file mode 100644 index 00000000..fff14d64 --- /dev/null +++ b/vllm/worker/neuron_worker.py @@ -0,0 +1,127 @@ +"""A Neuron worker class.""" +from typing import List, Optional, Tuple + +import torch +import torch.distributed + +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig, + ParallelConfig, SchedulerConfig) +from vllm.distributed import (ensure_model_parallel_initialized, + init_distributed_environment) +from vllm.model_executor import set_random_seed +from vllm.sequence import ExecuteModelRequest +from vllm.worker.neuron_model_runner import NeuronModelRunner +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, + LoraNotSupportedWorkerBase, WorkerInput) + + +class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase): + """A worker class that executes the model on a group of neuron cores. + """ + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + ) -> None: + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + if self.model_config.trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils import init_cached_hf_modules + init_cached_hf_modules() + + self.model_runner: NeuronModelRunner = NeuronModelRunner( + model_config, parallel_config, scheduler_config, device_config) + self.is_driver_worker = True + + def init_device(self) -> None: + self.init_distributed_environment() + + # Set random seed. + set_random_seed(self.model_config.seed) + + def load_model(self): + self.model_runner.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available KV blocks. + + Swapping is not yet supported, so always return num_cpu_blocks=0. + + We configure num_gpu_blocks to be equal to max_num_seqs. + """ + # Set the number of GPU blocks to be the same as the maximum number of + # sequences that can be processed in a single batch. This is equivalent + # to schedule without PagedAttention. + num_gpu_blocks = self.scheduler_config.max_num_seqs + + # Swap not yet supported with Neuron backend. + num_cpu_blocks = 0 + + return num_gpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache. + """ + + # Different values are not tested. + assert num_cpu_blocks == 0 + assert num_gpu_blocks == self.scheduler_config.max_num_seqs + + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + + @property + def do_metadata_broadcast(self) -> bool: + return False + + @property + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: + return None + + @torch.inference_mode() + def prepare_worker_input( + self, execute_model_req: ExecuteModelRequest) -> WorkerInput: + return WorkerInput(num_seq_groups=len( + execute_model_req.seq_group_metadata_list), ) + + def execute_worker(self, worker_input: WorkerInput) -> None: + pass + + def get_cache_block_size_bytes(self) -> int: + """Determine the size in bytes of a cache block. + + This is required for speculative decoding; it is not yet implemented. + """ + raise NotImplementedError + + def init_distributed_environment(self): + """Neuron uses transformers-neuronx for tensor parallelism. + + vLLM still needs the environment inited when TP/PP > 1 + """ + init_distributed_environment( + world_size=1, + rank=self.rank, + local_rank=self.local_rank, + distributed_init_method=self.distributed_init_method, + backend="gloo", + ) + ensure_model_parallel_initialized( + 1, + 1, + ) diff --git a/vllm/worker/openvino_model_runner.py b/vllm/worker/openvino_model_runner.py new file mode 100644 index 00000000..760b1842 --- /dev/null +++ b/vllm/worker/openvino_model_runner.py @@ -0,0 +1,353 @@ +from typing import List, NamedTuple, Optional, Tuple + +import openvino as ov +import torch +from torch import nn + +from vllm.attention import get_attn_backend +from vllm.attention.backends.openvino import OpenVINOAttentionMetadata +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, MultiModalConfig, ParallelConfig, + SchedulerConfig) +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader.openvino import get_model +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs, + MultiModalInputs) +from vllm.sequence import SequenceGroupMetadata + +logger = init_logger(__name__) + + +class ModelInput(NamedTuple): + input_tokens: torch.Tensor + input_positions: torch.Tensor + attn_metadata: Optional[OpenVINOAttentionMetadata] + seq_lens: List[int] + query_lens: List[int] + multi_modal_kwargs: BatchedTensorInputs + + @classmethod + def empty(cls, device): + return ModelInput(input_tokens=torch.empty(0, device=device), + input_positions=torch.empty(0, device=device), + attn_metadata=None, + seq_lens=[], + query_lens=[], + multi_modal_kwargs={}) + + +class OpenVINOModelRunner: + + def __init__( + self, + ov_core: ov.Core, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + multimodal_config: Optional[MultiModalConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + *args, + **kwargs, + ): + self.ov_core = ov_core + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.lora_config = lora_config + self.multimodal_config = multimodal_config + self.load_config = load_config + self.is_driver_worker = is_driver_worker + + self.device = self.device_config.device + + self.kv_cache_dtype = kv_cache_dtype + self.sliding_window = model_config.get_sliding_window() + self.block_size = cache_config.block_size + + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.kv_cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) + + # Multi-modal data support + self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \ + .create_input_mapper(self.model_config) + + # Lazy initialization. + self.model: nn.Module # Set after init_Model + + def load_model(self) -> None: + self.model = get_model(model_config=self.model_config, + device_config=self.device_config, + kv_cache_dtype=self.kv_cache_dtype, + ov_core=self.ov_core) + + def _prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> ModelInput: + """Prepare the model input based on a given sequence group. + + The API assumes seq_group_metadata_list is sorted by prefill -> decode. + + The result tensors and data structure also batches input in prefill + -> decode order. For example, + + - input_tokens[:num_prefill_tokens] contains prefill tokens. + - input_tokens[num_prefill_tokens:] contains decode tokens. + """ + input_tokens: List[int] = [] + input_positions: List[int] = [] + + seq_lens: List[int] = [] + past_lens: List[int] = [] + query_lens: List[int] = [] + multi_modal_inputs_list: List[MultiModalInputs] = [] + + subsequence_begins: List[int] = [] + block_indices: List[int] = [] + block_indices_begins: List[int] = [] + + # initialize beginning of prefix sums + subsequence_begins.append(0) + block_indices_begins.append(0) + + if len(seq_group_metadata_list) == 0: + return ModelInput.empty(self.device) + + for seq_group_metadata in seq_group_metadata_list: + seq_ids = list(seq_group_metadata.seq_data.keys()) + is_prompt = seq_group_metadata.is_prompt + + for seq_id in seq_ids: + computed_block_nums = seq_group_metadata.computed_block_nums + if (self.scheduler_config is not None + and self.scheduler_config.chunked_prefill_enabled + and not (computed_block_nums is None + or computed_block_nums == [])): + raise RuntimeError( + "chunked prefill cannot be used with prefix caching " + "now.") + + seq_data = seq_group_metadata.seq_data[seq_id] + if is_prompt: + computed_len = seq_data.get_num_computed_tokens() + else: + # get_num_computed_tokens is incorrect for spec decoding. + # So, we should have a special logic here. + # TODO(sang): Fix it. + computed_len = seq_data.get_len() - 1 + + seq_len = min( + seq_data.get_len(), + computed_len + seq_group_metadata.token_chunk_size, + ) + if is_prompt: + tokens = seq_data.get_token_ids()[computed_len:seq_len] + else: + # Optimization. get_token_ids requires the entire copy of + # tokens. + tokens = [seq_data.get_last_token_id()] + + # Prefix cache was hit. + # Prefix is not supported with sliding_window + prefix_cache_hit = (computed_block_nums is not None + and len(computed_block_nums) > 0 + and self.sliding_window is None + and is_prompt) + + mm_data = seq_group_metadata.multi_modal_data + if mm_data: + mm_kwargs = self.multi_modal_input_mapper( + mm_data, + mm_processor_kwargs=seq_group_metadata. + mm_processor_kwargs, + ) + multi_modal_inputs_list.append(mm_kwargs) + + block_table = seq_group_metadata.block_tables[seq_id] + # TODO(sang): Combine chunked prefill and prefix caching by + # only allowing multiple of block_size chunk size. + # NOTE: This only works for oooooooxxx style attention. + if prefix_cache_hit: + assert computed_block_nums is not None + computed_len = len(computed_block_nums) * self.block_size + tokens = tokens[computed_len:] + elif (self.scheduler_config.chunked_prefill_enabled + or not is_prompt): + if seq_group_metadata.block_tables is not None: + # chunked prefill or decode + block_table = seq_group_metadata.block_tables[seq_id] + if self.sliding_window is not None: + # chunked prefill doesn't support sliding window. + assert not self.scheduler_config.chunked_prefill_enabled # noqa: E501 + sliding_window_blocks = (self.sliding_window // + self.block_size) + block_table = block_table[-sliding_window_blocks:] + else: + # Only happens when memory profiling runs. + block_table = [] + else: + # prompt phase w/o prefix_caching, chunked_prefill + pass + + block_indices.extend(block_table) + block_indices_begins.append(block_indices_begins[-1] + + len(block_table)) + + # TODO(sang): This is a hack to make sliding window work with + # paged attn. We can remove it if we make paged attn kernel + # to properly handle slinding window attn. + if self.sliding_window is not None and not is_prompt: + seq_len = min(seq_len, self.sliding_window) + computed_len = seq_len - 1 + + seq_lens.append(seq_len) + + query_len = seq_len - computed_len + query_lens.append(query_len) + + input_tokens.extend(tokens) + input_positions.extend(list(range(computed_len, seq_len))) + + past_lens.append(computed_len) + subsequence_begins.append(subsequence_begins[-1] + query_len) + + if is_prompt: + assert len(seq_ids) == 1 + else: + assert ( + query_len == 1 + ), "seq_len: {}, computed_len: {}, query_len: {}".format( + seq_len, computed_len, query_len) + + max_query_len = max(query_lens) + assert max_query_len > 0, "query_lens: {}".format(query_lens) + + input_tokens = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) # type: ignore + input_positions = torch.tensor(input_positions, + dtype=torch.long, + device=self.device) # type: ignore + + past_lens_tensor = torch.tensor(past_lens, + dtype=torch.int32, + device=self.device) # type: ignore + subsequence_begins_tensor = torch.tensor( + subsequence_begins, dtype=torch.int32, + device=self.device) # type: ignore + block_indices_tensor = torch.tensor(block_indices, + dtype=torch.int32, + device=self.device) # type: ignore + block_indices_begins_tensor = torch.tensor( + block_indices_begins, dtype=torch.int32, + device=self.device) # type: ignore + + max_context_len = max(seq_lens) + max_context_len_tensor = torch.tensor( + max_context_len, dtype=torch.int32, + device=self.device) # type: ignore + + attn_metadata = self.attn_backend.make_openvino_metadata( + past_lens=past_lens_tensor, + subsequence_begins=subsequence_begins_tensor, + block_indices=block_indices_tensor, + block_indices_begins=block_indices_begins_tensor, + max_context_len=max_context_len_tensor, + ) + + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list) + + return ModelInput( + input_tokens, + input_positions, + attn_metadata, + seq_lens, + query_lens, + multi_modal_kwargs=multi_modal_kwargs, + ) + + def prepare_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, OpenVINOAttentionMetadata, + SamplingMetadata, BatchedTensorInputs]: + # Prepare input tensors. + ( + input_tokens, + input_positions, + attn_metadata, + seq_lens, + query_lens, + multi_modal_kwargs, + ) = self._prepare_model_input(seq_group_metadata_list) + + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, + seq_lens, + query_lens, + self.device, + pin_memory=False, + ) + + return ( + input_tokens, + input_positions, + attn_metadata, + sampling_metadata, + multi_modal_kwargs, + ) + + @torch.inference_mode() + def execute_model( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + kv_caches: List[Tuple["ov.Tensor", "ov.Tensor"]], + ) -> Optional[SamplerOutput]: + ( + input_tokens, + input_positions, + attn_metadata, + sampling_metadata, + multi_modal_kwargs, + ) = self.prepare_input_tensors(seq_group_metadata_list) + + model_executable = self.model + execute_model_kwargs = { + "input_ids": + input_tokens, + "positions": + input_positions, + "kv_caches": + kv_caches, + "attn_metadata": + attn_metadata, + **MultiModalInputs.as_kwargs(multi_modal_kwargs or {}, + device=self.device), + } + + hidden_states = model_executable(**execute_model_kwargs) + + # Compute the logits. + logits = self.model.compute_logits(hidden_states, sampling_metadata) + + # Sample the next token. + output = self.model.sample( + logits=logits, + sampling_metadata=sampling_metadata, + ) + return output diff --git a/vllm/worker/openvino_worker.py b/vllm/worker/openvino_worker.py new file mode 100644 index 00000000..24425fec --- /dev/null +++ b/vllm/worker/openvino_worker.py @@ -0,0 +1,611 @@ +"""An OpenVINO worker class.""" +from typing import Any, Dict, List, Optional, Tuple + +import openvino as ov +import torch +import torch.distributed + +import vllm.envs as envs +from vllm.attention import get_attn_backend +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, MultiModalConfig, ParallelConfig, + SchedulerConfig) +from vllm.distributed import (broadcast_tensor_dict, + ensure_model_parallel_initialized, + init_distributed_environment) +from vllm.executor.openvino_executor import is_openvino_cpu +from vllm.inputs import INPUT_REGISTRY +from vllm.logger import init_logger +from vllm.model_executor import set_random_seed +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sampling_params import SamplingParams +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata +from vllm.worker.openvino_model_runner import OpenVINOModelRunner +from vllm.worker.worker_base import LoraNotSupportedWorkerBase + +logger = init_logger(__name__) + + +class OpenVINOCacheEngine: + """Manages the KV cache for OpenVINO backend. + + This class is responsible for initializing and managing CPU KV + caches. It also provides methods for performing KV cache operations, such + as copying. + """ + + def __init__( + self, + cache_config: CacheConfig, + model_config: ModelConfig, + parallel_config: ParallelConfig, + device_config: DeviceConfig, + ov_core: ov.Core, + ov_device: str, + ) -> None: + assert device_config.device_type == "openvino" + self.cache_config = cache_config + self.model_config = model_config + self.parallel_config = parallel_config + + self.head_size = model_config.get_head_size() + if device_config.device.type == "cpu" and \ + cache_config.cache_dtype == ov.Type.u8: + # Scale, zero point and quantized data will be stored together. + # The layout for per token per head: + # |scale(f32)|zeropoint(f32)|quantized data(u8,idx_1)|quantized data(u8,idx_2)|...|quantized data(u8,idx_head_size)| # noqa: E501 + # so, we have to extend head_size by 8, which is sizeof(float) + # for scale and sizeof(float) for zeropoint + self.head_size += 8 + self.num_layers = model_config.get_num_layers(parallel_config) + self.num_kv_heads = model_config.get_num_kv_heads(parallel_config) + + self.block_size = cache_config.block_size + # Note: In CacheConfig, num_gpu_blocks actual is num_cpu_blocks + # for OpenVINO backend with a CPU target device, because we want + # to reuse KV cache management in the scheduler. + self.num_device_blocks = cache_config.num_gpu_blocks + self.num_swap_blocks = cache_config.num_cpu_blocks + + # Get attention backend. + self.attn_backend = get_attn_backend( + self.head_size, + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.cache_config.cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) + + # Initialize the cache. + self.kv_cache: List[Tuple[ov.Tensor, + ov.Tensor]] = self._allocate_kv_cache( + self.num_device_blocks, ov_core, + ov_device) + + # Initialize the swap. + self.swap_cache: List[Tuple[ov.Tensor, + ov.Tensor]] = self._allocate_swap_cache( + self.num_swap_blocks, ov_device) + + def _allocate_kv_cache( + self, + num_blocks: int, + ov_core: ov.Core, + ov_device: str, + ) -> List[Tuple[ov.Tensor, ov.Tensor]]: + """Allocates KV cache.""" + k_block_shape = v_block_shape = self.attn_backend.get_kv_cache_shape( + num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:] + kv_cache: List[Tuple[ov.Tensor, ov.Tensor]] = [] + + if is_openvino_cpu(): + for _ in range(self.num_layers): + key_blocks = ov.Tensor(self.cache_config.cache_dtype, + k_block_shape) + value_blocks = ov.Tensor(self.cache_config.cache_dtype, + v_block_shape) + kv_cache.append((key_blocks, value_blocks)) + else: + # Update key_cache shape: + k_block_shape = (v_block_shape[0], v_block_shape[1], + v_block_shape[3], v_block_shape[2]) + + remote_context = ov_core.get_default_context(ov_device) + + for _ in range(self.num_layers): + key_blocks = \ + remote_context.create_tensor(self.cache_config.cache_dtype, + ov.Shape(k_block_shape), + {}) + + value_blocks = \ + remote_context.create_tensor(self.cache_config.cache_dtype, + ov.Shape(v_block_shape), + {}) + + kv_cache.append((key_blocks, value_blocks)) + + return kv_cache + + def _allocate_swap_cache( + self, + num_blocks: int, + ov_device: str, + ) -> List[Tuple[ov.Tensor, ov.Tensor]]: + """Allocates swap cache.""" + k_block_shape = v_block_shape = self.attn_backend.get_kv_cache_shape( + num_blocks, self.block_size, self.num_kv_heads, self.head_size)[1:] + swap_cache: List[Tuple[ov.Tensor, ov.Tensor]] = [] + + if num_blocks == 0: + return swap_cache + + assert not is_openvino_cpu(), \ + "CPU device isn't supposed to have swap cache" + + # Update key_cache shape: + k_block_shape = (v_block_shape[0], v_block_shape[1], v_block_shape[3], + v_block_shape[2]) + + for _ in range(self.num_layers): + key_blocks = ov.Tensor(self.cache_config.cache_dtype, + k_block_shape) + value_blocks = ov.Tensor(self.cache_config.cache_dtype, + v_block_shape) + swap_cache.append((key_blocks, value_blocks)) + + return swap_cache + + def swap_in(self, src_to_dst: List[Tuple[int, int]]) -> None: + for i in range(self.num_layers): + for swap_tensor, kv_tensor in zip(self.swap_cache[i], + self.kv_cache[i]): + self.attn_backend.swap_blocks(swap_tensor, kv_tensor, + src_to_dst) + + def swap_out(self, src_to_dst: List[Tuple[int, int]]) -> None: + for i in range(self.num_layers): + for swap_tensor, kv_tensor in zip(self.swap_cache[i], + self.kv_cache[i]): + self.attn_backend.swap_blocks(kv_tensor, swap_tensor, + src_to_dst) + + def copy(self, src_to_dsts: List[Tuple[int, int]]) -> None: + if (len(src_to_dsts) > 0): + self.attn_backend.copy_blocks(self.kv_cache, src_to_dsts) + + @staticmethod + def get_cache_block_size( + block_size: int, + cache_dtype: ov.Type, + model_config: ModelConfig, + parallel_config: ParallelConfig, + ) -> int: + head_size = model_config.get_head_size() + num_kv_heads = model_config.get_num_kv_heads(parallel_config) + num_layers = model_config.get_num_layers(parallel_config) + + if cache_dtype == ov.Type.u8: + # Scale, zero point and quantized data will be stored together. + # The layout for per token per head: + # |scale(f32)|zeropoint(f32)|quantized data(u8,idx_1)|quantized data(u8,idx_2)|...|quantized data(u8,idx_head_size)| # noqa: E501 + # so, we have to extend head_size by 8, which is sizeof(float) + # for scale and sizeof(float) for zeropoint + head_size += 8 + + key_cache_block = block_size * num_kv_heads * head_size + value_cache_block = key_cache_block + total = num_layers * (key_cache_block + value_cache_block) + dtype_size = cache_dtype.size + return dtype_size * total + + +class OpenVINOWorker(LoraNotSupportedWorkerBase): + """A worker class that executes the model on OpenVINO backend. + + Each worker is associated with a single OpenVINO device. The worker is + responsible for maintaining the KV cache and executing the model on the + OpenVINO backend. + """ + + def __init__( + self, + ov_core: ov.Core, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + lora_config: Optional[LoRAConfig] = None, + multimodal_config: Optional[MultiModalConfig] = None, + kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined, + is_driver_worker: bool = False, + ) -> None: + self.ov_core = ov_core + self.model_config = model_config + self.parallel_config = parallel_config + self.parallel_config.rank = rank + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.load_config = load_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + self.lora_config = lora_config + self.multimodal_config = multimodal_config + self.is_driver_worker = is_driver_worker + if self.is_driver_worker: + assert self.rank == 0, "The driver worker must have rank 0." + + if self.model_config.trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils import init_cached_hf_modules + + init_cached_hf_modules() + self.model_runner = OpenVINOModelRunner( + self.ov_core, + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config=self.load_config, + lora_config=self.lora_config, + multimodal_config=self.multimodal_config, + kv_cache_dtype=kv_cache_dtype, + is_driver_worker=is_driver_worker, + ) + # Uninitialized cache engine. Will be initialized by + # initialize_cache. + self.cache_engine: OpenVINOCacheEngine + self.kv_cache: List[Tuple[ov.Tensor, ov.Tensor]] + + def init_device(self) -> None: + self.init_distributed_environment() + # Set random seed. + set_random_seed(self.model_config.seed) + + def load_model(self): + self.model_runner.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of blocks available for the KV cache. + + This determines how many KV blocks can fit into the configured + KV cache space. + """ + # For OpenVINO backend, in case of CPU device, the block number will be + # calculated based on the openvino_kvcache_space_bytes. + cache_block_size = self.get_cache_block_size_bytes() + kvcache_space_bytes = self.cache_config.openvino_kvcache_space_bytes + + if is_openvino_cpu(): + num_device_blocks = int(kvcache_space_bytes // cache_block_size) + num_swap_blocks = 0 + else: + if kvcache_space_bytes > 0: + logger.info("KV_CACHE size was explicitly configured via " + "VLLM_OPENVINO_KVCACHE_SPACE environment " + "variable, ignoring profiling run.") + kv_cache_size = kvcache_space_bytes + else: + try: + kv_cache_size = self.profile_run() + except Exception as err: + raise RuntimeError( + "The error occurred during profile run. This might be " + "due to insufficient GPU memory. Consider decreasing " + "`max_model_len` to limit the maximum simultaneously " + "processed tokens.") from err + + num_device_blocks = int(kv_cache_size // cache_block_size) + num_swap_blocks = int(self.cache_config.swap_space_bytes // + cache_block_size) + + return num_device_blocks, num_swap_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache. Swappable CPU memory is only + supported on GPU. + + For CPU, we use the num_gpu_blocks to + determine how many non-swappable CPU blocks to allocate. + """ + + num_device_blocks = num_gpu_blocks + num_swap_blocks = num_cpu_blocks + + if is_openvino_cpu(): + assert (num_swap_blocks == 0 + ), f"{type(self)} does not support swappable cache for CPU" + + self._validate_num_blocks(num_device_blocks) + self.cache_config.num_gpu_blocks = num_device_blocks + self.cache_config.num_cpu_blocks = num_swap_blocks + + # Initialize the cache. + self._init_cache_engine() + + def _validate_num_blocks(self, num_blocks: int) -> None: + """Raise errors if the num_blocks is invalid.""" + if num_blocks <= 0: + raise ValueError( + "No available memory for the cache blocks. " + "Try increasing `VLLM_OPENVINO_KVCACHE_SPACE` when " + "initializing the engine.") + + max_seq_len = self.cache_config.block_size * num_blocks + if self.model_config.max_model_len > max_seq_len: + raise ValueError( + f"The model's max seq len ({self.model_config.max_model_len}) " + "is larger than the maximum number of tokens that can be " + f"stored in KV cache ({max_seq_len}). Try increasing " + "`VLLM_OPENVINO_KVCACHE_SPACE` or decreasing `max_model_len` " + "when initializing the engine.") + + def _init_cache_engine(self) -> None: + ov_device = envs.VLLM_OPENVINO_DEVICE + self.cache_engine = OpenVINOCacheEngine( + self.cache_config, + self.model_config, + self.parallel_config, + self.device_config, + self.ov_core, + ov_device, + ) + self.kv_cache = self.cache_engine.kv_cache + self.model_runner.block_size = self.cache_engine.block_size + + assert self.kv_cache is not None + + # Populate the cache to warmup the memory + if is_openvino_cpu(): + for key_cache, value_cache in self.kv_cache: + key_cache.data[:] = 0 + value_cache.data[:] = 0 + + def cache_swap_in(self, src_to_dst: List[Tuple[int, int]]) -> None: + self.cache_engine.swap_in(src_to_dst) + + def cache_swap_out(self, src_to_dst: List[Tuple[int, int]]) -> None: + self.cache_engine.swap_out(src_to_dst) + + def cache_copy( + self, + blocks_to_copy: List[Tuple[int, int]], + ) -> None: + self.cache_engine.copy(blocks_to_copy) # type: ignore + + @torch.inference_mode() + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None, + ) -> List[SamplerOutput]: + if execute_model_req is None: + seq_group_metadata_list = None + else: + seq_group_metadata_list = execute_model_req.seq_group_metadata_list + + if self.is_driver_worker: + assert seq_group_metadata_list is not None + num_seq_groups: int = len(seq_group_metadata_list) + assert execute_model_req is not None + blocks_to_copy = execute_model_req.blocks_to_copy + blocks_to_swap_in = execute_model_req.blocks_to_swap_in + blocks_to_swap_out = execute_model_req.blocks_to_swap_out + data: Dict[str, Any] = { + "num_seq_groups": num_seq_groups, + "blocks_to_copy": execute_model_req.blocks_to_copy, + "blocks_to_swap_in": execute_model_req.blocks_to_swap_in, + "blocks_to_swap_out": execute_model_req.blocks_to_swap_out, + } + broadcast_tensor_dict(data, src=0) + else: + data = broadcast_tensor_dict(src=0) + num_seq_groups = data["num_seq_groups"] + blocks_to_copy = data["blocks_to_copy"] + blocks_to_swap_in = data["blocks_to_swap_in"] + blocks_to_swap_out = data["blocks_to_swap_out"] + + if is_openvino_cpu(): + assert len(execute_model_req.blocks_to_swap_in) == 0 + assert len(execute_model_req.blocks_to_swap_out) == 0 + else: + self.cache_swap_in(blocks_to_swap_in) + self.cache_swap_out(blocks_to_swap_out) + + self.cache_copy(blocks_to_copy) + + # If there is no input, we don't need to execute the model. + if num_seq_groups == 0: + return [] + + output = self.model_runner.execute_model(seq_group_metadata_list, + self.kv_cache) + + # OpenVINO worker only supports single-step execution. + return [output] + + def init_distributed_environment(self) -> None: + """Initialize the distributed environment.""" + + parallel_config = self.parallel_config + rank = self.rank + distributed_init_method = self.distributed_init_method + init_distributed_environment( + world_size=parallel_config.world_size, + rank=rank, + distributed_init_method=distributed_init_method, + backend="gloo", + ) + + # A small all_reduce for warmup. + torch.distributed.all_reduce(torch.zeros(1).cpu()) + + ensure_model_parallel_initialized( + parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size, + ) + + def get_cache_block_size_bytes(self) -> int: + """Return the size in bytes of a single KV cache block.""" + return OpenVINOCacheEngine.get_cache_block_size( + self.cache_config.block_size, + self.cache_config.cache_dtype, + self.model_config, + self.parallel_config, + ) + + def profile_run(self) -> int: + ov_device = envs.VLLM_OPENVINO_DEVICE + + assert not is_openvino_cpu(), \ + "CPU device isn't supposed to use profile run." + + import openvino.properties.device as device + import openvino.properties.intel_gpu as intel_gpu + + ov_core = self.ov_core + cache_config = self.cache_config + model_config = self.model_config + parallel_config = self.parallel_config + device_config = self.device_config + input_registry = INPUT_REGISTRY + mm_registry = MULTIMODAL_REGISTRY + mm_registry.init_mm_limits_per_prompt(model_config) + + # Execute a forward pass with dummy inputs to profile the memory usage + # of the model. + def model_profile_run(): + top_k = model_config.get_vocab_size() - 1 + sampling_params = SamplingParams(top_p=0.99, top_k=top_k) + + max_num_batched_tokens = \ + self.scheduler_config.max_num_batched_tokens + max_num_seqs = self.scheduler_config.max_num_seqs + tmp_cache_config = CacheConfig(cache_config.block_size, + cache_config.gpu_memory_utilization, + cache_config.swap_space_bytes, + "auto") + tmp_cache_config.num_gpu_blocks = 1 + tmp_cache_config.num_cpu_blocks = 0 + tmp_cache_config.cache_dtype = cache_config.cache_dtype + + profiling_cache_engine = OpenVINOCacheEngine( + tmp_cache_config, model_config, parallel_config, device_config, + ov_core, ov_device) + + # Profile memory usage with max_num_sequences sequences and the + # total # number of tokens equal to max_num_batched_tokens. + seqs: List[SequenceGroupMetadata] = [] + for group_id in range(max_num_seqs): + seq_len = (max_num_batched_tokens // max_num_seqs + + (group_id < max_num_batched_tokens % max_num_seqs)) + block_size = cache_config.block_size + seq_num_blocks = (seq_len + block_size - 1) // block_size + + seq_data, dummy_multi_modal_data = input_registry \ + .dummy_data_for_profiling(model_config, + seq_len, + mm_registry) + + block_tables = [[0] * seq_num_blocks] * max_num_seqs + seq = SequenceGroupMetadata( + request_id=str(group_id), + is_prompt=True, + seq_data={group_id: seq_data}, + sampling_params=sampling_params, + block_tables=block_tables, + lora_request=None, + multi_modal_data=dummy_multi_modal_data) + seqs.append(seq) + + self.model_runner.block_size = tmp_cache_config.block_size + + # Run the model with the dummy inputs. + self.model_runner.execute_model(seqs, + profiling_cache_engine.kv_cache) + + # explicitly delete temporary KV cache manager to free KV cache + # when real inputs will be passed to OV + del profiling_cache_engine + + logger.info( + "Start profiling run with dummy inputs to evaluate " + "memory usage for %s. It might take a while.", ov_device) + + model_profile_run() + + gpu_device_type = ov_core.get_property(ov_device, device.type) + memory_statistics = \ + ov_core.get_property(ov_device, intel_gpu.memory_statistics) + memory_utilization = cache_config.gpu_memory_utilization + + if gpu_device_type == device.Type.INTEGRATED and \ + memory_utilization >= 0.9: + logger.warning( + "iGPU is used with high gpu_memory_utilization=%f " + "value. This may cause low performance due to " + "occupying the majority of available system " + "memory. Please consider decreasing " + "gpu_memory_utilization or explicitly setting" + "`VLLM_OPENVINO_KVCACHE_SPACE` (GB) environment " + "variable.", memory_utilization) + + # sum up all used device memory + device_memory_types = ["cl_mem", "usm_device"] + used_device_mem = \ + sum(memory_statistics.get(key, 0) for key in device_memory_types) + + if gpu_device_type == device.Type.INTEGRATED: + used_device_mem += memory_statistics.get("usm_host", 0) + + # there could be unaccounted extra memory reserved by kernels, kept + # in memory pools, etc + # therefore, add a threshold to account for this + used_memory_threshold = 1.1 + used_device_mem *= used_memory_threshold + + total_device_memory = \ + ov_core.get_property(ov_device, intel_gpu.device_total_mem_size) + + def format_memory_size(size) -> str: + units = ["B", "KB", "MB", "GB"] + unit_index = 0 + + while size > 1024 and unit_index < len(units) - 1: + size /= 1024 + unit_index += 1 + + return f"{size:.2f} {units[unit_index]}" + + total_device_memory_str = \ + format(format_memory_size(total_device_memory)) + used_device_memory_str = \ + format(format_memory_size(used_device_mem)) + + logger.info( + "Total %s memory: %s. " + "Amount of memory required to run the model with " + "max_num_batched_tokens=%d: %s.", ov_device, + total_device_memory_str, + self.scheduler_config.max_num_batched_tokens, + used_device_memory_str) + + if used_device_mem >= total_device_memory: + raise RuntimeError( + f"The required memory size {used_device_memory_str} for model " + "is higher than the total available device " + "memory {total_device_memory_str}. Please consider to " + "decrease `max_num_batched_tokens` or increase " + "`gpu_memory_utilization`") + + return total_device_memory * memory_utilization - used_device_mem diff --git a/vllm/worker/tpu_model_runner.py b/vllm/worker/tpu_model_runner.py new file mode 100644 index 00000000..c13e95f6 --- /dev/null +++ b/vllm/worker/tpu_model_runner.py @@ -0,0 +1,842 @@ +import time +from dataclasses import dataclass +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, + Type, Union) +from unittest.mock import patch + +import numpy as np +import torch +import torch.nn as nn +import torch_xla.core.xla_model as xm +import torch_xla.runtime as xr + +from vllm.attention import AttentionMetadata, get_attn_backend +from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig, + ParallelConfig, SchedulerConfig) +from vllm.logger import init_logger +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader import get_model +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors, + Logprob, SequenceGroupMetadata, SequenceOutput) +from vllm.worker.model_runner_base import ( + ModelRunnerBase, ModelRunnerInputBase, + _add_attn_metadata_broadcastable_dict, + _init_attn_metadata_from_tensor_dict) + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + +# Here we utilize the behavior that out-of-bound index is ignored. +# FIXME(woosuk): Find a more reliable way to prevent possible bugs. +_PAD_SLOT_ID = 1_000_000_000 +# FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow. +_ENABLE_TOP_P = False +# FIXME(woosuk): A temporary hack to support `n > 1`. +# This can significantly affect the performance if too large. +_MAX_NUM_SAMPLES = 128 + + +@dataclass(frozen=True) +class ModelInputForTPU(ModelRunnerInputBase): + token_ids: torch.Tensor + position_ids: torch.Tensor + attn_metadata: AttentionMetadata + input_lens: torch.Tensor + t: torch.Tensor + p: torch.Tensor + num_samples: int + n: List[int] + seq_groups: List[List[int]] + is_first_multi_step: bool = True + is_last_step: bool = True + virtual_engine: int = 0 + async_callback: Optional[Callable] = None + + def as_broadcastable_tensor_dict( + self) -> Dict[str, Union[int, torch.Tensor]]: + tensor_dict = { + "token_ids": self.token_ids, + "position_ids": self.position_ids, + "input_lens": self.input_lens, + "t": self.t, + "p": self.p, + "num_samples": self.num_samples, + "n": self.n, + "seq_groups": self.seq_groups, + "is_first_multi_step": self.is_first_multi_step, + "is_last_step": self.is_last_step, + "virtual_engine": self.virtual_engine, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls: Type["ModelInputForTPU"], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "ModelInputForTPU": + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]): + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + is_driver_worker: bool = False, + ): + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.load_config = load_config + self.is_driver_worker = is_driver_worker + + self.block_size = self.cache_config.block_size + self.max_num_blocks_per_seq = (self.model_config.max_model_len // + self.block_size) + self.block_tables = np.zeros( + (self.scheduler_config.max_num_seqs, self.max_num_blocks_per_seq), + dtype=np.int32) + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.cache_config.cache_dtype, + self.block_size, + self.model_config.is_attention_free, + False, + ) + self.cached_step_outputs: List[torch.Tensor] = [] + + def load_model(self) -> None: + self.device = self.device_config.device + + # NOTE(woosuk): While the executor assigns the TP ranks to the worker + # process, the ranks can be different from the ranks internally assigned + # by the xm runtime. Therefore, there is a mismatch in the rank + # assignment between the gloo (cpu) runtime and the xm (tpu) runtime. + # This is not a problem in linear layers because all-reduce is + # rank-agnostic. However, it matters for all-gather as the ranks + # determine the order of concatenating the output tensors. + # As a workaround, we use the xm's rank assignment only when loading + # the embedding weights. + xm_tp_rank = xr.global_ordinal() + with patch( + "vllm.model_executor.layers.vocab_parallel_embedding." + "get_tensor_model_parallel_rank", + return_value=xm_tp_rank): + model = get_model( + model_config=self.model_config, + load_config=self.load_config, + device_config=self.device_config, + parallel_config=self.parallel_config, + cache_config=self.cache_config, + scheduler_config=self.scheduler_config, + lora_config=None, + ) + model = model.eval() + xm.wait_device_ops() + self.model = ModelWrapper(model) + + def _dummy_run( + self, + batch_size: int, + seq_len: int, + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]], + is_prompt: bool, + ) -> None: + if is_prompt: + seq_len = (seq_len + 15) // 16 * 16 + token_ids = torch.zeros((batch_size, seq_len), + dtype=torch.int32, + device=self.device) + position_ids = torch.zeros((batch_size, seq_len), + dtype=torch.int32, + device=self.device) + slot_mapping = torch.zeros((batch_size, seq_len), + dtype=torch.int64, + device=self.device) + attn_metadata = self.attn_backend.make_metadata( + num_prefills=batch_size, + num_prefill_tokens=batch_size * seq_len, + num_decode_tokens=0, + slot_mapping=slot_mapping, + block_tables=None, + context_lens=None, + ) + input_lens = torch.ones((batch_size, ), + dtype=torch.int32, + device=self.device) + else: + assert seq_len == 1 + token_ids = torch.zeros((batch_size, seq_len), + dtype=torch.int32, + device=self.device) + position_ids = torch.zeros((batch_size, seq_len), + dtype=torch.int32, + device=self.device) + slot_mapping = torch.zeros((batch_size, seq_len), + dtype=torch.int64, + device=self.device) + block_tables = torch.zeros( + (batch_size, self.max_num_blocks_per_seq), + dtype=torch.int32, + device=self.device) + context_lens = torch.ones((batch_size, ), + dtype=torch.int32, + device=self.device) + input_lens = torch.ones((batch_size, ), + dtype=torch.int32, + device=self.device) + attn_metadata = self.attn_backend.make_metadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=batch_size * seq_len, + slot_mapping=slot_mapping, + block_tables=block_tables, + context_lens=context_lens, + ) + t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device) + p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device) + num_samples = _MAX_NUM_SAMPLES if is_prompt else 1 + + # NOTE(woosuk): There are two stages of compilation: torch.compile and + # XLA compilation. Using `mark_dynamic` can reduce the torch.compile + # overhead by reusing the FX graph for different shapes. + # However, the XLA graph will still require static shapes and needs to + # be re-compiled for every different shapes. This overhead is inevitable + # in the first run, but can be skipped afterwards as we cache the XLA + # graphs in the disk (VLLM_XLA_CACHE_PATH). + if is_prompt: + # Prefll + torch._dynamo.mark_dynamic(token_ids, 1) + torch._dynamo.mark_dynamic(position_ids, 1) + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1) + else: + # Decode + torch._dynamo.mark_dynamic(token_ids, 0) + torch._dynamo.mark_dynamic(position_ids, 0) + torch._dynamo.mark_dynamic(input_lens, 0) + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0) + torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0) + torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0) + torch._dynamo.mark_dynamic(t, 0) + torch._dynamo.mark_dynamic(p, 0) + # Dummy run. + self.model(token_ids, + position_ids, + attn_metadata, + input_lens, + t, + p, + num_samples, + kv_caches, + is_prompt=is_prompt) + + def warmup_model( + self, + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]], + ) -> None: + # Prefill + logger.info("Compiling the model with different input shapes...") + start = time.time() + for batch_size in [1]: + seq_len = 16 + while True: + self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=True) + xm.wait_device_ops() + logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len) + + if seq_len >= self.model_config.max_model_len: + break + num_tokens = batch_size * seq_len + if num_tokens >= self.scheduler_config.max_num_batched_tokens: + break + seq_len = seq_len * 2 + + end = time.time() + logger.info("Compilation for prefill done in %.2f s.", end - start) + + # Decode + start = time.time() + seq_len = 1 + batch_size = 8 # Must be in sync with _get_padded_batch_size() + while True: + self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False) + xm.wait_device_ops() + logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len) + + if batch_size >= self.scheduler_config.max_num_seqs: + break + batch_size = batch_size + 16 if batch_size >= 16 else batch_size * 2 + + end = time.time() + logger.info("Compilation for decode done in %.2f s.", end - start) + + def _prepare_prompt( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[int] = [] + input_positions: List[int] = [] + prompt_lens: List[int] = [] + slot_mapping: List[int] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert seq_group_metadata.is_prompt + seq_ids = list(seq_group_metadata.seq_data.keys()) + assert len(seq_ids) == 1 + seq_id = seq_ids[0] + + seq_data = seq_group_metadata.seq_data[seq_id] + # Could include output tokens when a request is preempted. + prompt_tokens = seq_data.get_token_ids() + prompt_len = len(prompt_tokens) + prompt_lens.append(prompt_len) + + input_tokens.extend(prompt_tokens) + input_positions.extend(list(range(prompt_len))) + + assert seq_group_metadata.block_tables is not None + block_table = seq_group_metadata.block_tables[seq_id] + for i in range(prompt_len): + block_number = block_table[i // self.block_size] + block_offset = i % self.block_size + slot = block_number * self.block_size + block_offset + slot_mapping.append(slot) + + # Add paddings to EACH prompt to the smallest power of 2 that is + # greater than or equal to the prompt length. + # We pad the seq_len to reduce the compilation overhead. + # We execute each prompt individually (i.e., with batch_size 1) + # because the FlashAttention kernel does not support ragged inputs. + # TODO(woosuk): Use SplashAttention to support ragged inputs. + padded_prompt_len = _get_padded_prefill_len(prompt_len) + num_paddings = padded_prompt_len - prompt_len + input_tokens += [0] * num_paddings + input_positions += [0] * num_paddings + slot_mapping += [_PAD_SLOT_ID] * num_paddings + + assert len(prompt_lens) > 0 + num_prefills = len(prompt_lens) + input_tokens = torch.tensor(input_tokens, + dtype=torch.int32, + device="cpu") + input_positions = torch.tensor(input_positions, + dtype=torch.int32, + device="cpu") + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.int64, + device="cpu") + prompt_lens = torch.tensor(prompt_lens, + dtype=torch.int32, + device="cpu") + attn_metadata = self.attn_backend.make_metadata( + num_prefills=num_prefills, + num_prefill_tokens=0, # NOTE: This is not used. + num_decode_tokens=0, + slot_mapping=slot_mapping, + block_tables=None, + context_lens=None, + ) + return input_tokens, input_positions, attn_metadata, prompt_lens + + def _prepare_decode( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[List[int]] = [] + input_positions: List[List[int]] = [] + slot_mapping: List[List[int]] = [] + context_lens: List[int] = [] + + batch_idx = 0 + for seq_group_metadata in seq_group_metadata_list: + assert not seq_group_metadata.is_prompt + seq_ids = list(seq_group_metadata.seq_data.keys()) + for seq_id in seq_ids: + seq_data = seq_group_metadata.seq_data[seq_id] + generation_token = seq_data.get_last_token_id() + input_tokens.append([generation_token]) + + seq_len = seq_data.get_len() + position = seq_len - 1 + input_positions.append([position]) + context_lens.append(seq_len) + + assert seq_group_metadata.block_tables is not None + block_table = seq_group_metadata.block_tables[seq_id] + self.block_tables[batch_idx, :len(block_table)] = block_table + batch_idx += 1 + + block_number = block_table[position // self.block_size] + block_offset = position % self.block_size + slot = block_number * self.block_size + block_offset + slot_mapping.append([slot]) + + batch_size = _get_padded_batch_size(batch_idx) + num_paddings = batch_size - batch_idx + input_tokens = input_tokens + [[0]] * num_paddings + input_positions = input_positions + [[0]] * num_paddings + slot_mapping = slot_mapping + [[_PAD_SLOT_ID]] * num_paddings + context_lens = context_lens + [0] * num_paddings + + input_tokens = torch.tensor(input_tokens, + dtype=torch.int32, + device="cpu") + input_positions = torch.tensor(input_positions, + dtype=torch.int32, + device="cpu") + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.int64, + device="cpu") + context_lens = torch.tensor(context_lens, + dtype=torch.int32, + device="cpu") + block_tables = torch.tensor(self.block_tables[:batch_size], + dtype=torch.int32, + device="cpu") + input_lens = torch.tensor([1] * batch_size, + dtype=torch.int32, + device="cpu") + attn_metadata = self.attn_backend.make_metadata( + num_prefills=0, + num_prefill_tokens=0, + num_decode_tokens=batch_size, + slot_mapping=slot_mapping, + block_tables=block_tables, + context_lens=context_lens, + ) + return input_tokens, input_positions, attn_metadata, input_lens + + def _prepare_sample( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + padded_batch_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor, List[int]]: + assert len(seq_group_metadata_list) > 0 + t = [] + p = [] + n = [] + for seq_group_metadata in seq_group_metadata_list: + sampling_params = seq_group_metadata.sampling_params + t.append(sampling_params.temperature) + if sampling_params.top_p != 1 and not _ENABLE_TOP_P: + raise NotImplementedError( + "Top-p sampling is currently disabled for the TPU backend " + "due to performance issues.") + p.append(sampling_params.top_p) + if sampling_params.top_k != -1: + raise NotImplementedError( + "Top-k sampling is currently disabled for the TPU backend " + "due to performance issues.") + if sampling_params.n > _MAX_NUM_SAMPLES: + raise NotImplementedError( + f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU " + "backend.") + n.append(sampling_params.n) + if sampling_params.logprobs is not None: + raise NotImplementedError( + "logprobs is not currently supported by the TPU backend.") + if sampling_params.prompt_logprobs is not None: + raise NotImplementedError( + "prompt_logprobs is not currently supported by the TPU " + "backend.") + + # Repeat the sampling params if the seq group has multiple seqs. + num_seqs = len(seq_group_metadata.seq_data) + t += [t[-1]] * (num_seqs - 1) + p += [p[-1]] * (num_seqs - 1) + n += [n[-1]] * (num_seqs - 1) + + num_paddings = padded_batch_size - len(t) + t += [1.0] * num_paddings + p += [1.0] * num_paddings + + t = torch.tensor(t, dtype=torch.float32, device="cpu") + p = torch.tensor(p, dtype=torch.float32, device="cpu") + return t, p, n + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None, + ) -> ModelInputForTPU: + del finished_requests_ids # Unused. + assert virtual_engine == 0 + assert len(seq_group_metadata_list) > 0 + # NOTE: We assume that all sequences in the group are all prompts or + # all decodes. + is_prompt = seq_group_metadata_list[0].is_prompt + if is_prompt: + inputs = self._prepare_prompt(seq_group_metadata_list) + else: + inputs = self._prepare_decode(seq_group_metadata_list) + input_tokens, input_positions, attn_metadata, input_lens = inputs + padded_batch_size = input_tokens.shape[0] + t, p, n = self._prepare_sample(seq_group_metadata_list, + padded_batch_size) + num_samples = _MAX_NUM_SAMPLES if is_prompt else 1 + + seq_groups = [ + list(metadata.seq_data.keys()) + for metadata in seq_group_metadata_list + ] + return ModelInputForTPU(input_tokens, input_positions, attn_metadata, + input_lens, t, p, num_samples, n, seq_groups) + + def make_model_input_from_broadcasted_tensor_dict( + self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU: + model_input = ModelInputForTPU.from_broadcasted_tensor_dict( + tensor_dict, attn_backend=self.attn_backend) + return model_input + + @torch.no_grad() + def execute_model( + self, + model_input: ModelInputForTPU, + kv_caches: Optional[List[Any]], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> List[SamplerOutput]: + assert intermediate_tensors is None + if not model_input.is_first_multi_step: + if not model_input.is_last_step: + return [] + + use_async_out_proc = model_input.async_callback is not None + sampler_outputs = [] + num_outputs = len(self.cached_step_outputs) + for i in range(num_outputs): + next_token_ids = self.cached_step_outputs.pop(0) + next_token_ids = next_token_ids.cpu().tolist() + sampler_output = _make_decode_output(next_token_ids, + model_input.seq_groups) + sampler_outputs.append(sampler_output) + + if i < num_outputs - 1 and use_async_out_proc: + assert model_input.async_callback is not None + ctx = model_input.async_callback.keywords[ # type: ignore + "ctx"] + ctx.append_output( + outputs=[sampler_output], + seq_group_metadata_list=ctx.seq_group_metadata_list, + scheduler_outputs=ctx.scheduler_outputs, + is_async=False, + is_last_step=False, + is_first_step_output=i == 0) + model_input.async_callback() + if use_async_out_proc: + return [sampler_outputs[-1]] + else: + return sampler_outputs + + is_prompt = model_input.attn_metadata.num_prefills > 0 + if is_prompt: + assert num_steps == 1 + # NOTE(woosuk): Since the FlashAttention kernel does not support + # ragged inputs, we split the prompts into different batches and + # process them separately. This is a temporary hack that should be + # optimized by using SplashAttention. + orig_slot_mapping = model_input.attn_metadata.slot_mapping + batch_size = model_input.input_lens.shape[0] + start_idx = 0 + next_token_ids = [] + for i in range(batch_size): + # Get the actual prefill_len. + prefill_len = model_input.input_lens[i:i + 1].item() + prefill_len = _get_padded_prefill_len(prefill_len) + end_idx = start_idx + prefill_len + + token_ids = model_input.token_ids[None, start_idx:end_idx].to( + self.device) + position_ids = model_input.position_ids[None, + start_idx:end_idx].to( + self.device) + attn_metadata = model_input.attn_metadata + attn_metadata.num_prefills = 1 + attn_metadata.slot_mapping = orig_slot_mapping[ + None, start_idx:end_idx].to(self.device) + input_lens = model_input.input_lens[i:i + 1].to(self.device) + t = model_input.t[i:i + 1].to(self.device) + p = model_input.p[i:i + 1].to(self.device) + output_token_ids = self.model(token_ids, + position_ids, + attn_metadata, + input_lens, + t, + p, + model_input.num_samples, + kv_caches, + is_prompt=True) + next_token_ids.append(output_token_ids[0]) + start_idx = end_idx + + if model_input.async_callback is not None: + model_input.async_callback() + # Retrieve the outputs to CPU. + next_token_ids = [ + output_token_ids.cpu().tolist() + for output_token_ids in next_token_ids + ] + + # NOTE(woosuk): Minimal code to construct the sampler outputs. + # The TPU backend does not reuse the sampler, since the TPU backend + # does not support advanced sampling parameters such as logprobs. + zero_logprob = Logprob(0.0) + sampler_outputs = [] + for i, seq_group in enumerate(model_input.seq_groups): + seq_ids = seq_group + assert len(seq_ids) == 1 + seq_id = seq_ids[0] + seq_outputs = [] + for j in range(model_input.n[i]): + next_token_id = next_token_ids[i][j] + seq_outputs.append( + SequenceOutput(seq_id, next_token_id, + {next_token_id: zero_logprob})) + sampler_outputs.append( + CompletionSequenceGroupOutput(seq_outputs, None)) + return [SamplerOutput(sampler_outputs)] + else: + token_ids = model_input.token_ids.to(self.device) + position_ids = model_input.position_ids.to(self.device) + attn_metadata = model_input.attn_metadata + attn_metadata.slot_mapping = attn_metadata.slot_mapping.to( + self.device) + attn_metadata.block_tables = attn_metadata.block_tables.to( + self.device) + attn_metadata.context_lens = attn_metadata.context_lens.to( + self.device) + t = model_input.t.to(self.device) + p = model_input.p.to(self.device) + input_lens = model_input.input_lens.to(self.device) + for i in range(num_steps): + slot_mapping = attn_metadata.slot_mapping + output_token_ids = self.model(token_ids, + position_ids, + attn_metadata, + input_lens, + t, + p, + model_input.num_samples, + kv_caches, + is_prompt=False) + self.cached_step_outputs.append(output_token_ids) + + if i < num_steps - 1: + # Prepare the inputs for the next step. + token_ids = output_token_ids.unsqueeze(dim=1).int() + position_ids = position_ids + 1 + attn_metadata.context_lens = attn_metadata.context_lens + 1 + + block_tables = attn_metadata.block_tables + block_number = block_tables.gather( + 1, + position_ids.long() // self.block_size) + block_offset = position_ids % self.block_size + + is_padding = slot_mapping == _PAD_SLOT_ID + slot_mapping = block_number * self.block_size + block_offset + slot_mapping = slot_mapping.long() + slot_mapping = torch.where(is_padding, _PAD_SLOT_ID, + slot_mapping) + attn_metadata.slot_mapping = slot_mapping + + if model_input.async_callback is not None: + model_input.async_callback() + + if num_steps > 1: + return [] + # Retrieve the outputs to CPU. + next_token_ids = self.cached_step_outputs.pop(0) + next_token_ids = next_token_ids.cpu().tolist() + sampler_output = _make_decode_output(next_token_ids, + model_input.seq_groups) + return [sampler_output] + + +class ModelWrapper(TorchCompileWrapperWithCustomDispatcher): + + def __init__(self, model: nn.Module): + self.model = model + compiled_callable = torch.compile(self.forward, + backend="openxla", + fullgraph=True, + dynamic=False) + super().__init__(compiled_callable) + + def __call__(self, *args, is_prompt: bool, **kwargs): + if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher: + # not fully compiled yet, or not using the custom dispatcher, + # let PyTorch handle it + return self.compiled_callable(*args, **kwargs) + # the 3 compiled codes are: + # 0: for profiling + # 1: for prompt + # 2: for decode + # dispatch to the compiled code directly, skip PyTorch + if is_prompt: + with self.dispatch_to_code(1): + return self.forward(*args, **kwargs) + else: + with self.dispatch_to_code(2): + return self.forward(*args, **kwargs) + + def forward( + self, + token_ids: torch.Tensor, + position_ids: torch.Tensor, + attn_metadata: AttentionMetadata, + input_lens: torch.Tensor, + t: torch.Tensor, + p: torch.Tensor, + num_samples: int, + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]], + ) -> torch.Tensor: + """Executes the forward pass of the model and samples the next token. + + Args: + token_ids: The input token IDs of shape [batch_size, seq_len]. + position_ids: The input position IDs of shape [batch_size, seq_len]. + attn_metadata: The Pallas attention metadata. + input_lens: The actual input lengths of shape [batch_size]. + t: The sampling temperature of shape [batch_size]. + p: The top-p probability of shape [batch_size]. + num_samples: Number of samples to draw from each logits vector. + kv_caches: The key and value caches. They can be None during the + memory profiling at initialization. + """ + batch_size, seq_len = token_ids.shape + # Calculate the positions to sample from. + start_indicies = torch.arange( + batch_size, dtype=torch.int32, device=input_lens.device) * seq_len + logits_indices = start_indicies + input_lens - 1 + + # FIXME(woosuk): This is a temporary hack to avoid using the existing + # sampler and sampling metadata. + sampling_metadata = SamplingMetadata( + seq_groups=[], + selected_token_indices=logits_indices, + categorized_sample_indices={}, + num_prompts=attn_metadata.num_prefills, + ) + + # Skip this in memory profiling at initialization. + if kv_caches[0][0].numel() > 0: + # index_copy_(slot_mapping) only works when the inserted dimension + # is 0. However, the KV cache in the Pallas backend has the shape + # [num_kv_heads, num_blocks, block_size, head_size]. To make it + # work, we need to flatten the first three dimensions and modify + # the slot_mapping accordingly. + num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape + slot_mapping = attn_metadata.slot_mapping + slot_mapping = slot_mapping.flatten() + head_indicies = torch.arange(0, + num_kv_heads, + device=slot_mapping.device, + dtype=slot_mapping.dtype) + head_indicies *= block_size * num_blocks + slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view( + -1, num_kv_heads) + slot_mapping = slot_mapping + head_indicies.view(1, -1) + slot_mapping = slot_mapping.flatten() + attn_metadata.slot_mapping = slot_mapping + + hidden_states = self.model( + token_ids, + position_ids, + kv_caches, + attn_metadata, + ) + hidden_states = hidden_states.flatten(0, 1) + logits = self.model.compute_logits(hidden_states, sampling_metadata) + + # Argmax sampling. + argmax_token_ids = torch.argmax(logits, dim=-1, keepdim=True) + argmax_token_ids = argmax_token_ids.repeat(1, num_samples) + + # Zero temperature means greedy decoding. Avoid division by zero. + nonzero_t = torch.where(t != 0, t, 1.0) + logits = logits / nonzero_t.unsqueeze(dim=1) + if _ENABLE_TOP_P: + logits = _apply_top_p(logits, p.unsqueeze(dim=1)) + + # Random sampling. + probs = torch.softmax(logits, dim=-1, dtype=torch.float32) + sampled_token_ids = torch.multinomial(probs, + num_samples, + replacement=True) + if num_samples == 1: + argmax_token_ids = argmax_token_ids.squeeze(dim=-1) + sampled_token_ids = sampled_token_ids.squeeze(dim=-1) + next_token_ids = torch.where(t != 0, sampled_token_ids, + argmax_token_ids) + return next_token_ids + + +def _get_padded_prefill_len(x: int) -> int: + # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence + # length to be a multiple of 16. We pad the prompt length to the nearest + # multiple of 16. This is also good for performance. + if x <= 16: + return 16 + return 1 << (x - 1).bit_length() + + +def _get_padded_batch_size(batch_size: int) -> int: + # The GMM Pallas kernel requires num_tokens * topk to be a multiple of 16. + # To meet this requirement in the simplest way, we set the minimal batch + # size to 8. + if batch_size <= 8: + return 8 + else: + return ((batch_size + 15) // 16) * 16 + + +def _apply_top_p(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor: + logits_sorted = torch.sort(logits, dim=-1, descending=True).values + sorted_cum_probs = torch.cumsum(logits_sorted.softmax(dim=-1), dim=-1) + cutoff_index = torch.sum(sorted_cum_probs < p, dim=-1, keepdim=True) + cutoff_logit = torch.gather(logits_sorted, -1, cutoff_index) + logits = logits.masked_fill_(logits < cutoff_logit, -float("inf")) + return logits + + +def _make_decode_output( + next_token_ids: List[int], + seq_groups: List[List[int]], +) -> SamplerOutput: + zero_logprob = Logprob(0.0) + sampler_outputs = [] + batch_idx = 0 + for seq_group in seq_groups: + seq_ids = seq_group + seq_outputs = [] + for seq_id in seq_ids: + next_token_id = next_token_ids[batch_idx] + seq_outputs.append( + SequenceOutput(seq_id, next_token_id, + {next_token_id: zero_logprob})) + batch_idx += 1 + sampler_outputs.append(CompletionSequenceGroupOutput( + seq_outputs, None)) + return SamplerOutput(sampler_outputs) diff --git a/vllm/worker/tpu_worker.py b/vllm/worker/tpu_worker.py new file mode 100644 index 00000000..fe819b9f --- /dev/null +++ b/vllm/worker/tpu_worker.py @@ -0,0 +1,309 @@ +import os +from typing import List, Optional, Tuple, Union + +import torch +import torch_xla.core.xla_model as xm +import torch_xla.runtime as xr + +import vllm.envs as envs +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig, + ParallelConfig, SchedulerConfig) +from vllm.distributed import (ensure_model_parallel_initialized, + init_distributed_environment) +from vllm.logger import init_logger +from vllm.model_executor import set_random_seed +from vllm.sequence import ExecuteModelRequest +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size +from vllm.worker.tpu_model_runner import TPUModelRunner +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, + LoraNotSupportedWorkerBase, WorkerInput) + +logger = init_logger(__name__) + + +class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase): + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + is_driver_worker: bool, + ) -> None: + self.model_config = model_config + self.parallel_config = parallel_config + self.parallel_config.rank = rank + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.load_config = load_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + self.is_driver_worker = is_driver_worker + + assert self.device_config.device_type == "tpu" + if self.cache_config.cache_dtype == "auto": + self.cache_dtype = self.model_config.dtype + else: + self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[ + self.cache_config.cache_dtype] + + self.model_runner: TPUModelRunner = TPUModelRunner( + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config, + is_driver_worker=is_driver_worker) + + def init_device(self) -> None: + os.environ["PJRT_DEVICE"] = "TPU" + torch.set_grad_enabled(False) + torch.set_default_dtype(self.model_config.dtype) + + # NOTE(woosuk): This is just to initialize the TP group and broadcast + # the input objects on CPU. The all-reduce and all-gather ops on TPU + # are invoked by `xm.all_reduce` and `xm.all_gather` which use their + # own context. + init_distributed_environment( + world_size=self.parallel_config.world_size, + rank=self.rank, + local_rank=self.local_rank, + distributed_init_method=self.distributed_init_method, + backend="gloo", + ) + ensure_model_parallel_initialized( + self.parallel_config.tensor_parallel_size, + self.parallel_config.pipeline_parallel_size) + + # Device initialization should happen after initializing the distributed + # runtime. + self.device = xm.xla_device() + self.device_config.device = self.device + + # Set random seed. + set_random_seed(self.model_config.seed) + xm.set_rng_state(self.model_config.seed, self.device) + + # Increase the cache size limit, which is the maximum number of + # dynamo graphs that can be compiled. + # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and + # 30-40 graphs for decode. 128 is an arbitrary safe number. + torch._dynamo.config.cache_size_limit = 128 + # Use persistent cache to avoid XLA recompilation. + # NOTE(woosuk): Set per-rank cache path since different ranks + # can have slightly different XLA graphs. + world_size = self.parallel_config.world_size + rank = xr.global_ordinal() + per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH, + f"tp{world_size}_rank{rank}") + xr.initialize_cache(per_rank_path, readonly=False) + + def load_model(self): + self.model_runner.load_model() + + def determine_num_available_blocks(self) -> Tuple[int, int]: + num_layers = self.model_config.get_num_layers(self.parallel_config) + head_size = self.model_config.get_head_size() + num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config) + + # use an empty tensor instead of `None`` to force Dynamo to pass + # it by reference, rather by specializing on the value ``None``. + # the `dtype` argument does not matter, and we use `float32` as + # a placeholder (it has wide hardware support). + kv_caches = [(torch.tensor([], dtype=torch.float32, + device=self.device), + torch.tensor([], dtype=torch.float32, + device=self.device)) + for _ in range(num_layers)] + self.model_runner._dummy_run( + batch_size=1, + seq_len=self.scheduler_config.max_num_batched_tokens, + kv_caches=kv_caches, + is_prompt=True, + ) + # Synchronize before measuring the memory usage. + xm.wait_device_ops() + + dtype_btyes = get_dtype_size(self.cache_dtype) + block_size = self.cache_config.block_size + block_size_bytes = (dtype_btyes * block_size * num_layers * 2 * + head_size * num_kv_heads) + + # Calculate the TPU KV cache size based on profiling. + m = xm.get_memory_info(self.device) + total_memory_size = m["bytes_limit"] + usable_memory_size = int(total_memory_size * + self.cache_config.gpu_memory_utilization) + profiled = m["bytes_used"] # Weights + intermediate activations. + tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0) + num_tpu_blocks = tpu_kv_cache_bytes // block_size_bytes + num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8. + + # Calculate the CPU KV cache size based on the config. + num_cpu_blocks = int(self.cache_config.swap_space_bytes // + block_size_bytes) + num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8. + return num_tpu_blocks, num_cpu_blocks + + def initialize_cache( + self, + num_gpu_blocks: int, + num_cpu_blocks: int, + ) -> None: + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + self.block_size = self.cache_config.block_size + + dtype = self.cache_dtype + num_layers = self.model_config.get_num_layers(self.parallel_config) + num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config) + head_size = self.model_config.get_head_size() + + self.cpu_cache: List[Tuple[torch.Tensor, torch.Tensor]] = [] + self.tpu_cache: List[Tuple[torch.Tensor, torch.Tensor]] = [] + tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape( + num_gpu_blocks, self.block_size, num_kv_heads, head_size) + cpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape( + num_cpu_blocks, self.block_size, num_kv_heads, head_size) + for _ in range(num_layers): + tpu_k_cache = torch.zeros(tpu_cache_shape, + dtype=dtype, + device=self.device) + tpu_v_cache = torch.zeros_like(tpu_k_cache) + self.tpu_cache.append((tpu_k_cache, tpu_v_cache)) + cpu_k_cache = torch.zeros(cpu_cache_shape, + dtype=dtype, + device="cpu") + cpu_v_cache = torch.zeros_like(cpu_k_cache) + self.cpu_cache.append((cpu_k_cache, cpu_v_cache)) + self._warmup_model() + + def _warmup_model(self) -> None: + # FIXME(woosuk): Here we are abusing `enforce_eager` which is defined + # for CUDA graphs. We should refactor this part. + if not self.model_config.enforce_eager: + # Warm up the model with all possible input shapes so that + # compilation never happens during the actual execution. + # This may take ~30 mins for the first run and ~20 mins for the + # subsequent runs. + # If `enforce_eager` is True, the ahead-of-time compilation is + # skipped and the compilation happens during the actual execution, + # which is bad for performance but useful for development. + self.model_runner.warmup_model(self.tpu_cache) + + def get_cache_block_size_bytes(self) -> int: + head_size = self.model_config.get_head_size() + num_heads = self.model_config.get_num_kv_heads(self.parallel_config) + num_layers = self.model_config.get_num_layers(self.parallel_config) + + key_cache_block = self.cache_config.block_size * num_heads * head_size + value_cache_block = key_cache_block + total = num_layers * (key_cache_block + value_cache_block) + dtype_size = get_dtype_size(self.cache_dtype) + return dtype_size * total + + @property + def do_metadata_broadcast(self) -> bool: + return self.parallel_config.tensor_parallel_size > 1 + + @property + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: + # NOTE(woosuk): This assumes virtual_engine == 0, i.e., no pipeline + # parallelism. + return [self.tpu_cache] + + def prepare_worker_input( + self, + execute_model_req: ExecuteModelRequest, + ) -> WorkerInput: + virtual_engine = execute_model_req.virtual_engine + num_seq_groups = len(execute_model_req.seq_group_metadata_list) + blocks_to_swap_in = _make_src_to_dst( + execute_model_req.blocks_to_swap_in, "cpu", self.device) + blocks_to_swap_out = _make_src_to_dst( + execute_model_req.blocks_to_swap_out, self.device, "cpu") + blocks_to_copy = _make_src_to_dst(execute_model_req.blocks_to_copy, + self.device, self.device) + return WorkerInput( + num_seq_groups=num_seq_groups, + blocks_to_swap_in=blocks_to_swap_in, + blocks_to_swap_out=blocks_to_swap_out, + blocks_to_copy=blocks_to_copy, + virtual_engine=virtual_engine, + ) + + def execute_worker(self, worker_input: WorkerInput) -> None: + virtual_engine = worker_input.virtual_engine + assert virtual_engine == 0 + attn_backend = self.model_runner.attn_backend + num_layers = self.model_config.get_num_layers(self.parallel_config) + + # Issue cache operations. + if worker_input.blocks_to_swap_in is not None: + src_indices, dst_indices = worker_input.blocks_to_swap_in + if src_indices.numel() > 0: + # Swap from CPU to TPU. + for i in range(num_layers): + tpu_k_cache, tpu_v_cache = self.tpu_cache[i] + cpu_k_cache, cpu_v_cache = self.cpu_cache[i] + k = cpu_k_cache[:, src_indices].to(self.device) + v = cpu_v_cache[:, src_indices].to(self.device) + _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache) + + if worker_input.blocks_to_swap_out is not None: + src_indices, dst_indices = worker_input.blocks_to_swap_out + if src_indices.numel() > 0: + # Swap from TPU to CPU. + for i in range(num_layers): + tpu_k_cache, tpu_v_cache = self.tpu_cache[i] + cpu_k_cache, cpu_v_cache = self.cpu_cache[i] + cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices] + cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices] + + if worker_input.blocks_to_copy is not None: + src_indices, dst_indices = worker_input.blocks_to_copy + if src_indices.numel() > 0: + attn_backend.copy_blocks(self.tpu_cache, + (src_indices, dst_indices)) + + +def _make_src_to_dst( + mapping: List[Tuple[int, int]], + src_device: Union[torch.device, str], + dst_device: Union[torch.device, str], +) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + if not mapping: + return None + + src_indices = [i for i, _ in mapping] + dst_indices = [i for _, i in mapping] + src_indices = torch.tensor(src_indices, + device=src_device, + dtype=torch.int64) + dst_indices = torch.tensor(dst_indices, + device=dst_device, + dtype=torch.int64) + return src_indices, dst_indices + + +@torch.compile(backend="openxla") +def _insert_kv( + k: torch.Tensor, + v: torch.Tensor, + indices: torch.Tensor, + tpu_k_cache: torch.Tensor, + tpu_v_cache: torch.Tensor, +) -> None: + torch.ops.xla.dynamo_set_buffer_donor_(tpu_k_cache, True) + torch.ops.xla.dynamo_set_buffer_donor_(tpu_v_cache, True) + tpu_k_cache[:, indices] = k + tpu_v_cache[:, indices] = v diff --git a/vllm/worker/utils.py b/vllm/worker/utils.py new file mode 100644 index 00000000..f4363546 --- /dev/null +++ b/vllm/worker/utils.py @@ -0,0 +1,51 @@ +''' +Worker-related helper functions. +''' + +from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS +from vllm.worker.model_runner import GPUModelRunnerBase + + +def assert_enc_dec_mr_supported_scenario( + enc_dec_mr: GPUModelRunnerBase) -> None: + ''' + Asserted that the provided encoder/decoder model runner instance reflects + a supported scenario. + ''' + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + + if enc_dec_mr.cache_config.enable_prefix_caching: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE']) + + if enc_dec_mr.sliding_window is not None: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SWA']) + + if enc_dec_mr.scheduler_config.chunked_prefill_enabled: + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[ + 'STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL']) + + if getattr(enc_dec_mr.model_config.hf_config, 'attn_logit_softcapping', + None) is not None: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP'] + ) + + if enc_dec_mr.lora_config is not None: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LORA']) + + if enc_dec_mr.parallel_config.pipeline_parallel_size > 1: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP']) + + if enc_dec_mr.scheduler_config.num_lookahead_slots > 0: + raise NotImplementedError( + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC']) + + if enc_dec_mr.prompt_adapter_config is not None: + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[ + 'STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER']) diff --git a/vllm/worker/worker.py b/vllm/worker/worker.py new file mode 100644 index 00000000..ab61e437 --- /dev/null +++ b/vllm/worker/worker.py @@ -0,0 +1,497 @@ +"""A GPU worker class.""" +import gc +import os +from typing import Dict, List, Optional, Set, Tuple, Type, Union + +import torch +import torch.distributed + +import vllm.envs as envs +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig) +from vllm.distributed import (ensure_model_parallel_initialized, + init_distributed_environment, + set_custom_all_reduce) +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor import set_random_seed +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader.tensorizer import TensorizerConfig +from vllm.platforms import current_platform +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sequence import (ExecuteModelRequest, IntermediateTensors, + SequenceGroupMetadata, SequenceGroupMetadataDelta) +from vllm.worker.cache_engine import CacheEngine +from vllm.worker.embedding_model_runner import EmbeddingModelRunner +from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner +from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner +from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput + +logger = init_logger(__name__) + + +class Worker(LocalOrDistributedWorkerBase): + """A worker class that executes (a partition of) the model on a GPU. + + Each worker is associated with a single GPU. The worker is responsible for + maintaining the KV cache and executing the model on the GPU. In case of + distributed inference, each worker is assigned a partition of the model. + """ + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + lora_config: Optional[LoRAConfig] = None, + speculative_config: Optional[SpeculativeConfig] = None, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + is_driver_worker: bool = False, + model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None, + observability_config: Optional[ObservabilityConfig] = None, + ) -> None: + self.model_config = model_config + self.parallel_config = parallel_config + self.parallel_config.rank = rank + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + self.lora_config = lora_config + self.load_config = load_config + self.prompt_adapter_config = prompt_adapter_config + self.is_driver_worker = is_driver_worker + if parallel_config and is_driver_worker: + assert rank % parallel_config.tensor_parallel_size == 0, \ + "Driver worker should be rank 0 of tensor parallel group." + if self.model_config.trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils import init_cached_hf_modules + init_cached_hf_modules() + self.observability_config = observability_config + + # Return hidden states from target model if the draft model is an + # mlp_speculator + speculative_args = {} if speculative_config is None \ + or (speculative_config.draft_model_config.model == + model_config.model) \ + or (speculative_config.draft_model_config.hf_config.model_type + not in ["medusa", "mlp_speculator", "eagle"]) \ + else {"return_hidden_states": True} + + ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner + if model_runner_cls is not None: + ModelRunnerClass = model_runner_cls + elif self._is_embedding_model(): + ModelRunnerClass = EmbeddingModelRunner + elif self._is_encoder_decoder_model(): + ModelRunnerClass = EncoderDecoderModelRunner + self.model_runner: GPUModelRunnerBase = ModelRunnerClass( + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config=load_config, + lora_config=self.lora_config, + kv_cache_dtype=self.cache_config.cache_dtype, + is_driver_worker=is_driver_worker, + prompt_adapter_config=prompt_adapter_config, + observability_config=observability_config, + **speculative_args, + ) + # Uninitialized cache engine. Will be initialized by + # initialize_cache. + self.cache_engine: List[CacheEngine] + # Initialize gpu_cache as embedding models don't initialize kv_caches + self.gpu_cache: Optional[List[List[torch.Tensor]]] = None + self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {} + + # Torch profiler. Enabled and configured through env vars: + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace + if envs.VLLM_TORCH_PROFILER_DIR: + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR + logger.info("Profiling enabled. Traces will be saved to: %s", + torch_profiler_trace_dir) + self.profiler = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, + on_trace_ready=torch.profiler.tensorboard_trace_handler( + torch_profiler_trace_dir, use_gzip=True)) + else: + self.profiler = None + + def start_profile(self): + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.start() + + def stop_profile(self): + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() + + def _is_encoder_decoder_model(self): + return self.model_config.is_encoder_decoder_model + + def _is_embedding_model(self): + return self.model_config.is_embedding_model + + def init_device(self) -> None: + if self.device_config.device.type == "cuda": + # torch.distributed.all_reduce does not free the input tensor until + # the synchronization point. This causes the memory usage to grow + # as the number of all_reduce calls increases. This env var disables + # this behavior. + # Related issue: + # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" + + # This env var set by Ray causes exceptions with graph building. + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None) + self.device = torch.device(f"cuda:{self.local_rank}") + torch.cuda.set_device(self.device) + + _check_if_gpu_supports_dtype(self.model_config.dtype) + gc.collect() + torch.cuda.empty_cache() + self.init_gpu_memory = torch.cuda.mem_get_info()[0] + else: + raise RuntimeError( + f"Not support device type: {self.device_config.device}") + # Initialize the distributed environment. + init_worker_distributed_environment(self.parallel_config, self.rank, + self.distributed_init_method, + self.local_rank) + # Set random seed. + set_random_seed(self.model_config.seed) + + def load_model(self): + self.model_runner.load_model() + + def save_sharded_state( + self, + path: str, + pattern: Optional[str] = None, + max_size: Optional[int] = None, + ) -> None: + self.model_runner.save_sharded_state( + path, + pattern=pattern, + max_size=max_size, + ) + + def save_tensorized_model( + self, + tensorizer_config: TensorizerConfig, + ) -> None: + self.model_runner.save_tensorized_model( + tensorizer_config=tensorizer_config, ) + + @torch.inference_mode() + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Profiles the peak memory usage of the model to determine how many + KV blocks may be allocated without OOMs. + + The engine will first conduct a profiling of the existing memory usage. + Then, it calculate the maximum possible number of GPU and CPU blocks + that can be allocated with the remaining free memory. + + .. tip:: + You may limit the usage of GPU memory + by adjusting the `gpu_memory_utilization` parameter. + """ + # Profile the memory usage of the model and get the maximum number of + # cache blocks that can be allocated with the remaining free memory. + torch.cuda.empty_cache() + + # Execute a forward pass with dummy inputs to profile the memory usage + # of the model. + self.model_runner.profile_run() + + # Calculate the number of blocks that can be allocated with the + # profiled peak memory. + torch.cuda.synchronize() + free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info() + # NOTE(woosuk): Here we assume that the other processes using the same + # GPU did not change their memory usage during the profiling. + peak_memory = self.init_gpu_memory - free_gpu_memory + assert peak_memory > 0, ( + "Error in memory profiling. " + f"Initial free memory {self.init_gpu_memory}, current free memory" + f" {free_gpu_memory}. This happens when the GPU memory was " + "not properly cleaned up before initializing the vLLM instance.") + + cache_block_size = self.get_cache_block_size_bytes() + if cache_block_size == 0: + num_gpu_blocks = 0 + num_cpu_blocks = 0 + else: + num_gpu_blocks = int( + (total_gpu_memory * self.cache_config.gpu_memory_utilization - + peak_memory) // cache_block_size) + num_cpu_blocks = int(self.cache_config.swap_space_bytes // + cache_block_size) + num_gpu_blocks = max(num_gpu_blocks, 0) + num_cpu_blocks = max(num_cpu_blocks, 0) + if self.model_runner.lora_manager: + self.model_runner.remove_all_loras() + gc.collect() + torch.cuda.empty_cache() + return num_gpu_blocks, num_cpu_blocks + + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Allocate GPU and CPU KV cache with the specified number of blocks. + + This also warms up the model, which may record CUDA graphs. + """ + raise_if_cache_size_invalid(num_gpu_blocks, + self.cache_config.block_size, + self.cache_config.is_attention_free, + self.model_config.max_model_len) + + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + + self._init_cache_engine() + self._warm_up_model() + + def _init_cache_engine(self): + assert self.cache_config.num_gpu_blocks is not None + self.cache_engine = [ + CacheEngine(self.cache_config, self.model_config, + self.parallel_config, self.device_config) + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + self.gpu_cache = [ + self.cache_engine[ve].gpu_cache + for ve in range(self.parallel_config.pipeline_parallel_size) + ] + + def _warm_up_model(self) -> None: + if not self.model_config.enforce_eager: + self.model_runner.capture_model(self.gpu_cache) + # Reset the seed to ensure that the random state is not affected by + # the model initialization and profiling. + set_random_seed(self.model_config.seed) + + @property + def do_metadata_broadcast(self) -> bool: + return self.parallel_config.tensor_parallel_size > 1 + + @property + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: + return self.gpu_cache + + @torch.inference_mode() + def prepare_worker_input( + self, execute_model_req: ExecuteModelRequest) -> WorkerInput: + virtual_engine = execute_model_req.virtual_engine + num_steps = execute_model_req.num_steps + num_seq_groups = len(execute_model_req.seq_group_metadata_list) + # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors. + # they contain parameters to launch cudamemcpyasync. + blocks_to_swap_in = torch.tensor(execute_model_req.blocks_to_swap_in, + device="cpu", + dtype=torch.int64).view(-1, 2) + blocks_to_swap_out = torch.tensor(execute_model_req.blocks_to_swap_out, + device="cpu", + dtype=torch.int64).view(-1, 2) + # `blocks_to_copy` is a gpu tensor. The src and tgt of + # blocks to copy are in the same device, and `blocks_to_copy` + # can be used directly within cuda kernels. + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy, + device=self.device, + dtype=torch.int64).view(-1, 2) + + return WorkerInput( + num_seq_groups=num_seq_groups, + blocks_to_swap_in=blocks_to_swap_in, + blocks_to_swap_out=blocks_to_swap_out, + blocks_to_copy=blocks_to_copy, + virtual_engine=virtual_engine, + num_steps=num_steps, + ) + + @torch.inference_mode() + def execute_worker(self, worker_input: WorkerInput) -> None: + virtual_engine = worker_input.virtual_engine + # Issue cache operations. + if (worker_input.blocks_to_swap_in is not None + and worker_input.blocks_to_swap_in.numel() > 0): + self.cache_engine[virtual_engine].swap_in( + worker_input.blocks_to_swap_in) + if (worker_input.blocks_to_swap_out is not None + and worker_input.blocks_to_swap_out.numel() > 0): + self.cache_engine[virtual_engine].swap_out( + worker_input.blocks_to_swap_out) + if (worker_input.blocks_to_copy is not None + and worker_input.blocks_to_copy.numel() > 0): + self.cache_engine[virtual_engine].copy(worker_input.blocks_to_copy) + + def _get_cached_seq_group_metadata( + self, + seq_group_metadata_list: List[Union[SequenceGroupMetadata, + SequenceGroupMetadataDelta]], + finished_request_ids: List[str]) -> List[SequenceGroupMetadata]: + """Return a list of cached Sequence Group Metadata after updating its + state. + + It is used because scheduler only sends delta to workers to reduce + the data payload size. The function also cleans up cache based on + a given `finished_request_ids`. + """ + new_seq_group_metadata_list = [] + for metadata_or_delta in seq_group_metadata_list: + request_id = metadata_or_delta.request_id + if request_id not in self._seq_group_metadata_cache: + # The first prefill. + assert isinstance(metadata_or_delta, SequenceGroupMetadata) + self._seq_group_metadata_cache[request_id] = metadata_or_delta + else: + # The first prefill is already cached. + if isinstance(metadata_or_delta, SequenceGroupMetadataDelta): + self._seq_group_metadata_cache[request_id].apply_delta( + metadata_or_delta) + else: + # If metadata snapshot is sent again, it is + # preempted. Reset the cache because we need to start + # from scratch. + assert isinstance(metadata_or_delta, SequenceGroupMetadata) + self._seq_group_metadata_cache[ + request_id] = metadata_or_delta + + new_seq_group_metadata_list.append( + self._seq_group_metadata_cache[request_id]) + + # Clean up finished ids + for finished_id in finished_request_ids: + del self._seq_group_metadata_cache[finished_id] + + return new_seq_group_metadata_list + + def _execute_model_spmd( + self, + execute_model_req: ExecuteModelRequest, + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Optional[List[SamplerOutput]]: + if execute_model_req is not None: + new_seq_group_metadata_list = self._get_cached_seq_group_metadata( + execute_model_req.seq_group_metadata_list, + execute_model_req.finished_requests_ids) + + execute_model_req.seq_group_metadata_list = ( + new_seq_group_metadata_list) + output = super()._execute_model_spmd(execute_model_req, + intermediate_tensors) + return output + + def add_lora(self, lora_request: LoRARequest) -> bool: + return self.model_runner.add_lora(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + return self.model_runner.remove_lora(lora_id) + + def pin_lora(self, lora_id: int) -> bool: + return self.model_runner.pin_lora(lora_id) + + def list_loras(self) -> Set[int]: + return self.model_runner.list_loras() + + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + return self.model_runner.add_prompt_adapter(prompt_adapter_request) + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + return self.model_runner.remove_lora(prompt_adapter_id) + + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: + return self.model_runner.pin_prompt_adapter(prompt_adapter_id) + + def list_prompt_adapters(self) -> Set[int]: + return self.model_runner.list_prompt_adapters() + + @property + def max_model_len(self) -> int: + return self.model_config.max_model_len + + @property + def vocab_size(self) -> int: + return self.model_runner.vocab_size + + def get_cache_block_size_bytes(self) -> int: + """Get the size of the KV cache block size in bytes. + """ + return CacheEngine.get_cache_block_size(self.cache_config, + self.model_config, + self.parallel_config) + + +def init_worker_distributed_environment( + parallel_config: ParallelConfig, + rank: int, + distributed_init_method: Optional[str] = None, + local_rank: int = -1, +) -> None: + """Initialize the distributed environment.""" + set_custom_all_reduce(not parallel_config.disable_custom_all_reduce) + + init_distributed_environment(parallel_config.world_size, rank, + distributed_init_method, local_rank) + + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size) + + +def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype): + # Check if the GPU supports the dtype. + if torch_dtype == torch.bfloat16: # noqa: SIM102 + if not current_platform.has_device_capability(80): + capability = current_platform.get_device_capability() + gpu_name = current_platform.get_device_name() + + if capability is None: + compute_str = "does not have a compute capability" + else: + version_str = capability.as_version_str() + compute_str = f"has compute capability {version_str}" + + raise ValueError( + "Bfloat16 is only supported on GPUs with compute capability " + f"of at least 8.0. Your {gpu_name} GPU {compute_str}. " + "You can use float16 instead by explicitly setting the" + "`dtype` flag in CLI, for example: --dtype=half.") + + +def raise_if_cache_size_invalid(num_gpu_blocks, block_size, is_attention_free, + max_model_len) -> None: + if is_attention_free and num_gpu_blocks != 0: + raise ValueError("No memory should be allocated for the cache blocks " + f"for an attention-free model, but {num_gpu_blocks}" + "blocks are allocated.") + if not is_attention_free and num_gpu_blocks <= 0: + raise ValueError("No available memory for the cache blocks. " + "Try increasing `gpu_memory_utilization` when " + "initializing the engine.") + max_seq_len = block_size * num_gpu_blocks + if not is_attention_free and max_model_len > max_seq_len: + raise ValueError( + f"The model's max seq len ({max_model_len}) " + "is larger than the maximum number of tokens that can be " + f"stored in KV cache ({max_seq_len}). Try increasing " + "`gpu_memory_utilization` or decreasing `max_model_len` when " + "initializing the engine.") diff --git a/vllm/worker/worker_base.py b/vllm/worker/worker_base.py new file mode 100644 index 00000000..6ba4f272 --- /dev/null +++ b/vllm/worker/worker_base.py @@ -0,0 +1,485 @@ +import dataclasses +import importlib +import os +import time +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union + +import torch + +from vllm.config import ObservabilityConfig +from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.platforms import current_platform +from vllm.sequence import ExecuteModelRequest, IntermediateTensors +from vllm.utils import (enable_trace_function_call_for_thread, + update_environment_variables) +from vllm.worker.model_runner_base import (BroadcastableModelInput, + ModelRunnerBase, + ModelRunnerInputBase) + +logger = init_logger(__name__) + + +class WorkerBase(ABC): + """Worker interface that allows vLLM to cleanly separate implementations for + different hardware. Also abstracts control plane communication, e.g., to + communicate request metadata to other workers. + """ + + @abstractmethod + def init_device(self) -> None: + """Initialize device state, such as loading the model or other on-device + memory allocations. + """ + raise NotImplementedError + + @abstractmethod + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Determine the number of available blocks for the GPU KV cache and + swappable CPU KV cache. + + The implementation may run profiling or other heuristics to determine + the size of caches. + + Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks + are blocks that are "active" on the device and can be appended to. + num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be + appended to. + """ + raise NotImplementedError + + @abstractmethod + def initialize_cache(self, num_gpu_blocks: int, + num_cpu_blocks: int) -> None: + """Initialize the KV cache with the given size in blocks. + """ + raise NotImplementedError + + @current_platform.inference_mode() + def start_worker_execution_loop(self) -> None: + """Execute model loop in parallel worker. + + You can stop the loop by executing a driver worker with an empty output. + See `stop_remote_worker_execution_loop` for more details. + """ + while True: + output = self.execute_model(execute_model_req=None) + if output is None: + return None + + @abstractmethod + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> Optional[List[SamplerOutput]]: + raise NotImplementedError + + @abstractmethod + def get_cache_block_size_bytes(self) -> int: + """Return the size of a single cache block, in bytes. Used in + speculative decoding. + """ + raise NotImplementedError + + @abstractmethod + def add_lora(self, lora_request: LoRARequest) -> bool: + raise NotImplementedError + + @abstractmethod + def remove_lora(self, lora_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def pin_lora(self, lora_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def list_loras(self) -> Set[int]: + raise NotImplementedError + + +class LoraNotSupportedWorkerBase(WorkerBase): + """Partial implementation of WorkerBase that raises exceptions when LoRA + methods are invoked. + """ + + def add_lora(self, lora_request: LoRARequest) -> bool: + raise ValueError(f"{type(self)} does not support LoRA") + + def remove_lora(self, lora_id: int) -> bool: + raise ValueError(f"{type(self)} does not support LoRA") + + def pin_lora(self, lora_id: int) -> bool: + return ValueError( + f"{type(self)} does not support LoRA") # type: ignore + + def list_loras(self) -> Set[int]: + raise ValueError(f"{type(self)} does not support LoRA") + + +@dataclasses.dataclass(frozen=True) +class WorkerInput: + """Local inputs to each worker. May contain device-specific data. These + fields should be broadcastable to other workers. + """ + + num_seq_groups: Optional[int] = None + blocks_to_swap_in: Optional[torch.Tensor] = None + blocks_to_swap_out: Optional[torch.Tensor] = None + blocks_to_copy: Optional[torch.Tensor] = None + virtual_engine: int = 0 + num_steps: int = 1 + + @classmethod + def from_broadcasted_tensor_dict( + cls: Type["WorkerInput"], + tensor_dict: Dict[str, Any], + ) -> "WorkerInput": + """ + Pop fields from the given tensor_dict and populate a new instance of + WorkerInput. + """ + return cls( + num_seq_groups=tensor_dict.pop("num_seq_groups"), + blocks_to_swap_in=tensor_dict.pop("blocks_to_swap_in"), + blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"), + blocks_to_copy=tensor_dict.pop("blocks_to_copy"), + virtual_engine=tensor_dict["virtual_engine"], + num_steps=tensor_dict.pop("num_steps"), + ) + + def as_broadcastable_tensor_dict( + self) -> Dict[str, Union[int, torch.Tensor]]: + """ + Extract broadcastable fields. + """ + tensor_dict = { + "num_seq_groups": self.num_seq_groups, + "blocks_to_swap_in": self.blocks_to_swap_in, + "blocks_to_swap_out": self.blocks_to_swap_out, + "blocks_to_copy": self.blocks_to_copy, + "virtual_engine": self.virtual_engine, + "num_steps": self.num_steps, + } + + return tensor_dict + + +class LocalOrDistributedWorkerBase(WorkerBase): + """ + Partial implementation of WorkerBase that has a default `execute_model` + definition to perform metadata transfer between workers when in distributed + mode. Subclasses of this interface should use model runners that inherit + from ModelRunnerBase, and should only need to implement worker-local logic. + If custom control plane logic is needed to transfer metadata, or if the + model runner cannot inherit from ModelRunnerBase, use WorkerBase instead. + """ + is_driver_worker: bool + model_runner: ModelRunnerBase + observability_config: Optional[ObservabilityConfig] = None + + @property + @abstractmethod + def do_metadata_broadcast(self) -> bool: + """ + Used by the default `execute_model` to check whether broadcast is + needed to transfer request inputs from the driver worker to other + workers in the TP group. If WorkerBase subclass only supports + single-worker execution, then this method should return False. + """ + raise NotImplementedError + + @property + @abstractmethod + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: + """ + Gets the list of kv caches to pass to the worker's model runner. Each + element in the list is a kv cache corresponding to a particular virtual + engine (PP stream). Used by the default `execute_model`. If the worker's + model runner does not follow the ModelRunnerBase interface, then inherit + from WorkerBase instead. + """ + raise NotImplementedError + + @abstractmethod + def prepare_worker_input( + self, execute_model_req: ExecuteModelRequest) -> WorkerInput: + """ + Prepare the inputs to WorkerBase.execute_worker from an execution + request. This method may move data to the worker's local device. It is + not allowed to communicate with other workers or devices. + """ + raise NotImplementedError + + @abstractmethod + def execute_worker(self, worker_input: WorkerInput) -> None: + """ + Process an execution request. + """ + raise NotImplementedError + + def _get_worker_input_from_broadcast( + self + ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[ + str, torch.Tensor]]]: + """ Get the worker input from the broadcasted tensor dict. """ + assert self.do_metadata_broadcast + assert not self.is_driver_worker + broadcast_data = broadcast_tensor_dict(src=0) + if not broadcast_data: + return None + + worker_input = WorkerInput.from_broadcasted_tensor_dict(broadcast_data) + model_input = ( + self.model_runner.make_model_input_from_broadcasted_tensor_dict( + broadcast_data)) + + kwargs = extract_previous_hidden_states(broadcast_data) + + return model_input, worker_input, kwargs + + def _get_driver_input_and_broadcast( + self, execute_model_req: ExecuteModelRequest + ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]: + """ Get the driver input and broadcast it to other workers. """ + assert self.is_driver_worker + + worker_input: WorkerInput = self.prepare_worker_input( + execute_model_req=execute_model_req) + model_input: ModelRunnerInputBase = ( + self.model_runner.prepare_model_input( + execute_model_req.seq_group_metadata_list, + execute_model_req.virtual_engine, + execute_model_req.finished_requests_ids)) + + kwargs = extract_previous_hidden_states(execute_model_req) + + if self.do_metadata_broadcast: + broadcast_data = worker_input.as_broadcastable_tensor_dict() + broadcast_data.update(model_input.as_broadcastable_tensor_dict()) + broadcast_data.update(kwargs) + broadcast_tensor_dict(broadcast_data, src=0) + + if execute_model_req.async_callback: + model_input = dataclasses.replace( # type: ignore + model_input, + async_callback=execute_model_req.async_callback) + + return model_input, worker_input, kwargs + + def prepare_input( + self, + execute_model_req: Optional[ExecuteModelRequest] = None + ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[ + str, torch.Tensor]]]: + """ + Prepare the inputs to ModelRunner and workers. + """ + if self.is_driver_worker: + if execute_model_req is None: + if self.do_metadata_broadcast: + # This signals that there's no more requests to process for + # now. All workers are running infinite loop with + # broadcast_tensor_dict, and it stops the loop when the + # driver broadcasts an empty input. Send an empty input to + # notify all other workers to stop their execution loop. + broadcast_tensor_dict({}, src=0) + return None + return self._get_driver_input_and_broadcast(execute_model_req) + else: + return self._get_worker_input_from_broadcast() + + def execute_model( + self, + execute_model_req: Optional[ExecuteModelRequest] = None, + ) -> Optional[List[SamplerOutput]]: + """Executes at least one model step on the given sequences, unless no + sequences are provided.""" + start_time = time.perf_counter() + + inputs = self.prepare_input(execute_model_req) + if inputs is None: + return None + + model_input, worker_input, kwargs = inputs + num_steps = worker_input.num_steps + + self.execute_worker(worker_input) + + # If there is no input, we don't need to execute the model. + if worker_input.num_seq_groups == 0: + return [] + + intermediate_tensors = None + orig_model_execute_time = 0.0 + if not get_pp_group().is_first_rank: + intermediate_tensors = IntermediateTensors( + get_pp_group().recv_tensor_dict( + all_gather_group=get_tp_group())) + if (self.observability_config is not None + and self.observability_config.collect_model_execute_time): + orig_model_execute_time = intermediate_tensors.tensors.get( + "model_execute_time", torch.tensor(0)).item() + + output = self.model_runner.execute_model( + model_input=model_input, + kv_caches=self.kv_cache[worker_input.virtual_engine] + if self.kv_cache is not None else None, + intermediate_tensors=intermediate_tensors, + num_steps=num_steps, + **kwargs, + ) + + model_execute_time = time.perf_counter() - start_time + if not get_pp_group().is_last_rank: + # output is IntermediateTensors + if (self.observability_config is not None + and self.observability_config.collect_model_execute_time): + output.tensors["model_execute_time"] = torch.tensor( + model_execute_time + orig_model_execute_time) + get_pp_group().send_tensor_dict(output.tensors, + all_gather_group=get_tp_group()) + return [None] + if (self.observability_config is not None + and self.observability_config.collect_model_execute_time + and output is not None): + for o in output: + o.model_execute_time = (orig_model_execute_time + + model_execute_time) + + # output is List[SamplerOutput] + return output + + def _execute_model_spmd( + self, + execute_model_req: ExecuteModelRequest, + intermediate_tensors: Optional[IntermediateTensors] = None + ) -> Optional[List[SamplerOutput]]: + """ + Execute model in Single Program Multiple Data (SPMD) fashion. + All workers take the same request, prepare the input and + execute the model. + """ + assert execute_model_req is not None, ( + "_execute_model_spmd() requires each worker to take in an " + "ExecuteModelRequest") + worker_input: WorkerInput = self.prepare_worker_input( + execute_model_req=execute_model_req) + model_input: ModelRunnerInputBase = ( + self.model_runner.prepare_model_input( + execute_model_req.seq_group_metadata_list)) + + self.execute_worker(worker_input) + + # If there is no input, we don't need to execute the model. + if worker_input.num_seq_groups == 0: + return [] + + kwargs = extract_previous_hidden_states(execute_model_req) + + return self.model_runner.execute_model( + model_input=model_input, + kv_caches=self.kv_cache[worker_input.virtual_engine] + if self.kv_cache is not None else None, + intermediate_tensors=intermediate_tensors, + **kwargs, + ) + + +class WorkerWrapperBase: + """ + The whole point of this class is to lazily initialize the worker. + We first instantiate the WorkerWrapper, which remembers the worker module + and class name. Then, when we call `update_environment_variables`, and the + real initialization happens in `init_worker`. + + If worker_class_fn is specified, it will be executed to get the worker + class. + Otherwise, the worker class will be obtained by dynamically importing it + using worker_module_name and worker_class_name. + """ + + def __init__( + self, + worker_module_name: str, + worker_class_name: str, + trust_remote_code: bool = False, + worker_class_fn: Optional[Callable[[], + Type[WorkerBase]]] = None) -> None: + self.worker_module_name = worker_module_name + self.worker_class_name = worker_class_name + self.worker_class_fn = worker_class_fn + self.worker: Optional[WorkerBase] = None + if trust_remote_code: + # note: lazy import to avoid importing torch before initializing + from vllm.utils import init_cached_hf_modules + init_cached_hf_modules() + + @staticmethod + def update_environment_variables(envs: Dict[str, str]) -> None: + key = 'CUDA_VISIBLE_DEVICES' + if key in envs and key in os.environ: + # overwriting CUDA_VISIBLE_DEVICES is desired behavior + # suppress the warning in `update_environment_variables` + del os.environ[key] + update_environment_variables(envs) + + def init_worker(self, *args, **kwargs): + """ + Here we inject some common logic before initializing the worker. + Arguments are passed to the worker class constructor. + """ + enable_trace_function_call_for_thread() + + # see https://github.com/NVIDIA/nccl/issues/1234 + os.environ['NCCL_CUMEM_ENABLE'] = '0' + + from vllm.plugins import load_general_plugins + load_general_plugins() + + if self.worker_class_fn: + worker_class = self.worker_class_fn() + else: + mod = importlib.import_module(self.worker_module_name) + worker_class = getattr(mod, self.worker_class_name) + + self.worker = worker_class(*args, **kwargs) + assert self.worker is not None + + def execute_method(self, method, *args, **kwargs): + try: + target = self if self.worker is None else self.worker + executor = getattr(target, method) + return executor(*args, **kwargs) + except Exception as e: + # if the driver worker also execute methods, + # exceptions in the rest worker may cause deadlock in rpc like ray + # see https://github.com/vllm-project/vllm/issues/3455 + # print the error and inform the user to solve the error + msg = (f"Error executing method {method}. " + "This might cause deadlock in distributed execution.") + logger.exception(msg) + raise e + + +def extract_previous_hidden_states( + data: Union[ExecuteModelRequest, Dict[str, torch.Tensor]]) -> \ + Dict[str, torch.Tensor]: + """If data contains previous_hidden_states, extract it. This returns a dict + which can be used directly as additional kwargs in any following + execute_model calls. This is used in draft models like EAGLE.""" + output = {} + + # When called from non-driver worker, data is dict but when called from + # driver worker, data is ExecuteModelRequest. + if isinstance(data, dict): + if "previous_hidden_states" in data: + output["previous_hidden_states"] = data["previous_hidden_states"] + elif data.previous_hidden_states is not None: + output["previous_hidden_states"] = data.previous_hidden_states\ + .hidden_states + + return output diff --git a/vllm/worker/xpu_model_runner.py b/vllm/worker/xpu_model_runner.py new file mode 100644 index 00000000..5ff4626c --- /dev/null +++ b/vllm/worker/xpu_model_runner.py @@ -0,0 +1,605 @@ +import dataclasses +import time +import weakref +from dataclasses import dataclass +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, + Type, TypeVar) + +import torch +import torch.nn as nn + +from vllm.attention import get_attn_backend +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig) +from vllm.distributed import get_pp_group +from vllm.inputs import INPUT_REGISTRY, InputRegistry +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadataCache +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.model_executor.model_loader import get_model +from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs, + MultiModalInputs, MultiModalRegistry) +from vllm.sampling_params import SamplingParams +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata +from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad +from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata +from vllm.worker.model_runner_base import ( + ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase, + _add_attn_metadata_broadcastable_dict, + _add_sampling_metadata_broadcastable_dict, + _init_attn_metadata_from_tensor_dict, + _init_sampling_metadata_from_tensor_dict) + +if TYPE_CHECKING: + from vllm.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + +_PAD_SLOT_ID = -1 +_BATCH_SIZE_ALIGNMENT = 8 +_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [ + _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33) +] + +TModelInputForXPU = TypeVar('TModelInputForXPU', bound="ModelInputForXPU") + + +@dataclass(frozen=True) +class ModelInputForXPU(ModelRunnerInputBase): + """ + Used by the NeuronModelRunner. + """ + input_tokens: Optional[torch.Tensor] = None + input_positions: Optional[torch.Tensor] = None + attn_metadata: Optional["AttentionMetadata"] = None + multi_modal_kwargs: Optional[BatchedTensorInputs] = None + virtual_engine: Optional[int] = None + seq_lens: Optional[List[int]] = None + query_lens: Optional[List[int]] = None + async_callback: Optional[Callable] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls: Type[TModelInputForXPU], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> TModelInputForXPU: + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +@dataclass(frozen=True) +class ModelInputForXPUWithSamplingMetadata(ModelInputForXPU): + """ + Used by the ModelRunner. + """ + sampling_metadata: Optional["SamplingMetadata"] = None + + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + tensor_dict = { + "input_tokens": self.input_tokens, + "input_positions": self.input_positions, + } + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata) + _add_sampling_metadata_broadcastable_dict(tensor_dict, + self.sampling_metadata) + return tensor_dict + + @classmethod + def from_broadcasted_tensor_dict( + cls, + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> "ModelInputForXPUWithSamplingMetadata": + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict) + if attn_backend is not None: + tensor_dict = _init_attn_metadata_from_tensor_dict( + attn_backend, tensor_dict) + return cls(**tensor_dict) + + +class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]): + + def __init__(self, + runner: "XPUModelRunner", + finished_requests_ids: Optional[List[str]] = None) -> None: + super().__init__() + self.seq_group_metadata_list: List[SequenceGroupMetadata] = [] + self.runner = runner + self.model_input_cls = self.runner._model_input_cls + self.attn_backend = self.runner.attn_backend + self.sliding_window = self.runner.sliding_window + self.block_size = self.runner.block_size + self.device = self.runner.device + + def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata): + self.seq_group_metadata_list.append(seq_group_metadata) + + def build(self) -> ModelInputForXPU: + is_prompt = self.seq_group_metadata_list[0].is_prompt + # Prepare input tensors. + if is_prompt: + (input_tokens, input_positions, attn_metadata, seq_lens, + multi_modal_kwargs) = self._prepare_prompt( + self.seq_group_metadata_list) + else: + (input_tokens, input_positions, + attn_metadata) = self._prepare_decode( + self.seq_group_metadata_list) + seq_lens = None + multi_modal_kwargs = None + + return self.model_input_cls( + input_tokens=input_tokens, + input_positions=input_positions, + attn_metadata=attn_metadata, + multi_modal_kwargs=multi_modal_kwargs, + seq_lens=seq_lens, + query_lens=seq_lens, + ) + + def _prepare_prompt( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int], + BatchedTensorInputs]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[int] = [] + input_positions: List[int] = [] + slot_mapping: List[int] = [] + seq_lens: List[int] = [] + multi_modal_inputs_list: List[MultiModalInputs] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert seq_group_metadata.is_prompt + seq_ids = list(seq_group_metadata.seq_data.keys()) + assert len(seq_ids) == 1 + seq_id = seq_ids[0] + + seq_data = seq_group_metadata.seq_data[seq_id] + prompt_tokens = seq_data.get_token_ids() + computed_len = seq_data.get_num_computed_tokens() + seq_len = len(prompt_tokens) + + seq_lens.append(seq_len) # Prompt token num + input_tokens.extend(prompt_tokens) # Token ids + + # Token position ids + # NOTE(woosuk): Here we assume that the first token in the prompt + # is always the first token in the sequence. + input_positions.extend(list(range(computed_len, seq_len))) + + if seq_group_metadata.block_tables is None: + # During memory profiling, the block tables are not initialized + # yet. In this case, we just use a dummy slot mapping. + slot_mapping.extend([_PAD_SLOT_ID] * seq_len) + continue + + # Compute the slot mapping. + block_table = seq_group_metadata.block_tables[seq_id] + # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID, + # where start_idx is max(0, seq_len - sliding_window). + # For example, if the prompt len is 10, sliding window is 8, and + # block size is 4, the first two tokens are masked and the slot + # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1]. + start_idx = 0 + if self.sliding_window is not None: + start_idx = max(0, seq_len - self.sliding_window) + + for i in range(computed_len, seq_len): + if i < start_idx: + slot_mapping.append(_PAD_SLOT_ID) + continue + + block_number = block_table[i // + self.block_size] # type: ignore + block_offset = i % self.block_size # type: ignore + slot = block_number * self.block_size + block_offset + slot_mapping.append(slot) + + num_prompt_tokens = len(input_tokens) + + input_tokens = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) # type: ignore + input_positions = torch.tensor(input_positions, + dtype=torch.long, + device=self.device) # type: ignore + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.long, + device=self.device) # type: ignore + + max_seqlen = max(seq_lens) + tmp = [0] + tmp.extend(seq_lens) + seqlen = torch.tensor(tmp) + seqlen_q = torch.cumsum(seqlen, dim=0).to(device=self.device) + + attn_metadata = self.attn_backend.make_metadata( + is_prompt=True, + slot_mapping=slot_mapping, + seq_lens=seq_lens, + seqlen_q=seqlen_q, + max_seqlen=max_seqlen, + seq_lens_tensor=torch.tensor([]), + max_decode_seq_len=0, + num_prefills=len(seq_lens), + num_prefill_tokens=num_prompt_tokens, + num_decode_tokens=0, + block_tables=torch.tensor([], device=self.device, dtype=torch.int), + ) + + multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list) + + return (input_tokens, input_positions, attn_metadata, seq_lens, + multi_modal_kwargs) + + def _prepare_decode( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]: + assert len(seq_group_metadata_list) > 0 + input_tokens: List[int] = [] + input_positions: List[int] = [] + slot_mapping: List[int] = [] + seq_lens: List[int] = [] + block_tables: List[List[int]] = [] + + for seq_group_metadata in seq_group_metadata_list: + assert not seq_group_metadata.is_prompt + assert seq_group_metadata.token_chunk_size == 1 + + seq_ids = list(seq_group_metadata.seq_data.keys()) + + for seq_id in seq_ids: + seq_data = seq_group_metadata.seq_data[seq_id] + generation_token = seq_data.get_last_token_id() + input_tokens.append(generation_token) + + seq_len = seq_data.get_len() + position = seq_len - 1 + input_positions.append(position) + + seq_len = seq_len if self.sliding_window is None else min( + seq_len, self.sliding_window) + seq_lens.append(seq_len) + + block_table = seq_group_metadata.block_tables[seq_id] + block_number = block_table[position // self.block_size] + block_offset = position % self.block_size + slot = block_number * self.block_size + block_offset + slot_mapping.append(slot) + + if self.sliding_window is not None: + sliding_window_blocks = (self.sliding_window // + self.block_size) + block_table = block_table[-sliding_window_blocks:] + block_tables.append(block_table) + + max_decode_seq_len = max(seq_lens) + + input_tokens = torch.tensor(input_tokens, + dtype=torch.long, + device=self.device) + input_positions = torch.tensor(input_positions, + dtype=torch.long, + device=self.device) + slot_mapping = torch.tensor(slot_mapping, + dtype=torch.long, + device=self.device) + seq_lens_tensor = torch.tensor(seq_lens, + dtype=torch.int, + device=self.device) + + block_tables = make_tensor_with_pad( + block_tables, + pad=0, + dtype=torch.int, + device=self.device, + ) + + attn_metadata = self.attn_backend.make_metadata( + is_prompt=False, + slot_mapping=slot_mapping, + seq_lens=seq_lens, + seqlen_q=torch.tensor([]), + max_seqlen=0, + seq_lens_tensor=seq_lens_tensor, + max_decode_seq_len=max_decode_seq_len, + num_prefill_tokens=0, + num_decode_tokens=len(input_tokens), + num_prefills=0, + block_tables=block_tables, + ) + return ( + input_tokens, + input_positions, + attn_metadata, + ) + + +class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]): + _model_input_cls: Type[ModelInputForXPUWithSamplingMetadata] = ( + ModelInputForXPUWithSamplingMetadata) + _builder_cls: Type[ModelInputForXPUBuilder] = ModelInputForXPUBuilder + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + kv_cache_dtype: Optional[str] = "auto", + is_driver_worker: bool = False, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + return_hidden_states: bool = False, + observability_config: Optional[ObservabilityConfig] = None, + input_registry: InputRegistry = INPUT_REGISTRY, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, + ): + self.model_config = model_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.lora_config = lora_config + self.load_config = load_config + self.is_driver_worker = is_driver_worker + self.prompt_adapter_config = prompt_adapter_config + self.observability_config = observability_config + if self.observability_config is not None: + print(f"observability_config is {self.observability_config}") + self.return_hidden_states = return_hidden_states + + self.device = self.device_config.device + + self.kv_cache_dtype = kv_cache_dtype + self.sliding_window = model_config.get_sliding_window() + self.block_size = cache_config.block_size + + self.attn_backend = get_attn_backend( + self.model_config.get_head_size(), + self.model_config.get_sliding_window(), + self.model_config.dtype, + self.kv_cache_dtype, + self.block_size, + self.model_config.is_attention_free, + ) + + # Multi-modal data support + self.input_registry = input_registry + self.mm_registry = mm_registry + self.multi_modal_input_mapper = mm_registry \ + .create_input_mapper(model_config) + self.mm_registry.init_mm_limits_per_prompt(self.model_config) + + # Lazy initialization. + self.model: nn.Module # Set after init_Model + + self.sampling_metadata_cache: SamplingMetadataCache = \ + SamplingMetadataCache() \ + if self.parallel_config.pipeline_parallel_size == 1 else None + + def load_model(self) -> None: + with DeviceMemoryProfiler() as m: + self.model = get_model( + model_config=self.model_config, + device_config=self.device_config, + load_config=self.load_config, + lora_config=self.lora_config, + parallel_config=self.parallel_config, + scheduler_config=self.scheduler_config, + cache_config=self.cache_config, + ) + + self.model_memory_usage = m.consumed_memory + logger.info("Loading model weights took %.4f GB", + self.model_memory_usage / float(2**30)) + + @property + def vocab_size(self) -> int: + return self.model_config.get_vocab_size() + + @torch.inference_mode() + def profile_run(self) -> None: + # Enable top-k sampling to reflect the accurate memory usage. + sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1) + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens + max_num_seqs = self.scheduler_config.max_num_seqs + + # Profile memory usage with max_num_sequences sequences and the total + # number of tokens equal to max_num_batched_tokens. + seqs: List[SequenceGroupMetadata] = [] + # Additional GPU memory may be needed for multi-modal encoding, which + # needs to be accounted for when calculating the GPU blocks for + # vLLM blocker manager. + # To exercise the worst scenario for GPU memory consumption, + # the number of seqs (batch_size) is chosen to maximize the number + # of images processed. + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens( + self.model_config) + if max_mm_tokens > 0: + max_num_seqs_orig = max_num_seqs + max_num_seqs = min(max_num_seqs, + max_num_batched_tokens // max_mm_tokens) + if max_num_seqs < 1: + expr = (f"min({max_num_seqs_orig}, " + f"{max_num_batched_tokens} // {max_mm_tokens})") + logger.warning( + "Computed max_num_seqs (%s) to be less than 1. " + "Setting it to the minimum value of 1.", expr) + max_num_seqs = 1 + + batch_size = 0 + for group_id in range(max_num_seqs): + seq_len = (max_num_batched_tokens // max_num_seqs + + (group_id < max_num_batched_tokens % max_num_seqs)) + batch_size += seq_len + + seq_data, dummy_multi_modal_data = self.input_registry \ + .dummy_data_for_profiling(self.model_config, + seq_len, + self.mm_registry) + + seq = SequenceGroupMetadata( + request_id=str(group_id), + is_prompt=True, + seq_data={group_id: seq_data}, + sampling_params=sampling_params, + block_tables=None, + lora_request=None, + multi_modal_data=dummy_multi_modal_data, + ) + seqs.append(seq) + + # Run the model with the dummy inputs. + num_layers = self.model_config.get_num_layers(self.parallel_config) + # use an empty tensor instead of `None`` to force Dynamo to pass + # it by reference, rather by specializing on the value ``None``. + # the `dtype` argument does not matter, and we use `float32` as + # a placeholder (it has wide hardware support). + kv_caches = [ + torch.tensor([], dtype=torch.float32, device=self.device) + ] * num_layers + finished_requests_ids = [seq.request_id for seq in seqs] + model_input = self.prepare_model_input( + seqs, finished_requests_ids=finished_requests_ids) + intermediate_tensors = None + if not get_pp_group().is_first_rank: + intermediate_tensors = self.model.make_empty_intermediate_tensors( + batch_size=batch_size, + dtype=self.model_config.dtype, + device=self.device) + self.execute_model(model_input, kv_caches, intermediate_tensors) + torch.xpu.synchronize() + return + + def make_model_input_from_broadcasted_tensor_dict( + self, + tensor_dict: Dict[str, + Any]) -> ModelInputForXPUWithSamplingMetadata: + return ( + ModelInputForXPUWithSamplingMetadata.from_broadcasted_tensor_dict( + tensor_dict, + attn_backend=self.attn_backend, + )) + + def _prepare_model_input_tensors( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForXPUWithSamplingMetadata: + """Helper method to prepare the model input based on a given sequence + group. Prepares metadata needed for the base model forward pass but not + metadata for possible additional steps, e.g., sampling. + + """ + builder = self._builder_cls(weakref.proxy(self), finished_requests_ids) + for seq_group_metadata in seq_group_metadata_list: + builder.add_seq_group(seq_group_metadata) + + return builder.build() # type: ignore + + def prepare_model_input( + self, + seq_group_metadata_list: List[SequenceGroupMetadata], + virtual_engine: int = 0, + finished_requests_ids: Optional[List[str]] = None + ) -> ModelInputForXPUWithSamplingMetadata: + """Prepare the model input based on a given sequence group, including + metadata for the sampling step. + + """ + model_input = self._prepare_model_input_tensors( + seq_group_metadata_list, finished_requests_ids) + # Sampling metadata is only required for the final pp group + generators = self.get_generators(finished_requests_ids) + sampling_metadata = SamplingMetadata.prepare( + seq_group_metadata_list, + model_input.seq_lens, + model_input.query_lens, + self.device, + pin_memory=False, + generators=generators, + cache=self.sampling_metadata_cache) + + return dataclasses.replace(model_input, + sampling_metadata=sampling_metadata, + virtual_engine=virtual_engine) + + @torch.inference_mode() + def execute_model( + self, + model_input: ModelInputForXPUWithSamplingMetadata, + kv_caches: List[torch.Tensor], + intermediate_tensors: Optional[IntermediateTensors] = None, + num_steps: int = 1, + ) -> Optional[List[SamplerOutput]]: + if num_steps > 1: + raise ValueError( + "XPUModelRunner does not support multi-step execution.") + + model_executable = self.model + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_start_time = time.time() + + hidden_or_intermediate_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {}, + device=self.device)) + # Compute the logits in the last pipeline stage. + if not get_pp_group().is_last_rank: + return hidden_or_intermediate_states + + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time): + model_forward_end_time = time.time() + + # Compute the logits. + logits = self.model.compute_logits(hidden_or_intermediate_states, + model_input.sampling_metadata) + + # Only perform sampling in the driver worker. + if not self.is_driver_worker: + return [] + + if model_input.async_callback is not None: + model_input.async_callback() + + # Sample the next token. + output: SamplerOutput = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) + if (self.observability_config is not None + and self.observability_config.collect_model_forward_time + and output is not None): + model_forward_time = (model_forward_end_time - + model_forward_start_time) + # If there are multiple workers, we are still tracking the latency + # from the start time of the driver worker to the end time of the + # driver worker. The model forward time will then end up covering + # the communication time as well. + output.model_forward_time = model_forward_time + + return [output] diff --git a/vllm/worker/xpu_worker.py b/vllm/worker/xpu_worker.py new file mode 100644 index 00000000..9ad070d0 --- /dev/null +++ b/vllm/worker/xpu_worker.py @@ -0,0 +1,206 @@ +"""A XPU worker class.""" +import gc +import os +from typing import List, Optional, Tuple + +import intel_extension_for_pytorch # noqa: F401 +import oneccl_bindings_for_pytorch # noqa: F401 +import torch +import torch.distributed + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig) +from vllm.distributed import (ensure_model_parallel_initialized, + init_distributed_environment) +from vllm.distributed.parallel_state import get_pp_group +from vllm.logger import init_logger +from vllm.model_executor import set_random_seed +from vllm.utils import is_xpu +from vllm.worker.cache_engine import CacheEngine +from vllm.worker.worker import Worker +from vllm.worker.worker_base import LoraNotSupportedWorkerBase +from vllm.worker.xpu_model_runner import XPUModelRunner + +logger = init_logger(__name__) + + +class XPUWorker(LoraNotSupportedWorkerBase, Worker): + """A worker class that executes (a partition of) the model on a GPU. + + Each worker is associated with a single XPU device. The worker is + responsible for maintaining the KV cache and executing the model on the + XPU. In case of distributed inference, each worker is assigned a partition + of the model. + """ + + def __init__( + self, + model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + cache_config: CacheConfig, + load_config: LoadConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + lora_config: Optional[LoRAConfig] = None, + speculative_config: Optional[SpeculativeConfig] = None, + prompt_adapter_config: Optional[PromptAdapterConfig] = None, + is_driver_worker: bool = False, + observability_config: Optional[ObservabilityConfig] = None, + ) -> None: + assert device_config.device_type == "xpu" + assert is_xpu() + + self.model_config = model_config + self.parallel_config = parallel_config + self.parallel_config.rank = rank + self.scheduler_config = scheduler_config + self.device_config = device_config + self.cache_config = cache_config + self.load_config = load_config + self.local_rank = local_rank + self.rank = rank + self.distributed_init_method = distributed_init_method + self.lora_config = lora_config + self.prompt_adapter_config = prompt_adapter_config + self.is_driver_worker = is_driver_worker + self.observability_config = observability_config + if parallel_config and is_driver_worker: + assert rank % parallel_config.tensor_parallel_size == 0, \ + "Driver worker should be rank 0 of tensor parallel group." + + self.model_runner = XPUModelRunner( # type: ignore + model_config, + parallel_config, + scheduler_config, + device_config, + cache_config, + load_config=self.load_config, + lora_config=self.lora_config, + kv_cache_dtype=self.cache_config.cache_dtype, + is_driver_worker=is_driver_worker, + observability_config=self.observability_config, + ) + # Uninitialized cache engine. Will be initialized by + # initialize_cache. + self.cache_engine: List[CacheEngine] + self.gpu_cache: Optional[List[List[torch.Tensor]]] + + def init_device(self) -> None: + if self.device_config.device.type == "xpu" and is_xpu(): + self.device = torch.device(f"xpu:{self.local_rank}") + torch.xpu.set_device(self.device) + torch.xpu.empty_cache() + self.init_gpu_memory = torch.xpu.get_device_properties( + self.local_rank).total_memory + else: + raise RuntimeError( + f"Not support device type: {self.device_config.device}") + # Initialize the distributed environment. + self.init_worker_distributed_environment() + # Initialize the model. + set_random_seed(self.model_config.seed) + + # keep this method for `empty_cache` and `synchronize` api + @torch.inference_mode() + def determine_num_available_blocks(self) -> Tuple[int, int]: + """Profiles the peak memory usage of the model to determine how many + KV blocks may be allocated without OOMs. + + The engine will first conduct a profiling of the existing memory usage. + Then, it calculate the maximum possible number of GPU and CPU blocks + that can be allocated with the remaining free memory. + + .. tip:: + You may limit the usage of GPU memory + by adjusting the `gpu_memory_utilization` parameter. + """ + # Profile the memory usage of the model and get the maximum number of + # cache blocks that can be allocated with the remaining free memory. + torch.xpu.empty_cache() + + # Execute a forward pass with dummy inputs to profile the memory usage + # of the model. + self.model_runner.profile_run() + + # Calculate the number of blocks that can be allocated with the + # profiled peak memory. + torch.xpu.synchronize() + used_memory = torch.xpu.memory_allocated() + total_gpu_memory = torch.xpu.get_device_properties( + self.local_rank).total_memory + free_gpu_memory = total_gpu_memory - used_memory + + # NOTE(woosuk): Here we assume that the other processes using the same + # GPU did not change their memory usage during the profiling. + peak_memory = self.init_gpu_memory - free_gpu_memory + assert peak_memory > 0, ( + "Error in memory profiling. " + f"Initial free memory {self.init_gpu_memory}, current free memory" + f" {free_gpu_memory}. This happens when the GPU memory was " + "not properly cleaned up before initializing the vLLM instance.") + + cache_block_size = self.get_cache_block_size_bytes() + num_gpu_blocks = int( + (total_gpu_memory * self.cache_config.gpu_memory_utilization - + peak_memory) // cache_block_size) + num_cpu_blocks = int(self.cache_config.swap_space_bytes // + cache_block_size) + num_gpu_blocks = max(num_gpu_blocks, 0) + num_cpu_blocks = max(num_cpu_blocks, 0) + gc.collect() + torch.xpu.empty_cache() + return num_gpu_blocks, num_cpu_blocks + + def _warm_up_model(self) -> None: + # IPEX don't support capture graph yet + pass + + def init_worker_distributed_environment(self) -> None: + """Initialize the distributed environment.""" + + parallel_config = self.parallel_config + rank = self.rank + distributed_init_method = self.distributed_init_method + + if torch.distributed.is_initialized(): + torch_world_size = torch.distributed.get_world_size() + if torch_world_size != parallel_config.world_size: + raise RuntimeError( + "torch.distributed is already initialized but the torch " + "world size does not match parallel_config.world_size " + f"({torch_world_size} vs. {parallel_config.world_size}).") + elif not distributed_init_method: + raise ValueError( + "distributed_init_method must be set if torch.distributed " + "is not already initialized") + else: + # use sockets as default Level zero IPC exchange backend. By + # default oneccl will use `drmfd` as mechanism which need extra + # dependency (libdrm and drm headers) on your system. + ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE", + "sockets") + ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE", + str(parallel_config.world_size)) + os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE + os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE + os.environ["LOCAL_RANK"] = str(self.local_rank) + init_distributed_environment( + world_size=parallel_config.world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=self.local_rank, + backend="ccl") + + ensure_model_parallel_initialized( + parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size) + + if parallel_config.pipeline_parallel_size > 1: + # torch-ccl xpu need a collective API warm up + # before calling send/recv API + get_pp_group().all_reduce(torch.zeros(1).xpu())

(0$MFajY<|*frQ~4TA{B;6l0^bCPjUY8m+@hkse17Tj z%a>yFHMh}CB*V^l>rm;nN<5qJo0AAdvLYBBHSinNWQGy@2@!#rsI_XL<9r)q2ef%z zwZMG(js+&vcM3{@SW#A)dPh7vMXmSEJ%X}Zzb*`hjWuyxaSoOSh5^SF261k+=6Qb0 z31C!7$%H;{`d$r|D$$pBlegd$VujgsHa#*E49uG6 z|1ezf-Z67xOf$-*>&@+5@2*u?qDL;W(jHdQgG9lD_&Egur-W!&OWP!&x8+z9*K*;u zZ09*Vo*JAgsB_ng&FhONQsUOzEFiz6~D0%Rnf8dj!+)Ji6564e-IjDzC>+K-jX z#aM%TQCYR~)kfP5Po?fy%v!BDdy#m@2TR48-|U3z@eKD@;BvxG6Kq9QyojyDYP;nJ zc1DbG@Z9i5Oob!gB=36qob-1mTYp_!a>Y9sUG?QvmUde**$L z;rsEYsrDHH-z9K{z;MJdlaQFL3inACQZ@2DG!7a7kW59Fg*1N7RQ*?$X38X`yC@o@ zRaWmXMgQDVEe0HhM89Jxs$v2+nJMeQ^da4pO*SeOWF0MfkKJ3wCzmPA;L?OjpTeRI zJi>$dcStLE7$+6UP~K8k3i7}hVSGANJemBE5S~dL7(Xm9wjl?{LP+w4kcxVy4ymlm zo7pXdO(faiQkm4eptA`&3(XCQ|0IN5)WPX>nhl}N8%m$`mA)$I1<(>odVTa2c-?x&xiAy`=UH_4QN~J4VwQ8`kt2h1sd-Osh?Z>r?tOJ*Pb1& z9qWbx%`Wv`c!h$X#IG%_x<6WA)eS|HRy!W;G0R9+Nkul;k1kMpSPv8PTtYA()O-1tO65$te zFEqdz(ClM-te!)WBs5f5qi{6Vh3*Mrjk~QCKT$L!PqBh1*)9r2>IrvQ9Bg@NOe+a( zBPJ9&W;m7&4ahNHc;uZvs-69 z(BbV0qU(Xua@;CuT&QOc7x69@9{e$YuFNIsYR81{O&%`!;ks(d#tx|+K$W3JmDwF* zhiL=MfKoGd474VQo@1qPT0zt0q6}{yem#?q0K#oiX(n@LhUSB+W4|=AOtyA<-@Uhr z`_f%4Ux?)!aUPLOvelIud>dG%j&n86aetUMh>J*QBoPlqz3tA`xHxi$l8V?QdW!E) z^C2`cB?zkB7D6zj2}G_5$4Gy~ee?+Bx(viaSV&o=z6l2c?ohyc$;Vj8@4MJ@pbR}HJ14l`yQYf5Txx^z*MIv9Nw9LSVyKUI{ffq+9hk*Yc zmcW0Xzz+!61l}Y-p)3CbfaS6gTVk7o0Mihqng=Tk8E?21-?r$i22r4GTwhFG1Oq;R zU{c#~hcq{(mCg7eN137d~JpOox& zv{A8du~D&Q+HEqIOV||xyBu%L?Q|OJ|ALxe0l>(Ku zANRg~Z`Kv#-)5s|>$bLhVQFdUc5zpb%1_erhAJ%9UlQ4(Sb>V--=Vsf2)qV>;6vnE zV)aV9-QX0iafQGNfzb{HM!t*)_{dWVUIY-4Ei>;FX0aW}BHK|>+IN~gPurt^^<$l} zJF3DyS5@{arm@eM&OT)s_DkJhpXgckKf1|2R&#~_79&?M=)~V3F6NJdjCk`XenuxS z;hfOAX#+xjZUVV`yot05$r7SY4-!>Z4pFp-FoQyCU`U}rW+Vshc-;uT_5w1t{m|D1W5(E3o?P~U+|1bH*YBGBy;R@Hm#Wr{O{ z^(LHJUaL&l(NYcOb(oGJGl&4ScUZ)%B&{Qh7^Ly)r*f(H@Sa{|C#&s7qkWyM<`w9% zr%L6*u8=e~NNG`JobR}mt561281FXc#K4?Fve-=WIuzL1+2NY))W639k$2@ew6qH} z1SKOW1M7R7RGToXNb?B)`WTI;CyBE$))HJr9Q$8*0*aS)bw=cIU_B_8L-qu2GC8`5 z6@bWmj_l4i_KM6z#%~z1exYYS5s6Z2au|s=7}!VOgbKz*Lk<5G5*3LOuB%YRU=pZe zsjeS_*GkbTXvh~LIwh|ziB1hKlSZf6VRQ<18}_FJ2Kq=*wk#9FbGoJJOr9p6JaKq3o%m;D>aifA{HL4kYY>2v(nfy3!{<5 z&*S}QyzxF1T^e3KjV`lM8eIu>{rwl(IeTvak+Gpx6edxC6W(IwVqR(_xO#2k?(r0iZgFe%u&rCF( z_L;s*pWXAor^ni6qnWhNOwvch-^u^t2sK+l2k$C8{w!Eh z_^bRWV2MK;Pme|4iPXl$QTUxu?93n$yM&J0JNO&LsT0xqs@w6=e;DUG@D4%A5@iKm z0zcv6|x5h(m#hU!EOvlLS zFkW0dX^Wi;zl4)9;x`Da6X+56A%TcMpMbE70thumb``Jtq~KqHs2PgejXRb~zYOj$ z6cAa~b1C~R4UxWsk_GWIe_hpAxb+0zoeYDBnN1H-8wOOP!K9Ru*@)(wP0k5z z=BV9<)<>A(?TgG|yLq&lA0JuRqCsoe9&dk?qt-&l9hgie!NdwCAh`~{0T4>>V&aNHy@5H%a(Jl1ryMk<VU@un%`9zG4sT)=bQk11i&Y#DtB9mGjlTRot}c@2r2W#Lur=x} z^9@uB#dn*p`j_m z(hNptJ(_k~#guz?NW?um^uS|yy&#Y6@JX~wwsVuP_l8BG)^o#0#-G+DSY;!ja8stu*{(6=DJ6 zWT$rR{cYk}%2<(Sd=v%+`4Jcc9XXZWYyUsT9m_$i3xT_u{YIY@>2%| zNH%yB*$Sj&;a195C4MwQiukI6Ux^&;P>!?>xeB>Y(wmV+3C_OMni|&Y6ZLvhpVSg> zZ^@f%3q2x=yVw%Hj(N+9&=10?Q5t?LHW7e?wUutW8Es2E z+d!bUak4l?4W@=6Z$xEE8`H0HP^wZ8Ll5#rx0Jq)4+?Mxq)GZ2(!~*81e4ytjj0Rs>_Y{spVBqVp|z|kxP0RMSI^$CtSmK8uSH`_+SEN2;sT#xf2E#vkQJdgAQeD%!z)72K;LpCX>kd z2_%il!d_iGndGf$E%r=0JVga-NdgexswGn*BG^M3c=Jehk6H{z)3BQLse$FeMV)dq|=rKF8Jx_b9tRU^JA%6NAs ztDl(}Dvg)PBS)@IP^q5$bn>r7yUz-l-qXJk7cF1l_nC04kefXC%LAd;1!wUDl#cv0 zv%`a|utIs_L2ekoc!3Q^SE?g-e2YHqNadS8d`q*fLuTS{VfwOW>mn>%On*<<`SGZ5 zak6+qa9ufXf9rnIx`PDoXv)9Alp@c|tAyP}itZYpkuLwWa=l_cm!`TxK&YOf{t?;H zexKP>iP2ocF=~*DsiuRe`U?9AKX#^r=DL%ikPu=fWqT>R6 z%jP6wkVqyk1W7RMgLfM!X-R&8ZHoGYFil;_ZxdOKIO90GA)U+))Huol%1W_W@?CTkPrt0 zsAoWlDDYw_W#!;QPN_<2tE?Qh%7?w=lvDnP98#5<$}K17nnS9xWhuY!>j4Om^lnZ8 zIgOrve*HavU!y%WRm$P>-f#bT=bzrp<^F?SMnBa1A)dHzb>j4HT9e*ntL&9qQ{EJ}6~gJ(j5ot|JDhFJd2>eY zc1{$9^DHMEnSYx1=J7ruN_a2H1+-Y+DYQ(AGFr-6%OYB)#57u_vzF6nnGv&SnU#}R z<&2n<3u0cLZCGMKoO)(?=ft9Pb5F{G`||{r`FA8JxQHAiB!Q>C{t&>lQi2= z5T=C(5h|5@I`L5u%X@7h(YDrFY8-n-@BTmAIYJGI+&B4G3bmC(xZHY$=8!!w29<%Q{4BJUN1<=LX(6@~3NqUcQs z2M3u*=WeuH9VO!!^sTj3D@f9UkWnmq({U$Iv0U0}Caoae?af3%b5AZk>IPBL>`Bp^ zX@u<{8EKemMx(FOouIxu(ml}%RM?D`uDxHYlv2m{qo5^yKP~xwiw}%?+4r$QsC(2j z&f;Zs_!%9krh|rQI>14qW2AxTIDGY(j*<6_4*%Nw-{#aJw&3D_wwIGuXSvhtEG2>3 zmdVmjTiu=3d#%UIVRLJ_v!Cp=qvgFYY%POY8T#^xtal->13+e1T5t-(;YIi$nVJmFcchf@B-jz|UGI4|#>NSw-*_*d&8|(MI z;*F12YqeW<)57ih>kmHl3Lo8FzwwDz*j)eQR_#O%UBvX)@x-L&nnPL>z$fbaK|ZN7 z=!LZBSL;c0k7d2Cge2ki{^d9avO_mz>^9oU4Th84b~dZTlWxQ1@(GY1!y2s z36vCW($;f{LMB0PsE9|Yt+zO?kt_vu31d9_i(7YYe|)k!UdDiT@Eoe612$9ZF*aLh z{E&wJ8;XIkJwUwG&QunpE%MiMviQ*PCJ>)&VO`I`wU*>0>Ne^{uPmoz;am>y4!=+1 zeTLsB_dUZ=%0MbocM?~I(os&mvvF- z?VLQ1*4G*)k3WUJ3y8*x;`G<1ID?%Q&*d7XI4jOQvz{7=dqtSg`Eh5ls+#C?N@#M0>?Aw9RyLL#)ru?FUqMqcC&mO_b!9Wz0mblKv>p4d(Yf({t0fcF ztgpC_cO(zRFjCerUl`YeP`b_79gZJcB=a3BaHQ-!P)6Iy4h;sQ*;Hv&9!FcUhZ|jiy_1igZawKL zEOSk`+(x5y+tI~zH!m_+@v7VE#)-Sdl+%Wg9Z-OK>DrU`FI{yn{p`tmKf$+5>ebQB z#Xe#*F*KHmyQ#X8w(3Ylv)!z7oV!arouU!2P8ZJTaWf2A2-q7OYi`jIg)6$u-9ZbY zsoU8BYPzabixy}`aUz4^&}P(-3eH8kk$gP9sS2X)!B*I2NJ4uH8or0!9z%0nxhK`W z8wagUD68&G*$BEJ^c^Q!3&KnmaRNtMA1*MeI90dSPQZX2uuVpQexwm6@?bcf&racH zo90QVJMZacbSc_9a9gs~R{J5)8QiVp$(c0_T;!ob(*)A7Okqap1HZ6AUCAIO%CviL zH-^LR|vT;EZG%_@bcEWuD7%fosRIq-~Ic*P$I=shHSE zb2^gY6a`fcLo91!Iw7`KL)aUDAoEF&1M}QB3!E`Rm zbEH3z^A^!Xscp%eHcDpMu*}}2&qiP@MW*VpX_bp7p)qv@3#oIo(~R!}%(cmCAp_rM z=&3GHKcz^jLdDOi_%RhW6$B6%da0{aTcKi!iYgUFDlStoLB)G0Qj4@i{V}~0Y*55$ zMW<08P%+FdMu72K==d3)n8+vr{+Ac0jj}Pze`VveVe;SfOS6EQdHVZO@vNa}R%&bC ztbT%*I%Rx=&zO)2J0`h@#zPZjo=fYYxt%{S`Vi+HM6G^`A(aB>8fqSO^=DL^qGEx% zOtp$H&-GVtY~J~Nb@R@8&HwDdYHjn5r(f3A@84V9+!#3^i}#IYOoGio=^*D~5%`++ z7S?5P+oo;UhV{KuP#>W!EpZyfIbmuywv`N=^+ls|A68-w6+VuQAYke_YLkr zUKj$1>939F(C@vRx{;uEE_YylmHWy7{;@>inbS9KLaz_3V?Ht=C{iGp zcTves7}O<<%dEcZ&BQx_5`IUuA2P$fH5Hk+H!*BE)VH*BJ#5AaIm{2fqW#Ee+d0!L zy|Q2RuEaa-F0LOm!+n-P^*YiX#Jvwt8O(%6&<<%)=f-jGowarsnLz+o*x=}U3`_0E zR+e$}EQ&zAcdG8*-MjaYG(76UeXq{vjNo0#Lx9$`}KP3ji9T!_m4kwNZ-{uiak6r8MI?s#L7*lvsbn+?0_n2(DZ$Mp32e4|`u}Kab*LG0;N2Y+PRPw#29MNwO ztAMUDpc^3b!uy$8?Kf$&}5*1h8QVb0nWTa zLJJ}UKLkdgKggE!M8m_)wQq54<--r0B&BU2a-)u2f|oE~ zSthrqw!SH<6LSV1V$zUNUG$*)i+?( zg}rO3Z|Up$dl3X-9$pMpY+A_a&?kmO*ebIAJvc709$=7pRyAAR>1ds*LR8 z>-7C9>4B223_vI=0-$z8Xg|Xs0|?C09kFI!Dlij8AJ#6(=|#03nbm z#G*BrL@=WA*}*jRe5wrz(#6RWwR&BNd1tNiL8j+pnd{*OD9sVs*}MFJG7Sd9$i{^}JhM?-CBm)S`#n0DX%Tta zIiFTQ0e)sqb;9=_1zvW1%s*msOhCg~ zv;g-?gmvH#9c$LG^6Cq8tT7l#r?tz6s1iE)F^i}8d4m>v69v1*se?RQdma{Ue4SS_ z{k)ko0CxrsGWa`SCg$LM#>u!3qa(XAM9875gQ3e~+tY-{I$h25nP(4NC^=8EK+JuXEow9Oi#&}ao>VP1ZB2y zcMB;4sVj4txlu9HpF@^4^(jgy8)H(66x^)tQ|%B?bNqdPwtvN=uhOR|!Xm1^bn@o+ zmZhjSwaABzp)Uf?!+7x_UiyR{jTeTR5oRCp9Ga8+23S&9FA!LVy%$A+doOaYEsE5u ze?|%LYns-q8x(zq!IvbNU3C$-8E6zWYdFmDc6WIXUKyOzS*yN>hlGy@5>~8a2kL-- zTBXhxBRcJKag#=29q1cZN!nG6Y%ZU4kh4O6w)Po^jrlWT(#i;_ERYqw?#iKW8Cj0} z%O{_Y(oKClOdb3(<1_E|TK|b@3_Rv1%Emm>G60XipruIr zj`^UW7JZIhb~v z`;esVnyNB*uh_kRXNIz3jlw!M^#3EG)82&F`8~Mq|J!ZR9a1k3MEPAF{HtxQAr_T1 z+9Xk?rjluaZASem1`J#RI|A|)X|V&Xkt#YOX!RL7e}{*2s}dv$IU1I2{SH9}Va9oh z(%?}QW3A$*1@_?TFKJCn;yz0O{1HIi@4!Xrq@HB03RLh_0i~jz)7cmFD`Z_!Ql+m2 z2(@|+;}t}GHA$~Or=Jb0Km;Q{8&(-Y#MRNepsN0jkeht2bNZw6biR$CWKrKxT&CJv zRPZGR;a;6c5Ps%B%KlGlFEM0 scheme] + self.target_scheme_map = target_scheme_map + self.kv_cache_scheme = kv_cache_scheme + + def get_linear_method(self) -> "CompressedTensorsLinearMethod": + return CompressedTensorsLinearMethod(self) + + def get_scaled_act_names(self) -> List[str]: + return [] + + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def get_name(self) -> str: + return "compressed_tensors" + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> Optional["QuantizeMethodBase"]: + from vllm.attention.layer import Attention # Avoid circular import + + # Check if the layer is skipped for quantization. + # TODO (@robertgshaw2): support module names + if should_ignore_layer(prefix, ignore=self.ignore): + return UnquantizedLinearMethod() + if isinstance(layer, LinearBase): + scheme = self.get_scheme(layer=layer, layer_name=prefix) + layer.scheme = scheme + return CompressedTensorsLinearMethod(self) + if isinstance(layer, Attention): + return CompressedTensorsKVCacheMethod(self) + if isinstance(layer, FusedMoE): + return CompressedTensorsMoEMethod.get_moe_method(self) + return None + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig": + target_scheme_map: Dict[str, Any] = dict() + ignore = cast(List[str], config.get("ignore")) + quant_format = cast(str, config.get("format")) + + # The quant_config has multiple config_groups, each containing + # an input_activations key with details about how the activations are + # quantized, a weights key indicating how the weights are quantized, + # and a list of targets under the `targets` key, dictating which + # layers are impacted by the quantization details. The quantization + # details follow the structure defined by the QuantizationArgs + # pydantic model, which is used to verify the structure of the + # quant_config and also store the details for later use. + for _, quant_config in config["config_groups"].items(): + targets = quant_config.get("targets") + for target in targets: + target_scheme_map[target] = {} + target_scheme_map[target][ + "weights"] = QuantizationArgs.parse_obj( + quant_config.get("weights")) + try: + target_scheme_map[target][ + "input_activations"] = QuantizationArgs.parse_obj( + quant_config.get("input_activations")) + except Exception: + target_scheme_map[target]["input_activations"] = None + + return cls(target_scheme_map=target_scheme_map, + ignore=ignore, + quant_format=quant_format, + kv_cache_scheme=config.get("kv_cache_scheme")) + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + def _check_scheme_supported(self, + min_capability: int, + error: bool = True) -> bool: + capability_tuple = current_platform.get_device_capability() + + if capability_tuple is not None: + capability = capability_tuple.to_int() + supported = capability >= min_capability + if error and not supported: + raise RuntimeError( + "Quantization scheme is not supported for ", + f"the current GPU. Min capability: {min_capability}. ", + f"Current capability: {capability}.") + return supported + else: + return False + + def _is_static_tensor_w8a8(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8 + weight_strategy = ( + weight_quant.strategy == QuantizationStrategy.TENSOR.value + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value) + is_tensor = (weight_strategy and input_quant.strategy + == QuantizationStrategy.TENSOR.value) + is_static = not weight_quant.dynamic and not input_quant.dynamic + + # Both symmetric and asymmetric input quantization supported. + # Only symmetric weight quantization supported. + return is_8_bits and is_tensor and weight_quant.symmetric and is_static + + def _is_dynamic_token_w8a8(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8 + weight_strategy = ( + weight_quant.strategy == QuantizationStrategy.TENSOR.value + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value) + is_token = (weight_strategy and input_quant.strategy + == QuantizationStrategy.TOKEN.value) + is_dynamic = not weight_quant.dynamic and input_quant.dynamic + + # Both symmetric and asymmetric input quantization supported. + # Only symmetric weight quantization supported. + return is_8_bits and is_token and weight_quant.symmetric and is_dynamic + + def _is_fp8_w8a8(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + # Confirm weights and activations quantized. + if weight_quant is None or input_quant is None: + return False + + # Confirm weight scheme is supported. + is_floating_point = (weight_quant.type == QuantizationType.FLOAT + and input_quant.type == QuantizationType.FLOAT) + is_symmetric_weight = weight_quant.symmetric + is_static_weight = not weight_quant.dynamic + is_per_tensor_or_channel_weight = (weight_quant.strategy in [ + QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL + ]) + if not (is_floating_point and is_symmetric_weight and is_static_weight + and is_per_tensor_or_channel_weight): + return False + + # Dynamic quantization is always supported if weights supported. + if input_quant.dynamic: + return True + + # Confirm activation scheme is supported. + is_symmetric_activation = input_quant.symmetric + is_per_tensor_activation = ( + input_quant.strategy == QuantizationStrategy.TENSOR) + return is_symmetric_activation and is_per_tensor_activation + + def _is_fp8_w8a16(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + # Confirm weights quantized. + if weight_quant is None: + return False + + # Confirm we have floating points. + if weight_quant.type != QuantizationType.FLOAT: + return False + + # Confirm weight scheme is supported. + is_symmetric_weight = weight_quant.symmetric + is_static_weight = not weight_quant.dynamic + is_per_tensor_or_channel_weight = (weight_quant.strategy in [ + QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL + ]) + if not (is_symmetric_weight and is_static_weight # noqa: SIM103 + and is_per_tensor_or_channel_weight): + return False + + # All conditions satisfied. + return True + + def _is_wNa16_group_channel(self, weight_quant: BaseModel, + input_quant: BaseModel) -> bool: + input_quant_none = input_quant is None + is_symmetric = weight_quant.symmetric + is_channel_group = ( + weight_quant.strategy == QuantizationStrategy.CHANNEL.value + or weight_quant.strategy == QuantizationStrategy.GROUP.value) + is_static = not weight_quant.dynamic + + return (is_channel_group and input_quant_none and is_symmetric + and is_static) + + def _get_scheme_from_parts( + self, weight_quant: BaseModel, + input_quant: BaseModel) -> "CompressedTensorsScheme": + + # Detect If Mixed Precision + if self._is_wNa16_group_channel(weight_quant, input_quant): + if (self.quant_format == CompressionFormat.marlin_24.value + and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS): + return CompressedTensorsW4A16Sparse24( + strategy=weight_quant.strategy, + num_bits=weight_quant.num_bits, + group_size=weight_quant.group_size) + if (self.quant_format == CompressionFormat.pack_quantized.value + and weight_quant.num_bits in WNA16_SUPPORTED_BITS): + return CompressedTensorsWNA16( + num_bits=weight_quant.num_bits, + strategy=weight_quant.strategy, + group_size=weight_quant.group_size, + actorder=weight_quant.actorder) + + # Detect If Activation Quantization. + # TODO @dsikka: clean-up conditions + if is_activation_quantization_format(self.quant_format): + if self._is_fp8_w8a8(weight_quant, input_quant): + is_fp8_w8a8_supported = self._check_scheme_supported( + CompressedTensorsW8A8Fp8.get_min_capability(), error=False) + if is_fp8_w8a8_supported: + return CompressedTensorsW8A8Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=(input_quant + and not input_quant.dynamic)) + else: + return CompressedTensorsW8A16Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=(input_quant + and not input_quant.dynamic)) + + if self._is_fp8_w8a16(weight_quant, input_quant): + return CompressedTensorsW8A16Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=(input_quant + and not input_quant.dynamic)) + + if self._is_static_tensor_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Int8( + strategy=weight_quant.strategy, + is_static_input_scheme=True, + input_symmetric=input_quant.symmetric) + + if self._is_dynamic_token_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Int8( + strategy=weight_quant.strategy, + is_static_input_scheme=False, + input_symmetric=input_quant.symmetric) + + raise NotImplementedError( + "No compressed-tensors compatible scheme was found.") + + def get_scheme( + self, + layer: torch.nn.Module, + layer_name: Optional[str] = None) -> "CompressedTensorsScheme": + """ + compressed-tensors supports non uniform in the following way: + + ignore: List of layer_names or nn.Module names to be ignored. + targets of config_groups: There can be N config_groups which each + have a quantization scheme. Each config_group has a list of targets + which can be a full layer_name, a regex for a layer_name, or + an nn.Module name. + + We first check whether a layer is in the ignore group and use + CompressedTensorsUnquantized (i.e. fp16/bf16) scheme for the layer + + We then detect whether a layer_name is found in any target and + use the quantization scheme corresponding to the matched target + to select the CompressedTensorsScheme used for infernece. + """ + + # Find the "target" in the compressed-tensors config + # that our layer conforms to. + # TODO (@robertgshaw): add compressed-tensors as dep + # so we do not have to re-write these functions + # need to make accelerate optional in ct to do this + matched_target = find_matched_target( + layer_name=layer_name, + module=layer, + targets=self.target_scheme_map.keys()) + + # Find the quant_scheme + scheme_dict = self.target_scheme_map[matched_target] + scheme = self._get_scheme_from_parts( + weight_quant=scheme_dict["weights"], + input_quant=scheme_dict["input_activations"]) + + # Raise error if device does not support the scheme + # (e.g. fp8 needs ada lovelace) + self._check_scheme_supported(scheme.get_min_capability()) + + return scheme + + +class CompressedTensorsLinearMethod(LinearMethodBase): + + def __init__(self, quantization_config: CompressedTensorsConfig): + self.quantization_config = quantization_config + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.scheme.process_weights_after_loading(layer) + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + """ + Use the CompressedTensorsScheme associated with each layer to create + the necessary parameters for the layer. See LinearMethodBase for param + details + """ + weight_loader = extra_weight_attrs.get("weight_loader") + layer.scheme.create_weights( + layer=layer, + input_size=input_size, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + output_size=output_size, + params_dtype=params_dtype, + weight_loader=weight_loader) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None): + """ + Use the output of create_weights and the CompressedTensorsScheme + associated with the layer to apply the forward pass with the + layer input. See LinearMethodBase for param details + + """ + + scheme = layer.scheme + if scheme is None: + raise ValueError("A scheme must be defined for each layer") + return scheme.apply_weights(layer, x, bias=bias) + + +class CompressedTensorsKVCacheMethod(BaseKVCacheMethod): + """ + Supports loading kv-cache scaling factors from compressed-tensors + checkpoints. + """ + + def __init__(self, quant_config: CompressedTensorsConfig): + self.validate_kv_cache_scheme(quant_config.kv_cache_scheme) + super().__init__(quant_config) + + @staticmethod + def validate_kv_cache_scheme(kv_cache_scheme: Optional[Dict[str, Any]]): + """ + Validator for the kv cache scheme. Useful for controlling the + kv cache quantization schemes, that are being supported in vLLM + :param kv_cache_scheme: the compressed-tensors kv cache scheme + """ + if kv_cache_scheme is None: + return + + type_ = kv_cache_scheme.get("type") + num_bits = kv_cache_scheme.get("num_bits") + + if type_ != "float" and num_bits != 8: + raise NotImplementedError( + "Currently supported kv cache quantization is " + "num_bits=8, type=float, however " + f"received num_bits={num_bits}, type={type_}") + + strategy = kv_cache_scheme.get("strategy") + if strategy != "tensor": + raise NotImplementedError( + "Only support per-tensor scaling factor " + "for compressed-tensors KV cache. " + f"Expected strategy: tensor, found strategy: {strategy}") + + is_symmetric = kv_cache_scheme.get("symmetric") + if not is_symmetric: + raise NotImplementedError( + "Only support symmetric scaling factor " + "for compressed-tensors KV cache. " + f"However found symmetric: {is_symmetric}") diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py new file mode 100644 index 00000000..af04d725 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py @@ -0,0 +1,511 @@ +import enum +from enum import Enum +from typing import Callable, List, Optional + +import torch + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase, + FusedMoeWeightScaleSupported) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + WNA16_SUPPORTED_BITS) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + CompressionFormat, QuantizationStrategy) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize) +from vllm.model_executor.utils import set_weight_attrs +from vllm.utils import is_hip, print_warning_once + + +class GPTQMarlinState(Enum): + REPACK = enum.auto() + READY = enum.auto() + + +__all__ = [ + "CompressedTensorsMoEMethod", "CompressedTensorsW8A8Fp8MoEMethod", + "CompressedTensorsWNA16MoEMethod" +] + + +class CompressedTensorsMoEMethod(FusedMoEMethodBase): + + @staticmethod + def get_moe_method( + quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501 + ) -> "CompressedTensorsMoEMethod": + # TODO: @dsikka: refactor this to use schemes as other kernels + # are supported + check if the layer is being ignored. + weight_quant = quant_config.target_scheme_map["Linear"].get("weights") + input_quant = quant_config.target_scheme_map["Linear"].get( + "input_activations") + + if quant_config._is_wNa16_group_channel(weight_quant, input_quant): + return CompressedTensorsWNA16MoEMethod(quant_config) + elif quant_config._is_fp8_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Fp8MoEMethod(quant_config) + else: + raise RuntimeError( + f"Unsupported FusedMoe scheme: {weight_quant}, {input_quant}") + + +class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): + + def __init__( + self, + quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501 + ): + self.quant_config = quant_config + self.weight_quant = self.quant_config.target_scheme_map["Linear"].get( + "weights") + self.input_quant = self.quant_config.target_scheme_map["Linear"].get( + "input_activations") + + if not (self.weight_quant.strategy == QuantizationStrategy.TENSOR + and self.input_quant.strategy == QuantizationStrategy.TENSOR): + raise ValueError( + "For FP8 Fused MoE layers, only per-tensor scales" + "for weights and activations are supported. Found " + f"{self.weight_quant}, {self.input_quant}") + + self.static_input_scales = not self.input_quant.dynamic + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + params_dtype = torch.float8_e4m3fn + + # WEIGHTS + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + 2 * intermediate_size, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size, + intermediate_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + 2, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + + w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + if self.static_input_scales: + w13_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if self.static_input_scales: + if (layer.w13_input_scale is None or layer.w2_input_scale is None): + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None.") + if (not all_close_1d(layer.w13_input_scale) + or not all_close_1d(layer.w2_input_scale)): + print_warning_once( + "Found input_scales that are not equal for " + "fp8 MoE layer. Using the maximum across experts " + "for each layer. ") + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False) + + # If rocm, normalize the weights and scales to e4m3fnuz + if is_hip(): + # Normalize the weights and scales + w13_weight, w13_weight_scale, w13_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale, + layer.w13_input_scale) + w2_weight, w2_weight_scale, w2_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale, + layer.w2_input_scale) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, + requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter(w13_weight_scale, + requires_grad=False) + if w13_input_scale is not None: + layer.w13_input_scale = torch.nn.Parameter(w13_input_scale, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(w2_weight, + requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, + requires_grad=False) + if w2_input_scale is not None: + layer.w2_input_scale = torch.nn.Parameter(w2_input_scale, + requires_grad=False) + + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.num_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start:start + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][ + start:start + shard_size, :], _ = ops.scaled_fp8_quant( + dq_weight, max_w13_scales[expert_id]) + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, + requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe import fused_experts + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_experts(x, + layer.w13_weight, + layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=True, + use_fp8_w8a8=True, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale) + + +class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): + + def __init__( + self, + quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501 + ): + self.quant_config = quant_config + # TODO: @dsikka: refactor this to use schemes as other kernels + # are supported + check if the layer is being ignored. + config = self.quant_config.target_scheme_map["Linear"].get("weights") + self.num_bits = config.num_bits + self.packed_factor = 32 // config.num_bits + self.strategy = config.strategy.value + self.group_size = config.group_size + assert config.symmetric, ( + "Only symmetric quantization is supported for MoE") + + if not (self.quant_config.quant_format + == CompressionFormat.pack_quantized.value + and self.num_bits in WNA16_SUPPORTED_BITS): + raise ValueError("For Fused MoE layers, only ", + f"{CompressionFormat.pack_quantized.value} ", + "is supported for the following bits: ", + f"{WNA16_SUPPORTED_BITS}") + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + # Will transpose the loaded weight along the + # intermediate and hidden dim sizes. Will + # shard for TP along the transposed dims + extra_weight_attrs.update({ + "is_transposed": True, + "quant_method": self.strategy + }) + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size // + self.packed_factor, + 2 * intermediate_size, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w13_weight_packed", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + intermediate_size // + self.packed_factor, + hidden_size, + dtype=torch.int32), + requires_grad=False) + layer.register_parameter("w2_weight_packed", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + if self.strategy == "channel": + num_groups_w2 = num_groups_w13 = 1 + self.group_size = -1 + else: + num_groups_w2 = intermediate_size // self.group_size + num_groups_w13 = hidden_size // self.group_size + + w13_scale = torch.nn.Parameter(torch.ones(num_experts, + num_groups_w13, + 2 * intermediate_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_scale) + set_weight_attrs(w13_scale, extra_weight_attrs) + + w2_scale = torch.nn.Parameter(torch.ones(num_experts, + num_groups_w2, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_scale) + set_weight_attrs(w2_scale, extra_weight_attrs) + + w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2), + requires_grad=False) + layer.register_parameter("w2_weight_shape", w2_weight_shape) + set_weight_attrs(w2_weight_shape, extra_weight_attrs) + w13_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2), + requires_grad=False) + + layer.register_parameter("w13_weight_shape", w13_weight_shape) + set_weight_attrs(w13_weight_shape, extra_weight_attrs) + + w13_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx", w13_g_idx) + set_weight_attrs(w13_g_idx, extra_weight_attrs) + + w2_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx", w2_g_idx) + set_weight_attrs(w2_g_idx, extra_weight_attrs) + + w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx_sort_indices", + w13_g_idx_sort_indices) + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs) + + w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx_sort_indices", + w2_g_idx_sort_indices) + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs) + + layer.a13_scale = None + layer.a2_scale = None + layer.marlin_state = GPTQMarlinState.REPACK + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + + def replace_tensor(name, new_t): + # It is important to use resize_() here since it ensures + # the same buffer is reused + getattr(layer, name).resize_(new_t.shape) + getattr(layer, name).copy_(new_t) + del new_t + + def get_scale_perms(num_bits: int): + scale_perm: List[int] = [] + for i in range(8): + scale_perm.extend([i + 8 * j for j in range(8)]) + scale_perm_single: List[int] = [] + for i in range(4): + scale_perm_single.extend( + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]]) + return scale_perm, scale_perm_single + + def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int, + group_size: int, num_bits: int): + scale_perm, scale_perm_single = get_scale_perms(num_bits) + if group_size < size_k and group_size != -1: + s = s.reshape((-1, len(scale_perm)))[:, scale_perm] + else: + s = s.reshape((-1, len(scale_perm_single)))[:, + scale_perm_single] + s = s.reshape((-1, size_n)).contiguous() + return s + + def marlin_moe_permute_scales(s: torch.Tensor, size_k: int, + size_n: int, group_size: int, + num_bits: int): + num_experts = s.shape[0] + output = torch.empty((num_experts, s.shape[1], s.shape[2]), + device=s.device, + dtype=s.dtype) + for e in range(num_experts): + output[e] = marlin_permute_scales(s[e], size_k, size_n, + group_size, num_bits) + return output + + size_k2 = layer.w2_weight_packed.shape[2] + size_k13 = layer.w13_weight_packed.shape[2] + + num_experts = layer.w13_g_idx.shape[0] + device = layer.w13_g_idx.device + layer.w13_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + + marlin_w13_qweight = ops.gptq_marlin_moe_repack( + layer.w13_weight_packed, + layer.w13_g_idx_sort_indices, + layer.w13_weight_packed.shape[1] * self.packed_factor, + layer.w13_weight_packed.shape[2], + self.num_bits, + ) + replace_tensor("w13_weight_packed", marlin_w13_qweight) + marlin_w2_qweight = ops.gptq_marlin_moe_repack( + layer.w2_weight_packed, + layer.w2_g_idx_sort_indices, + layer.w2_weight_packed.shape[1] * self.packed_factor, + layer.w2_weight_packed.shape[2], + self.num_bits, + ) + replace_tensor("w2_weight_packed", marlin_w2_qweight) + # Repack scales + marlin_w13_scales = marlin_moe_permute_scales( + layer.w13_weight_scale, + size_k13, + layer.w13_weight_scale.shape[2], + self.group_size, + self.num_bits, + ) + replace_tensor("w13_weight_scale", marlin_w13_scales) + marlin_w2_scales = marlin_moe_permute_scales( + layer.w2_weight_scale, + layer.w2_weight_scale.shape[1] * self.packed_factor, + size_k2, + self.group_size, + self.num_bits, + ) + replace_tensor("w2_weight_scale", marlin_w2_scales) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + fused_marlin_moe) + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_marlin_moe( + x, + layer.w13_weight_packed, + layer.w2_weight_packed, + layer.w13_weight_scale, + layer.w2_weight_scale, + router_logits, + topk_weights, + topk_ids, + g_idx1=layer.w13_g_idx, + g_idx2=layer.w2_g_idx, + sort_indices1=layer.w13_g_idx_sort_indices, + sort_indices2=layer.w2_g_idx_sort_indices, + num_bits=self.num_bits, + ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py new file mode 100644 index 00000000..5d259ec7 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py @@ -0,0 +1,19 @@ +from .compressed_tensors_scheme import CompressedTensorsScheme +from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS, + CompressedTensorsW4A16Sparse24) +from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8 +from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8 +from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8 +from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS, + CompressedTensorsWNA16) + +__all__ = [ + "CompressedTensorsScheme", + "CompressedTensorsWNA16", + "CompressedTensorsW8A16Fp8", + "CompressedTensorsW4A16Sparse24", + "CompressedTensorsW8A8Int8", + "CompressedTensorsW8A8Fp8", + "WNA16_SUPPORTED_BITS", + "W4A16SPARSE24_SUPPORTED_BITS", +] diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d24a3fc0e0d884fce92eaf7b656e1444c43f194c GIT binary patch literal 796 zcmZ`%O>fgc5Z$%&;iOK>p@LRa+$~gAiLBxSst8b|UP4p{RazmfOuRIfc4M=v5h_9NuT<%MO!ZSMO$`g$Hgttl>rT0+!j6Ar+pW9#EuMUCxPJ1sl$GLI^L)max>P!yFSz89S07!*1r_&D?v$J>X z&G{^?pKV>-(5p;oK8iv5V4IPQllN5uwZq2R`O-XC9G=*lUcG-gfvN6K^LSg&(xO0F zPgT!C*Fwue$3ojeU;#b_nnZi_aQ@#5W;SLp?c8uG^l3x=CeC1*QS3yVdNVD_685D^ zTvvM3BxD%{7~rORhyuN=id}^sG^tEroXu8t=eo;SCIn+YklL}UTKKDokrkeyuJ zikst?a^kv9?xKV z+9pT3Ux5#?2fhNIf%9BBapuMao;~^3mR+m!;_;v7_xruy^R(#qy8*O^zx_V{qZI^y zx|iJ+!^>lM%p))?U`PfzN>LuBA#(4bjPf{*eLs?|w8i2XN|UDngWl_a#T;EnX`4ka zf?@I(_y~q!9Xv>uR&v6@hub2~5C5Y?UMr4@xD@~nPvTI8kX#&GtMD^qCmg3fu) zsq?VgI`Hxs9;3kUAVn-lLx$3b$GpWto=ig)K_u}tx<+Xm##$_S9YDVW{Wjk_@1F0o z4*d4m9`C!~4})pKx@_O=NeA0i;Hk&@e)ND10MCI2#)9RqT@3BX3Js=`7=tT_9NPjj zZm1R$j_sV|OD<+}i>c zvMjWP6DA9MhpCbGQYV_*N-K&W#&vUj_ngA>w@a-JenCUD{L*i6FrIRYbD>0DNLwZ*eWY36|oZUAQa`P*RL+)%eX^)#xcR2^KH{y5VYM41TC8uI4DCp1&p^*bNmX~JyifhFsl)}8IB-8=Nq%5W2 zY_#R=1lT4y_J~e^U`^*ErAFTutddZS!p?oPn$H9{)lg0=+I(SdP`Qu_C|?aV3jR1O zHCS`WFQ7D=ym2>mJFy!Ft2nMsuyDCvZ}D;6UgStW!Om_^+ahX|5Nju3eb1Tu&b;T0 ztGFE{^c^3IwGcjqN#hmYUw-yBDcsmW-RgC7{acw|PB)>S5-EW^G6iPVcbqH;6KdwN z>k3x_YwoH8W*2Zna-ymd$R~E=L~+i16%>?KdVA9}ki#9ww}5_J>Vm?TVhuI(ab&9i zMi6;Z0bjih_B;2V4G-&1mMO@4met)XgPRJ}1N~l>fh^@}rk!Q1pfJ4e@BikTrrjk` zM?zWMcX1rK*OnJz9V%5PjTF&^~gx RH|`&W!=Q&cVG`bt{sXXsN96zj literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w4a16_24.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w4a16_24.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..258d50402c07dac3c5c719153c1fabd8519dfecd GIT binary patch literal 4093 zcma)9UvJ#T5$7(KJRXmCI{mT!*$%FM(@Sj$QUo<Cc&;ZB`@Jm}njmVOIF7(U^KPs@Z)b1F=33dWhR1e-CoZEkV< zk>%T>#7ly0Q|`b?FgO(l&Z@lU?p1h+lWhhk!4lZqhe&+ z<70drbn9oj6EHT)FTtAd2K^Ggw;evor$BFv6FBb~IRehED<{oln2DW3<+OWAuoY#g za(8eKgETr4E&Sl&gS-XpCbaa=Kys2Bfsva)2}hphG&AoLIh8RE{8td&ne~OMzKzeD^-f0X!zoC_zB{>Rf%5fm7 zoJFzFyP$qjL;=L?<|@SR=4TrlpWL~-`hIZtPaCV75YHRR{X-b{#Hy4XsjmDl_4huy zyQ$oJ7`Hm$Aih~D;_YJ9vEkC+8S)D7|M_x0OS<#jsCzvN<&Ma%|0e0}CJV{Id>n1f zcMr4OPJ8}-94GThhl@B64@9e%b>w^;9txSx>xH7D5EFR5b(Wq%R^;b=suMJQ$xv{x z6fP`7uFiEKGJ+s#M_CZef>Y^hKx}3cn>dsbhfu;!*{RDWpohG3di45_XVKiyp0cIL zda~UjU~~8&k4;AzhMbO9x_+_&yxtecTC)|WnlH#z7*b=fLqQU?gI3rLx1u=84zGhe zr9hZ@wD1b2UYxBgy^k$4FnqKz=xFbL3rq%Cut)YNC!9W}ISGiCnCB#>mauaYzICM6yJYd5l4LG9l*&d+7Lbgia>+r!;DMR}hTKYMJ3&PJH zfXS@UcQTrr4@tkAnHk%&at0&zEL^y=MqWmK!pNOj1I9|+d~9+69&Y6hj^sACAD8+r z@Q#M1Ic~L?wg5Bw6>YBqRxF$e+NG~)-y3KXxt^8zC|G0M`I_e5H{`FRUu~B&7o=rw zMPw+;v?Wf#o(GmFT6#?TwV}t{E&MHf=5zwRtv>e_*1$RF>qt(HYrK-XPbh&<1m4SQ zXM8oUp7FJOMDyir1irpLVf5?SDA*m#>p;iR4?e={;Kyh_3V}SKgLG{Dqyd|Jf`Jc- zXX&Gnn4l*}05N|FFI6tilt+nj2geS)?-ubJAYUEu+hGeZ3aXb-#*nRNC@bgC9@X}S z{1S-dn@Es5agh535akW>Bi#)lgkDGauYi0Ye~v6BLi3VDyzWEY><65{|P&tMVf;0bgddlD7=ns&j){+9M zgdvb1?xlYL;@J+dsYg7*2zyE`yaC`2fM$PR^WbhcWkcChYTN&ID>jawQtCW&8EBn4 zCd`Ds{mf=HxKqyCL0k6so`?25LQUwO!t7^tJ=3K}*Ixlu7rj>Gfnr-xm|iGq5CQR| zHK6|u7=Ux5&k)r67Es!^ftvkNPLCO&4BfLhnK7QwzLPt^LrK#wYt97D&25k&SIO+- z5}*ppt=!H_h!KDrn5Shx4DXm}OMqKnVo~H3v$n~U@dPj3mvi$8fy}PL zg~8TOPT|qnClSA!uFeL1m_-uc=_z(U5(moCxuHxb&;9Y8ZgwviWE@m5J0eMn^k5<9 zL9=!)GH-whxq{>!Bp4|2T_kARXAgo!?%O&Llfip`@NWC%L7XmfWf$xZL#Y~eyYb;L zlD+_wX#I83f7=0;rIB zkO+bDf&l8Gcrao@0Vpw^C6*cziyKvG#bKJl?Yi6H@~0?%3&phER`zXeMmAAuB3VF! z;3$6&#J7uoA9)KU&$CE#Sf%K)dm5M&0nIU}1$l#UP3be+bf41kI+!d*+gom&8y0W>k)47Vp;8lb{@U-9gS>r?Y6EXe1_$rv~lHC9DWb`a~I3SxuRU0 zJ6|r&wO%L}=LY4X#EaNp{(lW!jJyIQ5ib3!FSDQv0Q|w!$Xj5rMq`9+zpkg8`(u-s zzN!=i@Y5OueyInDyR`L|gr9OqgBX8c*}gbTbkIn)(PyYN?J6PRZD3KzAR|&-iarE zWw{qc0{v3EA1I)gz6J&C$2|03=pVw@KKWk=BC9jYJ4yafkRUwlaydIQ9M1e^b}ntT zTn(=4|M~Ck)dfxaA3mJFOnA5hBmXB%Ok;Ya#k!}*#3Qlc89LHPWO^ptjmU~^&sH=u zs>P1ysI?WLIZ4vhR?FmcV(ndT9ubAu6X z4oqe+^Rek!+-4T94G6PY?Xmet_Z+qa{&r!n&KleuHAYR==Dt19WetFc;C!3dc-8K1vqi|()a*KZ<>R5NcyO_?m1!+k&!dIbKnCS1Ka~Y^B;s! zC=Y%B^c=ly8i&^|w&zvB%>bpQ7sd=NQ6~KZ7KJv_q&^~hL@Y_8&_^Z=>k)ZKq&=#Q zoC09V$Pp=YY9@^d4Y)@2Gx{mjebRUeA35Q7L3dN!KHY7dlWECzp(E^%2$mjji?#Vd zPxo4ae>x7KO3CIHwvBB`t-r3mPzIR{L@u9h|vKC+p8C|xcbQr>l z?uSh7=A{K#33f}8BxU_9u}$VnzqBjKQkM#(Wn+qWP$V4C-7~~2zQR&q^V<9Q`I7Q- z=&ocU4WP);N^MU40hAURr9KOj;ai|6|1nHW-PTRrJaGuo-IJzCbmye4X4_~2%{l3i z!&?`}`5WINeuVbAlMe|GdAXR5C1x?yOc`R4r8$uE^3oX(&A`ml{jv#5Kh7zW2N^G0 z6Zhh0a1qo%|MbGrhVl44jC>B@F15mVq(5{DKoivqz!N2f1G7 z6hUSbHekVgSTFP^b!HXyr+^EPH2{RjHOhYx-xws`oZv&~ z11exuhwQlouC4#}?&jvkM*_PZuckzA{9G)fQbbj0L_8_oKl#y^ZwQgX0loe4*4DlI zI~%{D+dKE~?rePcyNzwn8idIpEe$A`)j88t39GU|`~0beJ{%4pR+lDk&KZEE56Z)7y5)HkckJgK!~FX2or@FyMN>-s^U{0}`gwpweR|;?=g<`HrmQr=M2bs5 z7Ff6y+{G2-vY}lBZ353!@l%}eq!l+|^6aW(6c`GDfI0Ws*z$;ZoUrLa&H2w@a-LxX0v%d5km@Xb#X!qkRlnYeR8{Q>A*ZGDE!&Gt&Dt_DicT z_Dic(zqES3Us?@b>6ca~{gSXy(Vg$1Xe>kMZt!^U-v730l+5a+-&YUVo>fr8^VnPa zZknLOTcedBm=xZX*NLms4*vwGVpJ72RfB4j!hae{1%{}qO*e3bp?h6SOXqe4viGsZ X=Zi3PHHS3e*MZR`w&@t=N!R*6tDU@B literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_fp8.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_fp8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d03344a28c2d870d0b34fd407d07ac5b81614ce GIT binary patch literal 3690 zcmbss+m0MZwX3STr>Ez#v$I!29GoB}3?{P)xh;}~Y%B*9!X{)*B3ejId#Yxq$9am8rA8lQ>V_Us&k#Z)$%Q9 zZ+!Xp(LXjU>z_DTY&J};LofdVfLoj-R!Rbr(txHcU<7eAu>%{vEOAmda1G8*YN;1^ zX6z*O)DL_!c9TZh44P>xXr=9-oh}7S2471$>2k1~c7txZ608`Um#n61!5WO~dn8!@ zsl|O>zYjCv9kXDA(_O3Acns&VdQ5vaqa=y$BtqLC#HG^ShlPssEJ}8IMAtuxWRwae zBuqAL=4l~CSqgqzWMwYP-N8tt0+v>PIf*hAA4a%!SIJ0;y@Q^uJ5f<22jQ^T4wE<& zk<@F0iAtifG`Mn76uDG_>ld=sIZH%dMFK7z^7PoLXY2DpTC5 zZc;vHDcs+Vw$a}D#ew!hxOuF?5S-R3T>vg|Ns}DDbdkpU^BvliLQP~gAh2uRaE#r& z7CJpA@bGl!@vmWQZVpz(21#kg6MzUQ4!sVlUN&(m*H$vYiubprgy2lrr4B>ZRiekmy-wVH}r@eF;zRZzu+0J7TbAPa#H42(0=r0&{)<6k#1^>|x^(M6Vgf1`Iu&IoXYw+$PAKpwyX^-x?{^ z3C`p=r*gvcax!z;aqGx=u0&@+qNm#2L8*VW{zi|@NhLM!QxR(@-hJ3m^%>!o%ebqXkna+Aw-6I zGUA6X{wlkh<@?!{McchHcg8Ed<;qKH$jZHaYiIYvk7WaypGuG~;&cfC#u@n?1gKn1 zl&;j2isX**qM(6+bHBP-CB<;W%RFtL zmkXH>AlQf1Ym{L$gftK)dBo#v?#Q(~K+f8x57;Lj$_6VJ9`&C&<=B%5A@9^K0AEpLeu}oBgt3rqU-rYVPGpFsbb7VVF^j4 z)SEL^VTLK-9;fTN3BxEYL#_@Ap<6TM<2{8Opfk-Iss`sjhh9=}ECoUFm>kv6WiT`< z2Ch|`GWa^^S{&SA>VSLLN8Xe?gbXnC9>GR1%bN1Wb#RQgl#jbAPMi*rM+5d&JKYED zsGhCQ=)sZ7Ywug15!Et#+9-Wa4{~Y`=uus=y8C=v8*`Ca$~_3F)eoN9lj78uK`NQWj4 zzJzqLj{vbfS|JRDF-kPgURSenl2*RZ0)%!TFAPT75ow_gg4Pg}rM73~LeQ0B4+@z| z$5rui=4bL{l+^?f)XcUwe<-gYstrK1L}Y<8jI&`5-V{UK@_hiF1AfCQkFrlFcp4?{ zSIt_}p`?wz^aOp%`%lMpiEn1^*XvFb_AJc7|LlJszIvfgotLf2J#F8KqjG?`Y~FB; z+1;cR3`uCO5L5|t{4z)z*4X9*(nbxeg6-Vy)wE;O|McD8h3%$P2sS5W^-e0kE3z;i zhWsFl(s!qfvJxv43h!3MD*VD*{_F6S z3*Y4&3ouNqgm&Z#z}_lW5pYrwhT0E9s8=Qlj+1}Ek!XO0rO0GGEk%!y@ z&-h(G%S;XmeuCv%|9rXDugbOlV#@0eo+;P*vvN&hKnmYAkjEsr)ylQ!ulU1g9q0sa^Ck-*~s literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_int8.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_w8a8_int8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2aa37c893684988a8388b684e260208b9950e68 GIT binary patch literal 3896 zcmbtXOK%*<5uWaO?Ck85%jH9o9SF|DStTYRJF$qwNn{I-^N=XV5+Gm@WH8$9C1*J^ zyYB84MPl|6$+At*~Xre%j8K!lkSecCzJgIqQa9VsW3Z zJR;%hCl;%+;IYL59z3$cHAe4Pz1lOd%IewLzY(Wtd_U#dxtU0%{o7-ejPf|$?h#%6 zG!}8j6&En5C3&KvbhNW$hHE!Q*;sHXIlIeqITG?tf0t(*G`qi=#JNfiW8CSE60zbt z2R*8n;_)~=h>~1wMQM`rSm@RMD1X3(iqt6T@5Xt~)BQy9o}-sOiY0$=^X9D+8}!8+ zrH0x5PCrih>0I~Li1G9uPj+_IsdVjAE=ucOR`282FFtse*Ws0a1;VY6Fe{{taGO!? zFq=CAk2%bJ?1V1&n8zyv$|}r%?0|=TR^`=UFsw0;1p|lGSRM50C(;IQ@X7^?HF<@# zK(oo(yfp~e66=6o`$TUU=9XBOckmbVJ^UT`Y=y0Y-ZBf|-q!SmSNwi&>%*;|gHN?D zXTH{}iHxKI|M$V$l&_>}FT9{mnGF zzd1foyQ6&bL7HZp8Qx;VAM*Z0jl^aeA8;WzKVOKa&HhQ;MXC(E&GPHyYo?<8t$52s z+{XAo`w_4Z#L*g9B|RV>^+-St-#XW+8}lXF6I@M1-Y2kZ{veO9aT)rL5pWRQe)db? z&9?&_?0zZ@Uy>D=(jD+ZlqGr8kH_)-Bu&)8n;@6yf5$%j?)mn-x=_2uP1}83Jb!Ay zxbPawJ_8#IGPI^tk%B&K>3Otu0tzo-=cx@9!o;mW#o$(o`?)e~!QmzBaV zs+X*FYvF_dv`%eq<({!>W*gcr{|4t-!&T-o&)8Ktvcc{(41c3*xSqfL(;f*Mf`2|q zAQMJAB4*Dp9KIylW$Iwe1&;1QnzdPyg_S+d$H0iMfwZ@-y?v9e1m2S-O4}*VbrUjd z0=X^PPng=3C8B)P@a}%RedqRP0&|tFmumsokOsBGVnEgj95GjysK-e=iy!KCDLLd~ z6pcrag0%|(>+gnclH`Mtb}%bzszxBl)qC&4swCI8oMgHZAC9A#4SO}Q3`Yy7c9ysd zBy2&N8BCx8M1b2#%ERWqTukF3*a5j7vQnaC=)gWQtcp0_;Stz$!ooqW`Hs*Am9tQJ zb1X)Es54Rd#WIQqP>`Sy#VpBp-U2)1Z-97qhcvx5@rXm5<2LmvCH}EzBfgRIY29wa zoKNVngD;(Pf$b3QxJez*rN3TVKJ`isChymme!|Eqm^bu>|U7vor{1Pfv$ zlS3}``s|8e5J8}v&g(j;#+i&v1nI`?qVapV=ws5FKTTk8jLAm6j~K^PZ51{ntI%BR z*`kB|MQAYA0P$VGQBDhcT2W+30r&I~DUf#ncQ)Y3hgQV;0+M`Te?bTI0^-R9JV6mG zT;?7*)1V+vp!^lVQ%WpAN=5m@D&&dV3NSUSm3uOK0C{Fuhh3>z3Fw*D^3|DsV~@&v zzp#EwRMV_!p&fI5ILAMt)0#P_TGVFew8fv*5^6L1jAq?v21R|Q*^z%)(8Mo8+i}l^ z(X15>)GU#VXoh@U-+p!-Txh^5-arm>anBLof}gNjUKHR~2eZ3G41XW>E~B1p^u#q3 zptMIzfzkLTO5aA(!>MKQb>y!A30(xE6dNe`4w42+JA&^(BUVO4*?gVmfUt7oXCH2F ze|%HaL07vx8>@q`GJxO!%!+T|E*KR8VNvD-ECgkZZZ5$0U6j6uCailFmKXEQc*m*B3^C>7Wo>RLaIrG!3mcf_;}iC$bpi!rn{&2#NRM-DYXTt%sk z1k0YlCPn-ZNa&TFxj^TOpCG|zae=JhumuDei5vS?0PzqUQ%LlyPVfUv*bpHv9GgC; zcJRFYewoyp$7UVYSSZ6#Cdy-t73MetK;KlrjXXCMIILj@x>5dvBQURq9~sHZJ?Tc- zlWr{bq#ONLdeV(qPbw<7(e?jhqlrqOf5Z+T-1=X)F>5wV?^`Z6VHe;YZ@z91Q*rSB zJYmuWCKr>9%4BjKhT5As2Jd6xj?9(e{b>J#vI+Px_B;~hT?>jXshhV=*QxJz?f(FC Cn)H7F literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_wNa16.cpython-310.pyc b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__pycache__/compressed_tensors_wNa16.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c84fb933bbbb38959b65f73bf6f3016e574438d9 GIT binary patch literal 4503 zcmbtXOK%*<5uWLpot>S1aJhU_lC`p>!zT7Jl4V&*97hk!ierglOeseC(iv_y$sy;Z zo>@^`3=zQ6Rrvw76G%WVKIAV12#`a5!(4OXKL}!3OjXY=wUp?Zf5W$e zGA%2a>DxhtRuo_IouEpqinsh)P^a~vK^v-G_9ucSZ7SaOCxaGk1ygh?n5NUg44nyP z>8#RK{JCJB&g+^ei-kuzUF6P&PEUyWt&>nr@#@!vz9CMFH=eeiX`nxIQ{y%6Jl41) zDvu2MCMT;}r@jv>(K=>sUw3`qUGqh5-tv+(x9`NM7lp2$8>=E+?&x{d3%!*2(Z+_5 zP@TIT1+f%KBKTbqCXq~5dz&H$3VA&W z*S(E=uD2OQiC{s@e5GXDLWaTzD`x^%f{aNK3pZs+&-F#ZqL9HvnRLVLq*KZ#uephM zaO>9X6<4}}NQKN7uWz~`nD|ZK4bxTZ^-?!~Q%3#xIL*|G+uIiWsHx^>(asSGOsj?y zaV1N4W0B;uukFI;%U91|>X3c(m3OQIs|1&v$qcNVmDo0qx$xJ0X>#zC`AxWlWDO*rd1uZhOi z#8#8np#vFYrZZH>W)5q-?=*@)N@YWNWyeAy3$vfb(a4@RQ}-PEzTnVq-VA!+yZlyfn$3 z4LlQ;cmt8!ZZD0{WqKtrHu{japj?NNJOsk^j4W!I#tw?uAJ|jw6{0=sVol_AlCl zxo&&)I7P?Ft1%o9TUQ${XJJut9tnCZE#3U=&b^hqvgsylgL(X6?)=vE`{IU_k<1sA zi?h{xD=T+CyL;m!cK3HHH&)s0t1G#=Eq38V)sj_&K!}=&@YjnG<_?Eh0PY)2z6HPk ze6f@U@lx!?@1(BW5a~Pb2mQ_9e6X|RduvPaZn_zTOAmZMSPCL8d?p@>ULV|K$wweb zmc}=DsdtD0oE{A&*5lF@giv!J)|V{tEDFBLG~Co}VbbAZJ09 zw_r$1dAWsMo(-?HVXELw%%1dg)@@mvi<31oNMx5h?hy3hN5xX+= z5WQ8CB%goThQ&g%Lw#NY)(sNI5e@VDj_@`%@s4mYgwd5o-wZHSD4#+}zJz^9wG0Am zuaaWGl68`mvIk1hku;q`3%mw&8Z3~k%wTV4Z9q@4XVTFHK^&>?485C@B}~Sb3#zM(zcGoUe<7HI!4hA?WiFR)e1g4 zCT;An`xrQGz(Pn=u%3+9CwAB%9v4jRunj=oF;M{$^SP1q16je5&LB~+jXNtbc||0{ z0T`1QbGcO{Y`N)$>rrmFoJ)*DIf3LPwkQFV_BQj92;y{?mf)b?y&&tT#zaNdfGFvu zi?k)h1|)|C1Qh9(Y@!Won6=GF#83GnBtJ&-Hj;Ocbdf9}!M%kZI{@|xT^IwSqu@g4 zUQG&FPbH1CT8uX4Wi=V?sGahC;nU_zKP|FG+8Sn$aFgOTE9A3^q6nNKhV51S}enPdW842V^U z%T!sLJcli-J>N|dcnNJrTy{|ROB9nZ%&psMENP(h0unr9`4JL42Wk}$If?5UTdP|2 zS(IUhe;P`11&E`ybW#F`d=9P)u1H>hdwyq`&T}%?)WLs23QnmsCTNJ^d^h|1CI-QQ zsW%QfZ{-%;7~mkqJk+$*YH-PF+O&#kU$=lTiBAA}O-YU>g54q!?2adb-QH`7V0V}Z z$}$?d@_!iWryk^KDpgjAY4-=rXZScMfE$sT{_Ka&Kn>R&K8K635yR=J;|!=f1|Px- zxyOK9AwC8&ZK{4_?^rnrV&#(Z1K{#9cp6;(jF$W0Vi(sg(+V<|*Un$KEYSgS(}#ON zwt=P7uRaH^eR6kIxyo@ql-nN^F8?83> int: + """ + Get minimum device capability. + """ + raise NotImplementedError + + @abstractmethod + def create_weights(self, *args, **kwargs): + """ + Weight creation for the particular scheme. Inputs to this function + + """ + raise NotImplementedError + + @abstractmethod + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor, + bias: Optional[torch.Tensor]): + """ + Run the forward pass for the particular scheme. This is where + scheme-specific dequant/quant steps/kernels should be applied. + + :param layer: torch.nn.Module with the registered weights and + other parameters relevant to the particular scheme. + :param x: input to the layer + :param bias: bias parameter + + """ + raise NotImplementedError + + @abstractmethod + def process_weights_after_loading(self, layer: torch.nn.Module): + """ + Called after weight loading is complete for any cleanup that + needs to occur. + """ + raise NotImplementedError diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py new file mode 100644 index 00000000..9ad61a64 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py @@ -0,0 +1,153 @@ +from typing import Callable, List, Optional + +import torch +from torch.nn import Parameter + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme) +from vllm.model_executor.layers.quantization.gptq_marlin_24 import ( + GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N) +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter) +from vllm.scalar_type import scalar_types + +__all__ = ["CompressedTensorsW4A16Sparse24"] +W4A16SPARSE24_SUPPORTED_TYPES_MAP = { + 4: scalar_types.uint4b8, +} +W4A16SPARSE24_SUPPORTED_BITS = list(W4A16SPARSE24_SUPPORTED_TYPES_MAP.keys()) + + +class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme): + + def __init__(self, + strategy: str, + num_bits: int, + group_size: Optional[int] = None): + self.strategy = strategy + self.group_size = group_size + self.tile_size = 16 + + if num_bits not in W4A16SPARSE24_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Unsupported num_bits = {num_bits}. " + f"Supported num_bits = {W4A16SPARSE24_SUPPORTED_BITS}") + + self.quant_type = W4A16SPARSE24_SUPPORTED_TYPES_MAP[num_bits] + + if self.strategy == "group" and self.group_size is None: + raise ValueError( + "group_size must be given when using strategy group") + + @classmethod + def get_min_capability(cls) -> int: + # ampere + up + return 80 + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # required by torch.compile to be torch.nn.Parameter + layer.weight_packed = Parameter(layer.weight_packed.data, + requires_grad=False) + layer.scale_packed = Parameter(layer.scale_packed.data, + requires_grad=False) + layer.meta = Parameter(layer.meta.data, requires_grad=False) + + def create_weights(self, layer: torch.nn.Module, input_size: int, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, weight_loader: Callable, + **kwargs): + + pack_factor = 32 // self.quant_type.size_bits + output_size_per_partition = sum(output_partition_sizes) + + qweight = PackedvLLMParameter(data=torch.empty( + input_size_per_partition // self.tile_size // 2, + output_size_per_partition * self.tile_size // pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=pack_factor, + marlin_tile_size=self.tile_size, + weight_loader=weight_loader) + + input_groups = (1 if self.group_size is None else + input_size_per_partition // self.group_size) + + weight_scale_args = { + "data": + torch.empty( + input_groups, + output_size_per_partition, + dtype=params_dtype, + ), + "weight_loader": + weight_loader + } + + if self.group_size is not None: + scales = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **weight_scale_args) + else: + scales = ChannelQuantScaleParameter(output_dim=1, + **weight_scale_args) + + weight_shape = BasevLLMParameter(data=torch.empty(2, + dtype=torch.int64), + weight_loader=weight_loader) + + meta = PackedvLLMParameter(data=torch.empty( + input_size_per_partition // 8 // 2 // 2, + output_size_per_partition * 2, + dtype=torch.int16, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=1, + marlin_tile_size=2, + weight_loader=weight_loader) + + layer.register_parameter("weight_packed", qweight) + layer.register_parameter("weight_shape", weight_shape) + layer.register_parameter("scale_packed", scales) + layer.register_parameter("meta", meta) + + max_workspace_size = ( + output_size_per_partition // + GPTQ_MARLIN_24_MIN_THREAD_N) * GPTQ_MARLIN_24_MAX_PARALLEL + + workspace = Parameter(torch.zeros(max_workspace_size, dtype=torch.int), + requires_grad=False) + layer.workspace = workspace + + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + + qweight = layer.weight_packed + meta = layer.meta + scales = layer.scale_packed + workspace = layer.workspace + + x_2d = x.view(-1, x.shape[-1]) + + size_m = x_2d.shape[0] + size_k = x_2d.shape[1] + size_n = scales.shape[1] + + output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales, + workspace, self.quant_type, size_m, + size_n, size_k) + + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], )) + + if bias is not None: + output.add_(bias) # In-place add + + return output diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py new file mode 100644 index 00000000..3d55d55c --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py @@ -0,0 +1,118 @@ +from typing import Callable, List, Optional + +import torch + +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + QuantizationStrategy) +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + convert_to_channelwise) +from vllm.model_executor.parameter import (ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter) + +__all__ = ["CompressedTensorsW8A16Fp8"] + +SUPPORTED_STRATEGIES = [ + QuantizationStrategy.CHANNEL, QuantizationStrategy.TENSOR +] + + +class CompressedTensorsW8A16Fp8(CompressedTensorsScheme): + + def __init__(self, strategy: str, is_static_input_scheme: bool): + self.strategy = strategy + self.is_static_input_scheme = is_static_input_scheme + + @classmethod + def get_min_capability(cls) -> int: + # ampere and up + return 80 + + # W8A8-Fp8 kernels support only per-tensor and per-channel cases. + # So if we have a fused module (QKV, MLP) with per tensor scales, + # we expand each scale to its shard's channels. + def process_weights_after_loading(self, layer) -> None: + if self.strategy == QuantizationStrategy.TENSOR: + ws_channelwise = convert_to_channelwise(layer.weight_scale, + layer.logical_widths) + layer.weight_scale = torch.nn.Parameter(ws_channelwise, + requires_grad=False) + else: + # required by torch.compile to be torch.nn.Parameter + layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data, + requires_grad=False) + + # Weights must be transposed for marlin + layer.weight = torch.nn.Parameter(layer.weight.t(), + requires_grad=False) + + if self.is_static_input_scheme: + # required by torch.compile to be torch.nn.Parameter + layer.input_scale = torch.nn.Parameter(layer.input_scale.data, + requires_grad=False) + prepare_fp8_layer_for_marlin(layer, strategy="channel") + + def create_weights(self, layer: torch.nn.Module, input_size: int, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, weight_loader: Callable, + **kwargs): + + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight = ModelWeightParameter(data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + if self.strategy == QuantizationStrategy.CHANNEL: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), + dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader) + elif self.strategy == QuantizationStrategy.TENSOR: + weight_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + else: + raise ValueError( + f"Unsupported weight strategy={self.strategy}, " + f"supported strategies are {SUPPORTED_STRATEGIES}") + + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE (to deal with converted checkpoints) + if self.is_static_input_scheme: + input_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + layer.register_parameter("input_scale", input_scale) + + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + return apply_fp8_marlin_linear(input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py new file mode 100644 index 00000000..5931ec36 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -0,0 +1,143 @@ +from typing import Callable, List, Optional + +import torch +from torch.nn import Parameter + +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + QuantizationStrategy) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, + requantize_with_max_scale) +from vllm.model_executor.parameter import (ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter) +from vllm.utils import is_hip + +__all__ = ["CompressedTensorsW8A8Fp8"] + + +class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): + + def __init__(self, strategy: str, is_static_input_scheme: bool): + self.strategy = strategy + self.is_static_input_scheme = is_static_input_scheme + self.cutlass_fp8_supported = cutlass_fp8_supported() + + @classmethod + def get_min_capability(cls) -> int: + # lovelace and up + return 89 + + def process_weights_after_loading(self, layer) -> None: + # If per tensor, when we have a fused module (e.g. QKV) with per + # tensor scales (thus N scales being passed to the kernel), + # requantize so we can always run per tensor + if self.strategy == QuantizationStrategy.TENSOR: + max_w_scale, weight = requantize_with_max_scale( + weight=layer.weight, + weight_scale=layer.weight_scale, + logical_widths=layer.logical_widths, + ) + + if is_hip(): + weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=max_w_scale, + input_scale=layer.input_scale) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, + requires_grad=False) + + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(max_w_scale, requires_grad=False) + + # If channelwise, scales are already lined up, so just transpose. + elif self.strategy == QuantizationStrategy.CHANNEL: + weight = layer.weight + + if is_hip(): + weight, weight_scale, input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, + requires_grad=False) + else: + weight_scale = layer.weight_scale.data + + layer.weight = Parameter(weight.t(), requires_grad=False) + # required by torch.compile to be torch.nn.Parameter + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + + else: + raise ValueError(f"Unknown quantization strategy {self.strategy}") + + # INPUT SCALE + if self.is_static_input_scheme: + layer.input_scale = Parameter(layer.input_scale.max(), + requires_grad=False) + else: + layer.input_scale = None + + def create_weights(self, layer: torch.nn.Module, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, weight_loader: Callable, + **kwargs): + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + + # WEIGHT + weight = ModelWeightParameter(data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + # TODO: update create_xxx_parameter functions to return + # the newly added parameters + if self.strategy == QuantizationStrategy.CHANNEL: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), + dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader) + else: + assert self.strategy == QuantizationStrategy.TENSOR + weight_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + + # min requirement for fp8 kernels + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE + if self.is_static_input_scheme: + input_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + input_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", input_scale) + + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + use_per_token_if_dynamic=True) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py new file mode 100644 index 00000000..b93f35e2 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py @@ -0,0 +1,155 @@ +from typing import Callable, List, Optional + +import torch +from torch.nn import Parameter + +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + QuantizationStrategy) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + apply_int8_linear, convert_to_channelwise) +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter) + +logger = init_logger(__name__) + + +class CompressedTensorsW8A8Int8(CompressedTensorsScheme): + + def __init__(self, strategy: str, is_static_input_scheme: bool, + input_symmetric: bool): + self.strategy = strategy + self.is_static_input_scheme = is_static_input_scheme + self.input_symmetric = input_symmetric + + @classmethod + def get_min_capability(cls) -> int: + # turing and up + return 75 + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # WEIGHT + # Cutlass kernels need transposed weight. + weight = layer.weight + layer.weight = Parameter(weight.t(), requires_grad=False) + + # WEIGHT SCALE + # Cutlass kernels support only per-tensor and per-channel. + # If we have a fused module (QKV, MLP) with per tensor scales (thus N + # scales being passed to the kernel), convert to the per-channel case. + is_fused_module = len(self.logical_widths) > 1 + if is_fused_module and self.strategy == QuantizationStrategy.TENSOR: + ws_channelwise = convert_to_channelwise(layer.weight_scale, + self.logical_widths) + layer.weight_scale = Parameter(ws_channelwise, requires_grad=False) + else: + layer.weight_scale = Parameter(layer.weight_scale.data, + requires_grad=False) + # INPUT SCALE + if self.is_static_input_scheme: + if self.input_symmetric: + layer.input_scale = Parameter(layer.input_scale.max(), + requires_grad=False) + layer.input_zero_point = None + else: + # reconstruct the ranges + int8_traits = torch.iinfo(torch.int8) + azps = layer.input_zero_point.to(dtype=torch.int32) + range_max = (layer.input_scale * + (int8_traits.max - azps)).max() + range_min = (layer.input_scale * + (int8_traits.min - azps)).min() + + scale = (range_max - range_min) / (int8_traits.max - + int8_traits.min) + layer.input_scale = Parameter(scale, requires_grad=False) + + # AZP loaded as int8 but used as int32 + azp = (int8_traits.min - + range_min / scale).to(dtype=torch.int32) + layer.input_zero_point = Parameter(azp, requires_grad=False) + + else: + layer.input_scale = None + layer.input_zero_point = None + + # azp_adj is the AZP adjustment term, used to account for weights. + # It does not depend on scales or azp, so it is the same for + # static and dynamic quantization. + # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md + # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md + if not self.input_symmetric: + layer.azp_adj = layer.weight.sum(dim=0, + keepdim=True, + dtype=torch.int32) + else: + layer.azp_adj = None + + def create_weights(self, layer: torch.nn.Module, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, weight_loader: Callable, + **kwargs): + self.logical_widths = output_partition_sizes + + # WEIGHT + if input_size_per_partition % 64 != 0: + pad_input_size_per_partition = (input_size_per_partition // 64 + 1) * 64 + else: + pad_input_size_per_partition = input_size_per_partition + w_pad = torch.zeros( + sum(output_partition_sizes), + pad_input_size_per_partition, + dtype=torch.int8) + w = w_pad[:, :input_size_per_partition] + weight = ModelWeightParameter(data=w, + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + if self.strategy == QuantizationStrategy.CHANNEL: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), + dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader) + else: + assert self.strategy == QuantizationStrategy.TENSOR + weight_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE + if self.is_static_input_scheme: + input_scale = BasevLLMParameter(data=torch.empty( + 1, dtype=torch.float32), + weight_loader=weight_loader) + layer.register_parameter("input_scale", input_scale) + + if not self.input_symmetric: + # Note: compressed-tensors stores the zp using the same dtype + # as the weights + # AZP loaded as int8 but used as int32 + input_zero_point = BasevLLMParameter( + data=torch.empty(1, dtype=torch.int8), + weight_loader=weight_loader) + layer.register_parameter("input_zero_point", input_zero_point) + + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + + return apply_int8_linear(input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + input_zero_point=layer.input_zero_point, + azp_adj=layer.azp_adj, + bias=bias) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py new file mode 100644 index 00000000..cb65557b --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -0,0 +1,163 @@ +from typing import Callable, List, Optional, Set + +import torch + +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme) +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + ActivationOrdering) +from vllm.model_executor.layers.quantization.kernels import ( + MPLinearLayerConfig, choose_mp_linear_kernel) +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_repeat_scales_on_all_ranks) +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter, + RowvLLMParameter) +from vllm.scalar_type import scalar_types + +logger = init_logger(__name__) + +__all__ = ["CompressedTensorsWNA16"] +WNA16_SUPPORTED_TYPES_MAP = { + 4: scalar_types.uint4b8, + 8: scalar_types.uint8b128 +} +WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys()) + + +class CompressedTensorsWNA16(CompressedTensorsScheme): + _kernel_backends_being_used: Set[str] = set() + + def __init__(self, + strategy: str, + num_bits: int, + group_size: Optional[int] = None, + actorder: Optional[ActivationOrdering] = None): + + self.pack_factor = 32 // num_bits + self.strategy = strategy + self.group_size = -1 if group_size is None else group_size + self.has_g_idx = actorder == ActivationOrdering.GROUP + + if self.group_size == -1 and self.strategy != "channel": + raise ValueError("Marlin kernels require group quantization or " + "channelwise quantization, but found no group " + "size and strategy is not channelwise.") + + if num_bits not in WNA16_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Unsupported num_bits = {num_bits}. " + f"Supported num_bits = {WNA16_SUPPORTED_TYPES_MAP.keys()}") + + self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + @classmethod + def get_min_capability(cls) -> int: + # ampere and up + return 80 + + def create_weights(self, layer: torch.nn.Module, output_size: int, + input_size: int, output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, weight_loader: Callable, + **kwargs): + + output_size_per_partition = sum(output_partition_sizes) + + mp_linear_kernel_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=\ + (input_size_per_partition, output_size_per_partition), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=self.group_size, + zero_points=False, + has_g_idx=self.has_g_idx + ) + + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) + + if kernel_type.__name__ not in self._kernel_backends_being_used: + logger.info("Using %s for CompressedTensorsWNA16", + kernel_type.__name__) + self._kernel_backends_being_used.add(kernel_type.__name__) + + # If group_size is -1, we are in channelwise case. + group_size = self.group_size if self.group_size != -1 else input_size + row_parallel = (input_size != input_size_per_partition) + partition_scales = not marlin_repeat_scales_on_all_ranks( + self.has_g_idx, self.group_size, row_parallel) + + scales_and_zp_size = input_size // group_size + + if partition_scales: + assert input_size_per_partition % group_size == 0 + scales_and_zp_size = input_size_per_partition // group_size + + weight = PackedvLLMParameter(input_dim=1, + output_dim=0, + weight_loader=weight_loader, + packed_factor=self.pack_factor, + packed_dim=1, + data=torch.empty( + output_size_per_partition, + input_size_per_partition // + self.pack_factor, + dtype=torch.int32, + )) + + weight_scale_args = { + "weight_loader": + weight_loader, + "data": + torch.empty( + output_size_per_partition, + scales_and_zp_size, + dtype=params_dtype, + ) + } + if not partition_scales: + weight_scale = ChannelQuantScaleParameter(output_dim=0, + **weight_scale_args) + else: + weight_scale = GroupQuantScaleParameter(output_dim=0, + input_dim=1, + **weight_scale_args) + + # A 2D array defining the original shape of the weights + # before packing + weight_shape = BasevLLMParameter(data=torch.empty(2, + dtype=torch.int64), + weight_loader=weight_loader) + + layer.register_parameter("weight_packed", weight) + layer.register_parameter("weight_scale", weight_scale) + layer.register_parameter("weight_shape", weight_shape) + + # group index (for activation reordering) + if self.has_g_idx: + weight_g_idx = RowvLLMParameter(data=torch.empty( + input_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight_g_idx", weight_g_idx) + + self.kernel = kernel_type(mp_linear_kernel_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name=None, + w_gidx_param_name="weight_g_idx") + + # Checkpoints are serialized in compressed-tensors format, which is + # different from the format the kernel may want. Handle repacking here. + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/utils.py b/vllm/model_executor/layers/quantization/compressed_tensors/utils.py new file mode 100644 index 00000000..785f7ffa --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/utils.py @@ -0,0 +1,269 @@ +import regex as re +from enum import Enum +from typing import Any, Dict, Iterable, Optional, Union + +from pydantic import BaseModel, Field, field_validator +from torch.nn import Module + +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + FUSED_LAYER_NAME_MAPPING) + + +class CompressionFormat(Enum): + dense = "dense" + sparse_bitmask = "sparse-bitmask" + naive_quantized = "naive-quantized" + float_quantized = "float-quantized" + int_quantized = "int-quantized" + pack_quantized = "pack-quantized" + marlin_24 = "marlin-24" + + +class QuantizationType(str, Enum): + """ + Enum storing quantization type options + """ + + INT = "int" + FLOAT = "float" + + +class QuantizationStrategy(str, Enum): + """ + Enum storing quantization strategy options + """ + + TENSOR = "tensor" + CHANNEL = "channel" + GROUP = "group" + BLOCK = "block" + TOKEN = "token" + + +class ActivationOrdering(str, Enum): + """ + Enum storing strategies for activation ordering + + Group: reorder groups and weight\n + Weight: only reorder weight, not groups. Slightly lower latency and + accuracy compared to group actorder\n + """ + + GROUP = "group" + WEIGHT = "weight" + + +class QuantizationArgs(BaseModel): + """ + User facing arguments used to define a quantization config + for weights or activations + + :param num_bits: quantization bit depth + :param type: dtype to quantized to, either int or float + :param symmetric: whether or not quantization scale is symmetric + :param strategy: string determining the scope of scale/zero-point to apply + :param group_size: group length to use for the group strategy + :param block_structure: 2d block structure to use for the block + strategy, must be of the format "2x4", "8x16", etc. + :param dynamic: set True to perform dynamic quantization - + values will not be calibrated during calibration phase, + instead during inference new quantization ranges will be + observed with every sample. Defaults to False for static + quantization. Note that enabling dynamic quantization + will change the default observer to a memoryless one + :param actorder: whether to apply group quantization in decreasing order of + activation. Defaults to None for arbitrary ordering + """ + + num_bits: int = 8 + type: QuantizationType = QuantizationType.INT + symmetric: bool = True + group_size: Optional[int] = None + strategy: Optional[QuantizationStrategy] = None + block_structure: Optional[str] = None + dynamic: bool = False + actorder: Union[ActivationOrdering, bool, None] = None + observer: str = Field( + default="minmax", + description=("The class to use to compute the quantization param - " + "scale and zero-point'"), + ) + observer_kwargs: Dict[str, Any] = Field( + default_factory=dict, + description= + ("optional dict of kwargs to be passed directly to torch quantization " + "Observers constructor excluding quantization range or symmetry"), + ) + + @field_validator("actorder", mode="before") + def validate_actorder(cls, value) -> Optional[ActivationOrdering]: + if isinstance(value, bool): + return ActivationOrdering.GROUP if value else None + + if isinstance(value, str): + return ActivationOrdering(value.lower()) + + return value + + +def is_activation_quantization_format(format: str) -> bool: + _ACTIVATION_QUANTIZATION_FORMATS = [ + CompressionFormat.naive_quantized.value, + CompressionFormat.int_quantized.value, + CompressionFormat.float_quantized.value + ] + return format in _ACTIVATION_QUANTIZATION_FORMATS + + +def should_ignore_layer(layer_name: Optional[str], + ignore: Iterable[str]) -> bool: + if layer_name is None: + return False + + # layer_name = model.layers.0.self_attn.qkv_proj + # proj_name = qkv_proj + proj_name = layer_name.split(".")[-1] + + # Fused layers like gate_up_proj or qkv_proj will not be fused + # in the safetensors checkpoint. So, we convert the name + # from the fused version to unfused + check to make sure that + # each shard of the fused layer has the same scheme. + if proj_name in FUSED_LAYER_NAME_MAPPING: + shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name] + + # Convert fused_name --> [shard_names] + shard_names = [ + layer_name.replace(proj_name, shard_proj_name) + for shard_proj_name in shard_proj_names + ] + + # Layer should be ignored if shards are ignored. + should_ignore_layer = None + for shard_name in shard_names: + should_ignore_shard = check_equal_or_regex_match( + layer_name=shard_name, targets=ignore) + + # If shard_idx=0, set layer ignore to match shard. + if should_ignore_layer is None: + should_ignore_layer = should_ignore_shard + + # If shard_idx=1+ confirm scheme matches prior shards. + elif should_ignore_shard != should_ignore_layer: + raise ValueError(f"Found a different quantization schemes for " + f"{shard_proj_names} in {layer_name}. vLLM " + "requires all to use the same scheme.") + + # Unfused layers like down_proj and o_proj will match + # the safetensors checkpoint already. + else: + should_ignore_layer = check_equal_or_regex_match(layer_name=layer_name, + targets=ignore) + + assert should_ignore_layer is not None + return should_ignore_layer + + +def check_equal_or_regex_match(layer_name: str, + targets: Iterable[str]) -> bool: + """ + Checks whether a layer_name is exactly equal or a regex match for + if target starts with 're:' to any target in list. + """ + for target in targets: + if _is_equal_or_regex_match(layer_name, target): + return True + return False + + +def find_matched_target(layer_name: Optional[str], module: Module, + targets: Iterable[str]) -> str: + """ + Helper function to look up which "target" in the compressed-tensors + config that a layer corresponds to. + + Recall that a compressed-tensors configs has a concept of + config_groups, where each layer can be quantized with with a different + scheme. + + targets in each config_group will be a list of either layer names + (or regexes corresponding to layer names) or names of torch Modules. + + First, we try to match the layer_name with a target + Second, we try to match the module's name with a target + + :param layer_name: layer name + :param module: torch.nn.Module + :param targets: list of targets to match the layer against + """ + + if layer_name is None: + layer_name = "" + + matched_target = (_find_first_match(layer_name, targets) + or _find_first_match(module.__class__.__name__, targets, + True)) + + if matched_target is None: + raise ValueError(f"Unable to find matching target for {module} in the " + "compressed-tensors config.") + + return matched_target + + +def _find_first_match(value: str, + targets: Iterable[str], + check_contains: bool = False) -> Optional[str]: + """ + Returns first element of target that matches value either + exactly or as a regex after 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + + :param value: string to compare the list of targets against + :param targets: list of targets to match the layer against + :param check_contains: whether or not to do a substring match + """ + + for target in targets: + if _is_equal_or_regex_match(value, + target, + check_contains=check_contains): + return target + return None + + +def get_compressed_tensors_cache_scale(name: str) -> Optional[str]: + """ + Check whether the param name matches the format for k/v cache scales + in compressed-tensors. If this is the case, return its equivalent + param name expected by vLLM + + :param name: param name + :return: matching param name for KV cache scale in vLLM + """ + if name.endswith(".output_scale") and ".k_proj" in name: + return name.replace(".k_proj.output_scale", ".attn.k_scale") + if name.endswith(".output_scale") and ".v_proj" in name: + return name.replace(".v_proj.output_scale", ".attn.v_scale") + # If no matches, return None + return None + + +def _is_equal_or_regex_match(value: str, + target: str, + check_contains: bool = False) -> bool: + """ + Checks whether a value is exactly equal or a regex match for target + if target starts with 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + """ + + if target.startswith("re:"): + pattern = target[3:] + if re.match(pattern, value): + return True + elif check_contains: + if target.lower() in value.lower(): + return True + elif target == value: + return True + return False diff --git a/vllm/model_executor/layers/quantization/deepspeedfp.py b/vllm/model_executor/layers/quantization/deepspeedfp.py new file mode 100644 index 00000000..29484801 --- /dev/null +++ b/vllm/model_executor/layers/quantization/deepspeedfp.py @@ -0,0 +1,193 @@ +from typing import Any, Dict, List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.utils import set_weight_attrs + + +class DeepSpeedFPConfig(QuantizationConfig): + """Config for DeepSpeed FP quantizer. It supports fp6 and fp8. + + Args: + weight_bits: the target quantization bits, 6 or 8. + group_size: group size for quantizaiton, default to 128. + """ + + def __init__( + self, + weight_bits: int = 8, + group_size: int = 512, + ) -> None: + self.weight_bits = weight_bits + self.group_size = group_size + self.valid_types = [torch.bfloat16, torch.float16] + + if self.weight_bits not in (6, 8): + raise ValueError( + "Currently, only 6-bit or 8-bit weight quantization are " + f"supported for DeepSpeed FP quantizaiton, but got " + f"{self.weight_bits} bits.") + + def __repr__(self) -> str: + return (f"DeepSpeedFPConfig(weight_bits={self.weight_bits}), " + f"group_size={self.group_size}") + + @classmethod + def get_name(cls) -> str: + return "DeepSpeedFP" + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "DeepSpeedFPConfig": + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + return cls(weight_bits=weight_bits, group_size=group_size) + + def get_linear_method(self) -> "DeepSpeedFPLinearMethod": + return DeepSpeedFPLinearMethod(self) + + def get_scaled_act_names(self) -> List[str]: + return [] + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 60 + + @staticmethod + def get_config_filenames() -> List[str]: + return [ + "quant_config.json", + "quantize_config.json", + ] + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["DeepSpeedFPLinearMethod"]: + if isinstance(layer, LinearBase): + return DeepSpeedFPLinearMethod(self) + return None + + +class DeepSpeedFPLinearMethod(LinearMethodBase): + """Linear method for DeepSpeedFP quantizer. + + Args: + quant_config: the DeepSpeedFP quantization config. + """ + + def __init__(self, quant_config: DeepSpeedFPConfig): + self.quant_config = quant_config + self.weight = None + + def create_weights(self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + weight_loader=None, + **extra_weight_attrs): + del output_size + del input_size + output_size_per_partition = sum(output_partition_sizes) + weight = DeepSpeedFPParameter( + torch.Size((output_size_per_partition, input_size_per_partition)), + params_dtype=params_dtype, + quant_config=self.quant_config, + ) + set_weight_attrs(weight, { + "input_dim": 1, + "output_dim": 0, + }) + layer.register_parameter("weight", weight) + + def quant_weight_loader(param, loaded_weight, *args, **kwargs): + # Calls the original weight loader (if any), quantizes the result, + # and then loads the quantized parameter. + if weight_loader is not None: + orig_param_data = param.data + param.data = param.ds_dequantize() + weight_loader(param, loaded_weight, *args, **kwargs) + param.data, loaded_weight = orig_param_data, param.data + param.ds_quantize_(loaded_weight.cuda()) + + extra_weight_attrs["weight_loader"] = quant_weight_loader + set_weight_attrs(weight, extra_weight_attrs) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + weight = layer.weight + y = weight.ds_dequantize() + return F.linear(x, y, bias) + + +class DeepSpeedFPParameter(nn.Parameter): + """ + DeepSpeedFP quantized parameter class that implements fp8/fp6 + quantization deepspeed. Weights are stored in quantized form on + GPUs, and can be dequantized on-the-fly when needed by the model. + """ + + def __new__(cls, orig_shape: torch.Size, params_dtype: torch.dtype, + quant_config: DeepSpeedFPConfig): + try: + import deepspeed + if deepspeed.__version__ < "0.14.2": + raise ImportError("deepspeed version is wrong. Please " + "install deepspeed>=0.14.2.") + from deepspeed.ops.fp_quantizer import FP_Quantize + except ImportError as err: + raise ImportError("Please install deepspeed>=0.14.2 via " + "`pip install deepspeed>=0.14.2` to use " + "deepspeedfp quantizer.") from err + data = torch.empty(( + orig_shape.numel() // quant_config.group_size, + quant_config.group_size * quant_config.weight_bits // 8 + 4, + ), + dtype=torch.int8) + self = torch.Tensor._make_subclass(cls, data, data.requires_grad) + self.orig_shape = orig_shape + self.quant_config = quant_config + self.fp_quantizer = FP_Quantize(group_size=quant_config.group_size) + self.fp_quantizer.orig_shape = orig_shape + self.fp_quantizer.orig_dtype = params_dtype + return self + + def ds_quantize_(self, tensor: torch.Tensor): + assert tensor.device.type == "cuda" and tensor.dtype != torch.int8 + return self.data.copy_( + self.fp_quantizer.quantize( + tensor.data, + q_bits=self.quant_config.weight_bits, + )) + + def ds_dequantize(self, fp_out=None) -> torch.Tensor: + """ + Return a tensor containing the dequantized weights of this parameter. + """ + assert self.data.device.type == "cuda" and self.data.dtype == torch.int8 + return self.fp_quantizer.dequantize( + self.data, fp_out=fp_out, q_bits=self.quant_config.weight_bits) + + def ds_selective_dequantize(self, indices, fp_out=None) -> torch.Tensor: + """ + Return a tensor where only the weights at `indices` are dequantized + (to save HBM -> SRAM bandwidth). + """ + assert self.data.device.type == "cuda" and self.data.dtype == torch.int8 + return self.fp_quantizer.selective_dequantize( + self.data, + indices, + fp_out=fp_out, + q_bits=self.quant_config.weight_bits) diff --git a/vllm/model_executor/layers/quantization/experts_int8.py b/vllm/model_executor/layers/quantization/experts_int8.py new file mode 100644 index 00000000..116a4ea0 --- /dev/null +++ b/vllm/model_executor/layers/quantization/experts_int8.py @@ -0,0 +1,179 @@ +from typing import Any, Callable, Dict, List, Optional + +import torch + +from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group +from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase +from vllm.model_executor.layers.linear import (LinearBase, + UnquantizedLinearMethod) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.utils import set_weight_attrs + + +class ExpertsInt8Config(QuantizationConfig): + """Config class for Int8 experts quantization.""" + + def __init__(self) -> None: + pass + + @classmethod + def get_name(cls) -> str: + return "experts_int8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "ExpertsInt8Config": + return cls() + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + if isinstance(layer, LinearBase): + return UnquantizedLinearMethod() + elif isinstance(layer, FusedMoE): + return ExpertsInt8MoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class ExpertsInt8MoEMethod(FusedMoEMethodBase): + + def __init__(self, quant_config: ExpertsInt8Config): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + int8_dtype = torch.int8 + + assert 'weight_loader' in extra_weight_attrs + weight_loader = extra_weight_attrs['weight_loader'] + wrapped_weight_loader = ExpertsInt8MoEMethod.quantizing_weight_loader( + layer, weight_loader) + extra_weight_attrs['weight_loader'] = wrapped_weight_loader + + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + 2 * intermediate_size, + hidden_size, + dtype=int8_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size, + intermediate_size, + dtype=int8_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_scale = torch.nn.Parameter(torch.zeros(num_experts, + 2 * intermediate_size, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_scale", w13_scale) + + w2_scale = torch.nn.Parameter(torch.zeros(num_experts, + hidden_size, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_scale", w2_scale) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + from vllm.model_executor.layers.fused_moe import fused_experts + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_experts(x, + layer.w13_weight, + layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=True, + use_int8_w8a16=True, + w1_scale=layer.w13_scale, + w2_scale=layer.w2_scale) + + @staticmethod + def quantizing_weight_loader(layer, weight_loader): + + def quantize_and_call_weight_loader(param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, shard_id: int, + expert_id: int): + tp_rank = get_tensor_model_parallel_rank() + shard_size = layer.intermediate_size_per_partition + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + device = get_tp_group().device + loaded_weight = loaded_weight.to(device) + # w1, gate_proj case: Load into first shard of w13. + if shard_id == "w1": + scales = quantize_in_place_and_get_scales( + loaded_weight[shard, :]) + layer.w13_scale.data[expert_id, 0:shard_size].copy_(scales[:, + 0]) + # w3, up_proj case: Load into second shard of w13. + elif shard_id == "w3": + scales = quantize_in_place_and_get_scales( + loaded_weight[shard, :]) + layer.w13_scale.data[expert_id, shard_size:2 * + shard_size].copy_(scales[:, 0]) + # w2, down_proj case: Load into only shard of w2. + elif shard_id == "w2": + scales = quantize_in_place_and_get_scales(loaded_weight[:, + shard]) + layer.w2_scale.data[expert_id, :].copy_(scales[:, 0]) + else: + raise ValueError( + f"Shard id must be in [0,1,2] but got {shard_id}") + weight_loader(param, loaded_weight, weight_name, shard_id, + expert_id) + + return quantize_and_call_weight_loader + + +def quantize_in_place_and_get_scales(weight: torch.Tensor) -> torch.Tensor: + vmax = torch.iinfo(torch.int8).max + scales = (torch.max(torch.abs(weight), dim=1, keepdim=True)[0] / vmax) + + weight.div_(scales) + weight.round_() + weight.clamp_(-vmax, vmax) + + return scales diff --git a/vllm/model_executor/layers/quantization/fbgemm_fp8.py b/vllm/model_executor/layers/quantization/fbgemm_fp8.py new file mode 100644 index 00000000..f2690717 --- /dev/null +++ b/vllm/model_executor/layers/quantization/fbgemm_fp8.py @@ -0,0 +1,169 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, + UnquantizedLinearMethod) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.layers.quantization.fp8 import cutlass_fp8_supported +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + apply_fp8_linear, normalize_e4m3fn_to_e4m3fnuz) +from vllm.model_executor.parameter import (ChannelQuantScaleParameter, + ModelWeightParameter) +from vllm.platforms import current_platform +from vllm.utils import is_hip + +logger = init_logger(__name__) + + +class FBGEMMFp8Config(QuantizationConfig): + """Config class for FBGEMM Fp8.""" + + def __init__(self, ignore_list: List[str], input_scale_ub: float): + self.ignore_list = ignore_list if ignore_list else [] + self.input_scale_ub = input_scale_ub + + # For GPUs that lack FP8 hardware support, we can leverage the Marlin + # kernel for fast weight-only FP8 quantization + self.use_marlin = not current_platform.has_device_capability(89) + + @classmethod + def get_name(cls) -> str: + return "fbgemm_fp8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.float16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "FBGEMMFp8Config": + ignore_list = cls.get_from_keys(config, ["modules_to_not_convert"]) + input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"]) + return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.ignore_list): + return UnquantizedLinearMethod() + return FBGEMMFp8LinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class FBGEMMFp8LinearMethod(LinearMethodBase): + + def __init__(self, quant_config: FBGEMMFp8Config): + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + weight_loader = extra_weight_attrs.get("weight_loader") + del input_size, output_size + output_size_per_partition = sum(output_partition_sizes) + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight = ModelWeightParameter(data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + weight_scale = ChannelQuantScaleParameter(data=torch.empty( + (sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader) + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE UPPER BOUND + input_scale_ub = torch.nn.Parameter(torch.tensor( + (self.quant_config.input_scale_ub), dtype=torch.float32), + requires_grad=False) + layer.input_scale_ub = input_scale_ub + + def process_weights_after_loading(self, layer: Module) -> None: + # required by torch.compile + layer.weight_scale = Parameter(layer.weight_scale.data, + requires_grad=False) + layer.weight = Parameter(layer.weight.data, requires_grad=False) + + weight = layer.weight + + if is_hip(): + weight, weight_scale, input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=layer.weight_scale, + input_scale=None) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + + layer.weight = Parameter(weight.t(), requires_grad=False) + if self.quant_config.use_marlin: + prepare_fp8_layer_for_marlin(layer) + # Activations not quantized for marlin. + del layer.input_scale_ub + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + if self.quant_config.use_marlin: + return apply_fp8_marlin_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias) + + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=None, + input_scale_ub=layer.input_scale_ub, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + use_per_token_if_dynamic=True) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py new file mode 100644 index 00000000..b5feb55d --- /dev/null +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -0,0 +1,514 @@ +from typing import Any, Callable, Dict, List, Optional + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase, + FusedMoeWeightScaleSupported) +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, + UnquantizedLinearMethod) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped) +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + all_close_1d, apply_fp8_linear, convert_to_channelwise, + cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize, + requantize_with_max_scale) +from vllm.model_executor.parameter import (ModelWeightParameter, + PerTensorScaleParameter) +from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform +from vllm.utils import is_hip, print_warning_once + +ACTIVATION_SCHEMES = ["static", "dynamic"] + +logger = init_logger(__name__) + + +class Fp8Config(QuantizationConfig): + """Config class for FP8.""" + + def __init__( + self, + is_checkpoint_fp8_serialized: bool = False, + activation_scheme: str = "dynamic", + ignored_layers: Optional[List[str]] = None, + ) -> None: + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + if is_checkpoint_fp8_serialized: + logger.warning("Detected fp8 checkpoint. Please note that the " + "format is experimental and subject to change.") + if activation_scheme not in ACTIVATION_SCHEMES: + raise ValueError( + f"Unsupported activation scheme {activation_scheme}") + self.activation_scheme = activation_scheme + self.ignored_layers = ignored_layers or [] + + @classmethod + def get_name(cls) -> str: + return "fp8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "Fp8Config": + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_fp8_serialized = ("fp8" in quant_method) + activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) + ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) + return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, + activation_scheme=activation_scheme, + ignored_layers=ignored_layers) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + from vllm.attention.layer import Attention # Avoid circular import + + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.ignored_layers): + return UnquantizedLinearMethod() + return Fp8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return Fp8MoEMethod(self) + elif isinstance(layer, Attention): + return Fp8KVCacheMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class Fp8LinearMethod(LinearMethodBase): + """Linear method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + + Limitations: + 1. Only support per-tensor quantization due to torch._scaled_mm support. + 2. Only support float8_e4m3fn data type due to the limitation of + torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856) + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: Fp8Config): + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + # For GPUs that lack FP8 hardware support, we can leverage the Marlin + # kernel for fast weight-only FP8 quantization + self.use_marlin = (not current_platform.has_device_capability(89) + or envs.VLLM_TEST_FORCE_FP8_MARLIN) + # Disable marlin for rocm + if is_hip(): + self.use_marlin = False + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del input_size, output_size + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight_dtype = (torch.float8_e4m3fn + if self.quant_config.is_checkpoint_fp8_serialized else + params_dtype) + + weight = ModelWeightParameter(data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=weight_dtype), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + # If checkpoint is serialized fp8, load them. + # Otherwise, wait until process_weights_after_loading. + if self.quant_config.is_checkpoint_fp8_serialized: + # WEIGHT SCALE + scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", scale) + + # INPUT ACTIVATION SCALE + if self.quant_config.activation_scheme == "static": + scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", scale) + else: + layer.register_parameter("input_scale", None) + + def process_weights_after_loading(self, layer: Module) -> None: + layer.weight = torch.nn.Parameter(layer.weight.data, + requires_grad=False) + # If checkpoint not serialized fp8, quantize the weights. + if not self.quant_config.is_checkpoint_fp8_serialized: + qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, + scale=None) + + # If using marlin (w8a16), kernel uses channelwise weights, + # so extend the weight scales to be channelwise. + if self.use_marlin: + assert weight_scale.numel() == 1 + weight_scale = convert_to_channelwise( + weight_scale.expand(len(layer.logical_widths)), + layer.logical_widths) + + # Update the layer with the new values. + layer.weight = Parameter(qweight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + layer.input_scale = None + + # If checkpoint is fp8, handle that there are N scales for N + # shards in a fused module + else: + layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data, + requires_grad=False) + if self.quant_config.activation_scheme == "static": + layer.input_scale = torch.nn.Parameter(layer.input_scale.data, + requires_grad=False) + # If using marlin (w8a16), kernel uses channelwise weights, + # so extend the weight scales to be channelwise. + if self.use_marlin: + weight = layer.weight + weight_scale = convert_to_channelwise(layer.weight_scale, + layer.logical_widths) + + # If using w8a8, torch._scaled_mm needs per tensor, so + # requantize the logical shards as a single weight. + else: + # Dequant -> Quant with max scale so we can run per tensor. + weight = layer.weight + weight_scale = layer.weight_scale + + # If rocm, use float8_e4m3fnuz. + if is_hip(): + weight, weight_scale, input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=weight_scale, + input_scale=layer.input_scale) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, + requires_grad=False) + + weight_scale, weight = requantize_with_max_scale( + weight=weight, + weight_scale=weight_scale, + logical_widths=layer.logical_widths, + ) + + # Update layer with new values. + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + if self.quant_config.activation_scheme == "static": + layer.input_scale = Parameter(layer.input_scale.max(), + requires_grad=False) + + if self.use_marlin: + prepare_fp8_layer_for_marlin(layer) + # Activations not quantized for marlin. + del layer.input_scale + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + + if self.use_marlin: + return apply_fp8_marlin_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias) + + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + use_per_token_if_dynamic=False) + + +class Fp8MoEMethod(FusedMoEMethodBase): + """MoE method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: Fp8Config): + self.quant_config = quant_config + + def create_weights(self, layer: Module, num_experts: int, hidden_size: int, + intermediate_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + + if self.quant_config.is_checkpoint_fp8_serialized: + params_dtype = torch.float8_e4m3fn + + # WEIGHTS + w13_weight = torch.nn.Parameter(torch.empty(num_experts, + 2 * intermediate_size, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter(torch.empty(num_experts, + hidden_size, + intermediate_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + 2, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + + w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts, + dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) + # If loading fp8 checkpoint, pass the weight loaders. + # If loading an fp16 checkpoint, do not (we will quantize in + # process_weights_after_loading() + if self.quant_config.is_checkpoint_fp8_serialized: + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + if self.quant_config.activation_scheme == "static": + if not self.quant_config.is_checkpoint_fp8_serialized: + raise ValueError( + "Found static activation scheme for checkpoint that " + "was not serialized fp8.") + + w13_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: Module) -> None: + + # If checkpoint is fp16, quantize in place. + if not self.quant_config.is_checkpoint_fp8_serialized: + # If rocm, use float8_e4m3fnuz as dtype + fp8_dtype = torch.float8_e4m3fnuz \ + if is_hip() else torch.float8_e4m3fn + w13_weight = torch.empty_like(layer.w13_weight.data, + dtype=fp8_dtype) + w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype) + + # Re-initialize w13_scale because we directly quantize + # merged w13 weights and generate a single scaling factor. + layer.w13_weight_scale = torch.nn.Parameter(torch.ones( + layer.num_experts, + dtype=torch.float32, + device=w13_weight.device), + requires_grad=False) + for expert in range(layer.num_experts): + w13_weight[expert, :, :], layer.w13_weight_scale[ + expert] = ops.scaled_fp8_quant( + layer.w13_weight.data[expert, :, :]) + w2_weight[expert, :, :], layer.w2_weight_scale[ + expert] = ops.scaled_fp8_quant( + layer.w2_weight.data[expert, :, :]) + layer.w13_weight = torch.nn.Parameter(w13_weight, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(w2_weight, + requires_grad=False) + return + + # If checkpoint is fp8, we need to handle that the + # MoE kernels require single activation scale and single weight + # scale for w13 per expert. + else: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if self.quant_config.activation_scheme == "static": + if (layer.w13_input_scale is None + or layer.w2_input_scale is None): + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None.") + if (not all_close_1d(layer.w13_input_scale) + or not all_close_1d(layer.w2_input_scale)): + print_warning_once( + "Found input_scales that are not equal for " + "fp8 MoE layer. Using the maximum across experts " + "for each layer. ") + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False) + # If rocm, normalize the weights and scales to e4m3fnuz + if is_hip(): + # Normalize the weights and scales + w13_weight, w13_weight_scale, w13_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale, + layer.w13_input_scale) + w2_weight, w2_weight_scale, w2_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale, + layer.w2_input_scale) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, + requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter( + w13_weight_scale, requires_grad=False) + if w13_input_scale is not None: + layer.w13_input_scale = torch.nn.Parameter( + w13_input_scale, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(w2_weight, + requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, + requires_grad=False) + if w2_input_scale is not None: + layer.w2_input_scale = torch.nn.Parameter( + w2_input_scale, requires_grad=False) + + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.num_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start:start + + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][ + start:start + shard_size, :], _ = ops.scaled_fp8_quant( + dq_weight, max_w13_scales[expert_id]) + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, + requires_grad=False) + return + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + + from vllm.model_executor.layers.fused_moe import fused_experts + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function) + + return fused_experts(x, + layer.w13_weight, + layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=True, + use_fp8_w8a8=True, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale) + + +class Fp8KVCacheMethod(BaseKVCacheMethod): + """ + Supports loading kv-cache scaling factors from FP8 checkpoints. + """ + + def __init__(self, quant_config: Fp8Config): + super().__init__(quant_config) diff --git a/vllm/model_executor/layers/quantization/gguf.py b/vllm/model_executor/layers/quantization/gguf.py new file mode 100644 index 00000000..d73b9f6d --- /dev/null +++ b/vllm/model_executor/layers/quantization/gguf.py @@ -0,0 +1,178 @@ +from typing import Any, Dict, List, Optional + +import gguf +import torch +from torch.nn.parameter import Parameter, UninitializedParameter + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.model_executor.utils import set_weight_attrs + + +class GGUFConfig(QuantizationConfig): + """Config class for GGUF.""" + + def __init__(self, ) -> None: + pass + + def __repr__(self) -> str: + return ("GGUFConfig()") + + def get_name(self) -> str: + return "gguf" + + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 60 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] # no extra configs. + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "GGUFConfig": + return cls() + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + if isinstance(layer, LinearBase): + return GGUFLinearMethod(self) + elif isinstance(layer, VocabParallelEmbedding): + return GGUFEmbeddingMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor, + qweight_type: int) -> torch.Tensor: + # use dequantize mulmat for IQmatrix, mmq for k-quants + if x.shape[0] == 1: + # enable mmvq in contiguous batching + y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0]) + elif qweight_type >= 16: + block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type] + shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size) + weight = ops.ggml_dequantize(qweight, qweight_type, *shape) + y = x @ weight.T + else: + y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0]) + return y + + +class GGUFLinearMethod(LinearMethodBase): + """Linear method for GGUF. + + Args: + quant_config: The GGUF quantization config. + """ + + def __init__(self, quant_config: GGUFConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + output_size_per_partition = sum(output_partition_sizes) + + tensor_shape = (output_size_per_partition, input_size_per_partition) + qweight = GGUFUninitializedParameter(requires_grad=False) + set_weight_attrs( + qweight, { + "input_dim": 1, + "output_dim": 0, + "tensor_shape": tensor_shape, + "is_gguf_weight": True, + "data_container": [], + "shard_id": [], + "shard_id_map": {}, + }) + set_weight_attrs(qweight, extra_weight_attrs) + layer.register_parameter("qweight", qweight) + + qweight_type = Parameter(torch.empty(len(output_partition_sizes), + dtype=torch.uint8), + requires_grad=False) + set_weight_attrs( + qweight_type, { + "is_gguf_weight_type": True, + "weight_type": 0, + "shard_weight_type": {}, + "ignore_warning": True + }) + set_weight_attrs(qweight_type, extra_weight_attrs) + layer.register_parameter("qweight_type", qweight_type) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + shard_id = getattr(layer.qweight, "shard_id", None) + + if shard_id: + # dequantize shard weights respectively + shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id + qweight = layer.qweight.unbind(0) + result = [] + for id in shard_id: + q_idx = layer.qweight.shard_id_map[id] + qweight_type = layer.qweight_type.shard_weight_type[id] + result.append(_fuse_mul_mat(x, qweight[q_idx], qweight_type)) + out = torch.cat(result, axis=1) + else: + qweight = layer.qweight + qweight_type = layer.qweight_type.weight_type + out = _fuse_mul_mat(x, qweight, qweight_type) + if bias is not None: + out.add_(bias) + return out + + +class GGUFEmbeddingMethod(GGUFLinearMethod): + """Embedding method for GGUF. + + Args: + quant_config: The GGUF quantization config. + """ + + def embedding(self, layer: torch.nn.Module, + x: torch.Tensor) -> torch.Tensor: + qweight = layer.qweight + qweight_type = layer.qweight_type.weight_type + + block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type] + hidden_size = qweight.shape[1] // type_size * block_size + if qweight_type < 2: + return torch.embedding(qweight, x) + x_flat = x.flatten() + quant = torch.index_select(qweight, dim=0, index=x_flat) + dequant = ops.ggml_dequantize(quant, qweight_type, hidden_size, + x_flat.shape[0]) + return dequant.view(*x.shape, hidden_size) + + +class GGUFUninitializedParameter(UninitializedParameter): + cls_to_become = Parameter + data_container: List[torch.Tensor] + + def materialize_nested(self) -> Parameter: + nested_data = torch.nested.nested_tensor(self.data_container, + device=self.device, + dtype=torch.uint8) + self.data_container.clear() + param = torch.Tensor._make_subclass(self.cls_to_become, + nested_data, + require_grad=False) + for k, v in self.__dict__.items(): + setattr(param, k, v) + return param diff --git a/vllm/model_executor/layers/quantization/gptq.py b/vllm/model_executor/layers/quantization/gptq.py new file mode 100644 index 00000000..bdd99d74 --- /dev/null +++ b/vllm/model_executor/layers/quantization/gptq.py @@ -0,0 +1,248 @@ +import enum +from enum import Enum +from fractions import Fraction +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.parameter import (ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + RowvLLMParameter) + + +class GPTQConfig(QuantizationConfig): + """Config class for GPTQ. + + Reference: https://arxiv.org/abs/2210.17323 + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + desc_act: bool, + lm_head_quantized: bool, + ) -> None: + self.weight_bits = weight_bits + self.group_size = group_size + self.desc_act = desc_act + self.lm_head_quantized = lm_head_quantized + self.pack_factor = Fraction(32, self.weight_bits) + if self.weight_bits not in [2, 3, 4, 8]: + raise ValueError( + "Currently, only 2/3/4/8-bit weight quantization is " + f"supported for GPTQ, but got {self.weight_bits} bits.") + + def __repr__(self) -> str: + return (f"GPTQConfig(weight_bits={self.weight_bits}, " + f"group_size={self.group_size}, " + f"desc_act={self.desc_act})," + f"lm_head_quantized={self.lm_head_quantized}") + + @classmethod + def get_name(cls) -> str: + return "gptq" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 60 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "GPTQConfig": + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + desc_act = cls.get_from_keys(config, ["desc_act"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], + default=False) + return cls(weight_bits, group_size, desc_act, lm_head_quantized) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["GPTQLinearMethod"]: + if (isinstance(layer, LinearBase) or + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)): + return GPTQLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class ExllamaState(Enum): + + UNUSED = enum.auto() + UNINITIALIZED = enum.auto() + READY = enum.auto() + + +class GPTQLinearMethod(LinearMethodBase): + """Linear method for GPTQ. + + Args: + quant_config: The GPTQ quantization config. + """ + + def __init__(self, quant_config: GPTQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del output_size # Unused. + weight_loader = extra_weight_attrs.get("weight_loader") + if input_size_per_partition % self.quant_config.group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + output_size_per_partition = sum(output_partition_sizes) + if (output_size_per_partition % self.quant_config.pack_factor.numerator + != 0): + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size.") + + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + exllama_state = ExllamaState.UNINITIALIZED + scale_and_zero_size = input_size // group_size + scale_and_zero_input_dim = None + if (input_size != input_size_per_partition + and self.quant_config.group_size != -1): + # For act-order models, we cannot use Exllama for row parallel layer + if self.quant_config.desc_act: + exllama_state = ExllamaState.UNUSED + else: + # we need to partition qzeros and scales for exllama kernel + scale_and_zero_size = input_size_per_partition // group_size + scale_and_zero_input_dim = 0 + + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.pack_factor, + output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=0, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + g_idx = RowvLLMParameter(data=torch.tensor( + [ + i // self.quant_config.group_size + for i in range(input_size_per_partition) + ], + dtype=torch.int32, + ), + input_dim=0, + weight_loader=weight_loader) + qzeros_args = { + "data": + torch.empty( + scale_and_zero_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + "weight_loader": + weight_loader + } + weight_scale_args = { + "data": + torch.empty( + scale_and_zero_size, + output_size_per_partition, + dtype=params_dtype, + ), + "weight_loader": + weight_loader + } + if scale_and_zero_input_dim is None: + scales = ChannelQuantScaleParameter(output_dim=1, + **weight_scale_args) + qzeros = PackedColumnParameter( + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args) + + else: + scales = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **weight_scale_args) + qzeros = PackedvLLMParameter( + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("g_idx", g_idx) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + layer.exllama_state = exllama_state + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # for torch.compile + layer.qweight = Parameter(layer.qweight.data, requires_grad=False) + layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False) + layer.qweight = Parameter(layer.qweight.data, requires_grad=False) + layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False) + layer.scales = Parameter(layer.scales.data, requires_grad=False) + + # exllama needs to shuffle the weight after the weight is loaded + # here we do the shuffle on first forward pass + if layer.exllama_state == ExllamaState.UNINITIALIZED: + if self.quant_config.desc_act: + layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int) + else: + layer.g_idx.data = torch.empty((0, ), + dtype=torch.int, + device=layer.g_idx.device) + layer.exllama_state = ExllamaState.READY + ops.gptq_shuffle(layer.qweight, layer.g_idx, + self.quant_config.weight_bits) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + out_shape = x.shape[:-1] + (layer.qweight.shape[-1], ) + reshaped_x = x.reshape(-1, x.shape[-1]) + + output = ops.gptq_gemm(reshaped_x, layer.qweight, layer.qzeros, + layer.scales, layer.g_idx, + layer.exllama_state == ExllamaState.READY, + self.quant_config.weight_bits) + if bias is not None: + output.add_(bias) + return output.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/gptq_marlin.py b/vllm/model_executor/layers/quantization/gptq_marlin.py new file mode 100644 index 00000000..e7719179 --- /dev/null +++ b/vllm/model_executor/layers/quantization/gptq_marlin.py @@ -0,0 +1,570 @@ +from typing import Any, Callable, Dict, List, Optional, Set, Union + +import torch + +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.layer import ( + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported) +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase, + set_weight_attrs) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.quantization.kernels import ( + MPLinearLayerConfig, choose_mp_linear_kernel) +from vllm.model_executor.layers.quantization.utils import replace_parameter +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + check_marlin_supported, marlin_moe_permute_scales, + marlin_repeat_scales_on_all_ranks, verify_marlin_supported) +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.parameter import (ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + RowvLLMParameter) +from vllm.scalar_type import scalar_types + +logger = init_logger(__name__) + + +class GPTQMarlinConfig(QuantizationConfig): + """Config class for GPTQ Marlin""" + + # (num_bits, is_sym) -> quant_type + TYPE_MAP = { + (4, True): scalar_types.uint4b8, + (8, True): scalar_types.uint8b128, + } + + def __init__( + self, + weight_bits: int, + group_size: int, + desc_act: bool, + is_sym: bool, + lm_head_quantized: bool, + ) -> None: + if desc_act and group_size == -1: + # In this case, act_order == True is the same as act_order == False + # (since we have only one group per output channel) + desc_act = False + + self.pack_factor = 32 // weight_bits # packed into int32 + self.group_size = group_size + self.desc_act = desc_act + self.lm_head_quantized = lm_head_quantized + + if (weight_bits, is_sym) not in self.TYPE_MAP: + raise ValueError("Unsupported quantization config: " + f"bits={weight_bits}, sym={is_sym}") + + self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)] + + def __repr__(self) -> str: + return (f"GPTQMarlinConfig(quant_type={self.quant_type}, " + f"group_size={self.group_size}, " + f"desc_act={self.desc_act}, " + f"lm_head_quantized={self.lm_head_quantized})") + + @classmethod + def get_name(cls) -> str: + return "gptq_marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlinConfig": + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + desc_act = cls.get_from_keys(config, ["desc_act"]) + is_sym = cls.get_from_keys(config, ["sym"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], + default=False) + return cls(weight_bits, group_size, desc_act, is_sym, + lm_head_quantized) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg) + + is_valid_user_quant = (user_quant is None or user_quant == "marlin" + or user_quant == "gptq_marlin") + + if can_convert and is_valid_user_quant: + msg = ("The model is convertible to {} during runtime." + " Using {} kernel.".format(cls.get_name(), cls.get_name())) + logger.info(msg) + return cls.get_name() + + if can_convert and user_quant == "gptq": + logger.info("Detected that the model can run with gptq_marlin" + ", however you specified quantization=gptq explicitly," + " so forcing gptq. Use quantization=gptq_marlin for" + " faster inference") + return None + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod"]]: + if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead) + and self.lm_head_quantized): + return GPTQMarlinLinearMethod(self) + elif isinstance(layer, FusedMoE): + return GPTQMarlinMoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + @classmethod + def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]): + # Extract data from quant config. + quant_method = quant_config.get("quant_method", "").lower() + num_bits = quant_config.get("bits") + group_size = quant_config.get("group_size") + sym = quant_config.get("sym") + desc_act = quant_config.get("desc_act") + + if quant_method != "gptq": + return False + + # If we cannot find the info needed in the config, cannot convert. + if (num_bits is None or group_size is None or sym is None + or desc_act is None): + return False + + if (num_bits, sym) not in cls.TYPE_MAP: + return False + + return check_marlin_supported(quant_type=cls.TYPE_MAP[(num_bits, sym)], + group_size=group_size) + + +class GPTQMarlinLinearMethod(LinearMethodBase): + """Linear method for GPTQ Marlin. + + Args: + quant_config: The GPTQ Marlin quantization config. + """ + + _kernel_backends_being_used: Set[str] = set() + + def __init__(self, quant_config: GPTQMarlinConfig) -> None: + self.quant_config = quant_config + + # Verify supported on platform. + verify_marlin_supported(quant_type=self.quant_config.quant_type, + group_size=self.quant_config.group_size) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + is_row_parallel = input_size != input_size_per_partition + weight_loader = extra_weight_attrs.get("weight_loader") + + mp_linear_kernel_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=\ + (input_size_per_partition, output_size_per_partition), + weight_type=self.quant_config.quant_type, + act_type=params_dtype, + group_size=self.quant_config.group_size, + zero_points=False, + has_g_idx=self.quant_config.desc_act + ) + + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) + + if kernel_type.__name__ not in self._kernel_backends_being_used: + logger.info("Using %s for GPTQMarlinLinearMethod", + kernel_type.__name__) + self._kernel_backends_being_used.add(kernel_type.__name__) + + # Normalize group_size + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + # Determine sharding + if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act, + self.quant_config.group_size, + is_row_parallel): + # By setting scale_dim == None, weight_loader will + # repeat the scales on each GPU in TP>1 case. + scales_and_zp_input_dim = None + scales_and_zp_size = input_size // group_size + else: + # By setting scale_dim == 0, weight_loader will + # shard the scales in TP>1 case. + scales_and_zp_input_dim = 0 + scales_and_zp_size = input_size_per_partition // group_size + + # Quantized weights + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.pack_factor, + output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=0, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader) + + # Activation order + g_idx = RowvLLMParameter(data=torch.empty( + input_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + weight_loader=weight_loader) + + qzeros_args = { + "data": + torch.empty( + scales_and_zp_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + "weight_loader": + weight_loader + } + weight_scale_args = { + "data": + torch.empty( + scales_and_zp_size, + output_size_per_partition, + dtype=params_dtype, + ), + "weight_loader": + weight_loader + } + + if scales_and_zp_input_dim is None: + scales = ChannelQuantScaleParameter(output_dim=1, + **weight_scale_args) + qzeros = PackedColumnParameter( + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args) + + else: + scales = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **weight_scale_args) + qzeros = PackedvLLMParameter( + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("g_idx", g_idx) + layer.register_parameter("scales", scales) + layer.register_parameter("qzeros", qzeros) + + self.kernel = kernel_type(mp_linear_kernel_config, + w_q_param_name="qweight", + w_s_param_name="scales", + w_zp_param_name="qzeros", + w_gidx_param_name="g_idx") + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return self.kernel.apply_weights(layer, x, bias) + + +class GPTQMarlinMoEMethod(FusedMoEMethodBase): + """MoE Marlin method with quantization.""" + + def __init__(self, quant_config: GPTQMarlinConfig) -> None: + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + # Currently assuming is_k_full is always True + # (input size per partition is the same as full input size) + # Supports only sym for now (no zp) + if self.quant_config.group_size != -1: + scales_size13 = hidden_size // self.quant_config.group_size + scales_size2 = intermediate_size // self.quant_config.group_size + strategy = FusedMoeWeightScaleSupported.GROUP.value + else: + scales_size13 = 1 + scales_size2 = 1 + strategy = FusedMoeWeightScaleSupported.CHANNEL.value + + extra_weight_attrs.update({ + "quant_method": strategy, + "is_transposed": True + }) + # Fused gate_up_proj (column parallel) + w13_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size // self.quant_config.pack_factor, + 2 * intermediate_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qweight", w13_qweight) + set_weight_attrs(w13_qweight, extra_weight_attrs) + # down_proj (row parallel) + w2_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size // self.quant_config.pack_factor, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qweight", w2_qweight) + set_weight_attrs(w2_qweight, extra_weight_attrs) + # up_proj scales + w13_scales = torch.nn.Parameter( + torch.empty(num_experts, + scales_size13, + 2 * intermediate_size, + dtype=torch.half), + requires_grad=False, + ) + layer.register_parameter("w13_scales", w13_scales) + set_weight_attrs(w13_scales, extra_weight_attrs) + # down_proj scales + w2_scales = torch.nn.Parameter( + torch.empty(num_experts, + scales_size2, + hidden_size, + dtype=torch.half), + requires_grad=False, + ) + layer.register_parameter("w2_scales", w2_scales) + set_weight_attrs(w2_scales, extra_weight_attrs) + # up_proj scales + w13_qzeros = torch.nn.Parameter( + torch.empty(num_experts, + scales_size13, + 2 * intermediate_size // self.quant_config.pack_factor, + dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w13_qzeros", w13_qzeros) + set_weight_attrs(w13_qzeros, extra_weight_attrs) + # down_proj scales + w2_qzeros = torch.nn.Parameter( + torch.empty(num_experts, + scales_size2, + hidden_size // self.quant_config.pack_factor, + dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_qzeros", w2_qzeros) + set_weight_attrs(w2_qzeros, extra_weight_attrs) + w13_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx", w13_g_idx) + set_weight_attrs(w13_g_idx, extra_weight_attrs) + w2_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx", w2_g_idx) + set_weight_attrs(w2_g_idx, extra_weight_attrs) + w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx_sort_indices", + w13_g_idx_sort_indices) + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs) + w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx_sort_indices", + w2_g_idx_sort_indices) + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + + # Process act_order + if self.quant_config.desc_act: + # Get sorting based on g_idx + num_experts = layer.w13_g_idx.shape[0] + w13_g_idx_sort_indices = torch.empty_like(layer.w13_g_idx) + w2_g_idx_sort_indices = torch.empty_like(layer.w2_g_idx) + w13_sorted_g_idx = torch.empty_like(layer.w13_g_idx) + w2_sorted_g_idx = torch.empty_like(layer.w2_g_idx) + for e in range(num_experts): + w13_g_idx_sort_indices[e] = torch.argsort( + layer.w13_g_idx[e]).to(torch.int32) + w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_g_idx[e]).to( + torch.int32) + w13_sorted_g_idx[e] = layer.w13_g_idx[e][ + w13_g_idx_sort_indices[e]] + w2_sorted_g_idx[e] = layer.w2_g_idx[e][ + w2_g_idx_sort_indices[e]] + replace_parameter(layer, "w13_g_idx", w13_sorted_g_idx) + replace_parameter(layer, "w2_g_idx", w2_sorted_g_idx) + replace_parameter(layer, "w13_g_idx_sort_indices", + w13_g_idx_sort_indices) + replace_parameter(layer, "w2_g_idx_sort_indices", + w2_g_idx_sort_indices) + else: + # Reset g_idx related tensors + num_experts = layer.w13_g_idx.shape[0] + device = layer.w13_g_idx.device + layer.w13_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, + device=device), + requires_grad=False, + ) + layer.w2_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, + device=device), + requires_grad=False, + ) + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, + device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, + device=device), + requires_grad=False, + ) + # Repack weights + marlin_w13_qweight = ops.gptq_marlin_moe_repack( + layer.w13_qweight, + layer.w13_g_idx_sort_indices, + layer.w13_qweight.shape[1] * self.quant_config.pack_factor, + layer.w13_qweight.shape[2], + self.quant_config.quant_type.size_bits, + ) + replace_parameter(layer, "w13_qweight", marlin_w13_qweight) + marlin_w2_qweight = ops.gptq_marlin_moe_repack( + layer.w2_qweight, + layer.w2_g_idx_sort_indices, + layer.w2_qweight.shape[1] * self.quant_config.pack_factor, + layer.w2_qweight.shape[2], + self.quant_config.quant_type.size_bits, + ) + replace_parameter(layer, "w2_qweight", marlin_w2_qweight) + # Repack scales + marlin_w13_scales = marlin_moe_permute_scales( + s=layer.w13_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w13_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w13_scales", marlin_w13_scales) + marlin_w2_scales = marlin_moe_permute_scales( + s=layer.w2_scales, + size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor, + size_n=layer.w2_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w2_scales", marlin_w2_scales) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + ) -> torch.Tensor: + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + fused_marlin_moe) + + # The input must currently be float16 + orig_dtype = x.dtype + x = x.half() + + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=None) + + return fused_marlin_moe( + x, + layer.w13_qweight, + layer.w2_qweight, + layer.w13_scales, + layer.w2_scales, + router_logits, + topk_weights, + topk_ids, + g_idx1=layer.w13_g_idx, + g_idx2=layer.w2_g_idx, + sort_indices1=layer.w13_g_idx_sort_indices, + sort_indices2=layer.w2_g_idx_sort_indices, + num_bits=self.quant_config.quant_type.size_bits, + ).to(orig_dtype) diff --git a/vllm/model_executor/layers/quantization/gptq_marlin_24.py b/vllm/model_executor/layers/quantization/gptq_marlin_24.py new file mode 100644 index 00000000..a66f12ed --- /dev/null +++ b/vllm/model_executor/layers/quantization/gptq_marlin_24.py @@ -0,0 +1,295 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter) +from vllm.scalar_type import scalar_types + +logger = init_logger(__name__) + +GPTQ_MARLIN_24_TILE = 16 +GPTQ_MARLIN_24_MIN_THREAD_N = 128 +GPTQ_MARLIN_24_MIN_THREAD_K = 128 +GPTQ_MARLIN_24_MAX_PARALLEL = 64 + +GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES = [ + scalar_types.uint4b8, scalar_types.uint8b128 +] +GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128] + + +class GPTQMarlin24Config(QuantizationConfig): + """Config class for Marlin24. + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + ) -> None: + quant_type = { + 4: scalar_types.uint4b8, + 8: scalar_types.uint8b128, + }.get(weight_bits) + + self.group_size = group_size + + # Verify + if quant_type is None or \ + quant_type not in GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES: + raise ValueError( + f"Marlin_24 does not support quant_type = {quant_type}. " + f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES} " + "are supported.") + if self.group_size not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES: + raise ValueError( + f"Marlin_24 does not support group_size = {self.group_size}. " + f"Only group_sizes = {GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES} " + "are supported.") + + self.quant_type = quant_type + + # 4 Bits packed into 32 bit datatype. + self.pack_factor = 32 // self.quant_type.size_bits + + # Tile size used by marlin kernels. + self.tile_size = 16 + + # Min out_features dim + self.min_n_threads = GPTQ_MARLIN_24_MIN_THREAD_N + + # Min in_features dim + self.min_k_threads = GPTQ_MARLIN_24_MIN_THREAD_K + + # Max parallel problems to solve at once (improves large + # batch performance) + self.max_parallel = GPTQ_MARLIN_24_MAX_PARALLEL + + # Permutation length used by the marlin kernels. + self.perm_len = 1024 + + def __repr__(self) -> str: + return "Marlin24Config(quant_type={}, group_size={})".format( + self.quant_type, self.group_size) + + @classmethod + def get_name(cls) -> str: + return "gptq_marlin_24" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlin24Config": + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + return cls(weight_bits, group_size) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + is_marlin_24_format = ( + hf_quant_cfg.get("checkpoint_format") == "marlin_24") + + is_valid_user_quant = (user_quant is None or user_quant == "gptq" + or user_quant == "gptq_marlin_24") + + if is_marlin_24_format and is_valid_user_quant: + msg = ("The model is serialized in {} format. " + "Using {} kernel.".format(cls.get_name(), cls.get_name())) + logger.info(msg) + return cls.get_name() + + return None + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["GPTQMarlin24LinearMethod"]: + if isinstance(layer, LinearBase): + return GPTQMarlin24LinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class GPTQMarlin24LinearMethod(LinearMethodBase): + """Linear method for Marlin24. + + Args: + quant_config: The Marlin24 quantization config. + """ + + def __init__(self, quant_config: GPTQMarlin24Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del output_size # Unused. + weight_loader = extra_weight_attrs["weight_loader"] + if params_dtype != torch.float16: + raise ValueError( + f"The params dtype must be float16, but got {params_dtype}") + + # Validate output_size_per_partition + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.min_n_threads != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"min_n_threads = {self.quant_config.min_n_threads}.") + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"pack_factor = {self.quant_config.pack_factor}.") + + # Validate input_size_per_partition + if input_size_per_partition % self.quant_config.min_k_threads != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"min_k_threads = {self.quant_config.min_k_threads}.") + if (self.quant_config.group_size != -1 and + input_size_per_partition % self.quant_config.group_size != 0): + raise ValueError(f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"group_size = {self.quant_config.group_size}.") + + # Check that we have at least 4 tiles horizontally in the shard + num_tiles_per_perm = self.quant_config.perm_len // ( + self.quant_config.tile_size**2) + if output_size_per_partition % num_tiles_per_perm != 0: + raise ValueError( + "Each permutation group must reside on the same gpu") + + # Quantized 4Bit weights packed into Int32. + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.tile_size // 2, + output_size_per_partition * self.quant_config.tile_size // + self.quant_config.pack_factor, + device="cuda", + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + marlin_tile_size=self.quant_config.tile_size, + weight_loader=weight_loader) + + # Meta + meta = PackedvLLMParameter(data=torch.empty( + input_size_per_partition // 8 // 2 // 2, + output_size_per_partition * 2, + device="cuda", + dtype=torch.int16, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=1, + marlin_tile_size=2, + weight_loader=weight_loader) + + # Determine if channelwise or not + input_groups = (1 if self.quant_config.group_size == -1 else + input_size_per_partition // + self.quant_config.group_size) + + weight_scale_args = { + "data": + torch.empty( + input_groups, + output_size_per_partition, + device="cuda", + dtype=params_dtype, + ), + "weight_loader": + weight_loader + } + if input_groups == 1: + scales = ChannelQuantScaleParameter(output_dim=1, + **weight_scale_args) + else: + scales = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **weight_scale_args) + + # Allocate workspace (Used for internal locking mechanism) + max_workspace_size = ( + output_size_per_partition // + self.quant_config.min_n_threads) * self.quant_config.max_parallel + + workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size, + device="cuda", + dtype=torch.int), + weight_loader=weight_loader) + + layer.register_parameter("B_24", qweight) + layer.register_parameter("B_meta", meta) + layer.register_parameter("s", scales) + layer.register_parameter("workspace", workspace) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # required by torch.compile + layer.B_24 = Parameter(layer.B_24.data, requires_grad=False) + layer.s = Parameter(layer.s.data, requires_grad=False) + layer.B_meta = Parameter(layer.B_meta.data, requires_grad=False) + layer.workspace = Parameter(layer.workspace.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qweight = layer.B_24 + meta = layer.B_meta + scales = layer.s + workspace = layer.workspace + + x_2d = x.view(-1, x.shape[-1]) + + size_m = x_2d.shape[0] + size_k = x_2d.shape[1] + size_n = scales.shape[1] + + output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales, + workspace, + self.quant_config.quant_type, + size_m, size_n, size_k) + + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], )) + + if bias is not None: + output.add_(bias) # In-place add + + return output diff --git a/vllm/model_executor/layers/quantization/ipex_quant.py b/vllm/model_executor/layers/quantization/ipex_quant.py new file mode 100644 index 00000000..e5405263 --- /dev/null +++ b/vllm/model_executor/layers/quantization/ipex_quant.py @@ -0,0 +1,166 @@ +from typing import Any, Dict, List, Optional + +import torch + +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.awq import AWQLinearMethod +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.platforms import current_platform + + +class IPEXConfig(QuantizationConfig): + """INT8 quantization config class using IPEX for the CPU backend, + including AWQ. + """ + + IPEX_QUANT_METHOD_MAP = { + "awq": 1, + "gptq": 2, + } + + def __init__( + self, + method: str, + weight_bits: int, + group_size: int, + ) -> None: + self.method = method + self.weight_bits = weight_bits + self.group_size = group_size + self.pack_factor = 32 // self.weight_bits + + if self.weight_bits not in [4]: + raise ValueError(f"IPEX quantization supports weight bits [4], " + f"but got {self.weight_bits}.") + + if self.method == "awq": + self.quant_method = IPEXAWQLinearMethod + else: + raise ValueError(f"IPEX quantization supports [awq], " + f"but got {self.method}.") + + def __repr__(self) -> str: + return (f"IPEXConfig(method={self.method}" + f"weight_bits={self.weight_bits}, " + f"group_size={self.group_size}") + + def get_ipex_quant_method_id(self) -> int: + return IPEXConfig.IPEX_QUANT_METHOD_MAP[self.method] + + @classmethod + def get_name(cls) -> str: + return "ipex" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return -1 + + @staticmethod + def get_config_filenames() -> List[str]: + return [ + "quant_config.json", + "quantize_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "IPEXConfig": + method = cls.get_from_keys(config, ["quant_method"]).lower() + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"]) + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"]) + return cls(method, weight_bits, group_size) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + if not current_platform.is_cpu(): + return None + + quant_method = hf_quant_cfg.get("quant_method", "").lower() + + if quant_method in ["awq"]: + return cls.get_name() + + return None + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["LinearMethodBase"]: + if isinstance(layer, LinearBase): + return self.quant_method(self) + return None + + def get_scaled_act_names(self) -> List[str]: + if self.method == "awq": + return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"] + else: + return [] + + +class IPEXAWQLinearMethod(AWQLinearMethod): + """AWQ linear method using IPEX for the CPU backend. + """ + + def __init__(self, quant_config: IPEXConfig): + self.quant_config = quant_config # type: ignore + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer=layer) + + bias = layer.bias if not layer.skip_bias_add else None + + try: + import intel_extension_for_pytorch as ipex + if ipex.__version__ < "2.4.0": + raise ImportError("intel_extension_for_pytorch version is " + "wrong. Please install " + "intel_extension_for_pytorch>=2.4.0.") + except ImportError as err: + raise ImportError( + "Please install " + "intel_extension_for_pytorch>=2.4.0 via " + "`pip install intel_extension_for_pytorch>=2.4.0`" + " to use IPEX-AWQ linear method.") from err + + # Using the compute dtype (lowp_mode) as INT8 to leverage instructions + # with better performance. + lowp_mode = ipex.quantization.WoqLowpMode.INT8 + # The weight will be de-packed from INT4 to INT8. + weight_dtype = ipex.quantization.WoqWeightDtype.INT4 + # The float activation will be quantized (dynamic, per-token) to INT8. + act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH + + qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping( + weight_dtype=weight_dtype, + lowp_mode=lowp_mode, + act_quant_mode=act_quant_mode, + group_size=self.quant_config.group_size, + ) + + layer.ipex_output_size = layer.qweight.size( + 1) * self.quant_config.pack_factor + layer.ipex_qlinear = ipex.nn.modules.weight_only_quantization.\ + WeightOnlyQuantizedLinear.from_weight( + layer.qweight, + layer.scales, + layer.qzeros, + layer.qweight.size(0), + layer.ipex_output_size, + qconfig=qconfig, + bias=bias, + group_size=self.quant_config.group_size, + quant_method= + self.quant_config.get_ipex_quant_method_id() # type: ignore + ) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + reshaped_x = x.reshape(-1, x.shape[-1]) + out = layer.ipex_qlinear(reshaped_x) + + return out.reshape(x.shape[:-1] + (layer.ipex_output_size, )) diff --git a/vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py b/vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py new file mode 100644 index 00000000..fe50c493 --- /dev/null +++ b/vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py @@ -0,0 +1,83 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, Optional, Tuple + +import torch + +from vllm.model_executor.layers.quantization.utils import replace_parameter +from vllm.scalar_type import ScalarType + + +@dataclass +class MPLinearLayerConfig: + full_weight_shape: Tuple[int, int] # [in, out] + partition_weight_shape: Tuple[int, int] + weight_type: ScalarType + act_type: torch.dtype + group_size: int + zero_points: bool + has_g_idx: bool + + +class MPLinearKernel(ABC): + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + raise NotImplementedError + + @classmethod + @abstractmethod + def can_implement(cls, + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]: + raise NotImplementedError + + def __init__(self, + c: MPLinearLayerConfig, + w_q_param_name: str, + w_s_param_name: str, + w_zp_param_name: Optional[str] = None, + w_gidx_param_name: Optional[str] = None) -> None: + assert self.can_implement(c) + self.config = c + self.w_q_name = w_q_param_name + self.w_s_name = w_s_param_name + self.w_zp_name = w_zp_param_name + self.w_gidx_name = w_gidx_param_name + + @abstractmethod + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + raise NotImplementedError + + @abstractmethod + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + raise NotImplementedError + + def _transform_param(self, layer: torch.nn.Module, name: Optional[str], + fn: Callable) -> None: + if name is not None and getattr(layer, name, None) is not None: + + old_param = getattr(layer, name) + new_param = fn(old_param) + # replace the parameter with torch.nn.Parameter for TorchDynamo + # compatibility + replace_parameter( + layer, name, + torch.nn.Parameter(new_param.data, requires_grad=False)) + + def _get_weight_params( + self, layer: torch.nn.Module + ) -> Tuple[torch.Tensor, # w_q + torch.Tensor, # w_s + Optional[torch.Tensor], # w_zp, + Optional[torch.Tensor] # w_gidx + ]: + return ( + getattr(layer, self.w_q_name), + getattr(layer, self.w_s_name), + getattr(layer, self.w_zp_name or "", None), + getattr(layer, self.w_gidx_name or "", None), + ) diff --git a/vllm/model_executor/layers/quantization/kernels/__init__.py b/vllm/model_executor/layers/quantization/kernels/__init__.py new file mode 100644 index 00000000..47591c2a --- /dev/null +++ b/vllm/model_executor/layers/quantization/kernels/__init__.py @@ -0,0 +1,72 @@ +import os +from typing import List, Optional, Type + +from vllm.model_executor.layers.quantization.kernels.machete import ( + MacheteLinearKernel) +from vllm.model_executor.layers.quantization.kernels.marlin import ( + MarlinLinearKernel) +from vllm.model_executor.layers.quantization.kernels.MPLinearKernel import ( + MPLinearKernel, MPLinearLayerConfig) +from vllm.platforms import current_platform + +# in priority/performance order (when available) +_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [ + MacheteLinearKernel, + MarlinLinearKernel, +] + + +def choose_mp_linear_kernel( + config: MPLinearLayerConfig, + compute_capability: Optional[int] = None) -> Type[MPLinearKernel]: + """ + Choose an MPLinearKernel that can implement the given config for the given + compute capability. Attempts to choose the best kernel in terms of + performance. + + Args: + config (MPLinearLayerConfig): Description of the linear layer to be + implemented. + compute_capability (Optional[int], optional): The compute capability of + the target device, if None uses `current_platform` to get the compute + capability. Defaults to None. + + Raises: + ValueError: If no kernel can implement the given config. + + Returns: + Type[MPLinearKernel]: Chosen kernel. + """ + if compute_capability is None: + if current_platform is None: + raise ValueError("Cannot determine compute capability") + _cc = current_platform.get_device_capability() + compute_capability = _cc[0] * 10 + _cc[1] + + failure_reasons = [] + for kernel in _POSSIBLE_KERNELS: + if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\ + .split(","): + failure_reasons.append( + f' {kernel.__name__} disabled by environment variable') + continue + + if kernel.get_min_capability() > compute_capability: + failure_reasons.append( + f"{kernel.__name__} requires capability " + f"{kernel.get_min_capability()}, current compute capability " + f"is {compute_capability}") + continue + + can_implement, failure_reason = kernel.can_implement(config) + if can_implement: + return kernel + else: + failure_reasons.append( + f' {kernel.__name__} cannot implement due to: {failure_reason}' + ) + + raise ValueError( + "Failed to find a kernel that can implement the "\ + "WNA16 linear layer. Reasons: \n" + + '\n'.join(failure_reasons)) diff --git a/vllm/model_executor/layers/quantization/kernels/__pycache__/MPLinearKernel.cpython-310.pyc b/vllm/model_executor/layers/quantization/kernels/__pycache__/MPLinearKernel.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6690d8b234f62d732964f59c326c30c9740884e4 GIT binary patch literal 3003 zcma(TU5^|!vh8;F^z?k}><60-Byi~X8X!C5bP}A1PJsvzcd$fCPD1@+^|n2`z4TY? z9yU7~Uf6xT|G+)w;r_z?5Wn8jJ@ZCLLX~@FvO5bWy5(|}UAD_r<*F)sz0iU0=D+`Q zI0fv#NSyx|5VxV}ZvhBLkj%+RL~! zVcv;4cFfPZc`xb_N3zcx(GtO#BZ5%7)_Ulg;7*)n@qQ*v@Ksf(Wf5n_yFaNixd*K4N>*8% zNM6M%h7F{G!SG%ZXR*3}Qpp5QI{&#q+=iyV1|Xe?2q&WdaHK1lWMg{Ok)9y8F47ce zAI4qiTSoz43@`@02YnlQf8Mul!Oh`@ZL|61-7nHY#_EgsM5;SwF-{Mn_3~Q(j%Fx>3jKK-M?j&nJiZ&HQ+jrTe4m1dLgXzRj|H1Omv6 z%ZDQg%d+6vJsS26t(L!r~xAbmxqR^c?(DVR6=nUw91SEtOkS?*a zmu%bv&fB@EMT38V2jj5o*J!7W?{L{CiOtxSq0856lbJamF7&8G@opW{Dt#(Vi zB#*Nir6`0(F;osyaBAXZNA@P-!Zc{x+-3T z72Bc(9(vvMFY4O(O4buqB%rh9hd8!7(0;_wQ9Iw0M`Z62F`Ij3{do@0FqcIw#V1OY z`1af+)6bnmZNeI}0p7v$wBSix#rtWN)+akKt|IwqfbV${Uh+jjnbk;_WUJ@5PUC4YLI2sYy^xBFD7Z~?U|Cm2$B8+`jNbB?T+A*J2 z=Lgo0`2jq|b3=&_$1Og@F%Y3z^?Lwj&JWIv{EO(;-$21LGIKbYQGj&j0(55#Ae(sr zJ>lGI?mP4q-of~ZjV>lYDe-Y5rW>qnC9{K_xlBk5eZ#cO#*La@hF#Pv2>yWJRRk-l zgMgA2Y0dc`VN!Pi^nIT+e@LcRR`j>vHlBU#zt?z=sCp9*AO4C2*DF;fQtNq`)I1(T zwC7nFi?ldEB{s&S(|1>d`M(Pdc?hvBjdgL#{To4ES1;e%Vm%D(>sBk7ky;g(QAb#pv`i+J@rQb2|8wfbx zc{O?yt?7|_1padb$xYCI(o+``_q}Bsx_h4vNz_yF!6a1@GKq=>I7tiMVqDis1#qas z2yAFknD*U8wlxfMHF)B4yN!?)?P1zwCYmQ;+J!uB&{a(hI3)iQ6Y~e`zZO z^$rk`e_s~5RB9KeJp}I~_zQyPm$VoLceMooL$E#q5IB@lvH?b;#Qx9vr2mWtl(5G@ zKZDt)F!#h~^k?6rkEs_vroA2HXY7d&91J)D;9zd|$%b#&knkDp;+#*J&HL<0|EWE9 zI52L!pD4@+Rs*K9EFGm*_==4Wz=mwT2yBEuo-KIR*Jzt*ePU;lW y8a{$(VAp6{VcMF^R!fb-Q5V0aBoo|fim<<8P|>Kofzu^hB%oc|h1(CvR`?6+CDCXA literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/kernels/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/quantization/kernels/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2827fc5e6677dbd06aec539d64f8d8ab686ef21 GIT binary patch literal 2317 zcmb7F-EP}96ecNIR{WPVO;)7CI zve|HMwmrb^ivr2zF7^t05Z!J!eT816JCy9$>3)F9Fhw2m{C(#;WK^$L5Ii@2|Ks2) z%>L9L^N$T5U&2566^8GSk9-^;N54scL*kIoaE#D&Osr)_Pzo)_%I9WK4sFNI=cQmF ztT>gh>QwV@E2xEar;ZV`m^~m)gAr!3${JcjY|$^@Mhn=l`!=ik3(R0kYsjw@>wU|w z`n6w5&IKRup=RR^M1^*nMrL+-qB8q_tav26ATu8hW7fpk#T~DAz!d9p!9017Nx=dT zFYkCV;NqN=WR0C~=Ov&#`R;l{CO0F|=L1k)>ZMXLq1-s|R6mkocESDj{@&i*ue#gr zz3m6P+ugm)>gC(B`nM5f59-uK2DLm ztN?O&*A({2&-f>-&@nkdM`TQV1AsAC5X?$z=n^`@19W64V+%d~=ExkI&rMZ&ZXK6> z%P*ghW81e+@F9|4jE&<3e?hOm0$^VoCm5rbSH|XXGLjv7w@*bhEyDj-dulCjqB(QA|yV z5ZxVpzUc`O>C|?rlJyZ=E$ddW+ZJVmkVa z`aJRW1Lo8HA!Xu_%ShyGJ@h2k3!_yk*~^qmfOqDGj_5j_fc-B~P@d4yWzByb@2my< zltDo{bhPue$F)$@rA6YL|k!~$4-;Tbx AFK$6JRTc}H%qoRmkmR}j)?*ZAOcH(O#+6jL6KTfT0LDgb`O0S zs%pR<&Ez5W3U4HS!D=P@k%zoQ%4>c|zvhX*kO;^*)gEjIgR)9yRHsi>cb%#_m+zd) zrlzU}{FeUm_qA}=F#ds^gFg#8SE0#mAk1JSHWK0yf;^3BVtQs`d6w>*amg#exE0$; z*>m(*DXt_{ubR}nT2lAwdd!Zel7`nvroCz1FUQSf#+yl6UJLpTtHiU(tT&s?d2__L zWw0u%Z5gb_XE#mn7&BLlcKrqH#%Png+z*6`g1Aj{=Wbs`SsKK7>HeS}^JREl^TUBu zS>k7X**5c&cdlLk?8f~Y{>p=U_wIgv|Hd!MKfZnY&b>ec39K&iV|^}?0qg^g7$kliY-9uFzaV&Unm6v;yB(!G5WnFf z<#9ec>E8}ExVWCB-DovLB^~@#pmPQY9Txn*q`!=_kc ztK`kFDwwbdTP_rR=p44-Y9P`Ok2(1vkL@}%i4MsK2%yU^Jm80oDnUL;}CL36APZeP@*`9PG<#u2v z`?K~Y{cic%(s`FO zPlZA1M+ty)0v_`%7?R%yVw-csBDMvs1nqU5I@Ebpwd#cGr$emQ!;g;&szdVfk`CWP zF@w$UTv67ZB610tt>ZTyKSb4N!Jj&9OjvSTZ=SKcLGlBc8# z5y9w|vrT;m=2qZM6Y>NJhv^`q3=7a4@1e3Q+BE+i&D&H@I3 zzi>Iu!XTDcfGOl@hw!H}pU zL0=YFN#zy`AnAO7eN4Bx38e?rHhVP%K$I>5KFx{Bt6`R^XmyYcWNrfwDZ6b77iM|Y zb-ySVIP>gvA1(OYEMn@rDAkfY?jEq|loCNIyO~J*C;p?G@QFk@4bmhw{4GL>{i8TdFD3nFh9p1!G#V;wXO6+eLa z;u9e4k~oF@zSi5=Mfu_*ByKK*jv$!-mfmeItuL8{a zDU@+}35tqv%_~C{0vSo=WvBuIr9@6umYa~%awiCtHm|2S%m7}w3n8YK&U;R0j}!AK zA464~#zpFr+Az#w={5a*Q|MyRbHEAo4v>?YYg=Lw=buFKV1KJ3441yKPc3Vw&#g`r$PgiW@vK)DZz-o1f1m3t9ShUjM=Vhr}0`oR)%}IRa;MbWhi9youi~5 zh4nSWoCjs>xxfTcR(X;Xc=K&|=~+;k`0d&OW4ky9O{ek0Gr>2I$ej3|weFu|UIlL! z#fo-;e{aQ7(C^0^ZxR6r)A(XrEz+VNfHzd3QH?H0?SDDtDyh<=v-Tlb;yWO$-O3$b zyQ%NzRo{oQd=PWs>potuVhkqe)xtQC60S#U852Ln6CFn~jRf5WJSfXzfw?9(r4j;t zRh+{vP1D@Iqs58~FyPrm!6$x!pX?)t<}eflrq&iTiJ;>clr{-9DF7+BJgL8KeMC=N zFKy^ql)iFI`qDBRFZHbU@x0OWT-p{|8EG^zIBw^yx*~~k6lo0KuctkgXr0cHrrD%TbI$xXr)7Wv literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/kernels/__pycache__/marlin.cpython-310.pyc b/vllm/model_executor/layers/quantization/kernels/__pycache__/marlin.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b226a8ec60cbb29e3caf55429c966c36b270e5ae GIT binary patch literal 4209 zcmb_fOK%*<5uWaO?CgWf<%)VxwvvgHAYMn}T8UvNjv`2QEX4^$Q4hnwz(Hd;-6Ur@ zFLlq7;$nF*2}nBV7X$&MqYn8C`5|-2B`4i|GAt`qJxgk7Q2`QU7ya(8?yj%u>zZt4 zreVPE;@|(V@y}C+@h@yl|14-+fu!~!;|3?Okr1CG)Tad7G&X$`N;9?++jn%I71t8i zcayqbPa1wBY5L8i<+qXUpC-eS%a?(FZ zjOzw>d41pDbuqtZ`lmR(WAqx&;Uq?nmhR2Gh_W<@OZ)CHkHvD2luZ^6Rgonu%T=#l zzJ25B?WNC`*_|&|R&L(D``I7Z_1ibUSYdZQ_dmN+o(b|i-eK!`agQZ|jH8r65`iq= z3O7W!dC;kbd7jBaF|`roqU=oOMv85+wP75Wrzbs0uql{G@&bgih~FIM= z;V$&od1K%5XX`GXJsfGmj1F(XdMFpb?66#O`3!IG+x{GP!4NuS_l8F8$WzF=;q|l-{f%SCx`3lsZ8DKYN7%K_(Z|LmXzx0bF@8?l|p$O!YXrKK}tz*@%&|S=T$}Wgx z37A_LcM{xN-SNC*?&^IR#j&?4ynu6)wG8S4Qov2H~bi`L$8&7J7?^-r~JGy4ZW; zDb7!wP5YJ?Nb#{ZYF(GvF!xk6LIEEhBcM9l^!oL~C90AlJAjGQ4(OK65vYe%>X9=r zP&T(kw7yaKblK3W8Ez;MQ(40)r@)ts2yKc@CRcn8yyGS5n-jRoniTi zk>Witp|Pq{pp3$JNXN#2?9l-Q^E3wLBjcelusB)363lnO*fe*|!hu$8Z0^Av<`#85 z2Y6*nbx#9UHn}l0cHRplxfC4LmYwWwb_5%YX zBhR_J&f$J!Z0mgv_CGOpu+0r{qUN%;6m2u5T|Fk&HXG%~@|q?O?gy>j;5n9Aos!F`Jj<4c-+NX8{LaTf7u!@Mx*a zP^rYdb})ol9}G#hFvh?_dsNl_WPD8u01?QaLNXs)pipxFFbwP`&Qs$d8Pq_n?!*9G z4j@CjK4@_3u?B++ z$l9a>{%C0rbcp+%Wsk1^@30-&<2CFY*iU134JvQI&U|bla)6ul#0K09cn>P4btahA z05NnvNIyi3%3pwF`7UI@RqtW_mzbTy4p*MX`XKLmHd{x7LWzDI2|!n63q^F;D-?B@C*Q~AHC3HOV!$Tf z!kKU?r5E;UrKLo?R@vN#IM$NcHpAs{EMhG|{u&p4fEijP0HhVHiA39~Hq&$eksy8r z3l)lJ6Nk)G(7Z{B^Ri((fPq6CN)d~fXQ)fM6ndcDZM12t(W%dq(fj|cVN2Yvv{s`) zg-C=4KU4wQkq(NMpaSMC`-P*1Tltjn&*csQxE`~j5)?;g6L(*og zOMGwiD+ILF0+Y6;NvKuYnuJQHdYt{%!6S{)J%Cre@1wk8WQ0oo7Bb)7##qBYQ&E_v z2p}qiqQyF9BX;XYveMVJ#svKr24>HH<5^7P1AgVt^4HNdVpvU>< z*a+f?DETuG)9aQl)3(c4*bi$_mPn3skQ6eQ zYf#!D6bu~xFWaQ`^3+*+)q3GT%cAsq*Q75jv;9J^>YXkfz#HCBeHeW>`A3lATlhgC z(R$?tY=4aH#iJh>i`56l;`9f`;;@KfwU|_2u)0wNjT(1+hBf04n6&I u$8Dr-2R|jJco=V7sTA*joMKA%QU%aui0ur8uGcYK+M?%4$L^T(=Dz`fAYdr~ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/kernels/machete.py b/vllm/model_executor/layers/quantization/kernels/machete.py new file mode 100644 index 00000000..fa39cb51 --- /dev/null +++ b/vllm/model_executor/layers/quantization/kernels/machete.py @@ -0,0 +1,118 @@ +from functools import partial +from typing import Optional, Tuple + +import torch + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.quantization.utils.machete_utils import ( + MACHETE_SUPPORTED_GROUP_SIZES, check_machete_supports_shape, + query_machete_supported_quant_types) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + pack_weights_into_int32, unpack_weights_into_int32) +from vllm.model_executor.parameter import (BasevLLMParameter, + permute_param_layout_) + +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig + + +class MacheteLinearKernel(MPLinearKernel): + + @classmethod + def get_min_capability(cls) -> int: + return 90 + + @classmethod + def can_implement(cls, + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]: + if c.has_g_idx and\ + c.partition_weight_shape[0] != c.full_weight_shape[0]: + return False, "Act reordering currently not supported by Machete, "\ + "when the input features are partitioned across "\ + "devices" + + if c.zero_points: + return False, "Zero points currently not supported by "\ + " Compressed Tensors + Machete. (Kernel supports it"\ + " but CompressedTensorsWNA16 does not so support has"\ + " not been added to MacheteWNA16Kernel yet" + + if c.weight_type not in query_machete_supported_quant_types( + c.zero_points): + return False, f"Quant type ({c.weight_type}) not supported by "\ + "Machete, supported types are: "\ + f"{query_machete_supported_quant_types(c.zero_points)}" + + if c.group_size not in MACHETE_SUPPORTED_GROUP_SIZES: + return False, f"Group size ({c.group_size}) not supported by "\ + "Machete, supported group sizes are: "\ + f"{MACHETE_SUPPORTED_GROUP_SIZES}" + + return check_machete_supports_shape(c.partition_weight_shape[0], + c.partition_weight_shape[1]) + + # note assumes that + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} + # `weight_scale` is: {input_dim = 0, output_dim = 1} + def process_weights_after_loading(self, layer: torch.nn.Module): + c = self.config + + if c.has_g_idx: + assert self.w_gidx_name is not None + perm = torch.argsort(getattr(layer, self.w_gidx_name))\ + .to(torch.int) + + self.act_perm = lambda x: x[:, perm] + # use `ops.permute_cols` if possible + if c.act_type in [torch.float16, torch.bfloat16] \ + and c.partition_weight_shape[0] % 8 == 0: + self.act_perm = partial(ops.permute_cols, perm=perm) + + def transform_w_q(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + if c.has_g_idx: + x_unpacked = unpack_weights_into_int32(x.data, + c.weight_type, + packed_dim=0) + x_perm = x_unpacked[perm, :] + x.data = pack_weights_into_int32(x_perm, + c.weight_type, + packed_dim=0) + x.data = ops.machete_prepack_B(x.data.t().contiguous().t(), + self.config.weight_type) + return x + + def transform_w_s(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1) + x.data = x.data.contiguous() + return x + + # Repack weights and scales for Machete + self._transform_param(layer, self.w_q_name, transform_w_q) + self._transform_param(layer, self.w_s_name, transform_w_s) + + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + c = self.config + w_q, w_s, _, _ = self._get_weight_params(layer) + + x_2d = x.reshape(-1, x.shape[-1]) + out_shape = x.shape[:-1] + (c.partition_weight_shape[1], ) + + if c.has_g_idx: + x_2d = self.act_perm(x_2d) + + output = ops.machete_gemm(a=x_2d, + b_q=w_q, + b_type=c.weight_type, + b_zeros=None, + b_scales=w_s, + b_group_size=c.group_size) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/kernels/marlin.py b/vllm/model_executor/layers/quantization/kernels/marlin.py new file mode 100644 index 00000000..6969583d --- /dev/null +++ b/vllm/model_executor/layers/quantization/kernels/marlin.py @@ -0,0 +1,133 @@ +from typing import Optional, Tuple + +import torch + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + MARLIN_SUPPORTED_GROUP_SIZES, apply_gptq_marlin_linear, + check_marlin_supports_shape, marlin_is_k_full, marlin_make_empty_g_idx, + marlin_make_workspace, marlin_permute_scales, marlin_sort_g_idx, + query_marlin_supported_quant_types) +from vllm.model_executor.parameter import (BasevLLMParameter, + permute_param_layout_) + +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig + + +class MarlinLinearKernel(MPLinearKernel): + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def can_implement(cls, + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]: + if c.zero_points: + return False, "Zero points currently not supported by "\ + " MarlinLinearKernel. Will be added when AWQMarlin "\ + "is migrated over to using MPLinearKernel backend" + + quant_types = query_marlin_supported_quant_types(c.zero_points) + if c.weight_type not in quant_types: + return False, f"Quant type ({c.weight_type}) not supported by"\ + f" Marlin, supported types are: {quant_types}" + + if c.group_size not in MARLIN_SUPPORTED_GROUP_SIZES: + return False, f"Group size ({c.group_size}) not supported by "\ + "Marlin, supported group sizes are: "\ + f"{MARLIN_SUPPORTED_GROUP_SIZES}" + + return check_marlin_supports_shape( + c.partition_weight_shape[1], # out_features + c.partition_weight_shape[0], # in_features + c.full_weight_shape[0], # in_features + c.group_size) + + # note assumes that + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} + # `weight_scale` is: {input_dim = 0, output_dim = 1} + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + device = getattr(layer, self.w_q_name).device + c = self.config + + row_parallel = (c.partition_weight_shape[0] != c.full_weight_shape[0]) + self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel) + + # Allocate marlin workspace. + self.workspace = marlin_make_workspace(c.partition_weight_shape[1], + device) + + # Default names since marlin requires empty parameters for these, + # TODO: remove this requirement from marlin (allow optional tensors) + if self.w_gidx_name is None: + self.w_gidx_name = "g_idx" + if self.w_zp_name is None: + self.w_zp_name = "w_zp" + + if c.has_g_idx: + g_idx, g_idx_sort_indices = marlin_sort_g_idx( + getattr(layer, self.w_gidx_name)) + self._transform_param(layer, self.w_gidx_name, lambda _: g_idx) + layer.g_idx_sort_indices = g_idx_sort_indices + else: + setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device)) + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device) + + if c.zero_points: + pass + # TODO (lucas): add the following when AWQMarlin is migrated over to + # using MPLinearKernel backend + # self._transform_param(layer, self.w_zp_name, lambda x: \ + # marlin_zero_points( + # x, + # size_k=c.partition_weight_shape[0], + # size_n=c.partition_weight_shape[1], + # num_bits=c.weight_type.size_bits)) + else: + setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device)) + + def transform_w_q(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + x.data = ops.gptq_marlin_repack(x.data.contiguous(), + perm=layer.g_idx_sort_indices, + size_k=c.partition_weight_shape[0], + size_n=c.partition_weight_shape[1], + num_bits=c.weight_type.size_bits) + return x + + def transform_w_s(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1) + x.data = marlin_permute_scales(x.data.contiguous(), + size_k=c.partition_weight_shape[0], + size_n=c.partition_weight_shape[1], + group_size=c.group_size) + return x + + self._transform_param(layer, self.w_q_name, transform_w_q) + self._transform_param(layer, self.w_s_name, transform_w_s) + + def apply_weights(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + c = self.config + w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer) + + # `process_weights_after_loading` will ensure w_zp and w_gidx are not + # None for marlin + return apply_gptq_marlin_linear( + input=x, + weight=w_q, + weight_scale=w_s, + weight_zp=w_zp, # type: ignore + g_idx=w_gidx, # type: ignore + g_idx_sort_indices=layer.g_idx_sort_indices, + workspace=self.workspace, + wtype=c.weight_type, + input_size_per_partition=c.partition_weight_shape[0], + output_size_per_partition=c.partition_weight_shape[1], + is_k_full=self.is_k_full, + bias=bias) diff --git a/vllm/model_executor/layers/quantization/kv_cache.py b/vllm/model_executor/layers/quantization/kv_cache.py new file mode 100644 index 00000000..d79536d1 --- /dev/null +++ b/vllm/model_executor/layers/quantization/kv_cache.py @@ -0,0 +1,76 @@ +import torch + +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.utils import print_warning_once + + +class BaseKVCacheMethod(QuantizeMethodBase): + """ + Quant method that adds `_k_scale` and `_v_scale` attributes to the + Attention layer to support loading those scaling factors from checkpoints. + The k/v_scale will be used to: + - quantize k/v_cache entries before saving them to the cache + - dequantize k/v_cache entries before fetching them from the cache + + :param quant_config: the appropriate QuantizationConfig + """ + + def __init__(self, quant_config: QuantizationConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module): + """ + Create "weight" (aka k_scale and v_scale) for an attention layer. + """ + # Initialize the KV cache scales to -1.0, which is an invalid value. + # If the k/v_scale appears in the checkpoint, it will be + # overwritten when loading weights. + layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), + requires_grad=False) + layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), + requires_grad=False) + + def apply(self, layer: torch.nn.Module) -> torch.Tensor: + raise RuntimeError( + f"{self.__class__.__name__}.apply should not be called.") + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0 + # regardless whether the kv-scale is available in the checkpoint. + if layer.kv_cache_dtype != "auto": + if layer.k_scale > 0.0 and layer.v_scale > 0.0: + # We prefer to use separate k_scale and v_scale if present + k_scale = layer.k_scale.to("cpu").tolist() + v_scale = layer.v_scale.to("cpu").tolist() + elif layer.k_scale < 0.0 and layer.v_scale < 0.0: + # If no scales were loaded (both scales are invalid negative + # values), use the default value of 1.0 + k_scale = 1.0 + v_scale = 1.0 + else: + # If we find a single kv_scale in the checkpoint, we remap + # kv_scale to k_scale during weight loading, and duplicate + # k_scale to v_scale here + assert layer.k_scale > 0.0 + scale_to_duplicate = max(layer.k_scale, layer.v_scale) + k_scale = scale_to_duplicate.to("cpu").tolist() + v_scale = scale_to_duplicate.to("cpu").tolist() + + if not isinstance(k_scale, float) or not isinstance( + v_scale, float): + raise ValueError("Only support per-tensor scaling factor " + "for fp8 KV cache") + + # These are used in the final Attention.forward() + layer._k_scale = k_scale + layer._v_scale = v_scale + if (layer._k_scale == 1.0 and layer._v_scale == 1.0 + and "e5m2" not in layer.kv_cache_dtype): + print_warning_once( + "Using KV cache scaling factor 1.0 for fp8_e4m3. This " + "may cause accuracy issues. Please make sure k/v_scale " + "scaling factors are available in the fp8 checkpoint.") + + del layer.k_scale + del layer.v_scale diff --git a/vllm/model_executor/layers/quantization/marlin.py b/vllm/model_executor/layers/quantization/marlin.py new file mode 100644 index 00000000..c655a76f --- /dev/null +++ b/vllm/model_executor/layers/quantization/marlin.py @@ -0,0 +1,260 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter) + +logger = init_logger(__name__) + + +class MarlinConfig(QuantizationConfig): + """Config class for Marlin. + + Reference: https://github.com/IST-DASLab/marlin/tree/master + """ + + def __init__( + self, + group_size: int, + lm_head_quantized: bool, + ) -> None: + # Group size for the quantization. + self.group_size = group_size + self.lm_head_quantized = lm_head_quantized + if self.group_size != 128 and self.group_size != -1: + raise ValueError( + "Currently, only group size 128 and -1 (channelwise) " + "is supported for Marlin, but got group_size of " + f"{self.group_size}") + + # 4 Bits packed into 32 bit datatype. + self.pack_factor = 32 // 4 + + # Tile size used by marlin kernels. + self.tile_size = 16 + + # Min out_features dim + self.min_n_threads = 64 + + # Min in_features dim + self.min_k_threads = 128 + + # Max parallel problems to solve at once (improves large + # batch performance) + self.max_parallel = 16 + + # Permutation length used by the marlin kernels. + self.perm_len = 1024 + + def __repr__(self) -> str: + return (f"MarlinConfig(group_size={self.group_size}, " + f"lm_head_quantized={self.lm_head_quantized})") + + @classmethod + def get_name(cls) -> str: + return "marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "MarlinConfig": + group_size = cls.get_from_keys(config, ["group_size"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], + default=False) + return cls(group_size, lm_head_quantized) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, + user_quant) -> Optional[str]: + # compat: autogptq >=0.8.0 use checkpoint_format: str + # compat: autogptq <=0.7.1 is_marlin_format: bool + is_marlin_format = (hf_quant_cfg.get("checkpoint_format") == "marlin" + or hf_quant_cfg.get("is_marlin_format", False)) + + is_valid_user_quant = (user_quant is None or user_quant == "gptq" + or user_quant == "marlin") + + if is_marlin_format and is_valid_user_quant: + msg = ("The model is serialized in {} format. Using {} kernel.". + format(cls.get_name(), cls.get_name())) + logger.info(msg) + return cls.get_name() + + return None + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["MarlinLinearMethod"]: + if (isinstance(layer, LinearBase) or + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)): + return MarlinLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class MarlinLinearMethod(LinearMethodBase): + """Linear method for Marlin. + + Args: + quant_config: The Marlin quantization config. + """ + + def __init__(self, quant_config: MarlinConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del output_size # Unused. + weight_loader = extra_weight_attrs["weight_loader"] + + if params_dtype != torch.float16: + raise ValueError( + f"The params dtype must be float16, but got {params_dtype}") + + # Validate output_size_per_partition + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.min_n_threads != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"min_n_threads = {self.quant_config.min_n_threads}.") + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"pack_factor = {self.quant_config.pack_factor}.") + + # Validate input_size_per_partition + if input_size_per_partition % self.quant_config.min_k_threads != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"min_k_threads = {self.quant_config.min_k_threads}.") + if (self.quant_config.group_size != -1 and + input_size_per_partition % self.quant_config.group_size != 0): + raise ValueError(f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"group_size = {self.quant_config.group_size}.") + + # Check that we have at least 4 tiles horizontally in the shard + num_tiles_per_perm = self.quant_config.perm_len // ( + self.quant_config.tile_size**2) + if output_size_per_partition % num_tiles_per_perm != 0: + raise ValueError( + "Each permutation group must reside on the same gpu") + + # Quantized 4Bit weights packed into Int32. + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.tile_size, + output_size_per_partition * self.quant_config.tile_size // + self.quant_config.pack_factor, + device="cuda", + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + marlin_tile_size=self.quant_config.tile_size, + weight_loader=weight_loader) + + # Determine if channelwise or not + input_groups = (1 if self.quant_config.group_size == -1 else + input_size_per_partition // + self.quant_config.group_size) + + weight_scale_args = { + "data": + torch.empty( + input_groups, + output_size_per_partition, + device="cuda", + dtype=params_dtype, + ), + "weight_loader": + weight_loader + } + if input_groups == 1: + scales = ChannelQuantScaleParameter(output_dim=1, + **weight_scale_args) + else: + scales = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **weight_scale_args) + + # Allocate workspace (Used for internal locking mechanism) + max_workspace_size = ( + output_size_per_partition // + self.quant_config.min_n_threads) * self.quant_config.max_parallel + + workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size, + device="cuda", + dtype=torch.int), + weight_loader=weight_loader) + + layer.register_parameter("B", qweight) + layer.register_parameter("s", scales) + layer.register_parameter("workspace", workspace) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # required by torch.compile + layer.B = Parameter(layer.B.data, requires_grad=False) + layer.s = Parameter(layer.s.data, requires_grad=False) + layer.workspace = Parameter(layer.workspace.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qweight = layer.B + scales = layer.s + workspace = layer.workspace + + x_2d = x.view(-1, x.shape[-1]) + + size_m = x_2d.shape[0] + size_k = x_2d.shape[1] + size_n = scales.shape[1] + + output_2d = ops.marlin_gemm(x_2d, qweight, scales, workspace, size_m, + size_n, size_k) + + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], )) + + if bias is not None: + output.add_(bias) # In-place add + + return output diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py new file mode 100644 index 00000000..dc5f47eb --- /dev/null +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -0,0 +1,163 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase) +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod +from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( + apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale) +from vllm.model_executor.parameter import (ModelWeightParameter, + PerTensorScaleParameter) + +logger = init_logger(__name__) + +ACTIVATION_SCHEMES = ["static"] + + +class ModelOptFp8Config(QuantizationConfig): + """Config class for ModelOpt FP8.""" + + def __init__( + self, + is_checkpoint_fp8_serialized: bool = False, + ) -> None: + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + if is_checkpoint_fp8_serialized: + logger.warning("Detected ModelOpt fp8 checkpoint. Please note that" + " the format is experimental and could change.") + + @classmethod + def get_name(cls) -> str: + return "modelopt" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 89 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["hf_quant_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "ModelOptFp8Config": + quant_config = cls.get_from_keys(config, ["quantization"]) + quant_method = quant_config["quant_algo"] + is_checkpoint_fp8_serialized = ("FP8" in quant_method) + if not is_checkpoint_fp8_serialized: + raise ValueError("ModelOpt currently only supports static FP8" + "quantization in vLLM. Please check the " + "`hf_quant_config.json` file for your model's " + "quant configuration.") + return cls(is_checkpoint_fp8_serialized) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QuantizeMethodBase"]: + from vllm.attention.layer import Attention # Avoid circular import + if isinstance(layer, LinearBase): + return ModelOptFp8LinearMethod(self) + elif isinstance(layer, Attention): + return ModelOptFp8KVCacheMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class ModelOptFp8KVCacheMethod(BaseKVCacheMethod): + """ + Supports loading kv-cache scaling factors from FP8 checkpoints. + """ + + def __init__(self, quant_config: ModelOptFp8Config): + super().__init__(quant_config) + + +class ModelOptFp8LinearMethod(LinearMethodBase): + """Linear method for Model Optimizer static quantization. + Supports loading FP8 checkpoints with static weight scale and + activation scale. Future support might be added for dynamic + scales. + + Limitations: + 1. Only support per-tensor quantization due to torch._scaled_mm support. + 2. Only support float8_e4m3fn datatype + Args: quant_config: The ModelOpt quantization config. + """ + + def __init__(self, quant_config: ModelOptFp8Config): + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del input_size, output_size + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + weight_dtype = (torch.float8_e4m3fn + if self.quant_config.is_checkpoint_fp8_serialized else + params_dtype) + weight = ModelWeightParameter(data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=weight_dtype), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + if self.quant_config.is_checkpoint_fp8_serialized: + # WEIGHT SCALE + weight_scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + # INPUT SCALE + scale = PerTensorScaleParameter(data=torch.empty( + len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader) + + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", scale) + + def process_weights_after_loading(self, layer: Module) -> None: + max_w_scale, weight = requantize_with_max_scale( + layer.weight, layer.weight_scale, layer.logical_widths) + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(max_w_scale, requires_grad=False) + layer.input_scale = Parameter(layer.input_scale.max(), + requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported) diff --git a/vllm/model_executor/layers/quantization/neuron_quant.py b/vllm/model_executor/layers/quantization/neuron_quant.py new file mode 100644 index 00000000..2624981f --- /dev/null +++ b/vllm/model_executor/layers/quantization/neuron_quant.py @@ -0,0 +1,67 @@ +import os +from importlib.util import find_spec +from typing import Any, Dict, List, Optional + +from torch.nn import Module + +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) + +SUPPORTED_QUANT_DTYPE_LIST = ['s8', 'f8e4m3fn'] + + +class NeuronQuantConfig(QuantizationConfig): + """Int8 Quantization Config class for Neuron Backend.""" + + def __init__( + self, + dequant_dtype: str = "f16", + quantize_method: str = "vector_dynamic", + ) -> None: + self.quant_dtype = os.getenv("NEURON_QUANT_DTYPE", "s8") + if self.quant_dtype not in SUPPORTED_QUANT_DTYPE_LIST: + raise ValueError( + f"Neuron quantization datatype {self.quant_dtype} is not valid," + f"the quantization datatype should match one of the below types" + f"{SUPPORTED_QUANT_DTYPE_LIST}") + self.dequant_dtype = dequant_dtype + self.quantize_method = quantize_method + + def get_name(self) -> str: + return "neuron_quant" + + def get_supported_act_dtypes(self) -> List[str]: + return SUPPORTED_QUANT_DTYPE_LIST + + @classmethod + def get_min_capability(cls) -> int: + raise NotImplementedError( + "This function should not be called with Neuron Backend") + + @staticmethod + def get_config_filenames() -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "NeuronQuantConfig": + quantize_method = cls.get_from_keys(config, ["quantize_method"]) + dequant_dtype = cls.get_from_keys(config, ["dequant_dtype"]) + return cls(dequant_dtype=dequant_dtype, + quantize_method=quantize_method) + + def get_quant_method(self, layer: Module, prefix: str) -> Optional[Any]: + if find_spec("transformers_neuronx") is not None: + return self.get_quantization_config() + else: + raise NotImplementedError( + "Neuron Quantization is only supported through" + " transformers_neuronx.") + + def get_scaled_act_names(self) -> List[str]: + return [] + + def get_quantization_config(self): + from transformers_neuronx.config import QuantizationConfig + return QuantizationConfig(quant_dtype=self.quant_dtype, + dequant_dtype=self.dequant_dtype, + quantize_method=self.quantize_method) diff --git a/vllm/model_executor/layers/quantization/qqq.py b/vllm/model_executor/layers/quantization/qqq.py new file mode 100644 index 00000000..b88ecac7 --- /dev/null +++ b/vllm/model_executor/layers/quantization/qqq.py @@ -0,0 +1,273 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.parameter import (BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter) + +logger = init_logger(__name__) + +MARLIN_QQQ_TILE = 16 +MARLIN_QQQ_MIN_THREAD_N = 64 +MARLIN_QQQ_MIN_THREAD_K = 128 +MARLIN_QQQ_MAX_PARALLEL = 16 + +MARLIN_QQQ_SUPPORTED_NUM_BITS = [4] +MARLIN_QQQ_SUPPORTED_GROUP_SIZES = [-1, 128] +MARLIN_QQQ_SUPPORTED_SYM = [True] + + +class QQQConfig(QuantizationConfig): + """Config class for QQQ + + Reference: https://arxiv.org/pdf/2406.09904 + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + is_sym: bool = True, + ) -> None: + self.weight_bits = weight_bits + self.group_size = group_size + self.is_sym = is_sym + + # Verify + if self.weight_bits not in MARLIN_QQQ_SUPPORTED_NUM_BITS: + raise ValueError( + f"QQQ does not support weight_bits = {self.weight_bits}. " + f"Only weight_bits = {MARLIN_QQQ_SUPPORTED_NUM_BITS} " + "are supported.") + if self.group_size not in MARLIN_QQQ_SUPPORTED_GROUP_SIZES: + raise ValueError( + f"QQQ does not support group_size = {self.group_size}. " + f"Only group_sizes = {MARLIN_QQQ_SUPPORTED_GROUP_SIZES} " + "are supported.") + if self.is_sym not in MARLIN_QQQ_SUPPORTED_SYM: + raise ValueError( + f"QQQ does not support is_sym = {self.is_sym}. " + f"Only sym = {MARLIN_QQQ_SUPPORTED_SYM} are supported.") + + # 4 Bits packed into 32 bit datatype. + self.pack_factor = 32 // self.weight_bits + + # Tile size used by QQQ kernels. + self.tile_size = MARLIN_QQQ_TILE + + # Min out_features dim + self.min_n_threads = MARLIN_QQQ_MIN_THREAD_N + + # Min in_features dim + self.min_k_threads = MARLIN_QQQ_MIN_THREAD_K + + # Max parallel problems to solve at once (improves large + # batch performance) + self.max_parallel = MARLIN_QQQ_MAX_PARALLEL + + # Permutation length used by the QQQ kernels. + self.perm_len = 1024 + + def __repr__(self) -> str: + return "QQQConfig(weight_bits={}, group_size={})".format( + self.weight_bits, self.group_size) + + @classmethod + def get_name(cls) -> str: + return "qqq" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + """List of filenames to search for in the model directory.""" + return [ + "quant_config.json", + "quantize_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "QQQConfig": + weight_bits = cls.get_from_keys(config, ["wbits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + return cls(weight_bits, group_size) + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["QQQLinearMethod"]: + if isinstance(layer, LinearBase): + return QQQLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class QQQLinearMethod(LinearMethodBase): + """Linear method for QQQ. + + Args: + quant_config: The QQQ quantization config. + """ + + def __init__(self, quant_config: QQQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + weight_loader = extra_weight_attrs["weight_loader"] + if params_dtype != torch.float16: + raise ValueError( + f"The params dtype must be float16, but got {params_dtype}") + + # Validate output_size_per_partition + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.min_n_threads != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"min_n_threads = {self.quant_config.min_n_threads}.") + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"pack_factor = {self.quant_config.pack_factor}.") + + # Validate input_size_per_partition + if input_size_per_partition % self.quant_config.min_k_threads != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"min_k_threads = {self.quant_config.min_k_threads}.") + if (self.quant_config.group_size != -1 and + input_size_per_partition % self.quant_config.group_size != 0): + raise ValueError(f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"group_size = {self.quant_config.group_size}.") + + # Check that we have at least 4 tiles horizontally in the shard + num_tiles_per_perm = self.quant_config.perm_len // ( + self.quant_config.tile_size**2) + if output_size_per_partition % num_tiles_per_perm != 0: + raise ValueError( + "Each permutation group must reside on the same gpu") + + # Quantized 4Bit weights packed into Int32. + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.tile_size, + output_size_per_partition * self.quant_config.tile_size // + self.quant_config.pack_factor, + device="cuda", + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + marlin_tile_size=self.quant_config.tile_size, + weight_loader=weight_loader) + + s_channel = ChannelQuantScaleParameter(data=torch.empty( + 1, + output_size_per_partition, + device="cuda", + dtype=torch.float, + ), + weight_loader=weight_loader, + output_dim=1) + + if self.quant_config.group_size == -1: + s_group_data = torch.tensor( + [], + device="cuda", + dtype=torch.half, + ) + else: + s_group_data = torch.empty( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition, + device="cuda", + dtype=torch.half, + ) + + s_group_attr = {"data": s_group_data, "weight_loader": weight_loader} + + if self.quant_config.group_size == -1: + s_group = BasevLLMParameter(**s_group_attr) + else: + s_group = GroupQuantScaleParameter(output_dim=1, + input_dim=0, + **s_group_attr) + + # Allocate workspace (Used for internal locking mechanism) + max_workspace_size = ( + output_size_per_partition // + self.quant_config.min_n_threads) * self.quant_config.max_parallel + + workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size, + device="cuda", + dtype=torch.int), + weight_loader=weight_loader) + + layer.register_parameter("B", qweight) + layer.register_parameter("s_channel", s_channel) + layer.register_parameter("s_group", s_group) + layer.register_parameter("workspace", workspace) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # required by torch.compile + layer.B = Parameter(layer.B.data, requires_grad=False) + layer.s_channel = Parameter(layer.s_channel.data, requires_grad=False) + layer.s_group = Parameter(layer.s_group.data, requires_grad=False) + layer.workspace = Parameter(layer.workspace.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qweight = layer.B + s_ch = layer.s_channel + s_group = layer.s_group + workspace = layer.workspace + + x_2d = x.view(-1, x.shape[-1]) + + size_m = x_2d.shape[0] + size_k = x_2d.shape[1] + size_n = s_ch.shape[1] + + x_int8, s_tok, _ = ops.scaled_int8_quant(x_2d) + + output_2d = ops.marlin_qqq_gemm(x_int8, qweight, s_tok, s_ch, s_group, + workspace, size_m, size_n, size_k) + + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], )) + + if bias is not None: + output.add_(bias) # In-place add + + return output diff --git a/vllm/model_executor/layers/quantization/schema.py b/vllm/model_executor/layers/quantization/schema.py new file mode 100644 index 00000000..a26c5247 --- /dev/null +++ b/vllm/model_executor/layers/quantization/schema.py @@ -0,0 +1,84 @@ +""" +This file contains the Pydantic schemas for various quantization-related +parameters. When a relevant quantization technique is specified, these +parameters are loaded in the form of a JSON alongside the model weights +and augment the model with additional information needed for use of that +technique. The format of this JSON should be specified by one or more +schemas contained here. + +For example, when the KV cache is quantized to FP8-E4M3 (currently only +possible on ROCm), the model can be optionally augmented with KV cache +scaling factors. +""" + +from typing import Dict, Optional + +from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator + + +class KVCacheQuantSchema(BaseModel): + dtype: str + # Each key is a TP rank. Each value is a dictionary mapping a TP rank's + # layer indices to their per-tensor KV cache scaling factor. + # TODO: Consider pulling this and its validation methods out into its + # own schema class (tricky as its members are variable) + scaling_factor: Dict[int, Dict[int, float]] + + @model_validator(mode="after") + def check_is_fp8(self) -> "KVCacheQuantSchema": + assert self.dtype == "float8_e4m3fn", ( + "Loaded scaling factors intended for KV cache dtype = " + f"{self.dtype} rather than float8_e4m3fn!") + return self + + @model_validator(mode="after") + def check_tp_ranks(self, info: ValidationInfo) -> "KVCacheQuantSchema": + context = info.context + if context: + tp_size = context["tp_size"] + num_hidden_layers = context["num_hidden_layers"] + assert len(self.scaling_factor) == tp_size, ( + f"Loaded dictionary has TP size {len(self.scaling_factor)} " + f"but LLM engine is currently running with TP size {tp_size}.") + for tp_rank, layer_maps in self.scaling_factor.items(): + assert len(layer_maps) == num_hidden_layers, ( + f"KV cache scales map for TP rank {tp_rank} is malformed. " + f"Expected {num_hidden_layers} layers, got " + f"{len(layer_maps)}.") + for i in range(tp_size): + assert i in self.scaling_factor, ( + f"KV cache scales map for TP rank {i} not found.") + return self + + @model_validator(mode="after") + def check_current_rank(self, info: ValidationInfo) -> "KVCacheQuantSchema": + context = info.context + if context: + tp_rank = context["tp_rank"] + num_hidden_layers = context["num_hidden_layers"] + layer_scales_map = self.scaling_factor[tp_rank] + for i in range(num_hidden_layers): + assert i in layer_scales_map, ( + f"Could not find KV cache scales for layer {i} in " + f"TP rank {tp_rank}.") + return self + + +class QuantParamSchema(BaseModel): + # TODO: Generalize and extend with more fields + # (e.g. weights/activations params) once functionality is enabled + model_config = ConfigDict(protected_namespaces=()) + model_type: Optional[str] + kv_cache: KVCacheQuantSchema + + @model_validator(mode="after") + def check_model_type(self, info: ValidationInfo) -> "QuantParamSchema": + context = info.context + if context: + model_type = context.get("model_type", None) + if model_type is not None: + assert model_type == self.model_type, ( + f"Model type is {model_type} but loaded " + f"scaling factors belonging to different " + f"model type {self.model_type}!") + return self diff --git a/vllm/model_executor/layers/quantization/tpu_int8.py b/vllm/model_executor/layers/quantization/tpu_int8.py new file mode 100644 index 00000000..be8235b4 --- /dev/null +++ b/vllm/model_executor/layers/quantization/tpu_int8.py @@ -0,0 +1,119 @@ +from typing import Any, Dict, List, Optional, Tuple + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.parameter import ModelWeightParameter + +ACTIVATION_SCHEMES = ["none"] + + +class Int8TpuConfig(QuantizationConfig): + """Int8 Quantization Config class for TPU Backend.""" + + def __init__( + self, + activation_scheme: str = "none", + ) -> None: + if activation_scheme not in ACTIVATION_SCHEMES: + raise ValueError( + f"Unsupported activation scheme {activation_scheme}") + self.activation_scheme = activation_scheme + + def get_name(self) -> str: + return "tpu_int8" + + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + raise NotImplementedError( + "This function should not be called with TPU Backend") + + @staticmethod + def get_config_filenames() -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "Int8TpuConfig": + activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) + return cls(activation_scheme=activation_scheme) + + def get_quant_method(self, layer: Module, + prefix: str) -> Optional["TPUInt8LinearMethod"]: + if isinstance(layer, LinearBase): + return TPUInt8LinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class TPUInt8LinearMethod(LinearMethodBase): + """Int8 Linear method for TPU Quant. """ + + def __init__(self, quant_config: Int8TpuConfig): + self.quant_config = quant_config + + def create_weights(self, layer: Module, input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + + weight_loader = extra_weight_attrs.get("weight_loader") + weight = ModelWeightParameter(data=torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype), + input_dim=1, + output_dim=0, + weight_loader=weight_loader) + layer.register_parameter("weight", weight) + + def _quantize_weight( + self, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + weight_dtype = weight.dtype + weight = weight.cpu().to(torch.float32) + n_bit = 8 + eps = 1e-5 + max_int = 2**(n_bit - 1) - 1 + min_int = -(2**(n_bit - 1)) + max_val = weight.abs().amax(dim=-1, keepdim=True) + max_val = max_val.clamp(min=eps) + qscale = max_val / max_int + qweight = torch.clamp(torch.round(weight * (1.0 / qscale)), min_int, + max_int).to(torch.int8) + qscale = qscale.squeeze().to(weight_dtype) + return qweight, qscale + + def process_weights_after_loading(self, layer: Module) -> None: + layer.weight = Parameter(layer.weight.data, requires_grad=False) + device = layer.weight.device + qweight, qscale = self._quantize_weight(layer.weight) + qweight = qweight.to(device) + qscale = qscale.to(device) + layer.weight = Parameter(qweight, requires_grad=False) + layer.scale = Parameter(qscale, requires_grad=False) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + try: + import torch_xla.experimental.xla_quantized_matmul # noqa: F401 + except ImportError as err: + raise ImportError( + "Please install torch_xla by following the instructions at " + "https://docs.vllm.ai/en/latest/getting_started/tpu-installation.html " # noqa: E501 + "to run vLLM on TPU.") from err + weight = layer.weight + scale = layer.scale + out = torch.ops.xla.quantized_matmul(x, weight, scale) + if bias is not None: + out = out + bias + return out diff --git a/vllm/model_executor/layers/quantization/utils/__init__.py b/vllm/model_executor/layers/quantization/utils/__init__.py new file mode 100644 index 00000000..e60f0c79 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/__init__.py @@ -0,0 +1,3 @@ +from .layer_utils import replace_parameter, update_tensor_inplace + +__all__ = ['update_tensor_inplace', 'replace_parameter'] diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea126a2b734d5fcd8c493b4a1766972ae400eb41 GIT binary patch literal 300 zcmYjMF;2ul4BTvT0s$vMLB|JD9ILBn5aIi69I4=v5D**|EB)db1HrarTX*$o*Sw>*&hCYGs7D6gTkY*)Z% z5&?Xuw3_KN{5}{C(L^6NY||cja}{zMw&BH)e>8FAw(ra{Q81x*NPw@n%iJfkcnsz# j+m8L4WiPzN(STg#8Q?OG)8VzqGU3hq(NGDTsuTSOm042- literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/layer_utils.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/layer_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a710e9dd5ad6ec6d584db3cec9f3c111e88e07c GIT binary patch literal 1043 zcmY*YJ#W-N5Zw>k=kq1U5kdqK#Ysp=R|G2sqCp4=i6RmaQlN2ToZHQvFaFBxI^iOR zf9b>S?r&f*b=`jM&62ue39rRt~5mMxUA;!hyh2f z;2?Y<(81TQx1r9fuu7|Koxnuu?Zp`7-(#6Ll^9vYufMyMPv(F6L`>Y zw1Can5wo$iqBVhCMV8lY!?7MMvKDBsVY)N(XWa!;k5*B`1QqNvJ7;KG(9r)`Yi?^E z-+M8l(K^WEIt8iX2_$0FGac+~qBR)Dx(D{?L^tZfCIam%A(=?y6$f)I)v5_Ws?tK~ zq!>%%V!?Q7SW%ens|2tyT0*n|a*`^H;_8YUIbg35^eous8eV3iHF^d18kw6QTMPbu zw~4gcKnVCfdWZIT-y5JIed=Gr7Sf}@c&IfkCSXrK*wV;b9nw}V+(O29FH2Ep=o+3s nu>Wj7zP7w|3|bBT6EqoXpHigM0AY>zY>W5%1IjKL?O*-@7}pF} literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/machete_utils.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/machete_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d2369644c0f6fd34c8249000cd44b518955cfb1 GIT binary patch literal 1244 zcma)5&2G~`5caMe$IV|;sER5TDZt4{trdiz2ZUNmpjAr~HR;6{%l2*(Q#*Ed*M&qq zaNx`fAReFxUV#Vkl~Z1UOPO^@6;-OlO7puj_RM^H=JpXO%b=xy4e-eHk`CQit`Pp|=mpC6r6hy0QlR zOl2zs6F^UTzAHa)A27TGWQleY@facsGtWGN=&*#vVF5?lm4@Iu<~#*L{=m^^K#;y)8i+&eZ=_VB~*-hQjsayloi&hFk}>#g(VsJ(aS zboX~VEfcf*@LjelnOvoX!EV3;_-D5P=MEpb30;F&E{9pi8q$$*J(n7055R5KGacSc zFd8s9My5(8hsjoe*>$;$TMKWC735vM{5KVMPdlCVNe?Q1f6_kfINbxg)n)R}ndZfD zn6K9iQ57CWrgFjZpD%Qh_muC6@d|c74e4t+NG=;*RSaCm1~$-|x>hV{HB`I)3xH8K AEC2ui literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c4e2f53f393d7b2096406ebc4dbeae977536456 GIT binary patch literal 8301 zcmb_h%X8dDdIuT}3}!eSjwp%`S&~PzWqWLiv~2BK_R5N7i}cE-9u}#17g+}jPJ_b{ z=cNWl6lrkdSaQ~z#D`Q4Ii#FWTe+|*sZ`~VLk_v-ZwRV#%892OPM`|I!3Uw_jW9L#8V&i>6mF8|9pP5W1>?0$};@($kMnxSbz6M9vv>5g7w z4yzfCp;H^Ha)+z0>6rK$RjZbAY}LoB=~~9gsIpldsAZjOZO|D+-4dy4t~TTh)$&dr zb=x^4($28RIEPUlDQjZj*Sd2=WS!A86F0@6KPGZM^T#K&B`$_U{(-5;7C|*X-^J4cbT)ZMqqxDo@>lrbH))&Myt)+N`uUeOQ z#Ko)PEc#y*uZh+p9(u8-Vo=3S=^5S9=$2vLf^~cM|(zJATG5> zzb!5T;}!8+PsMmgTmr`F{TP`4UGX0J&Ism@E-`Ug%`v0ktb*?=nC$V#s(eMk51=a@ zxqS%E&Otx0{|QZ8ReUD?9~0Nabx3&X|2#WsRA=^sWtMyv|#gE@zyq3TG5F4 zTqOu&`+761H0oY8HWymWs(-Dh#{+Ju6@-nN+h_);=1VOp{d(v&t6sR&khLOdA$*rXObmRdz#Ef^D5y3vY_bMdO+r!kpVT0E)J>@QII< z{m|VHws_u2Q{nvL+1Jh&4S5KNa+qe!_x9$7B!MkW zj-lgQZ#t|sr<;}Lnb4D1_RJf#)^hD^ZFRa@S)6XJh0Bfl^xbN;HeG87zv}w;{8B4y z$my!L=F4FEPRpx@m35EAJKYK^)nK~j$!ewUs>)PzEuOs7^5vS_ZVp<_W2Oe zN@8E45E*+U#v}N2NHCEYR}wQq$SyMOvkf!SBJ-{;p~=FCv_IDbk4!)_(gCf=08B*; z(EfU02{KIdA$TLwLymFpVq99`W7``U_VTmkOfc2ZeegEVTJj zSoJIA<>;Q>P2- zO)9YB)cUEX5wLfzBklEjtqECsshHYg3)hOK(q2h<&0C!5zDL>E_&(YwrB58#k`s zT$s7+zJK%jtsCzA6=!C?Xvw35`3!-sy2?>houQtpsNE}tO~5$%<1x3i?8E%~73(ZI zgDC*p>L?Qf1rJ?hkC^C>z~7O&BGgEK#?hmDGdBKlNjGeOX!BZ`ijmD4@_Q53Xl*Tjx*Y3pCM0qZKw=p|V@I=}Wii$*@A6 zBOb5RpVE`*_3V)i?Vjvhao{cr5qL0)C~gIQFNF71mTQE{e!n(p@32qS#+U>~+nz(R zWZuSr4}jN(B}YaZ^UW@1N?E?0U=kncEvBzYI*itz`&b!n;i-Eh!#h=|B-Sn}cPl|< zvFaBV*9wIi+-0~deNVV`Lc2Ouc&||pD#Diq>5EpWQZE-mzaBIc&%A0Co|wE}0k$t& z^*Z&hR>I}NnKQc`a-rM|?@X;vwn^)8axeF3$>$HGY!_iGgf)2}R?jyO&fL`hBkhPF z#^1lO@R8egB3I!^7Jhtl=F(;NntUF7e`@!KMJ`_if_#ZUk$`eCbn5MLBF7F?;=m1- zy{7-Wzztpo$eO(EIWkYOsb1N(vS_#QT04A)KjFG0W89;tWU6EhN>OJ~SkH8f6~_(N zwU2Sln1?lo`r4$fZJ=d6D`2j4EG@B*q2)h?s%`LJz||z*RehBLhgnHi7@YI8X2Ok> z>jt58U6}F#n~!zB--w?%$OW~x|0$+br!wPkpR~y*+{`^0*X_^Pj+{nsY$XTHzIdr? znFU~xWgf-$Mi487>^5u+kmJ;Zm41ay(N-X6Q;?gNuhP7O(!bNHNI!7P(i7MXNuf{H zmww{jxODT<+}zAuY$7BnEj#AAFB?H@AO-?k6bEQJXSn9wb5|R3B|wbmE7R#`#F6$f zQu9{)9x4S72@4oQP(r7N1+Rv(ZatPiB%;z@DRdhWwi*vr0#`Ox$*(CJ-NP0NA<0uH zJ)i(JLX6T(MOq^j>Di83R%F39zX1e!o_*>g^i zm+Z!_bTAXwS8zf^S~DFPjdTRn+hR#nlB~^>wUx0D0ZtA}m6pAF-LI}zf_)lvvQ3cm zo4yC#*h=?0iHZDncPbFSnK(Bn8z(oS@66h1($o_@an}0=!mkG=O z#HLz$&oU%K1Fuq^gh!itq&?I(6`vmJxN$(X6L=}sOip;7Fe8P)9w$c_GZ+c+0GuN~ z6c4L)T7pl0#mKwF@#lHjcBiiFQPJFthDRrmgv=&TVs5YSd!9^50_}irR`$(h4eWVHP zOKk+i^+?;Mg%Ua_B$U^?RU%7%1n~D%xCwx(BiWHm6QC_^t5TLq6jyK4kZlUcacwqX zmStR>{Cfy01YNXy+lUYK%5aqRaYH$2A){b+i9JL(E3_|A zqa@gdRv+s2BjDTBj|&*}C%qA)o=pSSbhN-&-olcI78I!|oH=y#$41q!#~B3fVWr$^ zv;u_o6b?w2z|RRN5BWP(`Wb;fL+_j?q#CUdXO8lV{{YN@gpx7xdO<(IhH=T20u3cn ziq4Zlq3aTVf=Z;_h7Dm4JcMhE;1W^Cp48VsatjU}2P>Qls3e@4$_mMO5WdwmZIX^^*e2Cb z65A#>yG`5NCY4c!zr_;;nR&`Mj3$g zXr4uY*u)5=Z%j{0Z!J#G+$;Giix*pzp7TO-BX|9(B>yid!i65JP`PI!RjMiznM5hZ zT@J@_)yJ@Y9YBPpvPKlZxy=R;gMY@5fNtFxenPilFE*S?j!nX5a%45@H08h@WbhDh zl~#PQPyQVf3jPA9D(LQT8TVs~gXCyPCn%f&B+W>okhNJKMX!P0Q@eV3w>K4XVQ!jT ziStcMSYawUz~yX`e35M;jJCwn)DmW3!fkWhup&zxV@kHN22ztVNm3~wFa%H>kONc> z3A|6BMSyO;a+N?=aPl5il~jmDkjkgjuvaP$C&oELt-4T2#kN?SbVm-&ehE!@t9M*v zAUg;>LH!%1S3z>lvW-!QX|y9I20M+=`&7wDpGaE*0GVYfpCfRPr=fWc-iutOF%<#_ zxtHTL_?4#;Z|NN^1F>BfW7542Ss^AjFl}tAt8;86$+#*nWLdi9t}|F zg_0KK9_r9U#Mh*7%8SfySx$0v*OEg|Y0(4frl>;x0RVz-rF$g4*g!Vl$y45;qbOy+ zR&%%~gj>wZU(lf6Bk&o4-zP9jfH)^VCvcy@iv-loU9I&6RcWm_(>|YtdoNDoOi3>4 ziSYYQ;Bl|ntggAcX+d7v_v8qPoF*U6jp!C#>~{U=GT-@GkL;n0o?-Hjfn1_vyYmo( zezT0S2gqnp$tc?=qijz`xZ@a*4aq16c_lK+K}H!Rqa0*3xKBn4@|U1SehE+<`aT>! zO6WZfcbO~ykUIW|fHI~&evT#lBui@hfP|fIb=f)iJ=yscEl~g{vRR}~XN#}mzm2sf zf=#JHUW58M0p;z;B{-b^gpejR{)*~3s!w(PEDBWW*~zO$GD}aYe->nl2G}%UAo&A9 z{Vk!6E9Fm}{OZ$(74wL&~3{ZyEQ-K&fb8*-kiC7>%AHG^33d|TXPGr z=SHKdZdFOP+d%q7C013=R^>gD^vpEPMZuF}EvQVM${wg2ySfsqtEIaCsY_LI7z-Li zfsQ)tlq*#ZNhzdMZ0re1l~jOkU%;O=t*U5EfJTG@9sPpd~-t>C28L z;}gv6SZ6kN6e*fVY8{N@7{{>>?DBuO#lGb!_TS8A|HabmKUs$T2OD7jZfDsyY>++T ZgPBL{EJlCxeMj3EjX$Wq;o00b{|hkrnVA3p literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_fp8.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_fp8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fbd93ab7ac0b44e997b0e74c24dc6e490924118f GIT binary patch literal 2821 zcmZuzOK&4L5^k~|YPBR=){O13^VrS;8${T(l^2jXECxXWB)|f32FdI$(w9a{q@|Yo zWj5Q62CYkG_UJwC+0HG8{R?}@&*(!g0T$3d$ZN)_VoSCr2!$?^#Ue#k)%THUr{ic) z-umzVCOaFN_C0piOBXut!vmIh}4WN6D6wZ6!N=lWoAa$qDxNG}guXxXZR!@2TN+ zn|Iht>@MiGraiXJ@31bT#}?aRyH8E;F1x2zsplAMkL|;pZAL%W`Y$iw99o}L_NSQ) z(qi_*ppO z{wT7oVUA5Wt~zyIWLYXC zXKmO>PC+)H=h^VU&T9zN!NQq}cIWeme0uPni3A(Bx8 zmRc2b5q`>FLui!RB_v^~O~Gq>+7+EQR2y{vqI+6t9FtvbZc94V%SK5`TI!6J4Y1a6 zv^1Ge#f=aV4gm04GMn&sZ+nnTAtjy)q_%!hktij%5yhK zr8^1ExSIeLpjxK53sm5K^zgnrN@MV>!ihU9q|&W}#|?9L$blx@BIoST{ig!JL7GIf zoB0s59B}5IhfxH^%F>Pa85hAAXUYk8KT!yCPq;{Ubm%^sz`AiTa~U6tfN>X?1=%S8 zE>e4|_r(t;JUE-V2(Ty(SbkWiEv-x@XOMVSO(S9(V$wIoE>x9)ETvMkPOp2Q{HRa7 zj^L+72%GisFU4J)VpJr*HT9gaNQ=xz5cls011!9~I`@>$hn4*-5Hdtw@$M;7_??kD z%l`<|g@pp4BwoVRdgDx3#H0pnfgG?@4Q`^KxLyzW%4+avq}eLMU$jz zJ`6+H~@0f+WUssv{z; zAA9zOzQU*VY{w7>FtKuwKx$Nukfc@x!(qawK=^-X05~*Bru2m7_TRLxkUMqcPDXbC zo}j(6)B@ZX9V1M21RV0|*W@$p-`bp%sn?# z&_2*^#u_h-*)NPWUl?<649G&Cnz#Gi*T(hkU)bGc^9n)TdiT0&j|9(pq^=V225Rtf5q%Jq^-0Q#h*vsXEh@v+e%DT{o;~VDv;PXu8L)kT zd7nQ3(bfPyv`yNiOKqJh7y)FS8#;Yv81xrv(x0hCf3h3&N5gi0ps%&B0bKAo*ZZ&k E3RfWKoB#j- literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5929a46f3e0499b299431f8303efb1f1708bb5f GIT binary patch literal 4165 zcmZWs&2!tv6~`_<2!aqvNwmJwHmsAT!#0tew3+7MdYrZ%*QqV%!%1f_&R`%eNKpbv zY5`i73S25@+8%P~v6JZx)te8!_Si#@J@$xePx==!Nz~t4kczB=gXQkqw^;0ZzxR6& zqlE=W!}aFh|GE2@mZtre2A3}vgWI^1|DX|?rwJx_#A5F8Sod^BV?8oFQ~g_>g?}Te z#I|Q+&JT$zs#7(amFM5kiyQ>LXRG(?0dcwV>(CIPZ zh}tveEelsHJkz|EsE5~jTr@=Unc=Neb+I89Lu+qUEQPjM4zGz;xF)WJI?2|xo+DPo z>NC?@e^IMvi8Zl~Cu|7)Bdzn&@z%%bAR44c?cQYAO$Vc4(w-!tX!l04orX!8w1c5& z?}WqdZXC${ zj%0DIKTZ$)gGn$<2gjj*7!LZoX;L)*PvRycna=0C^v;(rj=^o*34E@Fn#bT;POgQa zb!8ar{Xo6wZ$~&ZULoun(4AixxeYDT zTUu_UY>#KuVjE$6u4kHW2o0Vvw(ji107&o#M<1AN)rO5^d? zcrbn=4J7>c#?Ru(Zv19^xD^d{w#G;4?r6C6B#Pp#cqGEe51)qJ30`h13XVdVY|UI5 z5U#c+0B5o_!-g981j^0vQDOW3U^qy9e+j!PN7k8vA3rzFUBlw&G54F}wJUDioG;r3 zzVM7;5Qn~BIKGe9nMBlgeL`4N&d7Bx@fMIJcSCTHlVt!HI!mVKP z4|FnZPXVLH0hH={ngWM01I{`DCu3h|LeGpZw7(h(Jb=q=zN6;(m3eJS05bqEBZDse z_l#4PW3MR}^WIzs8ZE8IXY)*$nLgD|l|+ExKI56*{%+J|K5o zeZ39egy=pu6mrifs3e{EOsD1?7ffs>y8!@M0h+1fRc3+jnZrpOM!=qa;2#!N`Q+LE zurNr?GiZw|b}v?DaF{d5j0C=iO%w9R5^LkPp*w7ay~YhLe+c0VZmB|0!L8gfMJI)^ zC`vQq#K^UjW%>!rO_+e^7R(3(O%2>;YVB3XDEPNATNP}bTv`<{S!Qm)jHDs-&v{-+ zoySh(S$HFN)Wz%|QVXsP7 zfdh=v0m{4`Ch3@+hB}y}!^lN6h29-c3Nq{X$g_e(5dj~LJ!^t!|H)gD$WDHrG))-{ z`=MN>zSEPV*e_R^O0nSSs8f}%kov3C{DAaqsQ5dBG?A}i=+!RKvR50)LEkSq$FmO< zWrw0*gR8!)&wMxY*&BFLvVx}0ZQWumzJai{&Rza0TO-fiz?9rX(K;ce9yr824J91 znXodeh4{j%N@ljSRYVBW4sIMYu{)?9n)N>cNK^g?4GEkQx}9PF6Sxz4cMhynpBfoP za4S~qbJ-(FUlY#XNNNw4A+JC#XY$%iUP(>iP65VKoD!^2nO4L?YG?KdqiCLkOT}*W z6o5*dX)PnLmH?`6pS5AGGvcZaR{^WQBq?mls3lOHkzHRfHbpfBHDM(o-s=PLpV~PH zMQDG7$yT*L^8*fyvNH|7P##Bpz&CFcZ2X)RY_9|Xtu36J5z2<{C?4N>NDri^uW84x zP@XR_p$db|o6!h$L2?V7xqink-&(!!TUd8S%vv7eWg05#p|FU22M^dVh z*J)0V!l8#SdJ;w|jO6Q(B4`M#UX`fHr#F{dG&X7cCgO7x73L0(wqNLSbf^NUa1e8$ zj6yuBG}Jp-B9*mN`E$uE?_wsQm@t@&OlRZg@H(rrC1!v19gBbIRyL6T9Np$k%r#k^ zU*`@qygZoZT)W=lwH3$)iMqtQ+6;FD~zWY;!MRN||FUZ-C2WSQ?2>CtyWNtk9B zkJBT0jReHIl0FiYg-gCt5F}K|Ur<9F*IAaorv5!@-lOI>)ZC}$f?*YQ3iHsHVNa27 z!4l8x`vdXRtAenFa*9`0wgVJ|UDVCBOYkV18M{(KwP*dLu#tMd1>;plZp_!rYihhx zLht057PECi8b>OvpSMf{h_ZE{iZ#)$Fc9>|Q0x*9&o3g83z+;1gBeWpocNajQpgfJY;%XAQukiuZK@4~zhh53@SpfzrH< zlPGvR>M*Z%`6H#KkRofOKv5!x|Mx#h!RMJadG75dElpJf{EbIPTRAH3lsx`TdL%Y1i*LdA<>vp5w;@6$8V3>2x S`8QnWU)zTBmHR3Tb^bpNvgKR= literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test_24.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/marlin_utils_test_24.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26930bc6fddad184ac11e26873e3ed3b224a0b46 GIT binary patch literal 10351 zcmd^FTW=#rcJ4cyO^Tu@>M|OQrfrXBhK@ZNjVEL8WY%$f1J-NH*tK_&wCyl0R#PHH zc2m_&X+$@(o7EWEAQ)ityf8xLDF%|ikh}%?4gHb;c@mJfeTuWT@}24?MaeRAfXM|3NR)e?)y9zwqCYFv*da%#@Do$atz9rK>us zoR(>hCTO~&*b-ZQqB#|Ig{^=-&#rn^l&-RCC|zJ{p59($*S#vU z@p}fpXQ`xVvFF$g)L3HAiyAk*98sGFyTxv!e3>a9O0^dT>pzKFomO;cH~W4gY6X61 z_d}1_&4AmH7e=A&`pn+){Kih#<-6etwnf^k$%*z}D~#~8K5V!hmw$BF^G;;YP*g(A zPRp+!c&+W7C`8G^b}!ni@AX|jY7M+Gp`lS{+20JFI(~2B7v4Zmk<`Yf$5N~^<*CYW z7@sRlYARQxk;2qiK30ZWR!2_jjFebnD#~@H4UJf7t4!Zj_$_AmGRqBfu@)OEQfySE zSY!Gkj;bmxQQky8zasK+j`BoXkr-+cJvQ)e;yq9ALoI79?<%n_`cZe4*QF87Mn9H^ zW=2P8Mn}yldLBKRaei5Ph5FLd{HqdXtq(1{TPxDAK+i>KSd0s?g!x)DXG$@j;*vBn zm~|}U01KO(HvLOEF_^dCYIvOX^yGC)6iRGL%9NOt2RYPm#@0^Zn3441q9wwD;( z>v?XJn61d`hB&4~iGoCH`O&M1w%_s&YBP!4O{}gLxplu0biyppBAh*^?j4hbQJn_| zVVa{#PHvkA{a*d?ICJQjQPAVW8jiIUL{ZS~c+DuWFp|eRp1bckCbg`0-S#-w^1qb$ zRSfY}Ydz}r)_bkqOOeaBz38P^y8WH*M)zR7)7o0^9Y#BWzrNq;bl1B9^E!3!vDfHh zrRyE{(Bt8H#)K~M`g$MCAFgNYDqiZu^!0o9@AM9n7wTZ`M!(}mUR|u3djV%2Z~5Ex zu;E6SZaru=LoW(fF}3gxk{imhtf>`wQL$v@w1Qtr(Y~=YTQ1^X(@oiwQHpW{KYk5O z8gzo=zw5~4@DpMn;kaUHBppNJ?5eyRK^LX(zXqd!p&rSPn4lN1f{JG)ktgW zU@NI@P!8<>Jf69s=C5RBCf@V-SybNC_9gxRxdLkB(Z6Vk-u0|^tz9HBU^y_Q4t6&8 zWxk1ec`>#|ePE2OSY}{XR=_M>P>ZpQK4#KB{1T|8jM|$}XR{F$mWv^X8T|?90=e*oulzbC003>KhcKfFh8`01%H{zm((uC z#k1O)gdH@87ScRK4}H>%u;LjK&ex@3DK=*%k(*6(voI3Fay&bni_61GT#CzYNO9o} zX*eIx#`#$ZxoSKYS4Oj#bNN_i^RH3aLOdT=FO)52=Yo~}O6DtZZZwB=EsQF(4k)vw ztYmRgvW!0G*vXk# zZ>q=gXg1cM#IEA3ak7|E6ki=HU!j5vP4$X&p=_SYSdRL;K#>K-xP-DGD2rkatNt#= zGsJlRcA%4)iri@*^7A?>l zRJnSj4cFppqeWJT*H8<*jME0Th_9fO&U5BiX_Ae;w#f;U3~Fyb?s<&}w)x&mtkw1W z5E{(w*bF%jw%QK^{-V7dMD|zn=PIP6ujYQ>@4KBAvzZ&Yq_XWn*$$dn`>q@D)?-dq zn3s^$6!8!Rg9qcLe!siracbptJWK)b0IJ_6}CpweyBK1bvZE$^a(vr->#c}Py zvfb^65rBrxTKlcAwbk+LtwVd`0WVPd?;yW?Z}99{&oCPOE&$AhMcISJ%RA?@G(R$l zu%9aS^h8Degl2t0P~@aai9yLSC8U>6NYmGr_%nDU#UJ+pXu96poCiG7!yPv>gq$oc zU!Xd=Fpr52>-O@!MB8yYO~>47b^-M5UVNeotY@KS?F+TAJU!u9lyTVNKEA+Am zOUSlbQCP>MQ~O4Tn&fuFp36he$%&Tr`;OARn<(84{O%$D@|bXs)2=wW_qfOJrjL#E zanI4v@m-N_h%_yE8B1Z0LsX{6hLcN2x;xHnBt|y2A?)paCw=;fM(+TYggr7V?6IUM z-ZK*{;6a+u86~x}t9s~lTlFyF{YKR19;-Kb(5Q@ujwS}7rg$tH?Qbu8qR)1sE<%8CM$uHjw76Xsv3pdDJ@u*l@&zlau9`~wPz zwiU$|{VS)`D}cZ^=uOvX6afn?wTga}(`7~ZW?419wzTPY3}9Wpoan|@g8cMv5s2#3=ZgRlL67xSHU1W1G;FZ`qR!2pm!^jpeD~uNB(L2Uys?>(fKUIO) z4FC%=oVB>Xs;>jDpEIA8ruqm_fysX;T|$LX6x70{R2Wl1UA&Z9&8XI;RG3lGcIi^; zaz?#!DRnWUu3Som!Fwe6zrngKtXtu$|DV>4wXR;eD^|8=*Dj^5Wz_4JQUN=}S=pCT zpUJ4tUP=Wp5pAEllzJnhK7T27C8OTFlzK~0i>W+-{{ghY{R*TE@)g5DfCSuzr(UQ^ zREI4bE2CMqzyM;?Ht?Ni&<^mP$!V*mOcI_+PU%eA!9}0MB>`e*0Q4&)fdr0xK5=g2}0E}2=&5uK&aGGoWoFj4H|QB`#Tzt-8ol%ur{eF zXepesqks@i;M;>G;4JdbQ-qe0iIB?(oIJn5EPi_{vDpdi>noW-b@Y!TOZQu}l@aRPwray(GZleL$ z7>+RwOVmq;o|hQkX{_*6MSmC@?w=Z~&_+e~J@CC{Qzb}>c!qjRbun~2@=ZPylILh5AK z^Lo?G-cOrRNRk%ej%Otrh>F5N7XE4)4js&XB-~-DE}9M2;FkKggnx-W@jHjY&16_2 ze5-)8u;?vJb&m~p;zCWRsiOxEjCqd}U2H_H=;%;yUMI1BN*g1*KcPe@2n&a7CsEz4 zkiSd4kTP*{h*ZKmj}jH5Ia=U*VJiHjDr%-7<-ON=Iz6 z+vD$1kLp2v>!Q#Xsa;)gM!?s=Vj<}~&4LF4IU-*La(POAN>#lHuZBDo{u5Mbs7S_t zqdu-YT#7!p9~SZav7DONqwA$j2WW5nvg?eb>L zNHr`CmB>+|HTuj=w5=e(4mJXK(A)i>AJ())K7a#A7X-W6)8`+d(Zz-4TYej62T|JB zXqBV|;CCrig@20d$vq_RoLdvw4bm54O$jp46}VLpG7mj5l8>dK6hXE}%?7U+Q^8G%eq$hX34sISL4_!|nY&h}Q@)7%aC$ZM&U5QZ+*D2dcep53bq_5Cie19fa)ddk^p16EFKt z&Eh}BKJ!08lIV!``%z-xPKjU#nOod8;9e;}w4t#R02YObE^Z-Gd6{Vh2&~loz$c4Z zD{%)+oC-+<8OA!ySSV)hn008ltpL#q>y7t5c=Lz#4}aji?aaLYM}J)3{9v>0y#3=3 zK6K1+!!$>C3r;!hfKi8aTDnt;g2?S4T(B(yTh1(o$IeX(61kPg?L-&b;1sCOY=Rg0 z(;V;mJH(y@F+>mmLTDj0ppkJ!AizXGfPNOz3c3D2hcj>TZLwnq6rekIKSumZK`Ft* z9GP#+CHP8h!!m)dwXtO4jtp-{a>~iEJ}HMMPPKu_bFl#nWwap`2;wSqLxwyBbb;+* z1&tUrge+^DseUBh5`2=0HidZH@a)7z{J|jzM8XUOr|n@U`N!1k6H0cFI2ILR59@=T zGkcb=Hv(umfq3$wRJ>mRIIE)23&Y8!BBRbygW}S*Q{=27BVGffqR8fHQL$88H5J|h zp@HAzkK!bxLsa}in%*{P1MJ3;47n}+6v1fN40UfxV z@HJ${O=@PU1%x7@iKQ4^96RB1E@1j?Uik{#Mm@!Qu+4g z>=P%^P!z*kf#_0d$~>g{Lak5$_1q@V6v-zM-1HFb&+&*eRPe6CJB6H)oH7+p+*gCs zUL6{^*9K(~n>CfwmnLsaS(>6{Z=g4+{SqWu9m%je6#2sS4c!(4g<=fq5qVr}(Npm5 z{kz&Vv{BS!akRGRnaQEsCA z&$IGkR-TJ=HZwvLk1Q&#D$P-jm4KgddDsLVPUB*6Ufk*DVS~-cLd4A!Bbl&evzsFN ze`e5d#MsfyLlhh#96^bgg+Ut~$JuHah1uwp>L$%2Um|7z*tM9m3(aGfj-vDw7Gzi;E? zj91f0!e>L_N!N-Q@4LMoc%S!Z4TMs0jjg+!yN7&}7V~G6yh}v2gU+T zki1?JDYAw1Zz0fLB3t;|HN$|)R=z5lD{u$lcESxb$q7}S#l5;F^PkbMa(HG^5=&rZ zh&HZ8@F@dg{r)-oX`sc#e)@Qcq$9farI<9nCfS91kFQARn*AE-dnN;6!ER3RsV0z` zrIgf6fl25fy z^kHs%1s|!1)T3rB9}%K10^_PogWf9vo5*eK;&3)H;v%lm#d|z^qz}tL+B(pYO<4e;WCKt zH-x(gi4k!hav3-+T>t~YSWaHx7LqwmER*hEgL2};XtRX>hA0y4Dt|(WII+nAI@$q_ z!^sJs3>lMNadp*njJ-PL#eqBeHi1tkPiR87SrmHb(Y-U%{q;9JTwC)Q zp4)%>`_Z>m#{NN_)yGBWBm9%!Lr5lh#QHqooZ2E1fiQ0?u%spJ$nHCVgE>n&k=ri> zg}xVf=)1D;gat+E$s*pq^uv-Yh2@Sd%X00B9jwWUtYW4jYoT>qHS_CXspHBGxrzCj z6u)MTtr_;f8a%f@>m}*ojL&;F`Yjdp2F*BB{c##LPs3jKC`}q7uXe+<8MoRe%}%SG z4pqK28jYHxachwF&aV{PbjIaF-YDRI5C7y_h?Hrjxfa@<*cnT?wkAAtH9HnrL9?lK z!KOC;PHG>!7aSV)F~*NQ$#23EZ-KYF%Q}KIr1fW!dD_wZM5te;MeR=befHp$tf)Or zK4o7CU6io+%Id&U+8;4(>Eaf<%RXnBH@Y*}xw6{)Uslgq{3`XQCGD9VmM{1PpRVZw zcIs&lK6MTixlvgV4IVf$J&VH`kiZ%hWz&oUk-IW{)N6;i)sDxxGZ^>dv%qa7q#U^8 z-XMMNPT-t}YMA6sa@2~$+)=GT7caQIeU$qhHS9O%c8*ccMLKNwxixB@=59jaI9a?0 zxd+AO!(N)?HU%~JyJ|R&o1_(#R^l0yhN{ug>OI_#&(fpeVE<7R_4oTj8Ai?Uao8SX%llF5EL6$<0xPYw z2fcCHi<12X+DxaJhDq84sP^Ksd}A>~7+YYF(3HFmQ4yZy@-49|Z0=rqqRijn;<9YL z&3A+?)J>?g@xOWmS_Df9OY&QYE(5fXeg-bJKLJ`({zul~Qb_BGx15sfJ7UG%Gqej#;aZ3=?EK(4^rn-=7H>gnPHIKSXaiY0QL7B4`lb9%fY#ErvQ9qvU^>r9#Ri%FH2WlBRlD}gz0wU1hZK^Q$NQpb?!`5S{_T2IDtJXC2Y>#GD;Coriy zSO^I~=QpRH(Rs}2XT(Tz7tj@{_8^w2LXMf+d_XPZBAQq4llTe5`R-}+()*puT z_us8|v76szvr9aU@*$-$O#zL`A$M zd?dH}8BE*c=jB8GrT#tqlW!nWHep|i%t7gaJUL*=Q)Hsgz(HT zznMj$v4Vfs%p$kQR-@EFS%6X|V}mN)75~S=%&u7A8;cu69I6ko(}q1~DTCUdVj%b1 zLlj)y@o=0p3V9)E8YJe$q`APSB2raEdFmAshKPSiEt3dW*^u8{xne1NRPGs$O+JTq zQit&2St3oJ{0(1k;eK7VH-snD&!Lp_Q?pW=aJqbm-mgl(zoLWkjeKr_F9m!pR7qQz zKq+XQ@MQf{26mygP}@=rxU@0{HT2y@%a5imHT{GD%ASGgXE;zR5F1u~Ag%DY`EjHT4O#>2jGz;W{WLf1J~%3OkCt zr0$dW7$PXGWI<3_2@h(ZV6`GzL_OGA(YXRZ;31E%bMw1l-(0a-R`0*T>IqSV?|2ra zU5yKrSmjIKrj)WT1;XxKdX9ZrvuvBq^>XUgB8 zG{0xfu&IUT4yDWA;5@fL6}bggFLQ%HErzY6HVUi2n$y^6&ow4n`MmqdpEzT~L37*fq_ln*kZoFdS*C$ zXtSF`C`@&Az3Z>P{_p$$R&!{`(eU@&-~DR-^h=ud6}=4prSWnBKmWH((}Y&g1QWc* z>b$_~dO>IOuGfr$p`K>J#M7u*^+X||>dcy5PZpAOr{FNna~4wgr7OHJRMCX>1uJCm zOnkwFEs_;3oL}(5aFU60BITJ2S&{aVV#wQ9wnRn@-!ls%d)8&eKGf|OBi^Xk?{RPR zkS0dGG4vV}2kse#1LB|ra+%2Wn)A)TKJ#6uuID@()d)7TKUO?SZ@uFJq_I^JQFQM+(p8d{> zmr-b8p3i(IeVw&MQLB<-2g9s-q_g0-$o8&Z(N|{VTxlB_ z|L;gD+I_9e=tA7H11(^ow!rVQyX1F6-!_ zwq~rdb=EP28L|a)LhG2qqTG30ka4YJ1y*R#xSa$&MdFYang=vtFLM7!Z)=~hFd-7A zMx}?bv~yaJSg^wc(j?a7h~#H_2dfKBn)AcaPBOGZLpWhlpzKq<%+b@jpwWzBa#TC7 zy{mQXrT0U7%?R~%hT~7olyLX^e;hdjzy0`qJ&vTJh1!R9hlSYkqAq#JKIcpPhW1dC z!>E49BR=*0h*cjlIfQpr_{~qIgL-SaRc)OPin8Jbr(deK=j+eam#1shx#`xOV7}Ry zzE!K$r|V7O)m-nkS84}MIbAE>@uWY!)GjuH>PnGxYq}j&YuzGO<)>PABIjHU`d4b! zTjwWfskH8KZRJ@yU3b~5R^|iWtu}%tCC{IkI#+8#-~98)^!n|&@8@Z-2_!%5?t4{R zgZS3jdm7W0xX`e3HuI20X0;)_+cJx$cr#j_tgF4tQItffmcZS)fLa3s|J2#(Eib&DNFN_x#G#DPkVukAZf%BI`|k#97GGB6z`;Q zlsU}d8T<~gEF0x%b`p7c29=M$y8nXO|M#ogzoakkbxG}QmvjV|)aYHv*W&95eB8RS z=d`7nfeY$9{({O3HXCal30Wzlq?@~?iE<3D&r#64OiS<2LdL{Tp~O98$UF#`2LVg_ zfUUBC-O}U_?rT5=-bDqJ9?}RkIxru=1P)7msAG^4+yWkiy3pF1d^$8c`qE^GcbM51 z0xrx|?dL#=U+Q<+Cv0i5VXVRsp@*(|F#m$W2ZHd#OvI*?sCL;jS}7w`PK#cJ zMPKK!Emr~sfG4EasufFKp38mcQy9j5mO`Pc+bGt(hg`lu-d%3xGkZUx9T(MTmsU`TiW;wTV zIaicku8R#ADR*&OeOWP=`0#Q;GdEwn<>i8TFXvH|-v@;c09t5yx7@yj=F zfT3Ku`2MBq?##t^F1hbqyn6MmnKvTmy<)BHU6QgX^OoAQe35pZ=}Ck%#`6mfw%6TQ zAw5`E$Wo0GU$;IM)yo(qHh|PXEw(qn4@r0@kz{n6rJ2R8M}}!L{Nol<4r^#^@=gBV zb}G#cCSOIX~ zQhY%V%`(Fi-pXgZlL*W%EwVc{Xc3>#!bE7px*mxykUSOY4I|)dR%n8vB~hC^q^%{^ zHIOll?6J^3W?B&V^c!Lh!AfLcq@Sx+#CnX)X z%FAac_ber+C^>u)YYRYQGjhO*a zW+h_Fwj4R?UMe~trHEFzttM!NA05Vty8{Jou^f2PCHn9AB{HVHKSGf{1%H#{ z%-Nc@4ub;S1*rxZrZYE@@{Wa6|A0SW4>*x*w3$o-^PeJ( zQF%>XM0-={c;jVI;0`ov%@QE!!dlnW8w{@ib*4H+w>8uy1W8P#)A>!gCj+7Nyj=xJ(FTL+-5 zr9;s8b-p%?G1HBssI`L(=_p3i<zX+YJ@^#)x0dmqbIy731 zOygmNUxw)h6Xrp!L-VG?AvK%c843;gYt;Wb99r8KTI(!KjcQ?fD-Wk7(-=Z~UH&E> zV_e&+?PRd0k#$Xeg%)gK|4SIZEBF5-P`nSBi=e1Ry4$}+!LQ@QV#yaUR)60iL{amV_OViEEXJ%H;U9L6ZPc$ky zrI$SzQ7%TVUc8fAZpuZ4Jo80gq0gM>HF9K~Jn_x&a<1O?gWQ~#E7LeJFHsDItnLPs zRhxat3W-I}Yr(aw{Kx!PtAGDr|8^dzs@JQHU3QxEo^bhV0#s-qG*z$xjRc_}`q0%# zXmNM{9CAT>SQB$*$0Fn`hI*(IiUW9XnV-hHe3J^9d{@;eTM=2x8(8c<8?hZwc0Aeb z>Ms8Z?Y|}@&*I*XFoUf;*#q95d$on+{a%s?Kz=MDSj6S`DxQq38(BSN?4o_BQX<4r z#3n)(s8TJKj3DGyyd7D$2!MV09+e~sOTo@O5t*f0vEG8u(`+|Hq!;J>NXJZ+l;vA! zT(EI|0Is56Fep_5$Y+%R@%bLKpE*UY zq3$H=KyiC@0T3Fv$?xhP7@b7N?j(WK<^t?3QlRXDElhYv&K_mB{qn<*hvuT@U&pA{ zT9V$@m5mP^SiA(@jk`v7WC!*F_#P%<$x>xg(PDW6`&k(uq&1aRu#}^N2Cw;Mdkv8i zx*EiI=vK=d^?v0tVqtyJoysZh(rxxpr*9SjW|2B%&a^6{?$u2qOpOx>+jyY>fU&(aOs9RH^mGh)2p|FJF-g4I!>cm2+`GH zhzNl5>7DA`q;|v!XYcsUG=EF~x;JH9!NV2WY^H12iB~Gyp!;fYG&x z1c<}{4KQLFAQBLK&%P4;R{rs6~lm5 zC$vy%ta9|)M=d8cRXe0=t;Q9S3dy7=nf`iqN(1W&4O&lE5*Vo`iC6-mktWlqZx_vIiO+LkkO24ceLvM`qa*{*co=ckO9Tt2ZEL8=d6Ct>E#qP7@1Ks#Ehx&IBGU2<*LJl}6v~`3a-4d*0 z!>eILA_r%8BIM{^GWjj~`U^;6blhopY(t?X&mv5iVQF|N6M7n`Y2z)+vpV771l|q< z4-c@6&S9M3OmAj&DjfmJrqMSGbRF<%?qE7)9=C;lNUVLFq{Ahg?C^lC0SnjJf*vsC zc~aP)g*v=O6IUHR3wfJ;Pm;>0CoocZjp*0S>AhSTAsu-M6Cm?{XIg#T*bfT{w-5Fbw&K$Pp+ek=?zatV=}eG> zwT0!v2&t|kMeaQ;vU0&=WT!inABw^G0`f7Y>ZMjYG7;p!2L`L?$AR`XbQQsI<(i+M zrq3XWoU&}zUDY%`4C0;-?f5X%9qEfs5^BaY(_BLQV<0RZA5ToZRrrKl?;rP$WyG?1 z7IC0aKF-tp1RKX8$-5YA*I|&!Qit&+GQZ<6^gr-0(0)SuU562CcJD~k=ceEDFv!fu zhax}1qVt&@hap3h$_^!ePKh#o+Ydp0OckdnA?YioFMmq89gHTaWLs+1{3gkk0>Zmx zOC(#~LskD1AtF}WBu(6@FX`Xl1s~pGCEt>3x~t^iykS>*Q2iXJ35T5ky@|`$q0%az zp{vWFE&?tCapMFjE1rm0sf&OTp;J6dAYhEoh6YH89pD>+x)fue1y*QZAh5@W#VaIJ zht7j64#lhJs+2jRUKayr4GdY$kyaODNX5NCJJvEWDFJ7OR|yemE3c8r5DSM4rj$3S z`0OCJJr%z+;tD0<%j$!WDo`k;LiqV%c>yEF#~V;i8MXi0@q$o)N<%#Eh7~ef>vnOQ z7ml=hKjjw1R+7(+o>_+ZK} zllReho0@C`iVSL@Cdvs4^}7g0sLv+(_#*eL_m1uCR9{8NX=8swtK%3lW69N>^Ab4q z0qU>uvfB`VufY<_F#;|8*9`szfU9FP_M=>d%(~@9w>%N(!o-(iw2IXgd`o??zKgFL z;GD6Nfg|@rq!e1SNNvIIC@7{uQ3v*pFBKakp1d8K;aTM610)*+CrVv5bb6(IY|vek z%Sa$5Qc6X|;?F{Vh1Zp1y>N8j340+2U+pY>G<-bxsH2!1LZv6EFIkevEJ|6t6D2R* zE_v!dE=Z-2iYI*!4GQ>{ddsWDmV|6U-rUQgBff#)hOBvLywld$#btaX#aVqxXGKZ} zQv;r1;}d3vABSh2;qr4Dn#G%q1aB|Qoh>9Ob9U~zGiT)z z)k;cUp(fU>B7iYpJVV9LBZ+iBz_;JIX0xWgM8ubzywTWeReqmhQi@0rY$@xi?u$}h zC4~X;^|^9npTqXrHSau~y8q8e48)-*4nk}#ji+T<%zDIa=MgbFd~15d$G7qmWA?%3Bt0cL- IB=hb600{KW&j0`b literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/__pycache__/w8a8_utils.cpython-310.pyc b/vllm/model_executor/layers/quantization/utils/__pycache__/w8a8_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e32cfbfeca811bbfd5d599eef217a9b3ffb4c3f2 GIT binary patch literal 4461 zcmaJ^UvJ#T5$7(KJl=^qon%Rt5;t+2G|8oj6(?z;wuVu|u^ab~Qp8Gt5D5tSmM2jn zd8Z}M`U4)2)(ILEF!B|o#{lh1fxhO0@U7^RKSATfiTj)7-C0u8@Nmo7-PxVpnfc8O zd(+bm1J9+uf3^9K7YyUy)H(i~LFd=_XC*TXX-F0u3G-OOJ)W4JnFvoX8t1X)S$LbV zozy%>kBPXRG`xnUt#~SNJ(n4wd)H7eht}b=w2#}^j;wzwyjj@@ zt*tpZ6^if-QC+@lfP*Mox^fyk^RlUVXq}KVvW2k)IjhI!LKZIS6(;rdI_!Y8XXHHi z&dLSN_v74-T$E?Qy;SG=w^U!2FUTkK%H!*w3^|QK-nj|&lL=K1TkjCVO|A@>%&Dc6*}EW&6aP9-pZePINHdp=&}Hj{K>79&4YJ z{Fc#v={{M#ElTnGD9cOd-F_bR(jYFa^+7)lOY8j<{k1kLr~J+!%X^96>u2b;Is>J` zH23>)kav43!I&Lo{$|v_50^1YTZQ>Rr5)OT{BRH^{@3u&UPY4|@Cr+^TSm?f`H?X+ z3V7APP&1OJW?>wf7-P9Ov$sZ@gp|m1uxjVF*FPFmhmN9Ynvs{d!>cb zb~e3QH|_=br8nC`wXjajqVe2rupRn4yJ57knR|7)<5#5CP1Vpz%Y}ZZ{OV+|%jnMG zGNv*cjcYngFttFeY_CJ^GJc4*@TWs(uc66}&y6j1mr1tGv-)q1KeL?w#poh-Nb^%C zH)&RiW#cY;8~?lHBi5R#W95l`Nhu`yb^NpE(EJIi_;?z=WgN24;d9H7Q?O;@h<{)( zV~3MZwz;D7tl>S6x90?L+=+Wx*ybg__-e_bFPU1z%(1K2FK>itxYt)#p2b8)2H>Au zFb0`iP4l+vwM8c zXU5{@*mr{SGHsjt#1gZXRB=gPegr5+MNIe>u9*7)j&EX%U_%25eVB8Xt z-!q3${Lst+NK*O7iy!f!7+Qr{SZ9r)T^I$70naiE`!kNjV0^~A{IXHlLvv@oppYXf z^FW7|%=d9kyMk_|0b8rOw8EsH@0VsaNXp6;Uw=)JM$3Ab4w6s>dDyP0^B7Tdb8r6m zUH#5xkftG&mjR}T0Fm4)r%&9!G{aP$_S?lyFTE40+(+aXZLu3=;qwrXkq0z|faBWV z3#-K(w!~~ExOy2ArCo{dkl&7avb_fTDTCw#2p=ibANX zk`FDYmTT3vp;~)bE3Bdh)jByT*D9RHYK5X%VZ-nFM>4k6Ya+?e@xP4gXa?d?R_in_@mn73u^-gI2kq%GM~7 zQlQchZ%?ULh~+#r+9*@l*59rC&pS6ryMv5EZ%+>55TO}54m;cZ9)h%bkruV7d5M~z zq4DY?o64P%`KN8QtR{g49-VLcBugJ@6>Bxd9no+E=fgPC6*y4o)QxeMRz>Wo+*+_{aZfSyBjJXSbD2@zi zZF3m4KB1?IhLXjUlDV~I|L<-?H?I}$U6wmYt{>)gqNTmfL0gB7!j&~Dn`8}Vm?HW} zK>ldv)ToOD%~YPdThj&Gn%*{vK5Pu7wq~Sr&pILfi(}His3c?l2V{*# z>nG1O#Mz7bJYvjY>&QUmA0a16G>>OdM2jo@H7ehu0;LrCc$O|k)RWl3v(rGS-mW@F+byiW z``)#8{I{=vaP!)A|K{7**Vb>Yf2gCRIs*npajRr`ds@9g-N)ekId$KphHk1W34JNg z-lmc@zAaTr=ylJ|#}9=>s-`8$fV;)y*U zdr)+%8eNFYRwmZ6D;s|WhU_95$F)Z`b-#10F+H*K1KSzfjx+mPbrVZYBvC2`sff}h zp~?j6)fFttL5Ipg`ba<;6{u)U$%d#=5ML2tx2ftd^_+z{q)KTbTCf6D38KY|cvGK} zCC0j_fnkv76MS1}KyN@4J->!ow8rMqQ7dRlaVH9QJwdniW=`bE!2!&15@ud=l*EFJ zK1No(Gw;27?GOHqKd$;~SJ(WjxBQ!H>!Wy7*nKFMcS?6j-ZbgMKxgM zPfhAy9Dm$p)rVF?59)6f{rTmIUK(bl=nQ0_u3{}xWUm+Nfa2Nf)u)N2Ra6xYx-2!J z2VFJk+^Yeq?XFLMOcmP6w0}95tD4~o@~Dyh4b76_aF^5FmqgR#Qp5@=lYi$3{?M`bw~o!f;WhpMHNw}T-uMqsMvnU6 W|8YCD#smJm_-;J!V7EoleE46bIw3p& literal 0 HcmV?d00001 diff --git a/vllm/model_executor/layers/quantization/utils/layer_utils.py b/vllm/model_executor/layers/quantization/utils/layer_utils.py new file mode 100644 index 00000000..edce6d19 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/layer_utils.py @@ -0,0 +1,37 @@ +from typing import Union + +import torch + + +def update_tensor_inplace(dst: torch.Tensor, src: torch.Tensor): + assert dst.dtype == src.dtype, "Tensors must have the same dtype" + + # update tensor shape and stride + dst.as_strided_(src.shape, src.stride()) + + # If not the same underlying storage move tensor data + if dst.data_ptr() != src.data_ptr(): + dst.copy_(src) + del src + + +# Newly generated tensors need to replace existing tensors that are +# already registered as parameters by vLLM (and won't be freed) +def replace_parameter(mod: torch.nn.Module, name: str, + new: Union[torch.Tensor, torch.nn.Parameter]) -> None: + + old = getattr(mod, name) + if type(old) is type(new) and old.dtype == new.dtype and \ + old.untyped_storage().nbytes() == new.untyped_storage().nbytes(): + # If we can just update in-place to avoid re-registering + # can be faster if the underlying storage is the same + update_tensor_inplace(old, new) + else: + # Fallback re-register parameter, convert to Parameter if necessary + # this not only ensures we don't register a tensor as a parameter, but + # also ensures that all parameter subclasses get re-registered as + # parameters for `torch.compile` compatibility + if not isinstance(new, torch.nn.Parameter): + new = torch.nn.Parameter(new, requires_grad=False) + mod.register_parameter(name, + torch.nn.Parameter(new, requires_grad=False)) diff --git a/vllm/model_executor/layers/quantization/utils/machete_utils.py b/vllm/model_executor/layers/quantization/utils/machete_utils.py new file mode 100644 index 00000000..18e13320 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/machete_utils.py @@ -0,0 +1,30 @@ +from typing import List, Optional, Tuple + +import torch + +from vllm.scalar_type import ScalarType, scalar_types + +MACHETE_SUPPORTED_GROUP_SIZES = [-1, 128] +MACHETE_PREPACKED_BLOCK_SHAPE = [64, 128] + + +def query_machete_supported_quant_types(zero_points: bool) -> List[ScalarType]: + if zero_points: + return [scalar_types.uint4, scalar_types.uint8] + else: + return [scalar_types.uint4b8, scalar_types.uint8b128] + + +def query_machete_supported_act_types(zero_points: bool) -> List[ScalarType]: + return [torch.float16, torch.bfloat16] + + +def check_machete_supports_shape(in_features: int, out_featrues: int) \ + -> Tuple[bool, Optional[str]]: + if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0: + return False, "Input features size must be divisible by "\ + f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}" + if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0: + return False, "Output features size must be divisible by "\ + f"{MACHETE_PREPACKED_BLOCK_SHAPE[1]}" + return True, None diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py new file mode 100644 index 00000000..9a1defa4 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -0,0 +1,348 @@ +from typing import List, Optional, Tuple + +import numpy +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform +from vllm.scalar_type import ScalarType, scalar_types + +from .quant_utils import pack_cols, unpack_cols + +GPTQ_MARLIN_TILE = 16 +GPTQ_MARLIN_MIN_THREAD_N = 64 +GPTQ_MARLIN_MIN_THREAD_K = 128 +GPTQ_MARLIN_MAX_PARALLEL = 16 + +MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + +# In case there is a performance issue with Marlin, the variable below can be +# changed to False, which allows Marlin to perform global reductions in fp16 +# precision (instead of fp32), and therefore, save on some memory movements. +USE_FP32_REDUCE_DEFAULT = True + + +# For binary size and compile time, we don't support the same types for with and +# without runtime zero-point. We support common cases, i.e. AWQ and GPTQ. +# TODO: we may want to move this into the C++ so its closer to the actual impl +def query_marlin_supported_quant_types(has_zp: bool, + device_capability: Optional[int] = None + ): + if device_capability is None: + capability_tuple = current_platform.get_device_capability() + device_capability = (-1 if capability_tuple is None else + capability_tuple.to_int()) + + if device_capability < 80: + return [] + + if has_zp: + # AWQ style, unsigned + runtime zero-point + return [scalar_types.uint4, scalar_types.uint8] + else: + # GPTQ style, unsigned + symmetric bias + # TODO: once fp8_marlin is merged into "gptq_marlin" we should be able + # to add `scalar_types.float8_e4m3fn` here + return [scalar_types.uint4b8, scalar_types.uint8b128] + + +def _check_marlin_supported( + quant_type: ScalarType, + group_size: Optional[int], + has_zp: bool, + device_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]: + + if device_capability is None: + capability_tuple = current_platform.get_device_capability() + device_capability = (-1 if capability_tuple is None else + capability_tuple.to_int()) + + supported_types = query_marlin_supported_quant_types( + has_zp, device_capability) + + if quant_type not in supported_types: + return (False, f"Marlin does not support weight_bits = {quant_type}. " + f"Only types = {supported_types} " + f"are supported (for group_size = {group_size}, " + f"device_capability = {device_capability}, zp = {has_zp}).") + if (group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES): + return (False, f"Marlin does not support group_size = {group_size}. " + f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} " + "are supported.") + + return True, None + + +def check_marlin_supported(quant_type: ScalarType, + group_size: int, + has_zp: bool = False, + device_capability: Optional[int] = None) -> bool: + cond, _ = _check_marlin_supported(quant_type, group_size, has_zp, + device_capability) + return cond + + +def verify_marlin_supported(quant_type: ScalarType, + group_size: int, + has_zp: bool = False) -> None: + cond, err_msg = _check_marlin_supported(quant_type, group_size, has_zp) + if not cond: + assert err_msg is not None + raise ValueError(err_msg) + + +def verify_marlin_supports_shape(output_size_per_partition: int, + input_size_per_partition: int, + input_size: int, group_size: int) -> None: + + # Validate output_size_per_partition + if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0: + raise ValueError(f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. " + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq.") + + # Validate input_size_per_partition + if input_size_per_partition % GPTQ_MARLIN_MIN_THREAD_K != 0: + raise ValueError(f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible " + f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. " + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq.") + + if (group_size < input_size + and input_size_per_partition % group_size != 0): + raise ValueError( + f"Weight input_size_per_partition = {input_size_per_partition}" + f" is not divisible by group_size = {group_size}." + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq.") + + +def check_marlin_supports_shape(output_size_per_partition: int, + input_size_per_partition: int, + input_size: int, group_size: int) \ + -> Tuple[bool, Optional[str]]: + try: + verify_marlin_supports_shape(output_size_per_partition, + input_size_per_partition, input_size, + group_size) + except ValueError as e: + return False, e.__str__() + return True, None + + +def marlin_make_workspace(output_size_per_partition: int, + device: torch.device) -> torch.Tensor: + max_workspace_size = (output_size_per_partition // + GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL + + return torch.zeros(max_workspace_size, + dtype=torch.int, + device=device, + requires_grad=False) + + +def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool: + return (not act_order) or (act_order and not is_row_parallel) + + +def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int, + is_row_parallel: bool) -> bool: + # Need to repeat scales on every rank if act_ordering or + # channelwise and RowParallelLinear + is_channelwise = group_size == -1 + return act_order or (is_channelwise and is_row_parallel) + + +def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor: + return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device), + requires_grad=False) + + +def marlin_make_empty_zp(device: torch.device) -> torch.Tensor: + return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device), + requires_grad=False) + + +def marlin_sort_g_idx( + g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + g_idx_sort_indices = torch.argsort(g_idx).to(torch.int) + return g_idx[g_idx_sort_indices], g_idx_sort_indices + + +def get_scale_perms(): + scale_perm: List[int] = [] + for i in range(8): + scale_perm.extend([i + 8 * j for j in range(8)]) + scale_perm_single: List[int] = [] + for i in range(4): + scale_perm_single.extend( + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]]) + return scale_perm, scale_perm_single + + +def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int, + group_size: int) -> torch.Tensor: + + scale_perm, scale_perm_single = get_scale_perms() + if group_size < size_k and group_size != -1: + s = s.reshape((-1, len(scale_perm)))[:, scale_perm] + else: + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] + s = s.reshape((-1, size_n)).contiguous() + + return s + + +def marlin_moe_permute_scales( + s: torch.Tensor, + size_k: int, + size_n: int, + group_size: int, +): + num_experts = s.shape[0] + output = torch.empty( + (num_experts, s.shape[1], s.shape[2]), + device=s.device, + dtype=s.dtype, + ) + + for e in range(num_experts): + output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size) + return output + + +def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int, + num_bits: int) -> torch.Tensor: + # Permute zero-points in a similar way to scales, but do not use the + # "single" permutation, since zero-points are applied on every MMA + scale_perm, _ = get_scale_perms() + zp = zp.reshape((-1, len(scale_perm)))[:, scale_perm] + + # Interleave column dim (for the dequantize code) and pack it to int32 + if num_bits == 4: + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = numpy.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + zp = zp.reshape((-1, len(interleave)))[:, interleave].ravel() + zp = zp.reshape((-1, size_n)).contiguous() + zp = pack_cols(zp, num_bits, size_k, size_n) + + return zp + + +def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int, + size_n: int, num_bits: int) -> torch.Tensor: + # AWQ zero-points are quantized and packed on the column dim. + # In addition, the values are permuted based on dequantizer. + # Here we undo both of these, and then apply marlin permutation + # and pack it back. + q_zp = unpack_cols(q_zp_packed, num_bits, size_k, size_n) + + # Undo interleaving (use argsort(..) to get inverse perm) + if num_bits == 4: + undo_interleave = numpy.argsort(numpy.array([0, 2, 4, 6, 1, 3, 5, 7])) + elif num_bits == 8: + undo_interleave = numpy.argsort(numpy.array([0, 2, 1, 3])) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + q_zp = q_zp.reshape((-1, len(undo_interleave)))[:, undo_interleave].ravel() + q_zp = q_zp.reshape((-1, size_n)).contiguous() + + marlin_zp = marlin_zero_points(q_zp, size_k, size_n, num_bits) + return marlin_zp + + +def moe_awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int, + size_n: int, num_bits: int): + num_experts = q_zp_packed.shape[0] + output = torch.empty( + (num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]), + device=q_zp_packed.device, + dtype=q_zp_packed.dtype, + ) + for e in range(num_experts): + output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n, + num_bits) + return output + + +def apply_gptq_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_zp: torch.Tensor, + g_idx: torch.Tensor, + g_idx_sort_indices: torch.Tensor, + workspace: torch.Tensor, + wtype: ScalarType, + output_size_per_partition: int, + input_size_per_partition: int, + is_k_full: bool, + bias: Optional[torch.Tensor] = None, + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor: + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (output_size_per_partition, ) + + output = ops.gptq_marlin_gemm(reshaped_x, + weight, + weight_scale, + weight_zp, + g_idx, + g_idx_sort_indices, + workspace, + wtype, + size_m=reshaped_x.shape[0], + size_n=output_size_per_partition, + size_k=input_size_per_partition, + is_k_full=is_k_full, + has_zp=False, + use_fp32_reduce=use_fp32_reduce) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) + + +def apply_awq_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_zp: torch.Tensor, + g_idx: torch.Tensor, + g_idx_sort_indices: torch.Tensor, + workspace: torch.Tensor, + quant_type: ScalarType, + output_size_per_partition: int, + input_size_per_partition: int, + bias: Optional[torch.Tensor] = None, + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor: + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (output_size_per_partition, ) + + output = ops.gptq_marlin_gemm(reshaped_x, + weight, + weight_scale, + weight_zp, + g_idx, + g_idx_sort_indices, + workspace, + quant_type, + size_m=reshaped_x.shape[0], + size_n=output_size_per_partition, + size_k=input_size_per_partition, + is_k_full=True, + has_zp=True, + use_fp32_reduce=use_fp32_reduce) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py new file mode 100644 index 00000000..8b3dfaae --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py @@ -0,0 +1,106 @@ +from typing import Optional + +import torch + +import vllm._custom_ops as ops +from vllm.platforms import current_platform +from vllm.utils import print_warning_once + +from .marlin_utils import marlin_make_workspace, marlin_permute_scales + + +def is_fp8_marlin_supported(): + return current_platform.has_device_capability(80) + + +def apply_fp8_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + workspace: torch.Tensor, + size_n: int, + size_k: int, + bias: Optional[torch.Tensor], +) -> torch.Tensor: + # For GPUs that lack FP8 hardware support, we can leverage the + # Marlin kernel for fast weight-only FP8 quantization + + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (size_n, ) + + output = ops.fp8_marlin_gemm( + a=reshaped_x, + b_q_weight=weight, + b_scales=weight_scale, + workspace=workspace, + num_bits=8, + size_m=reshaped_x.shape[0], + size_n=size_n, + size_k=size_k, + ) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) + + +def prepare_fp8_layer_for_marlin(layer: torch.nn.Module, + strategy: str = "tensor") -> None: + print_warning_once( + "Your GPU does not have native support for FP8 computation but " + "FP8 quantization is being used. Weight-only FP8 compression will " + "be used leveraging the Marlin kernel. This may degrade " + "performance for compute-heavy workloads.") + + part_size_n = layer.output_size_per_partition + part_size_k = layer.input_size_per_partition + + device = layer.weight.device + + # WORKSPACE + layer.workspace = marlin_make_workspace(part_size_n, device) + + # WEIGHT + # Repack weights to marlin format + marlin_qweight = ops.gptq_marlin_repack(b_q_weight=pack_fp8_to_int32( + layer.weight), + perm=torch.empty(0, + dtype=torch.int, + device=device), + size_k=part_size_k, + size_n=part_size_n, + num_bits=8) + layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False) + + # WEIGHT SCALES + scales = layer.weight_scale.to(layer.orig_dtype) + # Permute scales + marlin_scales = marlin_permute_scales(s=scales, + size_k=part_size_k, + size_n=part_size_n, + group_size=-1) + layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False) + + +def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor: + """ + Repack FP8 weights to gptq format (packed int32 elements) + """ + assert fp8_tensor.dtype == torch.float8_e4m3fn + assert fp8_tensor.shape[0] % 4 == 0 + + # Reshape to prepare for packing + reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:]) + + # Convert fp8 to uint8 (byte) representation + byte_tensor = reshaped.view(torch.uint8) + + # Pack 4 uint8 values into one int32 + packed = (byte_tensor[:, 0].to(torch.int32) | + (byte_tensor[:, 1].to(torch.int32) << 8) | + (byte_tensor[:, 2].to(torch.int32) << 16) | + (byte_tensor[:, 3].to(torch.int32) << 24)) + + return packed.view(fp8_tensor.shape[0] // 4, + *fp8_tensor.shape[1:]).contiguous() diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_test.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_test.py new file mode 100644 index 00000000..4a06c5d6 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_test.py @@ -0,0 +1,163 @@ +"""Utility functions used for tests and benchmarks""" + +from typing import List, Optional + +import numpy as np +import torch + +from vllm.scalar_type import ScalarType + +from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales, + marlin_zero_points) +from .quant_utils import (get_pack_factor, gptq_quantize_weights, + quantize_weights, sort_weights) + + +class MarlinWorkspace: + + def __init__(self, out_features, min_thread_n, max_parallel): + assert (out_features % min_thread_n == 0), ( + "out_features = {} is undivisible by min_thread_n = {}".format( + out_features, min_thread_n)) + + max_workspace_size = ((out_features // min_thread_n) * max_parallel) + + self.scratch = torch.zeros(max_workspace_size, + dtype=torch.int, + device="cuda") + + +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE): + assert q_w.shape == (size_k, size_n) + assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}" + assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}" + + # Permute weights to 16x64 marlin tiles + q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile)) + q_w = q_w.permute((0, 2, 1, 3)) + q_w = q_w.reshape((size_k // tile, size_n * tile)) + + q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape) + + return q_w + + +def marlin_weights(q_w, size_k, size_n, num_bits, perm): + # Permute + q_w = marlin_permute_weights(q_w, size_k, size_n, perm) + + # Pack + pack_factor = get_pack_factor(num_bits) + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(np.uint32) + + q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), + dtype=np.uint32) + for i in range(pack_factor): + q_packed |= q_w[:, i::pack_factor] << num_bits * i + + q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device) + + return q_packed + + +def get_weight_perm(num_bits: int): + perm_list: List[int] = [] + for i in range(32): + perm1: List[int] = [] + col = i // 4 + for block in [0, 1]: + for row in [ + 2 * (i % 4), + 2 * (i % 4) + 1, + 2 * (i % 4 + 4), + 2 * (i % 4 + 4) + 1, + ]: + perm1.append(16 * row + col + 8 * block) + for j in range(4): + perm_list.extend([p + 256 * j for p in perm1]) + + perm = np.array(perm_list) + + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +def marlin_quantize(w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + act_order: bool, + test_perm: Optional[torch.Tensor] = None): + size_k, size_n = w.shape + num_bits = quant_type.size_bits + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Quantize (and apply act_order if provided) + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights( + w, quant_type, group_size, act_order, test_perm) + + # For act_order, sort the "weights" and "g_idx" so that group ids are + # increasing + sort_indices = torch.empty(0, dtype=torch.int, device=w.device) + if act_order: + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx) + + # Reformat to marlin + weight_perm = get_weight_perm(num_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list + + +def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType, + group_size: int): + size_k, size_n = w.shape + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Detect num groups + assert size_k % group_size == 0 + num_groups = size_k // group_size + + # Quantize with zp + w_ref, q_w, s, zp = quantize_weights(w, + quant_type, + group_size, + zero_points=True) + + # Reformat to marlin + weight_perm = get_weight_perm(quant_type.size_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits, + weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + marlin_zp = marlin_zero_points(zp, num_groups, size_n, + quant_type.size_bits) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py new file mode 100644 index 00000000..17d09055 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py @@ -0,0 +1,463 @@ +"""Utility functions used for tests and benchmarks""" + +import random +from typing import List + +import numpy +import torch + +from vllm.scalar_type import ScalarType + +from .marlin_utils_test import marlin_weights +from .quant_utils import gptq_quantize_weights + + +# This is PyTorch implementation of main part of reorder_meta() +# function, from tools/util/include/cutlass/util/host_reorder.h file +# of CUTLASS source tree. Furthermore, CUTLASS template for sparse +# GEMM decides upon layout of this matrix, and at the moment for the +# sparse GEMM executed on tensor cores, this is layout described by +# ColumnMajorInterleaved<2> data structure, in +# include/cutlass/layout/matrix.h of CUTLASS source tree. The +# reordering of meta matrix into meta_reordered matrix calculated +# according to these segments of CUTLASS code is re-implemented here. +# Note that this calculation produces offsets for scattering metadata +# matrix elements into reordered metadata matrix elements (or, +# equivalently, for gathering reordered metadata matrix element back +# into metadata matrix elements). +def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype, + device): + dst_rows = torch.arange(0, m, device=device)[:, None].repeat(1, meta_ncols) + dst_cols = torch.arange(0, meta_ncols, device=device).repeat(m, 1) + + # Reorder the rows, then swizzle the 2x2 blocks. + group_x = 64 + group_y = 32 if meta_dtype.itemsize == 2 else 16 + + dst_rows = (dst_rows // group_x * group_x + (dst_rows % 2) * 2 + + (dst_rows % 8) // 4 + ((dst_rows % group_y) % 4) // 2 * 32 + + ((dst_rows % group_x) // 8) * 4) + + topright = ((dst_rows % 2 == 0) & (dst_cols % 2 == 1)).to(torch.int8) + bottomleft = ((dst_rows % 2 == 1) & (dst_cols % 2 == 0)).to(torch.int8) + dst_rows += topright - bottomleft + dst_cols -= topright - bottomleft + + # Assumed that meta tensor is to be stored in CUTLASS + # InterleavedColumnMajor layout, and reverse engineered + # corresponding code to store values into this tensor. + interleave = 2 + cols_maj = dst_cols // interleave + cols_min = dst_cols % interleave + return (cols_maj * m * interleave + dst_rows * interleave + + cols_min).view(-1) + + +# This function converts dense matrix into sparse semi-structured +# representation, producing "compressed" matrix, in the layout used by +# CUTLASS backend, and corresponding metadata matrix. +def sparse_semi_structured_from_dense_cutlass(dense): + if dense.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional dense tensor, got {dense.dim()}-dimensional tensor" # noqa: E501 + ) + + m, k = dense.shape + device = dense.device + + meta_dtype = torch.int8 + if dense.dtype == torch.int8: + meta_dtype = torch.int32 + elif dense.dtype in [torch.half, torch.bfloat16, torch.float, torch.int32]: + meta_dtype = torch.int16 + else: + raise RuntimeError(f"Invalid datatype {dense.dtype} of dense matrix") + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4 + if quadbits_per_meta_elem not in (4, 8): + raise RuntimeError( + "Invalid number of elements per meta element calculated") + + if meta_dtype == torch.int32: + if m % 16 != 0: + raise RuntimeError( + f"Number of rows of dense matrix {m} must be divisible by 16") + else: + if m % 32 != 0: + raise RuntimeError( + f"Number of rows of dense matrix {m} must be divisible by 32") + if k % (4 * quadbits_per_meta_elem) != 0: + raise RuntimeError( + f"Number of columns of dense matrix {k} must be divisible by {4 * quadbits_per_meta_elem}" # noqa: E501 + ) + + if dense.dtype != torch.float: + ksparse = 4 + dense_4 = dense.view(-1, k // ksparse, ksparse) + m0, m1, m2, m3 = (dense_4 != 0).unbind(-1) + else: + ksparse = 2 + dense_2 = dense.view(-1, k // ksparse, ksparse) + m0, m2 = m1, m3 = (dense_2 != 0).unbind(-1) + meta_ncols = k // (ksparse * quadbits_per_meta_elem) + + # Encoding quadruples of True/False values as follows: + # [True, True, False, False] -> 0b0100 + # [True, False, True, False] -> 0b1000 + # [False, True, True, False] -> 0b1001 + # [True, False, False, True ] -> 0b1100 + # [False, True, False, True ] -> 0b1101 + # [False, False, True, True ] -> 0b1110 + # Thus, lower two bits in the encoding are index of the True value + # at the lowest index in the quadruple, and the higher two bits in + # the encoding are index of the other True value in the quadruple. + # In case there are less than two True values, than False value or + # values at some index or indices are considered True for the + # encoding. In case there are more than two True values, then the + # excess True value(s) at some indices are considered False for + # the encoding. The exact encodings used for these cases are as + # follows: + # [False, False, False, False] -> 0b1110 + # [False, False, False, True ] -> 0b1110 + # [False, False, True, False] -> 0b1110 + # [False, True, False, False] -> 0b1001 + # [False, True, True, True ] -> 0b1101 + # [True, False, False, False] -> 0b1000 + # [True, False, True, True ] -> 0b1100 + # [True, True, False, True ] -> 0b0100 + # [True, True, True, False] -> 0b0100 + # [True, True, True, True ] -> 0b0100 + # These particular encodings are chosen, with the help of Espresso + # logic minimizer software, for the purpose of minimization of + # corresponding Boolean functions, that translate non-zero flags + # into encoding bits. Note also possible choices for the first + # and last of these encodings were limited only to (0b0100, + # 0b1110), in order to produce valid encodings for 1:2 sparsity + # case. + + expr0 = m0 & m1 + expr1 = ~m0 & m1 + expr2 = ~m0 & ~m1 + bit0 = expr1 + bit1 = expr2 + bit2 = expr0 | expr2 | m3 + bit3 = expr1 | ~m1 + idxs0 = bit0 | (bit1.to(torch.int64) << 1) + idxs1 = bit2 | (bit3.to(torch.int64) << 1) + + if dense.dtype != torch.float: + sparse0 = dense_4.gather( + -1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined] + sparse1 = dense_4.gather(-1, idxs1.unsqueeze(-1)) + sparse = torch.stack((sparse0, sparse1), dim=-1).view(m, k // 2) + else: + sparse = dense_2.gather(-1, + idxs0.unsqueeze(-1) // 2).view( + m, + k // 2) # type: ignore[possibly-undefined] + + meta_4 = idxs0 | (idxs1 << 2) + meta_n = meta_4.view( + (-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype) + + if quadbits_per_meta_elem == 4: + meta = (meta_n[:, :, 0] + | (meta_n[:, :, 1] << 4) + | (meta_n[:, :, 2] << 8) + | (meta_n[:, :, 3] << 12)) + elif quadbits_per_meta_elem == 8: + meta = (meta_n[:, :, 0] + | (meta_n[:, :, 1] << 4) + | (meta_n[:, :, 2] << 8) + | (meta_n[:, :, 3] << 12) + | (meta_n[:, :, 4] << 16) + | (meta_n[:, :, 5] << 20) + | (meta_n[:, :, 6] << 24) + | (meta_n[:, :, 7] << 28)) + + # Reorder meta tensor elements. + meta_reordered = meta.new_empty( + (m * meta_ncols, )) # type: ignore[possibly-undefined] + meta_offsets = _calculate_meta_reordering_scatter_offsets( + m, meta_ncols, meta_dtype, device) + meta_reordered.scatter_(0, meta_offsets, meta.view(-1)) + + return (sparse, meta_reordered.view(m, meta_ncols)) + + +# This function performs reverse of the function above - it +# reconstructs dense matrix from a pair of "compressed" matrix, given +# in the layout used by CUTLASS backend, and accompanying metadata +# matrix. +def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered): + if sparse.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional sparse tensor, got {sparse.dim()}-dimensional tensor" # noqa: E501 + ) + + m, k = sparse.shape + device = sparse.device + + if meta_reordered.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional meta tensor, got {meta_reordered.dim()}-dimensional tensor" # noqa: E501 + ) + if meta_reordered.device != device: + raise RuntimeError( + f"Expected meta matrix to be on {device} device, got matrix on {meta_reordered.device} device" # noqa: E501 + ) + + meta_dtype = meta_reordered.dtype + if meta_dtype not in (torch.int16, torch.int32): + raise RuntimeError(f"Invalid datatype {meta_dtype} of meta matrix") + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4 + + ksparse = 4 if sparse.dtype != torch.float else 2 + + meta_nrows, meta_ncols = meta_reordered.shape + if meta_nrows != m: + raise RuntimeError( + f"Number of rows of meta matrix {meta_nrows} must be equal to number of columns of spase matrix {m}" # noqa: E501 + ) + if meta_ncols * ksparse * quadbits_per_meta_elem != 2 * k: + raise RuntimeError( + f"Number of columns of sparse matrix {k} different from the {meta_ncols * ksparse * quadbits_per_meta_elem // 2}, " # noqa: E501 + "expected according to the number of columns of meta matrix") + + # Undo meta tensor elements reordering. + meta_offsets = _calculate_meta_reordering_scatter_offsets( + m, meta_ncols, meta_dtype, device) + meta = torch.gather(meta_reordered.view(-1), 0, + meta_offsets).view(m, meta_ncols) + + # Unpack sparse tensor back to original dense tensor, using + # information provided by meta tensor. Note that torch.float + # datatype is handled pretty much the same as + # torch.half/torch.bfloat16, as metadata for a pair of torch.float + # value is encoded as if underlying 8 bytes contain four + # torch.half/torch.bfloat16 values, where either first two or last + # two are zeros. + meta_2 = torch.empty( + (m, meta_ncols, 2 * quadbits_per_meta_elem), + dtype=meta_dtype, + device=device, + ) + if quadbits_per_meta_elem == 4: + meta_2[:, :, 0] = meta & 0b11 + meta_2[:, :, 1] = (meta >> 2) & 0b11 + meta_2[:, :, 2] = (meta >> 4) & 0b11 + meta_2[:, :, 3] = (meta >> 6) & 0b11 + meta_2[:, :, 4] = (meta >> 8) & 0b11 + meta_2[:, :, 5] = (meta >> 10) & 0b11 + meta_2[:, :, 6] = (meta >> 12) & 0b11 + meta_2[:, :, 7] = (meta >> 14) & 0b11 + elif quadbits_per_meta_elem == 8: + meta_2[:, :, 0] = meta & 0b11 + meta_2[:, :, 1] = (meta >> 2) & 0b11 + meta_2[:, :, 2] = (meta >> 4) & 0b11 + meta_2[:, :, 3] = (meta >> 6) & 0b11 + meta_2[:, :, 4] = (meta >> 8) & 0b11 + meta_2[:, :, 5] = (meta >> 10) & 0b11 + meta_2[:, :, 6] = (meta >> 12) & 0b11 + meta_2[:, :, 7] = (meta >> 14) & 0b11 + meta_2[:, :, 8] = (meta >> 16) & 0b11 + meta_2[:, :, 9] = (meta >> 18) & 0b11 + meta_2[:, :, 10] = (meta >> 20) & 0b11 + meta_2[:, :, 11] = (meta >> 22) & 0b11 + meta_2[:, :, 12] = (meta >> 24) & 0b11 + meta_2[:, :, 13] = (meta >> 26) & 0b11 + meta_2[:, :, 14] = (meta >> 28) & 0b11 + meta_2[:, :, 15] = (meta >> 30) & 0b11 + + dense_offsets = meta_2.view(-1) + ( + torch.arange(0, 2 * m * k // ksparse, device=device) * 4).view( + -1, 1).repeat(1, 2).view(-1) + + dense = torch.zeros((m * 2 * k, ), dtype=sparse.dtype, device=device) + if sparse.dtype != torch.float: + # dense.scatter_(0, dense_offsets, sparse.view(-1)) + dense.scatter_(0, dense_offsets, sparse.reshape(-1)) + else: + dense.view(torch.half).scatter_(0, dense_offsets, + sparse.view(torch.half).view(-1)) + + return dense.view(m, 2 * k) + + +def mask_creator(tensor): + """ + Class for creating N:M sparsity masks. + Masks will be created using the N:M ratio, where for every block of + M weights, N will be pruned based on ranked weight value. Each mask + will correspond to the given tensor. + + :param N: The number of weights in a group to keep + :param M: The size of a weight group + """ + N = 2 + M = 4 + + mask = None + # for i, tensor in enumerate(tensors): + if tensor.numel() % M != 0: + raise ValueError( + f"Tensor of size {tensor.shape} can't be evenly divided into " + f"{M} groups") + + num_groups = tensor.numel() // M + + # N:M sparsity for linear layers + tensor_temp = tensor.detach().abs().reshape(num_groups, M) + index = torch.argsort(tensor_temp, dim=1)[:, :int(M - N)] + + w_b = torch.ones(tensor_temp.shape, device=tensor_temp.device) + mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape) + + return mask + + +def inject_24(w, size_k, size_n): + assert w.shape == (size_k, size_n) + + mask = mask_creator(w.t()).t().cuda().bool() + + return (mask * w).contiguous(), mask.contiguous() + + +def check_24(w, num_rows_to_sample=50, _verbose=False): + BLOCK_SIZE = 4 + MAX_NON_ZEROS = 2 + + w = w.t().contiguous() + + print("check_24: w.shape = {}".format(w.shape)) + + num_rows, num_cols = w.shape + sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample) + if _verbose: + print(f"Sampled row idxs = {sampled_row_idxs}") + + total_segments = 0 + non_24_segments = 0 + for i in sampled_row_idxs: + for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE): + total_segments += 1 + block = w[i, j:j + BLOCK_SIZE] + num_nonzero = torch.count_nonzero(block) + if num_nonzero > MAX_NON_ZEROS: + print("i = {} j = {} block = {}".format(i, j, block)) + non_24_segments += 1 + + print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.") + + +def compress_quantized_24_weight(q_24, size_k, size_n, wtype: ScalarType): + assert q_24.shape == (size_k, size_n) + + # Remove bias to normalize over 0 + q_24_no_zp = q_24 - wtype.bias + + # Compress + q_24_no_zp = q_24_no_zp.t().contiguous() + q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass( + q_24_no_zp) + q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous() + + # Restore bias + q_24_comp = q_24_no_zp_comp + wtype.bias + + # Resize meta to its actual shape (without moving any data) + meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2) + + return q_24_comp, meta + + +def get_scale_perms_24(): + scale_perm: List[int] = [] + for i in range(8): + scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]]) + scale_perm_single: List[int] = [] + for i in range(8): + scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]]) + return scale_perm, scale_perm_single + + +def get_weight_perm_24(num_bits: int): + perm_list: List[int] = [] + for i in range(32): + perm1: List[int] = [] + col = i // 4 + col_o = col // 2 + for block in [0, 1]: + for row in [ + 2 * (i % 4), + 2 * (i % 4) + 1, + 2 * (i % 4 + 4), + 2 * (i % 4 + 4) + 1, + ]: + perm1.append(16 * row + col_o * 256 + 8 * (col % 2) + + 4 * block) + for j in range(4): + perm_list.extend([p + 1 * j for p in perm1]) + perm = numpy.array(perm_list) + + if num_bits == 4: + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = numpy.array([0, 2, 1, 3]) + else: + raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int, + group_size: int) -> torch.Tensor: + + scale_perm, scale_perm_single = get_scale_perms_24() + if group_size < size_k and group_size != -1: + s = s.reshape((-1, len(scale_perm)))[:, scale_perm] + else: + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] + s = s.reshape((-1, size_n)).contiguous() + + return s + + +def marlin_24_quantize( + w: torch.Tensor, + quant_type: ScalarType, + group_size: int, +): + size_k, size_n = w.shape + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Inject 2:4 sparsity + w_24, mask_24 = inject_24(w, size_k, size_n) + + # Quantize + w_24_ref, q_w_24, s, g_idx, rand_perm = gptq_quantize_weights( + w_24, quant_type, group_size, act_order=False) + + # Compress quantized weight + q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n, + quant_type) + size_k_comp = size_k // 2 + + # Reformat to marlin + weight_perm = get_weight_perm_24(quant_type.size_bits) + marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n, + quant_type.size_bits, weight_perm) + marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size) + + # Create result + res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_test_qqq.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_test_qqq.py new file mode 100644 index 00000000..cb58eb94 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_test_qqq.py @@ -0,0 +1,125 @@ +from typing import List + +import numpy +import torch + +from .marlin_utils_test import marlin_permute_weights +from .quant_utils import get_pack_factor, qqq_quantize_weights + + +def marlin_qqq_weights(q_w, size_k, size_n, num_bits, perm, group_size): + # Permute + q_w = marlin_permute_weights(q_w, size_k, size_n, perm) + + # Pack + pack_factor = get_pack_factor(num_bits) + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(numpy.uint32) + + q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), + dtype=numpy.uint32) + if group_size == size_k: + for i in range(pack_factor): + q_packed |= (q_w[:, i::pack_factor] & 0xF) << num_bits * i + else: + for i in range(pack_factor): + q_packed |= q_w[:, i::pack_factor] << num_bits * i + + q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device) + + return q_packed + + +def get_qqq_scale_perms(): + scale_perm: List[int] = [] + for i in range(8): + scale_perm.extend([i + 8 * j for j in range(8)]) + scale_perm_single: List[int] = [] + for i in range(4): + scale_perm_single.extend( + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]]) + return scale_perm, scale_perm_single + + +# NOTE(HandH1998): QQQ employs different perms for per-group and per-channel weight quantization. # noqa: E501 +def get_qqq_weight_perm(num_bits: int, quant_type: str): + perm_list: List[int] = [] + for i in range(32): + perm1: List[int] = [] + col = i // 4 + for block in [0, 1]: + for row in [ + 4 * (i % 4), + 4 * (i % 4) + 1, + 4 * (i % 4) + 2, + 4 * (i % 4) + 3, + ]: + perm1.append(16 * row + col + 8 * block) + for j in range(4): + perm_list.extend([p + 256 * j for p in perm1]) + + perm = numpy.array(perm_list) + + assert quant_type in ["per-channel", + "per-group"], "not supported quantization type" + if num_bits == 4: + if quant_type == "per-channel": + interleave = numpy.array([4, 0, 5, 1, 6, 2, 7, 3]) + else: + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7]) + else: + raise Exception("num_bits must be 4, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +def marlin_qqq_permute_scales(s_group, s_channel, size_k, size_n, group_size): + scale_perm, scale_perm_single = get_qqq_scale_perms() + if group_size < size_k and group_size != -1: + s_group = s_group.reshape((-1, len(scale_perm)))[:, scale_perm] + s_channel = s_channel.reshape( + (-1, len(scale_perm_single)))[:, scale_perm_single] + s_group = s_group.reshape((-1, size_n)).contiguous() + else: + s_channel = s_channel.reshape( + (-1, len(scale_perm_single)))[:, scale_perm_single] + s_channel = s_channel.reshape((-1, size_n)).contiguous() + + return s_group, s_channel + + +def marlin_qqq_quantize( + w: torch.Tensor, + num_bits: int, + group_size: int, +): + size_k, size_n = w.shape + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + quant_type = "per-channel" if group_size == size_k else "per-group" + + # Quantize + w_ref, q_w, s_group, s_channel = qqq_quantize_weights( + w, num_bits, group_size) + + # Reformat to marlin_qqq + weight_perm = get_qqq_weight_perm(num_bits, quant_type) + marlin_qqq_q_w = marlin_qqq_weights(q_w, size_k, size_n, num_bits, + weight_perm, group_size) + marlin_qqq_s_group, marlin_qqq_s_channel = marlin_qqq_permute_scales( + s_group, s_channel, size_k, size_n, group_size) + + # Create result + res_list = [ + w_ref, marlin_qqq_q_w, marlin_qqq_s_group, marlin_qqq_s_channel + ] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py new file mode 100644 index 00000000..833d0007 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -0,0 +1,451 @@ +"""This file is used for /tests and /benchmarks""" +from typing import List, Optional + +import numpy +import torch + +from vllm.model_executor.layers.quantization.qqq import ( + MARLIN_QQQ_SUPPORTED_NUM_BITS) +from vllm.scalar_type import ScalarType, scalar_types + +SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128] +SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + +# Note: this is a hack. We should update each model to register the +# stacked params and get it from there instead in a future PR. +# fused_name: List[shard_name] +FUSED_LAYER_NAME_MAPPING = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"] +} + + +def pack_weights_into_int32(w_q: torch.Tensor, + wtype: ScalarType, + packed_dim: int = 0): + # move dim to pack to the end + perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim) + inv_perm = tuple(perm.index(i) for i in range(len(perm))) + w_q_perm = w_q.permute(perm) + + pack_factor = 32 // wtype.size_bits + mask = (1 << wtype.size_bits) - 1 + + new_shape_perm = list(w_q_perm.shape) + assert w_q_perm.shape[-1] % pack_factor == 0 + new_shape_perm[-1] //= pack_factor + + res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device) + for i in range(pack_factor): + res |= (w_q_perm[..., i::pack_factor] & mask) << wtype.size_bits * i + + return res.permute(inv_perm) + + +def unpack_weights_into_int32(w_q: torch.Tensor, + wtype: ScalarType, + packed_dim: int = 0): + # move dim to pack to the end + perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim) + inv_perm = tuple(perm.index(i) for i in range(len(perm))) + w_q_perm = w_q.permute(perm) + + pack_factor = 32 // wtype.size_bits + mask = (1 << wtype.size_bits) - 1 + + new_shape_perm = list(w_q_perm.shape) + new_shape_perm[-1] *= pack_factor + + res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device) + for i in range(pack_factor): + res[..., i::pack_factor] = (w_q_perm >> wtype.size_bits * i) & mask + + return res.permute(inv_perm) + + +def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool: + # prefix: model.layers.0.self_attn.q_proj + # proj_name: q_proj + proj_name = prefix.split(".")[-1] + if proj_name in FUSED_LAYER_NAME_MAPPING: + shard_prefixes = [ + prefix.replace(proj_name, shard_proj_name) + for shard_proj_name in FUSED_LAYER_NAME_MAPPING[proj_name] + ] + + is_skipped = None + for shard_prefix in shard_prefixes: + is_shard_skipped = shard_prefix in ignored_layers + + if is_skipped is None: + is_skipped = is_shard_skipped + elif is_shard_skipped != is_skipped: + raise ValueError( + f"Detected some but not all shards of {prefix} " + "are quantized. All shards of fused layers " + "to have the same precision.") + else: + is_skipped = prefix in ignored_layers + + assert is_skipped is not None + return is_skipped + + +def get_pack_factor(num_bits): + assert 32 % num_bits == 0, f"Unsupported num_bits = {num_bits}" + return 32 // num_bits + + +def permute_rows(q_w: torch.Tensor, + w_ref: torch.Tensor, + group_size: int, + test_perm: Optional[torch.Tensor] = None): + assert q_w.shape == w_ref.shape + + orig_device = q_w.device + k_size, _ = q_w.shape + + g_idx = torch.zeros((k_size, ), dtype=torch.int32) + for i in range(k_size): + g_idx[i] = i // group_size + + # Simulate act_order by doing a random permutation on K + rand_perm = test_perm if test_perm is not None else torch.randperm(k_size) + + g_idx = g_idx[rand_perm].contiguous() + q_w = q_w[rand_perm, :].contiguous() + w_ref = w_ref[rand_perm, :].contiguous() + + return ( + w_ref.to(device=orig_device), + q_w.to(device=orig_device), + g_idx.to(device=orig_device), + rand_perm.to(device=orig_device), + ) + + +def quantize_weights(w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + zero_points: bool = False, + ref_zero_points_after_scales: bool = False): + assert quant_type.is_integer(), \ + "Floating point quantization may work but has not been tested" + + orig_device = w.device + orig_type = w.dtype + size_k, size_n = w.shape + + assert w.is_floating_point(), "w must be float" + + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Reshape to [groupsize, -1] + if group_size < size_k: + w = w.reshape((-1, group_size, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((group_size, -1)) + + # Compute scale for each group + max_val = torch.max(w, 0, keepdim=True).values + min_val = torch.min(w, 0, keepdim=True).values + + max_q_val = quant_type.max() + min_q_val = quant_type.min() + + if zero_points: + assert not quant_type.is_signed() and quant_type.max() > 0 + w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max() + maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \ + .clamp(min_q_val, max_q_val).int() + else: + # If the bias is such that there are no possible negative/positive + # values, set the max value to inf to avoid divide by 0 + w_s = torch.max( + abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)), + abs(min_val / (min_q_val if min_q_val != 0 else torch.inf))) + maybe_w_zp = None + + # Quantize + w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0) + w_q = torch.clamp(w_q, min_q_val, max_q_val) + + # Compute ref (dequantized) + # For some kernels (namely Machete) the zero-points are applied after the + # scales are applied, for this case computing the reference in similar way + # allows us to use tighter error tolerances in our unit tests. + if ref_zero_points_after_scales and zero_points: + w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s + else: + w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s + + if quant_type.has_bias(): + w_q += quant_type.bias + + # Restore original shapes + if group_size < size_k: + + def reshape_w(w): + w = w.reshape((group_size, -1, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((size_k, size_n)).contiguous() + return w + + w_q = reshape_w(w_q) + w_ref = reshape_w(w_ref) + + w_s = w_s.reshape((-1, size_n)).contiguous() + + if zero_points: + maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous() + maybe_w_zp = maybe_w_zp.to(device=orig_device) + + return ( + w_ref.to(device=orig_device), + w_q.to(device=orig_device), + w_s.to(device=orig_device), + maybe_w_zp, + ) + + +def gptq_quantize_weights(w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + act_order: bool, + test_perm: Optional[torch.Tensor] = None): + size_k, _ = w.shape + + assert w.is_floating_point(), "w must be float" + assert quant_type in SUPPORTED_GPTQ_QUANT_TYPES, \ + f"Unsupported gptq type = {quant_type}" + assert group_size in SUPPORTED_GROUP_SIZES + [ + size_k + ], f"Unsupported groupsize = {group_size}" + + w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size) + + # Apply act_order + g_idx = torch.empty(0, dtype=torch.int, device=w.device) + rand_perm = torch.empty(0, dtype=torch.int, device=w.device) + if act_order: + assert ( + group_size < size_k + ), "For act_order, groupsize = {} must be less than size_k = {}".format( + group_size, size_k) + + w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, + test_perm) + + return w_ref, w_q, w_s, g_idx, rand_perm + + +# QQQ employs different quant schemes for per-group and +# per-channel quantization. +def qqq_quantize_weights(w: torch.Tensor, num_bits: int, group_size: int): + orig_device = w.device + size_k, size_n = w.shape + + assert w.is_floating_point(), "w must be float" + assert num_bits in MARLIN_QQQ_SUPPORTED_NUM_BITS, \ + f"Unsupported num_bits = {num_bits}" + assert group_size in SUPPORTED_GROUP_SIZES + [ + size_k + ], f"Unsupported groupsize = {group_size}" + + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + if group_size < size_k: + # Reshape to [groupsize, -1] + w = w.reshape((-1, group_size, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((group_size, -1)) + + max_q_val = 2**num_bits - 1 + half_q_val = (max_q_val + 1) // 2 + + # Compute scale for each group + s_group = torch.max(torch.abs(w), 0, keepdim=True)[0] + s_group *= 2 / max_q_val # 2 => symmetric + + # Quantize + q_w = torch.round(w / s_group).int() + q_w += half_q_val + q_w = torch.clamp(q_w, 0, max_q_val) + # Compute ref (dequantized) + w_ref = (q_w - half_q_val).half() * s_group + + # Restore original shapes + def reshape_w(w): + w = w.reshape((group_size, -1, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((size_k, size_n)).contiguous() + return w + + q_w = reshape_w(q_w) + w_ref = reshape_w(w_ref) + + # Compute int8 quantization scale for each channel + s_channel = torch.max(torch.abs(w_ref), 0, keepdim=True)[0] + s_channel /= 127.0 + t_int8 = (w_ref / s_channel).round().clamp(-128, 127).to(torch.int8) + w_ref = t_int8.half() * s_channel + s_channel = s_channel.reshape(1, -1).to(dtype=torch.float) + + # Fuse scales + s_group = (s_group.reshape(-1, size_n).contiguous() / + s_channel).to(dtype=torch.half) + else: + max_q_val = 2**(num_bits - 1) - 1 + + # Compute scale for each channel + s_channel = torch.max(torch.abs(w), 0, keepdim=True)[0] + s_channel /= max_q_val + + # Quantize + q_w = torch.round(w / s_channel).int() + q_w = torch.clamp(q_w, -max_q_val, max_q_val) + # Compute ref (dequantized) + w_ref = q_w.half() * s_channel + + s_group = torch.tensor([], dtype=torch.half) + # div 2 ** (8 - self.bits)) to offset right shift in unpacking + s_channel /= (2**(8 - num_bits)) + s_channel = s_channel.reshape(-1, size_n).contiguous().to(torch.float) + + return ( + w_ref.to(device=orig_device), + q_w.to(device=orig_device), + s_group.to(device=orig_device), + s_channel.to(device=orig_device), + ) + + +def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor): + orig_device = q_w.device + + sort_indices = torch.argsort(g_idx).to( + dtype=torch.int32) # Sort based on g_idx + + g_idx = g_idx[sort_indices].contiguous() + q_w = q_w[sort_indices, :].contiguous() + + return ( + q_w.to(device=orig_device), + g_idx.to(device=orig_device), + sort_indices.to(device=orig_device), + ) + + +def pack_rows( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + pack_factor = get_pack_factor(num_bits) + assert size_k % pack_factor == 0 + + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(numpy.uint32) + + q_res = numpy.zeros((size_k // pack_factor, size_n), dtype=numpy.uint32) + + for i in range(pack_factor): + q_res |= q_w[i::pack_factor, :] << num_bits * i + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + return q_res + + +def pack_cols( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + pack_factor = get_pack_factor(num_bits) + assert size_n % pack_factor == 0 + + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(numpy.uint32) + + q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32) + + for i in range(pack_factor): + q_res |= q_w[:, i::pack_factor] << num_bits * i + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + q_res = q_res.contiguous() + + return q_res + + +def unpack_cols( + packed_q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + pack_factor = get_pack_factor(num_bits) + assert size_n % pack_factor == 0 + assert packed_q_w.shape == ( + size_k, size_n // pack_factor + ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format( + packed_q_w.shape, size_k, size_n, pack_factor) + + orig_device = packed_q_w.device + + packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32) + q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32) + + mask = (1 << num_bits) - 1 + for i in range(pack_factor): + vals = packed_q_w_cpu & mask + packed_q_w_cpu >>= num_bits + q_res[:, i::pack_factor] = vals + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + q_res = q_res.contiguous() + + return q_res + + +def gptq_pack( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + return pack_rows(q_w, num_bits, size_k, size_n) + + +def awq_pack( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + # Interleave column dim (for the dequantize code) and pack it to int32 + if num_bits == 4: + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = numpy.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + q_w = q_w.reshape((-1, len(interleave)))[:, interleave].ravel() + q_w = q_w.reshape((-1, size_n)).contiguous() + + return pack_cols(q_w, num_bits, size_k, size_n) diff --git a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py new file mode 100644 index 00000000..411af922 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py @@ -0,0 +1,246 @@ +from typing import List, Optional, Tuple, Union + +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform +from vllm.utils import is_hip + +# Input scaling factors are no longer optional in _scaled_mm starting +# from pytorch 2.5. Allocating a dummy tensor to pass as input_scale +TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() if is_hip() else None + + +def cutlass_fp8_supported() -> bool: + # cutlass is not supported on Rocm + if is_hip(): + return False + + capability_tuple = current_platform.get_device_capability() + capability = -1 if capability_tuple is None else capability_tuple.to_int() + + return ops.cutlass_scaled_mm_supports_fp8(capability) + + +def per_tensor_dequantize( + tensor: torch.Tensor, inv_scale: Union[float, + torch.Tensor]) -> torch.Tensor: + fake_qweight = tensor.to(torch.float16) + dq_weight = fake_qweight * inv_scale + return dq_weight + + +def all_close_1d(x: torch.Tensor) -> bool: + assert len(x.shape) == 1 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0])) + + +def convert_to_channelwise( + weight_scale: torch.Tensor, + logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]: + # Create channelwise buffer + weight_scale_channel = torch.empty((sum(logical_widths), 1), + dtype=torch.float32, + device=weight_scale.device) + + # Expand each scale to match the size of each logical matrix. + start = 0 + for idx, logical_width in enumerate(logical_widths): + end = start + logical_width + weight_scale_channel[start:end, :] = weight_scale[idx] + start = end + + return weight_scale_channel + + +def requantize_with_max_scale( + weight: torch.Tensor, weight_scale: torch.Tensor, + logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]: + # Max scale to be used for requanitzation. + max_w_scale = weight_scale.max() + + # QKV / MLP is fused in the on disk checkpoint if any of the + # weight scales are still set to the default since we initialize + # N weight scales for N shards but we only load 1 weight scale + # from disk in this case. Skip requantization in this case (since) + # we already are quantized with the single scale. + # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8 + unfused_module_in_checkpoint = (weight_scale[-1] > torch.finfo( + torch.float8_e4m3fn).min) + + # If unfused checkpoint, need requanize with the single scale. + if unfused_module_in_checkpoint: + start = 0 + for idx, logical_width in enumerate(logical_widths): + end = start + logical_width + weight_dq = per_tensor_dequantize(weight[start:end, :], + weight_scale[idx]) + weight[start:end, :], _ = ops.scaled_fp8_quant( + weight_dq, max_w_scale) + start = end + + return max_w_scale, weight + + +def apply_fp8_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + input_scale_ub: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + cutlass_fp8_supported: bool = True, + use_per_token_if_dynamic: bool = False, +) -> torch.Tensor: + # ops.scaled_fp8_quant supports both dynamic and static quant. + # If dynamic, layer.input_scale is None and x_scale computed from x. + # If static, layer.input_scale is scalar and x_scale is input_scale. + + # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A + if cutlass_fp8_supported: + qinput, x_scale = ops.scaled_fp8_quant( + input, + input_scale, + scale_ub=input_scale_ub, + use_per_token_if_dynamic=use_per_token_if_dynamic) + + # Fused GEMM_DQ + return ops.cutlass_scaled_mm(qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias) + + # torch.scaled_mm supports per tensor weights + activations only + # so fallback to naive if per channel or per token + else: + # Note: we pad the input because torch._scaled_mm is more performant + # for matrices with batch dimension > 16. + # This could change in the future. + qinput, x_scale = ops.scaled_fp8_quant( + input, + input_scale, + num_token_padding=17, + use_per_token_if_dynamic=use_per_token_if_dynamic) + + per_tensor_weights = (weight_scale.numel() == 1) + per_tensor_activations = (x_scale.numel() == 1) + + if per_tensor_weights and per_tensor_activations: + # Fused GEMM_DQ + output = torch._scaled_mm(qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias) + # A fix for discrepancy in scaled_mm which returns tuple + # for torch < 2.5 and a single value in torch >= 2.5 + if type(output) is tuple and len(output) == 2: + return torch.narrow(output[0], 0, 0, input.shape[0]) + return torch.narrow(output, 0, 0, input.shape[0]) + + else: + # Fallback for channelwise case, where we use unfused DQ + # due to limitations with scaled_mm + + # Symmetric quantized GEMM by definition computes the following: + # C = (s_x * X) (s_w * W) + bias + # This is equivalent to dequantizing the weights and activations + # before applying a GEMM. + # + # In order to compute quantized operands, a quantized kernel + # will rewrite the above like so: + # C = s_w * s_x * (X * W) + bias + # + # For the scaled_mm fallback case, we break this down, since it + # does not support s_w being a vector. + + # Making sure the dummy tensor is on the same device as the weight + global TORCH_DEVICE_IDENTITY + if (TORCH_DEVICE_IDENTITY is not None + and TORCH_DEVICE_IDENTITY.device != weight.device): + TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device) + + # GEMM + # This computes C = (X * W). + # Output in fp32 to allow subsequent ops to happen in-place + output = torch._scaled_mm(qinput, + weight, + scale_a=TORCH_DEVICE_IDENTITY, + scale_b=TORCH_DEVICE_IDENTITY, + out_dtype=torch.float32) + # A fix for discrepancy in scaled_mm which returns tuple + # for torch < 2.5 and a single value in torch >= 2.5 + if type(output) is tuple and len(output) == 2: + output = output[0] + # Unpad (undo num_token_padding) + output = torch.narrow(output, 0, 0, input.shape[0]) + x_scale = torch.narrow(x_scale, 0, 0, input.shape[0]) + + # DQ + # C = sw * sx * (X * W) + bias + output = output * x_scale * weight_scale.t() + if bias is not None: + output = output + bias + return output.to(dtype=input.dtype) + + +def apply_int8_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + input_zero_point: Optional[torch.Tensor] = None, + azp_adj: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +): + # ops.scaled_int8_quant supports both dynamic and static quant. + # * dynamic, layer.input_scale is None and x_scale computed from x. + # * static, layer.input_scale is scalar and x_scale is input_scale. + symmetric = azp_adj is None + x_q, x_scale, x_zp = ops.scaled_int8_quant(input, + input_scale, + input_zero_point, + symmetric=symmetric) + + if x_zp is not None: + return ops.cutlass_scaled_mm_azp(x_q, + weight, + scale_a=x_scale, + scale_b=weight_scale, + out_dtype=input.dtype, + azp_adj=azp_adj, + azp=x_zp, + bias=bias) + return ops.cutlass_scaled_mm(x_q, + weight, + scale_a=x_scale, + scale_b=weight_scale, + out_dtype=input.dtype, + bias=bias) + + +def normalize_e4m3fn_to_e4m3fnuz( + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + assert weight.dtype == torch.float8_e4m3fn + # The bits pattern 10000000(-128) represents zero in e4m3fn + # but NaN in e4m3fnuz. So here we set it to 0. + # https://onnx.ai/onnx/technical/float8.html + weight_as_int8 = weight.view(torch.int8) + ROCM_FP8_NAN_AS_INT = -128 + weight_as_int8[weight_as_int8 == ROCM_FP8_NAN_AS_INT] = 0 + weight = weight_as_int8.view(torch.float8_e4m3fnuz) + + # For the same bits representation, e4m3fnuz value is half of + # the e4m3fn value, so we should double the scaling factor to + # get the same dequantized value. + # https://onnx.ai/onnx/technical/float8.html + weight_scale = weight_scale * 2.0 + if input_scale is not None: + input_scale = input_scale * 2.0 + return weight, weight_scale, input_scale diff --git a/vllm/model_executor/layers/quantization/w8a16.py b/vllm/model_executor/layers/quantization/w8a16.py new file mode 100644 index 00000000..6c42ce7f --- /dev/null +++ b/vllm/model_executor/layers/quantization/w8a16.py @@ -0,0 +1,114 @@ +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.parameter import (GroupQuantScaleParameter, + PackedvLLMParameter) +from vllm.model_executor.utils import set_weight_attrs + + +class W8a16Config(QuantizationConfig): + """Config class for W8a16. + + """ + + def __init__( + self, + ) -> None: + pass + + def __repr__(self) -> str: + return ("W8a16Config") + + def get_name(self) -> str: + return "w8a16" + + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + def get_min_capability(self) -> int: + return 75 + + @staticmethod + def get_config_filenames(): + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "W8a16Config": + return cls() + + def get_quant_method(self, layer: torch.nn.Module, + prefix: str) -> Optional["W8a16LinearMethod"]: + if isinstance(layer, LinearBase): + return W8a16LinearMethod(self) + return None + + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class W8a16LinearMethod(LinearMethodBase): + """Linear method for w8a16. + + """ + + def __init__(self, quant_config: W8a16Config): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + output_size_per_partition = sum(output_partition_sizes) + weight = Parameter( + torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.int8, + ), + requires_grad=False, + ) + set_weight_attrs( + weight, { + "input_dim": 1, + "output_dim": 0, + }) + + scales = Parameter( + torch.empty( + 1, + output_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(scales, { + "input_dim": None, + "output_dim": 1, + }) + + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + layer.register_parameter("scales", scales) + set_weight_attrs(scales, extra_weight_attrs) + + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + qweight = layer.weight + scales = layer.scales + out_shape = (x.shape[:-1] + (qweight.shape[-2],)) + reshaped_x = x.reshape(-1, x.shape[-1]) + out = ops.linear_w8a16(reshaped_x, qweight, scales, format="TN") + if bias is not None: + out = out + bias + return out.reshape(out_shape) \ No newline at end of file diff --git a/vllm/model_executor/layers/rejection_sampler.py b/vllm/model_executor/layers/rejection_sampler.py new file mode 100644 index 00000000..1137b519 --- /dev/null +++ b/vllm/model_executor/layers/rejection_sampler.py @@ -0,0 +1,401 @@ +from functools import cached_property +from importlib.util import find_spec +from typing import Dict, List, Optional, Tuple + +import torch +import torch.jit + +import vllm.envs as envs +from vllm.logger import init_logger +from vllm.model_executor.layers.spec_decode_base_sampler import ( + SpecDecodeStochasticBaseSampler) + +logger = init_logger(__name__) + +if find_spec("flashinfer"): + """ + Consider utilizing the FlashInfer rejection sampling kernel initially, + as it employs a dedicated kernel rather than relying on + Torch tensor operations. This design choice helps to fuse operations, + reduce memory I/O, and consequently enhances performance. + """ + from flashinfer.sampling import chain_speculative_sampling +else: + chain_speculative_sampling = None + + +class RejectionSampler(SpecDecodeStochasticBaseSampler): + """Apply modified rejection sampling as described in "Accelerating Large + Language Model Decoding with Speculative Sampling" + https://arxiv.org/pdf/2302.01318.pdf. + """ + + def __init__(self, + strict_mode: bool = False, + use_flashinfer: Optional[bool] = None): + """Create a rejection sampler. + + Args: + strict_mode: Whether or not to perform shape/device/dtype checks + during sampling. This catches correctness issues but adds + nontrivial latency. + use_falshinfer: We will use this parameter to determine whether + to use the FlashInfer rejection sampling kernel or not. If it's + None, we will use the default value from the environment variable. + This parameter is only used for testing purposes. + """ + super().__init__(strict_mode=strict_mode) + if use_flashinfer is None: + self.use_flashinfer = envs.VLLM_USE_FLASHINFER_SAMPLER and ( + chain_speculative_sampling is not None) + else: + self.use_flashinfer = use_flashinfer + + if self.use_flashinfer: + logger.info("Use flashinfer for rejection sampling.") + else: + logger.info("Use pytorch for rejection sampling.") + + def forward( + self, + target_with_bonus_probs: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + seeded_seqs: Optional[Dict[int, torch.Generator]] = None, + ) -> torch.Tensor: + """Sample token ids using rejection sampling. This accepts or rejects + tokens proposed by the draft model using the probability of each token + according to the draft and target models. + + In the worst case where all draft tokens are rejected, it is guaranteed + one correct token will be emitted. + + In the case where all draft tokens are accepted, a bonus token will be + accepted as its cheap to have the target model score this speculative + sequence. + + Args: + target_with_bonus_probs: The probability distribution + over token ids given context according to the target model. + shape = [batch_size, num_speculative_tokens + 1, vocab_size] + + bonus_token_ids: The "bonus" token ids that are accepted iff all + speculative tokens in a sequence are accepted. + shape = [batch_size, num_bonus_tokens] + + draft_probs: The probability distribution over token ids given + context according to the draft model. + shape = [batch_size, num_speculative_tokens, vocab_size] + + draft_token_ids: The token ids that were sampled from the draft + probabilities. + shape = [batch_size, num_speculative_tokens] + + seeded_seqs: Dict of batch row index to torch generator, for + sequences using seeded generation. + + Returns: + output_token_ids: The token ids sampled via rejection sampling, + or -1 if unable to sample a token because the previous token + was rejected. + shape = [batch_size, num_speculative_tokens + num_bonus_tokens] + """ + # Only perform shape/dtype/device checking in strict mode, as it adds + # overhead. + if self._strict_mode: + self._raise_if_incorrect_input(target_with_bonus_probs, + draft_token_ids, bonus_token_ids, + draft_probs) + + batch_size, k, _ = draft_probs.shape + + # batch_size = 0 when all requests in the batch are + # non_spec requests. In this case, output_token_ids is + # just an empty tensor. + if batch_size == 0: + return torch.empty(0, k + 1, device=draft_probs.device, dtype=int) + + # If use Flashinfer chain_speculative_sampling kernel + # for rejection sampling + if self.use_flashinfer: + batch_size, k, _ = draft_probs.shape + uniform_samples = self._create_uniform_samples( + seeded_seqs, batch_size, k, draft_probs.device) + output_token_ids, accepted_token_num, emitted_token_num \ + = chain_speculative_sampling( + draft_probs, draft_token_ids, uniform_samples, + target_with_bonus_probs) + + # num_emitted_tokens returned by flashinfer + # does not include the bonus token + # Flashinfer stops at the first token that violates + # the condition p >= q and does not include recovery/bonus token. + # Therefore, we need to add batch_size here. + self.num_accepted_tokens += accepted_token_num.sum() + self.num_emitted_tokens += emitted_token_num.sum() + batch_size + self.num_draft_tokens += batch_size * k + else: + accepted, recovered_token_ids = ( + self._batch_modified_rejection_sampling( + target_with_bonus_probs[:, :-1], + draft_probs, + draft_token_ids, + seeded_seqs, + )) + + output_token_ids = self._create_output( + accepted, + recovered_token_ids, + draft_token_ids, + bonus_token_ids, + ) + + return output_token_ids + + def _batch_modified_rejection_sampling( + self, + target_probs: torch.Tensor, # [batch_size, k, vocab_size] + draft_probs: torch.Tensor, # [batch_size, k, vocab_size] + draft_token_ids: torch.Tensor, # [batch_size, k] + seeded_seqs: Optional[Dict[int, torch.Generator]], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Perform modified rejection sampling on each sequence. + + Returns: + A tuple of two tensors: + 0: A bool tensor of which tokens in each sequence is accepted. + shape = [batch_size, k] + 1: Token ids sampled from a recovered distribution, to be used + when a token is rejected. + shape = [batch_size, k] + """ + + batch_size, k, vocab_size = draft_probs.shape + + # shape [batch_size, k] + accepted = self._get_accepted(target_probs, draft_probs, + draft_token_ids, seeded_seqs) + + recovered_probs = self._get_recovered_probs( + target_probs, draft_probs).reshape(batch_size * k, vocab_size) + + # NOTE: the recovered_probs are overwritten by this method. + recovered_token_ids = _multinomial( + recovered_probs, + num_samples=1, + k=k, + seeded_seqs=seeded_seqs or {}, + ).reshape(batch_size, k) + + return accepted, recovered_token_ids + + def _create_uniform_samples(self, + seeded_seqs: Optional[Dict[int, + torch.Generator]], + batch_size: int, k: int, + device: torch.device) -> torch.Tensor: + """ + Generates a batch of uniform random samples, with optional seeding + for specific sequences. + + This method creates a tensor of shape `(batch_size, k + 1)` filled + with uniform random values in the range [0, 1). If `seeded_seqs` + is provided, the sequences corresponding to specific indices + will be generated using the provided `torch.Generator` for + reproducibility. The other sequences will be generated without + a seed. + + Args: + seeded_seqs : Optional[Dict[int, torch.Generator]] + A dictionary mapping indices in the batch to + `torch.Generator` objects. If `None`, all samples are + generated without a seed. + batch_size : int + The number of sequences to generate. + k : int + The number of random samples per sequence. + device : torch.device + The device on which to allocate the tensor. + + Returns: + uniform_rand : torch.Tensor + A tensor of shape `(batch_size, k + 1)` containing uniform + random values in the range [0, 1). + """ + if not seeded_seqs: + return torch.rand(batch_size, k + 1, device=device) + + uniform_rand = torch.empty(batch_size, k + 1, device=device) + + non_seeded_indices = [] + for idx in range(batch_size): + generator = seeded_seqs.get(idx) + if generator is None: + non_seeded_indices.append(idx) + else: + uniform_rand[idx, :] = torch.rand(1, + k + 1, + dtype=self.probs_dtype, + device=device, + generator=generator) + if non_seeded_indices: + uniform_rand[non_seeded_indices, :] = torch.rand( + len(non_seeded_indices), + k + 1, + dtype=self.probs_dtype, + device=device) + return uniform_rand + + def _get_accepted( + self, + target_probs: torch.Tensor, # [batch_size, k, vocab_size] + draft_probs: torch.Tensor, # [batch_size, k, vocab_size] + draft_token_ids: torch.Tensor, # [batch_size, k] + seeded_seqs: Optional[Dict[int, torch.Generator]], + ) -> torch.Tensor: + r"""Create bool matrix over the proposed draft tokens. If + True, then a token can be accepted, else it should be + rejected. + + Given :math:`q(\hat{x}_{n+1}|x_1, \dots, x_n)`, the probability of + :math:`\hat{x}_{n+1}` given context :math:`x_1, \dots, x_n` according + to the target model, and :math:`p(\hat{x}_{n+1}|x_1, \dots, x_n)`, the + same conditional probability according to the draft model, the token + is accepted with probability: + + .. math:: + \min\left(1, \frac{q(\hat{x}_{n+1}|x_1, \dots, x_n)} + {p(\hat{x}_{n+1}|x_1, \dots, x_n)}\right) + + This implementation does not apply causality. When using the output, + if a token is rejected, subsequent tokens should not be used. + + Returns a bool tensor of shape [batch_size, k] specifying which tokens + are accepted. + """ + batch_size, k, _ = draft_probs.shape + batch_indices = torch.arange(batch_size, + device=target_probs.device)[:, None] + probs_indicies = torch.arange(k, device=target_probs.device) + + # shape [batch_size, k] + selected_draft_probs = draft_probs[batch_indices, probs_indicies, + draft_token_ids] + + # shape [batch_size, k] + selected_target_probs = target_probs[batch_indices, probs_indicies, + draft_token_ids] + + uniform_rand = self._create_uniform_samples(seeded_seqs, batch_size, + k - 1, target_probs.device) + + capped_ratio = torch.minimum( + selected_target_probs / selected_draft_probs, + torch.full((1, ), 1, device=target_probs.device)) + accepted = uniform_rand < capped_ratio + + return accepted + + def _get_recovered_probs( + self, + target_probs: torch.Tensor, # [k, vocab_size] + draft_probs: torch.Tensor, # [k, vocab_size] + ) -> torch.Tensor: + r"""Create a probability distribution for each proposed token which can + be sampled if the proposed token is rejected. + + When this routine is applied sequentially, the true distribution of the + target model is recovered (within hardware numerics). + + The probability distribution used in this rejection case is constructed + as follows. Given :math:`q(x|x_1, \dots, x_n)`, the probability of + :math:`x` given context :math:`x_1, \dots, x_n` according to the target + model and :math:`p(x|x_1, \dots, x_n)`, the same conditional probability + according to the draft model: + + .. math:: + x_{n+1} \sim (q(x|x_1, \dots, x_n) - p(x|x_1, \dots, x_n))_+ + + where :math:`(f(x))_+` is defined as: + + .. math:: + (f(x))_+ = \frac{\max(0, f(x))}{\sum_x \max(0, f(x))} + + See https://github.com/vllm-project/vllm/pull/2336 for a visualization + of the draft, target, and recovered probability distributions. + + Returns a tensor of shape [batch_size, k, vocab_size]. + + Note: This batches operations on GPU and thus constructs the recovered + distribution for all tokens, even if they are accepted. This causes + division-by-zero errors, so we use self._smallest_positive_value to + avoid that. This introduces some drift to the distribution. + """ + _, k, _ = draft_probs.shape + + # shape [batch_size, k, vocab_size] + difference = target_probs - draft_probs + + # TODO(cade): Can we use logprobs instead of probs, and avoid the + # division-by-zero errors without introducing distribution drift? + + # shape [batch_size, k, vocab_size] + f = torch.clamp(difference, min=self._smallest_positive_value) + + # shape [batch_size, k, vocab_size] + recovered_probs = f / torch.sum(f, dim=-1).reshape(-1, k, 1) + + return recovered_probs + + @cached_property + def _smallest_positive_value(self) -> float: + """Return the smallest positive value representable by the probs dtype. + This value is used when constructing a distribution from which to sample + recovered tokens in the first rejection case. + + See _get_recovered_probs for more details + + Note that this isn't actually the smallest positive value representable + by float32, but the smallest positive normal value. + See https://en.wikipedia.org/wiki/Subnormal_number for more information. + """ + return torch.finfo(self.probs_dtype).tiny + + +# torch.multinomial forces a GPU<->CPU sync. +# Therefore, we use an optimized implementation instead that skips the sync. +# Note that we always sample with replacement. +# probs will be modified in place, but this is fine, as we pass +# in a copy already. +# @torch.jit.script +def _multinomial( + probs: torch.Tensor, + num_samples: int, + k: int, + seeded_seqs: Dict[int, torch.Generator], +) -> torch.Tensor: + + if num_samples > 1: + # This is equivalent to torch.repeat_interleaved (which also + # forces a GPU<->CPU sync). + probs = probs[:, None, :].expand(probs.shape[0], num_samples, + probs.shape[1]).contiguous().view( + -1, probs.shape[1]) + q = torch.empty_like(probs) + if not seeded_seqs: + q.exponential_(1.0) + else: + non_seeded_indices: List[int] = [] + start = 0 + for idx in range(len(q) // k): + end = start + k + generator = seeded_seqs.get(idx) + if generator is None: + non_seeded_indices.extend(list(range(start, end))) + else: + q[start:end].exponential_(1.0, generator=generator) + start = end + q[non_seeded_indices].exponential_(1.0) + + return probs.div_(q).argmax(dim=1).view(-1, num_samples) diff --git a/vllm/model_executor/layers/resampler.py b/vllm/model_executor/layers/resampler.py new file mode 100644 index 00000000..1a70d0f5 --- /dev/null +++ b/vllm/model_executor/layers/resampler.py @@ -0,0 +1,273 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py +# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20 +# +# Copyright 2023 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Shared resampler perceiver network used in multimodal models and +related helpers for sincos positional embeddings. + +Example models: Qwen (Qwen-VL), Minicpmv2.0 +""" +import math +from functools import partial +from typing import Callable, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.init import normal_ + +from vllm.model_executor.layers.linear import ReplicatedLinear + +DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6) + + +def get_abs_pos(abs_pos: torch.Tensor, tgt_size: Union[torch.Tensor, + int]) -> torch.Tensor: + # abs_pos: L, C + # tgt_size: (H, W) + # return: M, C + src_size = int(math.sqrt(abs_pos.size(0))) + dtype = abs_pos.dtype + if isinstance(tgt_size, int): + tgt_size = (tgt_size, tgt_size) + if (src_size == tgt_size[0] and src_size == tgt_size[1]): + return abs_pos + return (F.interpolate( + abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2), + size=(tgt_size[0], tgt_size[1]), + mode="bicubic", + align_corners=False, + ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)) + + +# sin/cos positional embedding helpers are adapted from: +# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20 +def get_1d_sincos_pos_embed_from_grid( + embed_dim: int, pos: np.ndarray, + version: Tuple[int, int] = (2, 0)) -> torch.Tensor: + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) / (H, W) + out: (M, D) / (H, W, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float32) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + if version == (2, 0): + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + else: + out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product + emb_sin = np.sin(out) # (H, W, D/2) + emb_cos = np.cos(out) # (H, W, D/2) + emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D) + return emb + + +def get_2d_sincos_pos_embed_from_grid( + embed_dim: int, grid: np.ndarray, + version: Tuple[int, int] = (2, 0)) -> torch.Tensor: + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid( + embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid( + embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2) + + if version == (2, 0): + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + else: + emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D) + return emb + + +def get_2d_sincos_pos_embed( + embed_dim: int, + grid_size: Union[int, Tuple[int, int]], + cls_token: bool = False, + version: Tuple[int, int] = (2, 0), +) -> torch.Tensor: + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or + [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + if isinstance(grid_size, int): + grid_h_size, grid_w_size = grid_size, grid_size + else: + grid_h_size, grid_w_size = grid_size[0], grid_size[1] + + grid_h = np.arange(grid_h_size, dtype=np.float32) + grid_w = np.arange(grid_w_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + assert isinstance(grid, np.ndarray) and \ + grid.shape == (2, grid_h_size, grid_w_size) + + if version == (2, 0): + grid = grid.reshape([2, 1, grid_h_size, grid_w_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], + axis=0) + else: + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version) + return pos_embed + + +class BaseResampler(nn.Module): + """ + A 2D perceiver-resampler network with one cross attention layers by + (grid_size**2) learnable queries and 2d sincos pos_emb. + Outputs: + A tensor with the shape of (grid_size**2, embed_dim) + """ + + def __init__( + self, + num_queries: int, + embed_dim: int, + num_heads: int, + kv_dim: Optional[int] = None, + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN, + do_post_projection: bool = True, + ) -> None: + super().__init__() + + self.num_queries = num_queries + self.embed_dim = embed_dim + self.num_heads = num_heads + + self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim)) + normal_(self.query, std=0.02) + if kv_dim is not None and kv_dim != embed_dim: + self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False) + else: + # Maintain the same return value with ReplicatedLinear.forward + self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa + nn.Identity()(*args, **kwargs), + None, + ) + self.attn = nn.MultiheadAttention(embed_dim, num_heads) + self.ln_q = norm_layer(embed_dim) + self.ln_kv = norm_layer(embed_dim) + self.do_post_projection = do_post_projection + self.ln_post = norm_layer(embed_dim) if do_post_projection else None + self.proj = nn.Parameter( + (embed_dim**-0.5) * + torch.randn(embed_dim, embed_dim)) if do_post_projection else None + + def _init_weights(self, m: nn.Module) -> None: + if isinstance(m, nn.Linear): + normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _repeat(self, query, N: int): + return query.unsqueeze(1).repeat(1, N, 1) + + +class Resampler2(BaseResampler): + """Resampler-perceiver network to be used for a variety of model types, + e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the + do_post_projection arg, which indicates whether or not there should be + a post layer normalization and projector after the attention. This is + present in minicpmv2.0, but not qwen-vl. + """ + + def __init__( + self, + grid_size: int, + embed_dim: int, + num_heads: int, + kv_dim: Optional[int] = None, + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN, + adaptive: bool = False, + do_post_projection: bool = True, + ) -> None: + super().__init__(grid_size**2, + embed_dim, + num_heads, + kv_dim, + norm_layer, + do_post_projection=do_post_projection) + + self.adaptive = adaptive + pos_embed_arr = get_2d_sincos_pos_embed(embed_dim, + grid_size, + version=(2, 0)) + + self.pos_embed = nn.Parameter( + torch.from_numpy(pos_embed_arr).requires_grad_(False)) + + self.apply(self._init_weights) + + def forward( + self, + x: torch.Tensor, + tgt_sizes: Optional[torch.Tensor] = None, + attn_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if tgt_sizes is None: + tgt_sizes = int(math.sqrt(x.size(1))) + if self.adaptive: + pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim, + tgt_sizes, + version=(2, 0)) + pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device, + dtype=x.dtype) + else: + pos_embed = get_abs_pos(self.pos_embed, + tgt_sizes).to(device=x.device, + dtype=x.dtype) + + x, _ = self.kv_proj(x) + x = self.ln_kv(x).permute(1, 0, 2) + + N = x.shape[1] + q = self.ln_q(self.query) + out = self.attn( + self._repeat(q, N) + self.pos_embed.unsqueeze(1), + x + pos_embed.unsqueeze(1), + x, + attn_mask=attn_mask, + )[0] + x = out.permute(1, 0, 2) + if self.do_post_projection: + x = self.ln_post(x) + x = x @ self.proj + return x diff --git a/vllm/model_executor/layers/rotary_embedding.py b/vllm/model_executor/layers/rotary_embedding.py new file mode 100644 index 00000000..847e834c --- /dev/null +++ b/vllm/model_executor/layers/rotary_embedding.py @@ -0,0 +1,1006 @@ +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rotary Positional Embeddings.""" +import math +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from vllm.model_executor.custom_op import CustomOp + + +def _rotate_neox(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., :x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=-1) + + +def _rotate_gptj(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., ::2] + x2 = x[..., 1::2] + x = torch.stack((-x2, x1), dim=-1) + return x.flatten(-2) + + +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + cos = cos.unsqueeze(-2).to(x.dtype) + sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_neox_style: + return torch.cat((o1, o2), dim=-1) + else: + return torch.stack((o1, o2), dim=-1).flatten(-2) + + +class RotaryEmbedding(CustomOp): + """Original rotary positional embedding.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + cache = cache.to(dtype) + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + """Compute the inverse frequency.""" + # NOTE(woosuk): To exactly match the HF implementation, we need to + # use CPU to compute the cache and then move it to GPU. However, we + # create the cache on GPU for faster initialization. This may cause + # a slight numerical difference between the HF implementation and ours. + inv_freq = 1.0 / (base**(torch.arange( + 0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim)) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + if offsets is not None: + positions = positions + offsets + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., :self.rotary_dim] + query_pass = query[..., self.rotary_dim:] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., :self.rotary_dim] + key_pass = key[..., self.rotary_dim:] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + return query, key + + def forward_cuda( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + from vllm import _custom_ops as ops + + self.cos_sin_cache = self.cos_sin_cache.to(query.device, + dtype=query.dtype) + # ops.rotary_embedding()/batched_rotary_embedding() + # are in-place operations that update the query and key tensors. + if offsets is not None: + ops.batched_rotary_embedding(positions, query, key, self.head_size, + self.cos_sin_cache, + self.is_neox_style, self.rotary_dim, + offsets) + else: + ops.rotary_embedding(positions, query, key, self.head_size, + self.cos_sin_cache, self.is_neox_style) + return query, key + + def forward_xpu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + from vllm._ipex_ops import ipex_ops as ops + + self.cos_sin_cache = self.cos_sin_cache.to(positions.device, + dtype=query.dtype) + # ops.rotary_embedding()/batched_rotary_embedding() + # are in-place operations that update the query and key tensors. + if offsets is not None: + ops.batched_rotary_embedding(positions, query, key, self.head_size, + self.cos_sin_cache, + self.is_neox_style, self.rotary_dim, + offsets) + else: + ops.rotary_embedding(positions, query, key, self.head_size, + self.cos_sin_cache, self.is_neox_style) + return query, key + + def extra_repr(self) -> str: + s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}" + s += f", max_position_embeddings={self.max_position_embeddings}" + s += f", base={self.base}, is_neox_style={self.is_neox_style}" + return s + + +class LinearScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with linear scaling. + + It supports multiple scaling factors. Since multiple LoRA adapters may have + different scaling factors, we need multiple cos/sin caches. In this way, + instead of running rotary embedding kernel per lora, we can run multiple + lora in a batched way. + + In addition to that, we also keep the cos/sin cache for the scaling factor + of 1 (default) at all times. + + Exemplary for two scaling factors x=1, y and z with embeddings + [[x11, x12, ... x1m], ..., [xn1, xn2, ..., xnm]] and + [[y11, y12, ... y1o], ..., [yn1, yn2, ..., yno]], and + [[z11, z12, ... z1p], ..., [zn1, zn2, ..., znp]], + + we construct the cos/sin cache as follows: + [[x11, x12, ... x1m, y11, y12, ... y1o, z11, z12, ... z1p], + ... + [xn1, xn2, ... xnm, yn1, yn2, ... yno, zn1, zn2, ... znp]] + + We then use offsets to index into the cos/sin cache for + the respective scaling factors. + + The offset to cache can be accessed via `scaling_factor_to_offset` API. + + Credits to the Reddit user /u/kaiokendev + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factors: Union[List[float], float], + dtype: torch.dtype, + ) -> None: + if isinstance(scaling_factors, float): + scaling_factors = [scaling_factors] + self.scaling_factors: List[float] = scaling_factors # noqa + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + # Lazy initialized. + self._scaling_factor_to_offset: Dict[float, int] + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.base) + cache_list: List[torch.Tensor] = [] + # offsets to the next cache in a tensor. + # Each offset corresponds to the same index in scaling_factors. + offsets: List[int] = [] + for scaling_factor in self.scaling_factors: + # NOTE(woosuk): self.max_position_embeddings is the original + # maximum length before applying the rope scaling. + # Thus, the maximum length after applying the rope scaling is + # self.max_position_embeddings * self.scaling_factor. + max_len = self.max_position_embeddings * scaling_factor + t = torch.arange(max_len, dtype=torch.float) + t = t / scaling_factor + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + if not cache_list: + offset = 0 + else: + last_offset = offsets[-1] + next_max_len = cache_list[-1].shape[0] + offset = last_offset + next_max_len + offsets.append(offset) + cache_list.append(cache) + self._scaling_factor_to_offset = { + float(scaling_factor): offsets[i] + for i, scaling_factor in enumerate(self.scaling_factors) + } + assert len(self.scaling_factors) == len(offsets) + return torch.cat(cache_list, dim=0) + + @property + def scaling_factor_to_offset(self) -> Dict[float, int]: + return self._scaling_factor_to_offset + + +class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with Dynamic NTK scaling. + + Credits to the Reddit users /u/bloc97 and /u/emozilla + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + ) -> None: + self.scaling_factor = scaling_factor + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + + def _compute_cos_sin_cache(self) -> torch.Tensor: + # NOTE(woosuk): self.max_position_embeddings is the original + # maximum length before applying the rope scaling. + # Thus, the maximum length after applying the rope scaling is + # self.max_position_embeddings * self.scaling_factor. + max_len = self.max_position_embeddings * self.scaling_factor + base = self.base * ( + (self.scaling_factor * max_len / self.max_position_embeddings) - + (self.scaling_factor - 1))**(self.rotary_dim / + (self.rotary_dim - 2)) + inv_freq = self._compute_inv_freq(base) + t = torch.arange(max_len, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + +# Inverse dim formula to find dim based on number of rotations +def _yarn_find_correction_dim(num_rotations: int, + dim: int, + base: float = 10000, + max_position_embeddings: int = 2048) -> float: + return (dim * math.log(max_position_embeddings / + (num_rotations * 2 * math.pi))) / (2 * + math.log(base)) + + +# Find dim range bounds based on rotations +def _yarn_find_correction_range( + low_rot: int, + high_rot: int, + dim: int, + base: float = 10000, + max_position_embeddings: int = 2048) -> Tuple[int, int]: + low = math.floor( + _yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings)) + high = math.ceil( + _yarn_find_correction_dim(high_rot, dim, base, + max_position_embeddings)) + return max(low, 0), min(high, dim - 1) # Clamp values just in case + + +def _yarn_linear_ramp_mask(low: float, high: float, dim: int, + dtype: torch.dtype) -> torch.Tensor: + if low == high: + high += 0.001 # Prevent singularity + + linear_func = (torch.arange(dim, dtype=dtype) - low) / (high - low) + ramp_func = torch.clamp(linear_func, 0, 1) + return ramp_func + + +def _yarn_get_mscale(scale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + +class YaRNScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with YaRN method. + + Credits to Peng et al. github.com/jquesnelle/yarn + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + *, + extrapolation_factor: float = 1, + attn_factor: float = 1, + beta_fast: int = 32, + beta_slow: int = 1, + ) -> None: + self.scaling_factor = scaling_factor + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + # Get n-d magnitude scaling corrected for interpolation + self.mscale = float( + _yarn_get_mscale(self.scaling_factor) * attn_factor) + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor: + pos_freqs = self.base**( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / + self.rotary_dim) + inv_freq_extrapolation = 1.0 / pos_freqs + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs) + + low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow, + self.rotary_dim, self.base, + self.max_position_embeddings) + # Get n-d rotational scaling corrected for extrapolation + inv_freq_mask = (1 - _yarn_linear_ramp_mask( + low, high, self.rotary_dim // 2, + dtype=torch.float)) * self.extrapolation_factor + inv_freq = inv_freq_interpolation * ( + 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.scaling_factor) + t = torch.arange(self.max_position_embeddings * self.scaling_factor, + dtype=torch.float32) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = (freqs.cos() * self.mscale) + sin = (freqs.sin() * self.mscale) + cache = torch.cat((cos, sin), dim=-1) + return cache + + +class Phi3LongRoPEScaledRotaryEmbedding(nn.Module): + """Phi3 family of models scaled rotary embedding. + + Based on the original RotaryEmbedding implementation. + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + original_max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + short_factor: List[float], + long_factor: List[float], + short_mscale: Optional[float] = None, + long_mscale: Optional[float] = None, + ): + super().__init__() + + if rotary_dim != head_size: + raise ValueError( + f"`Phi3LongRoPEScaledRotaryEmbedding` does not support \ + rotary_dim != head_size ({rotary_dim}!={head_size}).") + if is_neox_style is False: + raise ValueError( + "`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style." + ) + + self.head_size = head_size + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.base = base + self.short_factor = short_factor + self.long_factor = long_factor + + scale = self.max_position_embeddings / \ + self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = math.sqrt( + 1 + math.log(scale) / + math.log(self.original_max_position_embeddings)) + if short_mscale is None: + short_mscale = scaling_factor + if long_mscale is None: + long_mscale = scaling_factor + + self.short_mscale = short_mscale + self.long_mscale = long_mscale + + short_cache = self._compute_cos_sin_cache( + original_max_position_embeddings, short_factor, short_mscale) + short_cache = short_cache.to(dtype) + self.register_buffer("short_cos_sin_cache", + short_cache, + persistent=False) + + long_cache = self._compute_cos_sin_cache(max_position_embeddings, + long_factor, long_mscale) + long_cache = long_cache.to(dtype) + self.register_buffer("long_cos_sin_cache", + long_cache, + persistent=False) + + long_short_cache = torch.cat( + [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0) + self.register_buffer("long_short_cos_sin_cache", + long_short_cache, + persistent=False) + + def _compute_inv_freq(self, rescale_factors: List[float]) -> torch.Tensor: + rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32) + inv_freq = 1.0 / (rescale_factors * (self.base**(torch.arange( + 0, self.head_size, 2, dtype=torch.float) / self.head_size))) + return inv_freq + + def _compute_cos_sin_cache( + self, + max_position_embeddings: int, + rescale_factors: List[float], + mscale: float, + ) -> torch.Tensor: + inv_freq = self._compute_inv_freq(rescale_factors) + t = torch.arange(max_position_embeddings, dtype=torch.float) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() * mscale + sin = freqs.sin() * mscale + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + query = query.view(*query.shape[:-1], -1, self.head_size) + key = key.view(*key.shape[:-1], -1, self.head_size) + + k = self.original_max_position_embeddings + long_prompt_offset = (torch.any(positions > k).float() * + torch.full_like(positions, k)).long() + idx = (torch.add(positions, long_prompt_offset) + if long_prompt_offset is not None else positions) + self.long_short_cos_sin_cache: torch.Tensor = ( + self.long_short_cos_sin_cache.to(idx.device)) + idx = torch.add(idx, offsets) if offsets is not None else idx + cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx) + + cos, sin = cos_sin.chunk(2, dim=-1) + cos = cos.repeat(1, 2).unsqueeze(-2) + sin = sin.repeat(1, 2).unsqueeze(-2) + + query = query * cos + _rotate_neox(query) * sin + key = key * cos + _rotate_neox(key) * sin + + return query.flatten(-2), key.flatten(-2) + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +class DeepseekScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with YaRN method. + + Credits to Peng et al. github.com/jquesnelle/yarn + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + *, + extrapolation_factor: float = 1, + attn_factor: float = 1, + beta_fast: int = 32, + beta_slow: int = 1, + mscale: float = 1, + mscale_all_dim: float = 0, + ) -> None: + self.scaling_factor = scaling_factor + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + # Get n-d magnitude scaling corrected for interpolation. + self.mscale = float( + yarn_get_mscale(self.scaling_factor, float(mscale)) / + yarn_get_mscale(self.scaling_factor, float(mscale_all_dim)) * + attn_factor) + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor: + pos_freqs = self.base**(torch.arange( + 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") / + self.rotary_dim) + inv_freq_extrapolation = 1.0 / pos_freqs + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs) + + low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow, + self.rotary_dim, self.base, + self.max_position_embeddings) + # Get n-d rotational scaling corrected for extrapolation + inv_freq_mask = (1 - _yarn_linear_ramp_mask( + low, high, self.rotary_dim // 2, + dtype=torch.float)) * self.extrapolation_factor + inv_freq = inv_freq_interpolation * ( + 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.scaling_factor) + t = torch.arange(self.max_position_embeddings * self.scaling_factor, + device="cuda", + dtype=torch.float32) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = (freqs.cos() * self.mscale) + sin = (freqs.sin() * self.mscale) + cache = torch.cat((cos, sin), dim=-1) + print("Cache shape", cache.shape) + return cache + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """PyTorch-native implementation equivalent to forward().""" + query_rot = query[..., :self.rotary_dim] + key_rot = key[..., :self.rotary_dim] + if self.rotary_dim < self.head_size: + query_pass = query[..., self.rotary_dim:] + key_pass = key[..., self.rotary_dim:] + + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to( + positions.device) + cos_sin = self.cos_sin_cache[torch.add(positions, offsets) + if offsets is not None else positions] + cos, sin = cos_sin.chunk(2, dim=-1) + if self.is_neox_style: + # NOTE(woosuk): Here we assume that the positions tensor has the + # shape [batch_size, seq_len]. + cos = cos.repeat(1, 1, 2).unsqueeze(-2) + sin = sin.repeat(1, 1, 2).unsqueeze(-2) + else: + cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2) + sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2) + + rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj + query_rot = query_rot * cos + rotate_fn(query_rot) * sin + key_rot = key_rot * cos + rotate_fn(key_rot) * sin + + if self.rotary_dim < self.head_size: + query = torch.cat((query_rot, query_pass), dim=-1) + key = torch.cat((key_rot, key_pass), dim=-1) + else: + query = query_rot + key = key_rot + return query, key + + +class Llama3RotaryEmbedding(RotaryEmbedding): + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + scaling_factor: float, + low_freq_factor: float, + high_freq_factor: float, + orig_max_position: int, + ) -> None: + self.scaling_factor = scaling_factor + self.low_freq_factor = low_freq_factor + self.high_freq_factor = high_freq_factor + self.orig_max_position = orig_max_position + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freqs = super()._compute_inv_freq(base) + low_freq_wavelen = self.orig_max_position / self.low_freq_factor + high_freq_wavelen = self.orig_max_position / self.high_freq_factor + + wave_len = 2 * math.pi / inv_freqs + if self.low_freq_factor != self.high_freq_factor: + smooth = (self.orig_max_position / wave_len - self.low_freq_factor + ) / (self.high_freq_factor - self.low_freq_factor) + else: + smooth = 0 + new_freqs = torch.where( + wave_len < high_freq_wavelen, + inv_freqs, + torch.where( + wave_len > low_freq_wavelen, + inv_freqs / self.scaling_factor, + (1 - smooth) * inv_freqs / self.scaling_factor + + smooth * inv_freqs, + ), + ) + return new_freqs + + +class MRotaryEmbedding(RotaryEmbedding): + """Rotary Embedding with Multimodal Sections.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + mrope_section: Optional[List[int]] = None, + ) -> None: + super().__init__(head_size, rotary_dim, max_position_embeddings, base, + is_neox_style, dtype) + + self.mrope_section = mrope_section + if self.mrope_section: + assert sum(self.mrope_section) == rotary_dim // 2 + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """PyTorch-native implementation equivalent to forward(). + + Args: + positions: + [num_tokens,] (text only) or + [3, num_tokens] (T/H/W positions with multimodal inputs) + query: [num_tokens, num_heads * head_size] + key: [num_tokens, num_kv_heads * head_size] + """ + assert positions.ndim == 1 or positions.ndim == 2 + + num_tokens = positions.shape[-1] + cos_sin = self.cos_sin_cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + if positions.ndim == 2: + assert self.mrope_section + + cos = torch.cat([ + m[i] + for i, m in enumerate(cos.split(self.mrope_section, dim=-1)) + ], + dim=-1) + sin = torch.cat([ + m[i] + for i, m in enumerate(sin.split(self.mrope_section, dim=-1)) + ], + dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., :self.rotary_dim] + query_pass = query[..., self.rotary_dim:] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., :self.rotary_dim] + key_pass = key[..., self.rotary_dim:] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + return query, key + + @staticmethod + def get_input_positions( + input_tokens: List[int], + image_grid_thw: Union[List[List[int]], torch.Tensor], + video_grid_thw: Union[List[List[int]], torch.Tensor], + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, + vision_end_token_id: int, + spatial_merge_size: int, + context_len: int = 0, + ) -> Tuple[List[List[int]], int]: + """Get mrope input positions and delta value.""" + + if isinstance(image_grid_thw, torch.Tensor): + image_grid_thw = image_grid_thw.tolist() + if isinstance(video_grid_thw, torch.Tensor): + video_grid_thw = video_grid_thw.tolist() + + input_tokens_tensor = torch.tensor(input_tokens) + vision_start_indices = torch.argwhere( + input_tokens_tensor == vision_start_token_id).squeeze(1) + vision_tokens = input_tokens_tensor[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + llm_pos_ids_list: list = [] + + st = 0 + remain_images, remain_videos = image_nums, video_nums + + image_index, video_index = 0, 0 + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = \ + t, h // spatial_merge_size, w // spatial_merge_size + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len( + llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand( + -1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand( + llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand( + llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len( + llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + llm_positions = llm_positions[:, context_len:] + mrope_position_delta = (llm_positions.max() + 1 - + len(input_tokens)).item() + + return llm_positions.tolist(), mrope_position_delta + + @staticmethod + def get_next_input_positions( + mrope_position_delta: int, + context_len: int, + seq_len: int, + ) -> List[List[int]]: + return [ + list( + range(context_len + mrope_position_delta, + seq_len + mrope_position_delta)) for _ in range(3) + ] + + +_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {} + + +def get_rope( + head_size: int, + rotary_dim: int, + max_position: int, + base: int, + is_neox_style: bool = True, + rope_scaling: Optional[Dict[str, Any]] = None, + dtype: Optional[torch.dtype] = None, + partial_rotary_factor: float = 1.0, +) -> RotaryEmbedding: + if dtype is None: + dtype = torch.get_default_dtype() + if rope_scaling is not None: + # Transforms every value that is a list into a tuple for caching calls + rope_scaling_tuple = { + k: tuple(v) if isinstance(v, list) else v + for k, v in rope_scaling.items() + } + rope_scaling_args = tuple(rope_scaling_tuple.items()) + else: + rope_scaling_args = None + if partial_rotary_factor < 1.0: + rotary_dim = int(rotary_dim * partial_rotary_factor) + key = (head_size, rotary_dim, max_position, base, is_neox_style, + rope_scaling_args, dtype) + if key in _ROPE_DICT: + return _ROPE_DICT[key] + + if rope_scaling is None: + rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base, + is_neox_style, dtype) + else: + scaling_type = rope_scaling[ + "type"] if "type" in rope_scaling else rope_scaling["rope_type"] + scaling_type = "mrope" if scaling_type == "default" else scaling_type + # The correct one should be "longrope" but keep "su" here + # for backward compatible + if scaling_type not in {"su", "longrope"}: + scaling_factor = rope_scaling.get("factor", 1.0) + if scaling_type == "llama3": + low_freq_factor = rope_scaling["low_freq_factor"] + high_freq_factor = rope_scaling["high_freq_factor"] + original_max_position = rope_scaling[ + "original_max_position_embeddings"] + rotary_emb = Llama3RotaryEmbedding(head_size, rotary_dim, + max_position, base, + is_neox_style, dtype, + scaling_factor, low_freq_factor, + high_freq_factor, + original_max_position) + elif scaling_type == "linear": + rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim, + max_position, base, + is_neox_style, + scaling_factor, dtype) + elif scaling_type == "dynamic": + rotary_emb = DynamicNTKScalingRotaryEmbedding( + head_size, rotary_dim, max_position, base, is_neox_style, + scaling_factor, dtype) + elif scaling_type == "yarn": + original_max_position = rope_scaling[ + "original_max_position_embeddings"] + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k in ("extrapolation_factor", "attn_factor", "beta_fast", + "beta_slow") + } + rotary_emb = YaRNScalingRotaryEmbedding(head_size, rotary_dim, + original_max_position, + base, is_neox_style, + scaling_factor, dtype, + **extra_kwargs) + elif scaling_type == "deepseek_yarn": + original_max_position = rope_scaling[ + "original_max_position_embeddings"] + # assert max_position == original_max_position * scaling_factor + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k in ("extrapolation_factor", "attn_factor", "beta_fast", + "beta_slow", "mscale", "mscale_all_dim") + } + rotary_emb = DeepseekScalingRotaryEmbedding( + head_size, rotary_dim, original_max_position, base, + is_neox_style, scaling_factor, dtype, **extra_kwargs) + # The correct one should be "longrope" but keep "su" here + # for backward compatible + elif scaling_type == "su" or scaling_type == "longrope": + short_factor = rope_scaling["short_factor"] + long_factor = rope_scaling["long_factor"] + original_max_position = rope_scaling[ + "original_max_position_embeddings"] + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k in ("short_mscale", "long_mscale") + } + rotary_emb = Phi3LongRoPEScaledRotaryEmbedding( + head_size, rotary_dim, max_position, original_max_position, + base, is_neox_style, dtype, short_factor, long_factor, + **extra_kwargs) + elif scaling_type == "mrope": + rotary_emb = MRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + dtype, + mrope_section=rope_scaling["mrope_section"], + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + _ROPE_DICT[key] = rotary_emb + return rotary_emb diff --git a/vllm/model_executor/layers/sampler.py b/vllm/model_executor/layers/sampler.py new file mode 100644 index 00000000..42a6a0e6 --- /dev/null +++ b/vllm/model_executor/layers/sampler.py @@ -0,0 +1,1317 @@ +"""A layer that samples the next tokens from the model's outputs.""" +import itertools +import warnings +from dataclasses import dataclass +from importlib.util import find_spec +from math import inf +from typing import Dict, List, Optional, Tuple, Union + +import msgspec +import torch +import torch.nn as nn + +import vllm.envs as envs +from vllm.model_executor.sampling_metadata import (SamplingMetadata, + SamplingTensors, + SequenceGroupToSample) +from vllm.sampling_params import SamplingType +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, + CompletionSequenceGroupOutput, Logprob, + PromptLogprobs, SampleLogprobs, SequenceOutput) +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics + +if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"): + import flashinfer.sampling + # yapf: disable + from flashinfer.sampling import ( + top_k_top_p_sampling_from_probs as flashinfer_top_k_top_p_sampling) + + # yapf: enable +else: + flashinfer_top_k_top_p_sampling = None + +# (num_token_ids, num_parent_ids) per sequence group. +SampleResultType = List[Tuple[List[int], List[int]]] + +# Types of temporary data structures used for +# computing sample_result +SampleMetadataType = Dict[SamplingType, Tuple[List[int], + List[SequenceGroupToSample]]] +MultinomialSamplesType = Dict[SamplingType, torch.Tensor] +SampleResultsDictType = Dict[int, Tuple[List[int], List[int]]] + + +# Encapsulates temporary data structures for computing +# sample_result. +# +# * For multi-step scheduling: must be returned +# by `Sampler.forward()` and used later to compute the pythonized +# sample_result +# +# * For single-step scheduling: consumed immediately +# inside `Sampler.forward()` to compute pythonized sample_result. +@dataclass +class SampleResultArgsType: + sample_metadata: SampleMetadataType + multinomial_samples: MultinomialSamplesType + sample_results_dict: SampleResultsDictType + sampling_metadata: SamplingMetadata + greedy_samples: Optional[torch.Tensor] + beam_search_logprobs: Optional[torch.Tensor] + + +# Union of non-deferred (single-step scheduling) +# vs deferred (multi-step scheduling) +# sample result types +MaybeDeferredSampleResultType = Union[SampleResultType, SampleResultArgsType] + +# Abbreviation of the _sample() return type +SampleReturnType = Tuple[MaybeDeferredSampleResultType, Optional[torch.Tensor]] + + +class SamplerOutput( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + array_like=True): # type: ignore[call-arg] + """For each sequence group, we generate a list of SequenceOutput object, + each of which contains one possible candidate for the next token. + + This data structure implements methods, so it can be used like a list, but + also has optional fields for device tensors. + """ + + outputs: List[CompletionSequenceGroupOutput] + + # On-device tensor containing probabilities of each token. + sampled_token_probs: Optional[torch.Tensor] = None + + # On-device tensor containing the logprobs of each token. + logprobs: Optional["torch.Tensor"] = None + + # Holds either (1) the pythonized sampler result (single-step scheduling) + # or (2) what will be arguments for later deferred pythonization of the + # sampler result (muliti-step scheduling) + deferred_sample_results_args: Optional[SampleResultArgsType] = None + + # On-device tensor containing the sampled token ids. + sampled_token_ids: Optional[torch.Tensor] = None + # CPU tensor containing the sampled token ids. Used during multi-step to + # return the sampled token ids from last rank to AsyncLLMEngine to be + # 'broadcasted' to all other PP ranks for next step. + sampled_token_ids_cpu: Optional[torch.Tensor] = None + + # Spec decode metrics populated by workers. + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None + + # Optional last hidden states from the model. + hidden_states: Optional[torch.Tensor] = None + + # Optional prefill hidden states from the model + # (used for models like EAGLE). + prefill_hidden_states: Optional[torch.Tensor] = None + + # Time taken in the forward pass for this across all workers + model_forward_time: Optional[float] = None + + # Time taken in the model execute function. This will include model forward, + # block/sync across workers, cpu-gpu sync time and sampling time. + model_execute_time: Optional[float] = None + + def __getitem__(self, idx: int): + return self.outputs[idx] + + def __setitem__(self, idx: int, value): + self.outputs[idx] = value + + def __len__(self): + return len(self.outputs) + + def __eq__(self, other: object): + return isinstance(other, + self.__class__) and self.outputs == other.outputs + + def __repr__(self) -> str: + """Show the shape of a tensor instead of its values to reduce noise. + """ + sampled_token_probs_repr = ("None" if self.sampled_token_probs is None + else self.sampled_token_probs.shape) + sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else + self.sampled_token_ids.shape) + return ( + f"SamplerOutput(outputs={self.outputs}, " + f"sampled_token_probs={sampled_token_probs_repr}, " + f"sampled_token_ids={sampled_token_ids_repr}, " + f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})") + + +class Sampler(nn.Module): + """Samples the next tokens from the model's outputs. + + This layer does the following: + 1. Discard the hidden states that are not used for sampling (i.e., all + tokens except the final one in each prompt). + 2. Compute the logits for the next tokens. + 3. Apply presence, frequency and repetition penalties. + 4. Apply temperature scaling. + 5. Apply top-p and top-k truncation. + 6. Sample the next tokens. + Here, each sequence group within the batch can have different sampling + parameters (e.g., sampling method, temperature, top-p, top-k, etc.). + + The structure of the logits tensor is coupled with the seq_groups in + sampling_metadata. Typically, each sequence in each seq_group has one row in + logits for the next token to be sampled; however, for a seq_group with a + prompt request with the prompt_logprobs sampling parameter, there are rows + in logits for each token in the input prompt. + """ + + def __init__(self): + super().__init__() + + # Whether or not the SamplerOutput should have on-device tensors + # containing the sampled token ids and probabilities. This is used by + # speculative decoding. + self.include_gpu_probs_tensor = False + self.should_modify_greedy_probs_inplace = False + + def _init_sampling_tensors( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ): + """The goal here is to reuse sampling tensors between similar decode + runs. This is possible because sampling logic does not change between + decodes of the same sequences. + """ + _, vocab_size = logits.shape + + # First free any existing stored sampling tensors. + # This is necessary because some sampling tensors may + # have pinned memory. + self._sampling_tensors = None + + # Initialize new sampling tensors + (sampling_tensors, do_penalties, do_top_p_top_k, + do_min_p) = SamplingTensors.from_sampling_metadata( + sampling_metadata, vocab_size, logits.device, logits.dtype) + + self._sampling_tensors = sampling_tensors + self._do_penalties = do_penalties + self._do_top_p_top_k = do_top_p_top_k + self._do_min_p = do_min_p + + def forward( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + """ + Single-step scheduling: + * Perform GPU-side sampling computation & compute + GPU-side logprobs tensor + * Pythonize sampling result & logprobs tensor + + Multi-step scheduling: + * Perform GPU-side sampling computation & compute + GPU-side logprobs tensor + * Defer Pythonization of sampling result & logprobs + tensor + * Encapsulate arguments required for deferred Pythonization + in the :class:`SamplerOutput` structure + + Args: + logits: (num_tokens, vocab_size). + sampling_metadata: Metadata for sampling. + """ + assert logits is not None + _, vocab_size = logits.shape + + # Prepare sampling tensors with pinned memory to avoid blocking. + if not sampling_metadata.reuse_sampling_tensors: + self._init_sampling_tensors(logits, sampling_metadata) + elif self._do_penalties: + # In this case, the sampling tensors logic depends on + # "output_tokens" of a sequence. As a result, we cannot + # reuse sampling tensors, since "output_tokens" changes + # between decode runs. + self._init_sampling_tensors(logits, sampling_metadata) + + assert self._sampling_tensors is not None + sampling_tensors = self._sampling_tensors + do_penalties = self._do_penalties + do_top_p_top_k = self._do_top_p_top_k + do_min_p = self._do_min_p + + logits = _apply_min_tokens_penalty(logits, sampling_metadata) + + # Apply presence and frequency penalties. + if do_penalties: + logits = _apply_penalties(logits, sampling_tensors.prompt_tokens, + sampling_tensors.output_tokens, + sampling_tensors.presence_penalties, + sampling_tensors.frequency_penalties, + sampling_tensors.repetition_penalties) + + # Use float32 to apply temperature scaling. + # Use in-place division to avoid creating a new tensor. + logits = logits.to(torch.float) + logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1)) + + if do_top_p_top_k and flashinfer_top_k_top_p_sampling is None: + logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps, + sampling_tensors.top_ks) + + if do_min_p: + logits = _apply_min_p(logits, sampling_tensors.min_ps) + + # We use float32 for probabilities and log probabilities. + # Compute the probabilities. + probs = torch.softmax(logits, dim=-1, dtype=torch.float) + # Compute the log probabilities. + logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float) + + # Sample the next tokens. + maybe_deferred_sample_results, maybe_sampled_tokens_tensor = _sample( + probs, + logprobs, + sampling_metadata, + sampling_tensors, + include_gpu_probs_tensor=self.include_gpu_probs_tensor, + modify_greedy_probs=self._should_modify_greedy_probs_inplace, + ) + + if self.include_gpu_probs_tensor: + # Since we will defer sampler result Pythonization, + # preserve GPU-side tensors in support of later + # deferred pythonization of logprobs + assert maybe_sampled_tokens_tensor is not None + on_device_tensors = (probs, logprobs, maybe_sampled_tokens_tensor) + else: + # Since Pythonization has already happened, don't preserve + # GPU-side tensors. + on_device_tensors = None + + # Get the logprobs query results. + prompt_logprobs = None + sample_logprobs = None + if not sampling_metadata.skip_sampler_cpu_output: + # Pythonize logprobs now (GPU -> CPU); do not defer. + assert not isinstance(maybe_deferred_sample_results, + SampleResultArgsType) + prompt_logprobs, sample_logprobs = get_logprobs( + logprobs, sampling_metadata, maybe_deferred_sample_results) + + return _build_sampler_output( + maybe_deferred_sample_results, + sampling_metadata, + prompt_logprobs, + sample_logprobs, + on_device_tensors=on_device_tensors, + skip_sampler_cpu_output=sampling_metadata.skip_sampler_cpu_output) + + @property + def _should_modify_greedy_probs_inplace(self) -> bool: + """Whether or not the sampler should modify the probability distribution + of greedily-sampled tokens such that multinomial sampling would sample + the greedily-sampled token. + + In other words, if True then we set the probability of the greedily- + sampled token to 1. + + This is used by speculative decoding, which requires that the sampling + method be encoded into the probability distribution. + """ + return self.should_modify_greedy_probs_inplace + + +def _get_bin_counts_and_mask( + tokens: torch.Tensor, + vocab_size: int, + num_seqs: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + # Compute the bin counts for the tokens. + # vocab_size + 1 for padding. + bin_counts = torch.zeros((num_seqs, vocab_size + 1), + dtype=torch.long, + device=tokens.device) + bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens)) + bin_counts = bin_counts[:, :vocab_size] + mask = bin_counts > 0 + + return bin_counts, mask + + +def _apply_min_tokens_penalty( + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> torch.Tensor: + """Apply min_tokens penalty which sets stop tokens to -inf if min_tokens + have not been generated yet + """ + # list of indices in logits that will be set to -inf + logits_to_penalize: List[Tuple[int, int]] = [] + logits_applied = 0 + for seq_group in sampling_metadata.seq_groups: + seq_ids = seq_group.seq_ids + sampling_params = seq_group.sampling_params + + sample_indices = seq_group.sample_indices + logits_applied += len(sample_indices) + len( + seq_group.prompt_logprob_indices) + if not seq_group.do_sample: + continue + + start_idx = sample_indices[0] + min_tokens = sampling_params.min_tokens + token_ids_to_penalize = sampling_params.all_stop_token_ids + if min_tokens > 0 and token_ids_to_penalize: + seqs_to_penalize: List[int] = [] + for j, seq_id in enumerate(seq_ids): + seq_data = seq_group.seq_data[seq_id] + if len(seq_data.output_token_ids_array) < min_tokens: + seqs_to_penalize.append(j) + + if seqs_to_penalize: + # convert to the index into logits + seqs_to_penalize = [start_idx + j for j in seqs_to_penalize] + # itertools.product pairs each seq index with every token id + logits_to_penalize.extend( + itertools.product(seqs_to_penalize, token_ids_to_penalize)) + + if logits_to_penalize: + # use zip and * to group indices along each dimension + # eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) ) + logits[tuple(zip(*logits_to_penalize))] = -float("inf") + + # verifies that no rows in logits were missed unexpectedly + assert logits_applied == logits.shape[0] + return logits + + +def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor, + output_tokens_tensor: torch.Tensor, + presence_penalties: torch.Tensor, + frequency_penalties: torch.Tensor, + repetition_penalties: torch.Tensor) -> torch.Tensor: + num_seqs, vocab_size = logits.shape + _, prompt_mask = _get_bin_counts_and_mask(prompt_tokens_tensor, vocab_size, + num_seqs) + output_bin_counts, output_mask = _get_bin_counts_and_mask( + output_tokens_tensor, vocab_size, num_seqs) + + repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size) + repetition_penalties[~(prompt_mask | output_mask)] = 1.0 + logits = torch.where(logits > 0, logits / repetition_penalties, + logits * repetition_penalties) + + # We follow the definition in OpenAI API. + # Refer to https://platform.openai.com/docs/api-reference/parameter-details + logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts + logits -= presence_penalties.unsqueeze_(dim=1) * output_mask + return logits + + +def _apply_top_k_top_p( + logits: torch.Tensor, + p: torch.Tensor, + k: torch.Tensor, +) -> torch.Tensor: + logits_sort, logits_idx = logits.sort(dim=-1, descending=False) + + # Apply top-k. + top_k_mask = logits_sort.size(1) - k.to(torch.long) + # Get all the top_k values. + top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1)) + top_k_mask = logits_sort < top_k_mask + logits_sort.masked_fill_(top_k_mask, -float("inf")) + + # Apply top-p. + probs_sort = logits_sort.softmax(dim=-1) + probs_sum = probs_sort.cumsum(dim=-1) + top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1) + # at least one + top_p_mask[:, -1] = False + logits_sort.masked_fill_(top_p_mask, -float("inf")) + + # Re-sort the probabilities. + logits = torch.empty_like(logits_sort).scatter_(dim=-1, + index=logits_idx, + src=logits_sort) + return logits + + +def _apply_min_p( + logits: torch.Tensor, + min_p: torch.Tensor, +) -> torch.Tensor: + """ + Adapted from + https://github.com/oobabooga/text-generation-webui/blob/3146124ec01f02c8fb1650a6517cf1b60b537aaf/modules/sampler_hijack.py#L16C17-L16C17 + """ + probs = torch.softmax(logits, dim=-1) + top_probs, _ = probs.max(dim=-1, keepdim=True) + scaled_min_p = min_p.unsqueeze_(dim=1) * top_probs + tokens_to_remove = probs < scaled_min_p + logits = logits.masked_fill_(tokens_to_remove, -float("inf")) + + return logits + + +def _greedy_sample( + selected_seq_groups: List[SequenceGroupToSample], + samples: torch.Tensor, +) -> SampleResultType: + """Run greedy sampling on a given samples. + + Args: + selected_seq_groups: A list of sequence groups batched. + samples: (num_selected_samples,) A tensor of samples. The length of + samples could be smaller than selected_seq_groups if + seq_group.do_sample is False. + Returns: + Tuple of (next_token_ids, parent_ids). The length of returned list is + same as the length of selected_seq_groups. If the corresponding + seq_group has do_sample=False, tuple contains ([], []) + """ + samples_lst = samples.tolist() + sample_idx = 0 + results: SampleResultType = [] + for seq_group in selected_seq_groups: + if not seq_group.do_sample: + results.append(([], [])) + continue + + seq_ids = seq_group.seq_ids + num_parent_seqs = len(seq_ids) + assert num_parent_seqs == 1, ( + "Greedy sampling should have only one seq.") + parent_ids = list(range(num_parent_seqs)) + next_token_ids = [samples_lst[sample_idx]] + results.append((next_token_ids, parent_ids)) + sample_idx += num_parent_seqs + return results + + +def _random_sample( + selected_seq_groups: List[SequenceGroupToSample], + random_samples: torch.Tensor, +) -> SampleResultType: + """Run random sampling on a given samples. + + Args: + selected_seq_groups: A list of sequence groups batched. + random_samples: (num_selected_samples,) A tensor of samples. The + length of samples could be smaller than selected_seq_groups if + seq_group.do_sample is False. + Returns: + Tuple of (next_token_ids, parent_ids). The length of returned list is + same as the length of selected_seq_groups. If the corresponding + seq_group has do_sample=False, tuple contains ([], []) + """ + # Find the maximum n value of the prompt phase requests. + random_samples = random_samples.cpu() + sample_idx = 0 + results: SampleResultType = [] + for seq_group in selected_seq_groups: + if not seq_group.do_sample: + results.append(([], [])) + continue + + seq_ids = seq_group.seq_ids + sampling_params = seq_group.sampling_params + is_prompt = seq_group.is_prompt + num_parent_seqs = len(seq_ids) + if is_prompt: + # Prompt phase. + parent_ids = [0] * sampling_params.n + next_token_ids = random_samples[ + sample_idx, :sampling_params.n].tolist() + else: + # Generation phase. + parent_ids = list(range(num_parent_seqs)) + next_token_ids = random_samples[sample_idx:sample_idx + + num_parent_seqs, 0].tolist() + results.append((next_token_ids, parent_ids)) + sample_idx += num_parent_seqs + return results + + +def _beam_search_sample( + selected_seq_groups: List[SequenceGroupToSample], + logprobs: torch.Tensor, +) -> SampleResultType: + """Run beam sampling on a given samples. + + Args: + selected_seq_groups: A list of sequence groups batched. + logprobs: (num_selected_samples, vocab_size,) A tensor of logprob + on selected sample indices. + Returns: + Tuple of (next_token_ids, parent_ids). The length of returned list is + same as the length of selected_seq_groups. If the corresponding + seq_group has do_sample=False, tuple contains ([], []) + """ + # We sample 2 * beam_width candidates to make sure that with high + # probability we can get `beam_width` candidates in addition to + # the finished sequences for the next iteration. See + # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563 + # for details. See also HF reference: + # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065 + # + # NOTE: Beam search is not vectorized, so its speed can be slower than + # other sampling methods. + sample_idx = 0 + results: SampleResultType = [] + for seq_group in selected_seq_groups: + if not seq_group.do_sample: + results.append(([], [])) + continue + + is_prompt = seq_group.is_prompt + seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params + num_parent_seqs = len(seq_ids) + beam_width = sampling_params.n + seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs] + if is_prompt: + # Prompt phase. + assert num_parent_seqs == 1, ( + "Prompt input should have only one seq.") + parent_ids = [0] * (2 * beam_width) + _, next_token_ids = torch.topk(seq_group_logprobs[0], + 2 * beam_width) + next_token_ids = next_token_ids.tolist() + else: + # Generation phase. + cumulative_logprobs: List[float] = [ + seq_group.seq_data[seq_id].cumulative_logprob + for seq_id in seq_ids + ] + cumulative_logprobs_tensor = torch.tensor( + cumulative_logprobs, + dtype=torch.float, + device=seq_group_logprobs.device) + seq_group_logprobs = (seq_group_logprobs + + cumulative_logprobs_tensor.unsqueeze(dim=1)) + _, topk_ids = torch.topk(seq_group_logprobs.flatten(), + 2 * beam_width) + topk_ids = topk_ids.tolist() + vocab_size = seq_group_logprobs.size(-1) + parent_ids = [i // vocab_size for i in topk_ids] + next_token_ids = [i % vocab_size for i in topk_ids] + results.append((next_token_ids, parent_ids)) + sample_idx += num_parent_seqs + assert sample_idx == logprobs.size(0) + return results + + +# torch.multinomial forces a GPU<->CPU sync. +# Therefore, we use an optimized implementation instead. +# Note that we always sample with replacement. +# probs will be modified in place, but this is fine, as we pass +# in a copy already. +def _multinomial( + probs: torch.Tensor, + num_samples: int, + seq_groups: Optional[List[SequenceGroupToSample]] = None, +) -> torch.Tensor: + if num_samples > 1: + probs = probs.repeat_interleave(num_samples, dim=0) + q = torch.empty_like(probs) + if seq_groups is None: + q.exponential_() + else: + sample_idx = 0 + for seq_group in seq_groups: + seq_ids = seq_group.seq_ids + stride = len(seq_ids) * num_samples + assert seq_group.generator is not None + q[sample_idx:sample_idx + + stride].exponential_(generator=seq_group.generator) + sample_idx += stride + return probs.div_(q).argmax(dim=1).view(-1, num_samples) + + +def _top_k_top_p_multinomial_with_flashinfer( + probs: torch.Tensor, top_ks: torch.Tensor, top_ps: torch.Tensor, + num_samples: int, seq_groups: Optional[List[SequenceGroupToSample]]): + max_top_k_round = 32 + if num_samples > 1: + probs = probs.repeat_interleave(num_samples, dim=0) + top_ks = top_ks.repeat_interleave(num_samples) + top_ps = top_ps.repeat_interleave(num_samples) + batch_size = probs.shape[0] + uniform_samples = torch.empty((max_top_k_round, batch_size), + device=probs.device) + if seq_groups is None: + uniform_samples.uniform_() + else: + sample_idx = 0 + for seq_group in seq_groups: + seq_ids = seq_group.seq_ids + stride = len(seq_ids) * num_samples + assert seq_group.generator is not None + uniform_samples[:, sample_idx:sample_idx + + stride].uniform_(generator=seq_group.generator) + sample_idx += stride + batch_next_token_ids, success = flashinfer_top_k_top_p_sampling( + probs, + uniform_samples, + top_ks, + top_ps, + ) + if not success.all(): + warnings.warn("FlashInfer rejection sampling failed, fallback.", + stacklevel=1) + probs = flashinfer.sampling.top_k_renorm_prob(probs, top_ks) + probs = flashinfer.sampling.top_p_renorm_prob(probs, top_ps) + batch_next_token_ids = flashinfer.sampling.sampling_from_probs( + probs, uniform_samples[0]) + return batch_next_token_ids.view(-1, num_samples) + + +def get_pythonized_sample_results( + sample_result_args: SampleResultArgsType) -> SampleResultType: + '''This function consumes GPU-side sampler results and computes + Pythonized CPU-side sampler results (GPU -> CPU sync.) + + Single-step scheduling: this function is invoked at sampling-time + for immediate Pythonization. + + Multi-step scheduling: Pythonization is deferred until after multiple + GPU-side steps have been completed. + + Args: + sample_result_args: GPU-side inputs to the Pythonization process + + Returns: + Pythonized sampler results + ''' + + ( + sample_metadata, + sampling_metadata, + greedy_samples, + multinomial_samples, + beam_search_logprobs, + sample_results_dict, + ) = ( + sample_result_args.sample_metadata, + sample_result_args.sampling_metadata, + sample_result_args.greedy_samples, + sample_result_args.multinomial_samples, + sample_result_args.beam_search_logprobs, + sample_result_args.sample_results_dict, + ) + + for sampling_type in SamplingType: + if sampling_type not in sample_metadata: + continue + (seq_group_id, seq_groups) = sample_metadata[sampling_type] + if sampling_type == SamplingType.GREEDY: + sample_results = _greedy_sample(seq_groups, greedy_samples) + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED): + sample_results = _random_sample(seq_groups, + multinomial_samples[sampling_type]) + elif sampling_type == SamplingType.BEAM: + sample_results = _beam_search_sample(seq_groups, + beam_search_logprobs) + sample_results_dict.update(zip(seq_group_id, sample_results)) + + return [ + sample_results_dict.get(i, ([], [])) + for i in range(len(sampling_metadata.seq_groups)) + ] + + +def _sample_with_torch( + probs: torch.Tensor, + logprobs: torch.Tensor, + sampling_metadata: SamplingMetadata, + sampling_tensors: SamplingTensors, + include_gpu_probs_tensor: bool, + modify_greedy_probs: bool, +) -> SampleReturnType: + '''Torch-oriented _sample() implementation. + + Single-step scheduling: + * Perform GPU-side sampling computation + * Immediately Pythonize sampling result + + Multi-step scheduling: + * Perform GPU-side sampling computation + * Defer Pythonization & preserve GPU-side + tensors required for Pythonization + ''' + + categorized_seq_group_ids: Dict[SamplingType, + List[int]] = {t: [] + for t in SamplingType} + categorized_sample_indices = sampling_metadata.categorized_sample_indices + for i, seq_group in enumerate(sampling_metadata.seq_groups): + sampling_params = seq_group.sampling_params + sampling_type = sampling_params.sampling_type + categorized_seq_group_ids[sampling_type].append(i) + + sample_results_dict: SampleResultsDictType = {} + sample_metadata: SampleMetadataType = {} + multinomial_samples: MultinomialSamplesType = {} + greedy_samples: Optional[torch.Tensor] = None + beam_search_logprobs: Optional[torch.Tensor] = None + + # Create output tensor for sampled token ids. + if include_gpu_probs_tensor: + sampled_token_ids_tensor = torch.full((logprobs.shape[0], 1), + VLLM_INVALID_TOKEN_ID, + dtype=torch.long, + device=logprobs.device) + else: + sampled_token_ids_tensor = None + + # Counterintiutively, having two loops here is actually faster. + # The first loop can run without waiting on GPU<->CPU sync. + for sampling_type in SamplingType: + sample_indices = categorized_sample_indices[sampling_type] + num_tokens = len(sample_indices) + if num_tokens == 0: + continue + + seq_group_id = categorized_seq_group_ids[sampling_type] + seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id] + sample_metadata[sampling_type] = (seq_group_id, seq_groups) + long_sample_indices = sample_indices.long() + if sampling_type == SamplingType.GREEDY: + greedy_samples = torch.argmax(logprobs[long_sample_indices], + dim=-1) + + if sampled_token_ids_tensor is not None: + # Store sampled tokens in output tensor. + sampled_token_ids_tensor[ + long_sample_indices] = greedy_samples.unsqueeze(-1) + + if modify_greedy_probs: + # If required, modify the probabilities such that sampling from + # the modified distribution would always sample the argmax + # token id. + _modify_greedy_probs_inplace(logprobs, probs, + long_sample_indices, + greedy_samples) + + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED): + max_n_in_batch = 1 + for seq_group in seq_groups: + if seq_group.is_prompt: + sampling_params = seq_group.sampling_params + max_n_in_batch = max(max_n_in_batch, sampling_params.n) + seq_groups_arg = (None if sampling_type == SamplingType.RANDOM else + seq_groups) + + if flashinfer_top_k_top_p_sampling is not None: + multinomial_samples[ + sampling_type] = _top_k_top_p_multinomial_with_flashinfer( + probs[long_sample_indices], + sampling_tensors.top_ks[long_sample_indices], + sampling_tensors.top_ps[long_sample_indices], + max_n_in_batch, + seq_groups_arg, + ) + else: + multinomial_samples[sampling_type] = _multinomial( + probs[long_sample_indices], + max_n_in_batch, + seq_groups=seq_groups_arg) + + if sampled_token_ids_tensor is not None: + # Store sampled tokens in output tensor. + sampled_token_ids_tensor[long_sample_indices] = \ + multinomial_samples[sampling_type].to(torch.long) + + elif sampling_type == SamplingType.BEAM: + beam_search_logprobs = logprobs[sample_indices] + else: + raise ValueError(f"Unsupported sampling type: {sampling_type}") + + # Encapsulate arguments for computing Pythonized sampler + # results, whether deferred or otherwise. + maybe_deferred_args = SampleResultArgsType( + sampling_metadata=sampling_metadata, + sample_metadata=sample_metadata, + multinomial_samples=multinomial_samples, + greedy_samples=greedy_samples, + beam_search_logprobs=beam_search_logprobs, + sample_results_dict=sample_results_dict) + + if not sampling_metadata.skip_sampler_cpu_output: + # GPU<->CPU sync happens here. + # This also converts the sampler output to a Python object. + # Return Pythonized sampler result & sampled token ids + return get_pythonized_sample_results( + maybe_deferred_args), sampled_token_ids_tensor + else: + # Defer sampler result Pythonization; return deferred + # Pythonization args & sampled token ids + return ( + maybe_deferred_args, + sampled_token_ids_tensor, + ) + + +def _sample( + probs: torch.Tensor, + logprobs: torch.Tensor, + sampling_metadata: SamplingMetadata, + sampling_tensors: SamplingTensors, + include_gpu_probs_tensor: bool, + modify_greedy_probs: bool, +) -> SampleReturnType: + """ + Args: + probs: (num_query_tokens_in_batch, num_vocab) + logprobs: (num_query_tokens_in_batch, num_vocab) + sampling_metadata: The metadata for a batch for sampling. + sampling_tensors: Tensors that include sampling related metadata. + + Returns: + (next_token_ids, parent_seq_ids) for each seq group in a batch. + If sampling is skipped, it returns ([], []) + sampled_token_ids_tensor: A tensor of sampled token ids. + """ + return _sample_with_torch( + probs, + logprobs, + sampling_metadata, + sampling_tensors, + include_gpu_probs_tensor=include_gpu_probs_tensor, + modify_greedy_probs=modify_greedy_probs, + ) + + +def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor: + """ + This function calculates the ranks of the chosen tokens in a logprob tensor. + + Args: + x (torch.Tensor): 2D logprob tensor of shape (N, M) + where N is the no. of tokens and M is the vocab dim. + indices (torch.Tensor): List of chosen token indices. + + Returns: + torch.Tensor: 1D tensor of shape (N,) where N is the no. of tokens. + Each element in the returned tensor represents the rank + of the chosen token in the input logprob tensor. + """ + vals = x[torch.arange(0, len(x), device=x.device, dtype=indices.dtype), + indices] + result = (x > vals[:, None]) + del vals + return result.sum(1).add_(1) + + +def get_logprobs( + logprobs: torch.Tensor, + sampling_metadata: SamplingMetadata, + sample_results: SampleResultType, +) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]: + """Return sample logprobs and prompt logprobs. + + The logic consists of 3 parts. + - Select indices to compute logprob from, ranks of token ids, and + the top k token ids from logprobs. + - Compute prompt logprobs if required. + - Compute sample logprobs if required. + + Args: + logprobs: (num_query_tokens_across_batch, num_vocab). Each query token's + logprob per vocab. Sequence groups' query tokens are batched in a + single flattened tensor. For example, assuming there are N + seq groups, it is sorted by prefill tokens for seq_group_1 (if + prompt logprob is enabled), decode tokens for seq_group_1 (if + sampling is required), prefill tokens for seq_group_2, ... + sampling_metadata: The sampling metadata. + sample_results: (num_seq_groups) The tuple of (next_token_ids, + parent_ids) for each sequence group. When beam search is enabled, + sample_results can contain different number of seq_ids from + sampling_metadata.seq_groups. It is because beam search creates + 2 * BEAM_WIDTH number of samples (whereas there are only up to + BEAM_WIDTH number of seq_ids). + + Returns: + A tuple of prompt and sample logprobs per sequence group in a batch. + """ + # The index of query token to calculate logprobs. It includes both + # prompt and sample logprob indices. + query_indices: List[int] = [] + # The next token ids to get the logprob value from. + next_token_ids: List[int] = [] + # The largest requested number of logprobs. We find logprobs as many as the + # largest num logprobs in this API. If every logprobs is None, it will be + # set to -1. + largest_num_logprobs = -1 + + # Select indices to compute logprob from, ranks of token ids, and the top + # k token ids from logprobs. + for (seq_group, sample_result) in zip(sampling_metadata.seq_groups, + sample_results): + sampling_params = seq_group.sampling_params + + # Update indices and tokens for prompt logprobs. + if (seq_group.is_prompt + and sampling_params.prompt_logprobs is not None): + largest_num_logprobs = max(largest_num_logprobs, + sampling_params.prompt_logprobs) + next_prompt_tokens = _get_next_prompt_tokens(seq_group) + query_indices.extend(seq_group.prompt_logprob_indices) + next_token_ids.extend(next_prompt_tokens) + + # Update indices and next tokenes for sample logprob. + if seq_group.do_sample: + token_ids, parent_seq_ids = sample_result + # NOTE: We cannot directly use sample_indices because + # sample_indices only contain parent seq_ids of a previous step. + # The current step may have different number of seq_ids, and + # we can obtain it from `sample_result[1]`. + query_idx = seq_group.sample_indices[0] + query_indices.extend( + [query_idx + parent_id for parent_id in parent_seq_ids]) + next_token_ids.extend(token_ids) + + if sampling_params.logprobs is not None: + largest_num_logprobs = max(largest_num_logprobs, + sampling_params.logprobs) + + assert len(next_token_ids) == len(query_indices) + + if len(query_indices) == 0: + empty_sampled_logprob: SampleLogprobs = [] + empty_prompt_logprob: Optional[PromptLogprobs] = None + return [empty_prompt_logprob], [empty_sampled_logprob] + + selected_logprobs, ranks = None, None + top_logprobs, top_token_ids = None, None + + # If largest_num_logprobs == -1, i.e. no logprobs are requested, we can + # skip the whole logprob calculation. + if largest_num_logprobs >= 0: + query_indices_gpu = torch.tensor(query_indices, device=logprobs.device) + next_token_ids_gpu = torch.tensor(next_token_ids, + device=logprobs.device) + + # (num_selected_query_tokens, num_logprobs). Note that query_indices can + # contain duplicates if beam search is enabled. + selected_logprobs = logprobs[[ + query_indices_gpu, + next_token_ids_gpu, + ]] + ranks = _get_ranks( + logprobs[query_indices_gpu], + next_token_ids_gpu, + ) + assert selected_logprobs.shape[0] == ranks.shape[0] + + # We need to compute top k only if there exists logprobs > 0. + if largest_num_logprobs > 0: + # Logprobs of topk tokens for a batch of sequence groups. + # (num_query_tokens_across_batch). + top_logprobs, top_token_ids = torch.topk(logprobs, + largest_num_logprobs, + dim=-1) + top_logprobs = top_logprobs.to('cpu') + top_token_ids = top_token_ids.to('cpu') + + selected_logprobs = selected_logprobs.to('cpu') + ranks = ranks.to('cpu') + + # Find prompt/sample logprobs. + prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = [] + sample_logprobs_per_seq_group: List[SampleLogprobs] = [] + top_logprob_idx = 0 + selected_logprobs_idx = 0 + + for seq_group, sample_result in zip(sampling_metadata.seq_groups, + sample_results): + (prompt_logprobs, top_logprob_idx, + selected_logprobs_idx) = _get_prompt_logprob_if_needed( + seq_group, selected_logprobs, ranks, top_token_ids, top_logprobs, + selected_logprobs_idx, top_logprob_idx) + prompt_logprobs_per_seq_group.append(prompt_logprobs) + + (sampled_logprobs, top_logprob_idx, + selected_logprobs_idx) = _get_sampled_logprob_if_needed( + seq_group, sample_result, selected_logprobs, ranks, top_token_ids, + top_logprobs, selected_logprobs_idx, top_logprob_idx) + sample_logprobs_per_seq_group.append(sampled_logprobs) + + return prompt_logprobs_per_seq_group, sample_logprobs_per_seq_group + + +def _get_prompt_logprob_if_needed( + seq_group: SequenceGroupToSample, + selected_logprobs: torch.Tensor, + ranks: torch.Tensor, + top_token_ids: torch.Tensor, + top_logprobs: torch.Tensor, + selected_logprobs_idx: int, + top_logprob_idx: int, +): + """Compute the prompt logprob from a sequence group if needed.""" + sampling_params = seq_group.sampling_params + is_prompt = seq_group.is_prompt + + # Find prompt logprobs + prompt_logprobs: Optional[PromptLogprobs] = None + if is_prompt and sampling_params.prompt_logprobs is not None: + prompt_logprobs = [] + num_logprobs = sampling_params.prompt_logprobs + next_prompt_tokens = _get_next_prompt_tokens(seq_group) + # Pre-select indexes and create a list. It is faster than calling .item + # repetitively. + selected_logprob_items = selected_logprobs[ + selected_logprobs_idx:selected_logprobs_idx + + len(next_prompt_tokens)].tolist() + rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx + + len(next_prompt_tokens)].tolist() + + for idx, token_id in enumerate(next_prompt_tokens): + # Calculate the prompt logprob of the real prompt tokens. + # {token_id: (logprob, rank_from_vocab)} + prompt_logprobs_dict: Dict[int, Tuple[float, int]] = { + token_id: (selected_logprob_items[idx], rank_items[idx]) + } + + # Add top K prompt logprobs along with its rank. + if num_logprobs > 0: + top_ids = top_token_ids[ + top_logprob_idx, :num_logprobs].tolist() + top_probs = top_logprobs[ + top_logprob_idx, :num_logprobs].tolist() + # Top K is already sorted by rank, so we can use 1 ~ + # num_logprobs + 1 for rank. + top_ranks = range(1, num_logprobs + 1) + prompt_logprobs_dict.update({ + top_id: (top_prob, rank) + for top_id, top_prob, rank in zip(top_ids, top_probs, + top_ranks) + }) + prompt_logprobs.append({ + token_id: Logprob(*logprob_and_rank) + for token_id, logprob_and_rank in prompt_logprobs_dict.items() + }) + # + 1 to go to the next prompt token. + top_logprob_idx += 1 + + # + len(next_prompt_tokens) to go to the next prompt. + selected_logprobs_idx += len(next_prompt_tokens) + return prompt_logprobs, top_logprob_idx, selected_logprobs_idx + + +def _get_sampled_logprob_if_needed( + seq_group: SequenceGroupToSample, + sample_result: Tuple[List[int], List[int]], + selected_logprobs: torch.Tensor, + ranks: torch.Tensor, + top_token_ids: torch.Tensor, + top_logprobs: torch.Tensor, + selected_logprobs_idx: int, + top_logprob_idx: int, +): + """Compute the sample logprob if needed.""" + seq_ids = seq_group.seq_ids + num_logprobs = seq_group.sampling_params.logprobs + sampled_logprobs: SampleLogprobs = [] + next_token_ids, parent_seq_ids = sample_result + + if seq_group.do_sample: + assert len(next_token_ids) > 0 + if num_logprobs is None: + for next_token_id in next_token_ids: + # Use a dummy logprob + sampled_logprobs.append({next_token_id: Logprob(inf)}) + else: + # Pre-select items from tensor. tolist() is faster than repetitive + # `.item()` calls. + selected_logprob_items = selected_logprobs[ + selected_logprobs_idx:selected_logprobs_idx + + len(next_token_ids)].tolist() + rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx + + len(next_token_ids)].tolist() + for idx, (next_token_id, parent_id) in enumerate( + zip(next_token_ids, parent_seq_ids)): + # Get the logprob of a sampled token. + sampled_logprobs_dict = { + next_token_id: + (selected_logprob_items[idx], rank_items[idx]) + } + if num_logprobs is not None and num_logprobs > 0: + # Get top K logprobs. + top_ids = top_token_ids[top_logprob_idx + + parent_id, :num_logprobs].tolist() + top_probs = top_logprobs[ + top_logprob_idx + parent_id, :num_logprobs].tolist() + # Top K is already sorted by rank, so we can use 1 ~ + # num_logprobs + 1 for rank. + top_ranks = range(1, num_logprobs + 1) + sampled_logprobs_dict.update({ + top_id: (top_prob, rank) + for top_id, top_prob, rank in zip( + top_ids, top_probs, top_ranks) + }) + + sampled_logprobs.append({ + token_id: Logprob(*logprob_and_rank) + for token_id, logprob_and_rank in + sampled_logprobs_dict.items() + }) + + # NOTE: This part of code is not intuitive. `selected_logprobs` include + # logprobs for the current step, which has len(next_token_ids) tokens + # per sequence group. `logprobs` includes logprobs from the previous + # steps, which has len(seq_ids) tokens per sequence group. + + # Iterate to the next sequence group in a batch. + selected_logprobs_idx += len(next_token_ids) + # Iterate to the next sequence group in a batch. + top_logprob_idx += len(seq_ids) + return sampled_logprobs, top_logprob_idx, selected_logprobs_idx + + +def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor, + sample_indices: torch.Tensor, + greedy_samples: torch.Tensor) -> None: + """Modify the probability distributions of the greedily-sampled tokens such + that each sampled token has a "probability" of 1.0. This is required by + speculative decoding, which depends on the sampling method being encoded + within the probability distribution for correctness. + + # Why do we only need to do this for greedy sampling? + + vLLM's sampler performs the following steps for greedy or multinomial + (random) sampling: + 1. Get logits from model. + 2. Modify logits according to per-sequence sampling parameters. + - Multiply by temperature, top-k and top-p masking, penalize tokens + according to their frequency, etc. + 3. Sample a token. + - Random sampling simply samples from the modified probability + distribution. + - Greedy sampling performs `argmax` to obtain the token with the + highest likelihood. + + Ignoring greedy sampling for a moment, we find that the computed probability + distribution has the following property: we can sample from it independently + and find that the token sampled by the Sampler has a frequency corresponding + to how often we see it in our sampling. In other words, for tokens sampled + with vLLM's random SamplingType, the computed probability distribution + encodes the sampling methodology completely. + + Greedy sampling does not normally have this property. vLLM modifies logits + according to sampling params, then performs `argmax`, then returns the + sampled token and the computed probability distribution. If we sample from + the distribution, we'll find the likelihood of the greedily-sampled token + is not always 1.0. + + Since lossless speculative decoding requires that the sampling methodology + be encoded within the probability distribution, we are motivated to modify + the probability distribution such that the sampled token has probability 1 + when speculative decoding is used. + + NOTE: Alternatively, we could use an extremely low temperature to achieve + greedy sampling using multinomial computation and unite the codepaths. This + has implications on the overall design of the sampler, e.g. how to record + accurate logprobs for the user, so this improvement is deferred to later. + """ + # NOTE: logprobs are not modified so they can be returned to the user. + probs[sample_indices, :] = 0 + probs[sample_indices, greedy_samples] = 1.0 + + +def _build_sampler_output( + maybe_deferred_sample_results: MaybeDeferredSampleResultType, + sampling_metadata: SamplingMetadata, + prompt_logprobs: Optional[List[Optional[PromptLogprobs]]], + sample_logprobs: Optional[List[SampleLogprobs]], + on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor, + torch.Tensor]], + skip_sampler_cpu_output: bool = False, +) -> SamplerOutput: + """Construct Python objects with the output of sampling. + + Args: + on_device_tensors: Tuple containing on-device tensors with the + probabilities used in sampling and the sampled token ids. This + allows post-processing without copies to CPU/serialization, e.g. in + speculative decoding rejection sampling. + """ + sampler_output: List[CompletionSequenceGroupOutput] = [] + + if skip_sampler_cpu_output: + assert isinstance(maybe_deferred_sample_results, SampleResultArgsType) + deferred_sample_results_args = maybe_deferred_sample_results + else: + assert prompt_logprobs is not None + assert sample_logprobs is not None + assert not isinstance(maybe_deferred_sample_results, + SampleResultArgsType) + deferred_sample_results_args = None + + for (seq_group, sample_result, group_prompt_logprobs, + group_sample_logprobs) in zip(sampling_metadata.seq_groups, + maybe_deferred_sample_results, + prompt_logprobs, sample_logprobs): + seq_ids = seq_group.seq_ids + next_token_ids, parent_ids = sample_result + seq_outputs: List[SequenceOutput] = [] + for parent_id, next_token_id, logprobs in zip( + parent_ids, next_token_ids, group_sample_logprobs): + seq_outputs.append( + SequenceOutput(seq_ids[parent_id], next_token_id, + logprobs)) + sampler_output.append( + CompletionSequenceGroupOutput(seq_outputs, + group_prompt_logprobs)) + + # If not specified, store None values in SamplerOutput. + if on_device_tensors is not None: + (sampled_token_probs, logprobs_tensor, + sampled_token_ids) = on_device_tensors + else: + sampled_token_probs, logprobs_tensor, sampled_token_ids = (None, None, + None) + + return SamplerOutput( + outputs=sampler_output, + sampled_token_probs=sampled_token_probs, + sampled_token_ids=sampled_token_ids, + logprobs=logprobs_tensor, + deferred_sample_results_args=deferred_sample_results_args) + + +def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]: + """Get a list of next prompt tokens to compute logprob from a + given sequence group. + + It is used to compute prompt logprob. Imagine you have logprob for each + query token. Query token needs to know the next prompt token id to compute + prompt logprob. This is a helper to obtain next prompt token ids. + + This API has to be used only when the caller knows seq_group is in prefill + stage. + + Returns: + A list of next prompt tokens to compute logprob. + """ + assert seq_group.is_prompt, ( + "Caller should ensure the sequence group is in a prefill stage.") + seq_ids = seq_group.seq_ids + query_len = seq_group.query_len + assert query_len is not None + # prompt has only 1 seq id. + assert len(seq_ids) == 1 + seq_data = seq_group.seq_data[seq_ids[0]] + computed_len = seq_data.get_num_computed_tokens() + prompt_tokens = seq_data.prompt_token_ids + # +1 because we are looking for a next prompt token. + next_token_index_start = computed_len + 1 + next_token_index_end = min(computed_len + query_len + 1, + len(prompt_tokens)) + next_prompt_tokens = prompt_tokens[ + next_token_index_start:next_token_index_end] + return next_prompt_tokens diff --git a/vllm/model_executor/layers/spec_decode_base_sampler.py b/vllm/model_executor/layers/spec_decode_base_sampler.py new file mode 100644 index 00000000..7e750a74 --- /dev/null +++ b/vllm/model_executor/layers/spec_decode_base_sampler.py @@ -0,0 +1,239 @@ +from abc import abstractmethod +from typing import Dict, Optional, Union + +import torch +import torch.jit +import torch.nn as nn + + +class SpecDecodeBaseSampler(nn.Module): + """Base class for samplers used for Speculative Decoding verification + step. + """ + + def __init__(self, strict_mode: bool = False): + """Base class constructor. + Args: + strict_mode: Whether or not to perform shape/device/dtype checks + during sampling. This catches correctness issues but adds + nontrivial latency. + """ + super().__init__() + self._strict_mode = strict_mode + + # NOTE: A "bonus token" is accepted iff all proposal tokens are + # accepted. There is always only one possible bonus token. We store this + # value in a variable for readability. + self._num_bonus_tokens = 1 + + self.num_accepted_tokens: Optional[torch.Tensor] = None + self.num_emitted_tokens: Optional[torch.Tensor] = None + self.num_draft_tokens: int = 0 + + def init_gpu_tensors(self, device: Union[int, str]) -> None: + assert self.num_accepted_tokens is None + if isinstance(device, int): + device = f"cuda:{device}" + elif not isinstance(device, str): + raise ValueError(f"Device must be int or str, get {type(device)}") + self.num_accepted_tokens = torch.tensor(0, + dtype=torch.long, + device=device) + self.num_emitted_tokens = torch.tensor(0, + dtype=torch.long, + device=device) + + @property + def probs_dtype(self): + return torch.float32 + + @property + def token_id_dtype(self): + return torch.int64 + + def _create_output( + self, + accepted: torch.Tensor, # [batch_size, k] + substitute_token_ids: torch.Tensor, # [batch_size, k] + draft_token_ids: torch.Tensor, # [batch_size, k] + bonus_token_ids: torch.Tensor, # [batch_size] + ) -> torch.Tensor: + """Format output. Returns a matrix of token ids. When + a token is rejected via sampling, all subsequent token ids are + set to -1 for the sequence. + + Args: + accepted: A boolean tensor indicating if the corresponding + draft token in draft_token_ids should be accepted or not. + substitute_token_ids: A tensor of token_ids that can be used + as substitutes for the draft token ids if the proposed token + is rejected. + draft_token_ids: A tensor of token ids speculated by the + draft model. + bonus_token_ids: Token ids to use as the bonus token if + all the draft tokens are accepted. + Returns: + A tensor containing the accepted token ids. The shape of the + tensor is [batch_size, k + num_bonus_tokens] + """ + batch_size, k = substitute_token_ids.shape + bonus_token_ids = bonus_token_ids.squeeze() + # Determine the index of the first False value for each row. + limits = (accepted == 0).max(1).indices + limits[~(accepted == 0).any(1)] = k + + # Create masks using the indices. + indices = torch.arange(k, device=accepted.device).unsqueeze(0) + accepted_mask = indices < limits.unsqueeze(1) + after_false_mask = indices == limits.unsqueeze(1) + + # Create an extended output tensor + output_with_bonus_tokens = -torch.ones( + (batch_size, k + self._num_bonus_tokens), + dtype=self.token_id_dtype, + device=accepted.device) + output = output_with_bonus_tokens[:, :k] + + # Fill in the first k columns of the output tensor using masks and data + # tensors. + output[:, :k] = torch.where(accepted_mask, draft_token_ids, + -torch.ones_like(draft_token_ids)) + + # Fill the last column. + # We check output directly as accepted may have True values inconsistent + # with causal acceptance. + output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1, + bonus_token_ids, -1) + + # Fill the recovered token ids. + output.mul_(~after_false_mask).add_( + substitute_token_ids.mul(after_false_mask)) + + self.num_accepted_tokens += accepted.sum() + self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum() + self.num_draft_tokens += batch_size * k + + return output_with_bonus_tokens + + def _raise_if_incorrect_input( + self, + target_with_bonus_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: Optional[torch.Tensor] = None, + ) -> None: + self._raise_if_incorrect_shape(target_with_bonus_probs, + draft_token_ids, bonus_token_ids, + draft_probs) + self._raise_if_incorrect_dtype(target_with_bonus_probs, + draft_token_ids, bonus_token_ids, + draft_probs) + self._raise_if_inconsistent_device(target_with_bonus_probs, + draft_token_ids, bonus_token_ids, + draft_probs) + self._raise_if_out_of_bounds_vocab(target_with_bonus_probs.shape[-1], + draft_token_ids, bonus_token_ids) + + def _raise_if_incorrect_shape( + self, + target_with_bonus_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: Optional[torch.Tensor] = None, + ) -> None: + (target_batch_size, num_target_probs, + target_vocab_size) = target_with_bonus_probs.shape + + # Does not count the extra token + num_target_probs -= 1 + + # validate the shape of draft token ids. + draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape + assert draft_token_ids_batch_size == target_batch_size + assert num_draft_token_ids == num_target_probs + + # validate the shape of bonus token ids + bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape + assert bonus_batch_size == target_batch_size + assert num_bonus_tokens == self._num_bonus_tokens + + # validate the shape of draft probs if it is set + if draft_probs is not None: + (draft_batch_size, num_draft_probs, + draft_vocab_size) = draft_probs.shape + assert draft_batch_size == target_batch_size + assert num_draft_probs == num_target_probs + assert (draft_vocab_size == target_vocab_size + ), f"{draft_vocab_size=} {target_vocab_size=}" + + def _raise_if_incorrect_dtype( + self, + target_with_bonus_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: Optional[torch.Tensor] = None, + ) -> None: + assert target_with_bonus_probs.dtype == self.probs_dtype + assert draft_token_ids.dtype == self.token_id_dtype + assert bonus_token_ids.dtype == self.token_id_dtype + if draft_probs is not None: + assert draft_probs.dtype == self.probs_dtype + + def _raise_if_inconsistent_device( + self, + target_with_bonus_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: Optional[torch.Tensor] = None, + ) -> None: + devices = [ + t.device for t in [ + target_with_bonus_probs, bonus_token_ids, draft_probs, + draft_token_ids + ] if t is not None + ] + assert all([devices[0] == device for device in devices]) + + def _raise_if_out_of_bounds_vocab( + self, + vocab_size: int, + draft_token_ids: torch.Tensor, + bonus_token_ids: torch.Tensor, + ) -> None: + assert torch.all(bonus_token_ids < vocab_size) + assert torch.all(bonus_token_ids >= 0) + assert torch.all(draft_token_ids < vocab_size) + assert torch.all(draft_token_ids >= 0) + + +class SpecDecodeDeterministicBaseSampler(SpecDecodeBaseSampler): + """Base class for samplers used for Speculative Decoding verification + step which are deterministic. + """ + + @abstractmethod + def forward( + self, + target_with_bonus_probs: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + ) -> torch.Tensor: + raise NotImplementedError + + +class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler): + """Base class for samplers used for Speculative Decoding verification + step which are stochastic + """ + + @abstractmethod + def forward( + self, + target_with_bonus_probs: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + seeded_seqs: Optional[Dict[int, torch.Generator]] = None, + ) -> torch.Tensor: + raise NotImplementedError diff --git a/vllm/model_executor/layers/typical_acceptance_sampler.py b/vllm/model_executor/layers/typical_acceptance_sampler.py new file mode 100644 index 00000000..584cf971 --- /dev/null +++ b/vllm/model_executor/layers/typical_acceptance_sampler.py @@ -0,0 +1,170 @@ +import torch +import torch.jit + +from vllm.model_executor.layers.spec_decode_base_sampler import ( + SpecDecodeDeterministicBaseSampler) + + +class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler): + """Apply typical acceptance sampling as described in section 3.3.1 in + "MEDUSA: Simple LLM Inference Acceleration Framework with + Multiple Decoding Heads" + https://arxiv.org/pdf/2401.10774 + """ + + def __init__( + self, + posterior_threshold: float, + posterior_alpha: float, + strict_mode: bool = False, + ): + """Create a Typical Acceptance Sampler. + + Args: + strict_mode: Whether or not to perform shape/device/dtype checks + during sampling. This catches correctness issues but adds + nontrivial latency. + posterior_threshold : A threshold value that sets a lower bound + on the posterior probability of a token in target model for it + to be accepted. + posterior_alpha : A scaling factor for the entropy-based + threshold in typical acceptance sampling. + """ + self._posterior_threshold = posterior_threshold + self._posterior_alpha = posterior_alpha + super().__init__(strict_mode=strict_mode) + + def forward( + self, + target_with_bonus_probs: torch.Tensor, + bonus_token_ids: torch.Tensor, + draft_probs: torch.Tensor, + draft_token_ids: torch.Tensor, + ) -> torch.Tensor: + """Sample token ids using typical acceptance sampling. This accepts + or rejects tokens proposed by the draft model using the probability + of each token according to the draft and target models. + + In the worst case where all draft tokens are rejected, it is guaranteed + one token will be emitted. + + In the case where all draft tokens are accepted, the bonus token will be + accepted. + + Args: + target_probs: The probability distribution over token ids given + context according to the target model. + shape = [batch_size, num_speculative_tokens, vocab_size] + + bonus_token_ids: The "bonus" token ids that are accepted iff all + speculative tokens in a sequence are accepted. + shape = [batch_size, num_bonus_tokens] + + draft_probs: This parameter is unused by the acceptance sampler. + + draft_token_ids: The token ids that were sampled from the draft + probabilities. + shape = [batch_size, num_speculative_tokens] + + Returns: + output_token_ids: The token ids sampled via rejection sampling, + or -1 if unable to sample a token because the previous token + was rejected. + shape = [batch_size, num_speculative_tokens + num_bonus_tokens] + """ + # Only perform shape/dtype/device checking in strict mode, as it adds + # overhead. + if self._strict_mode: + self._raise_if_incorrect_input(target_with_bonus_probs, + draft_token_ids, bonus_token_ids) + target_probs = target_with_bonus_probs[:, :-1] + accepted = self._evaluate_accepted_tokens(target_probs, + draft_token_ids) + recovered_token_ids = self._get_recovered_token_ids(target_probs) + output_token_ids = self._create_output(accepted, recovered_token_ids, + draft_token_ids, + bonus_token_ids) + return output_token_ids + + def _evaluate_accepted_tokens(self, target_probs, draft_token_ids): + r""" + Evaluates and returns a mask of accepted tokens based on the + posterior probabilities. + + Parameters: + ---------- + target_probs : torch.Tensor + A tensor of shape (batch_size, k, vocab_size) representing + the probabilities of each token in the vocabulary for each + position in the proposed sequence. This is the distribution + generated by the target model. + draft_token_ids : torch.Tensor + A tensor of shape (batch_size, k) representing the proposed + token ids. + + A draft token_id x_{n+k} is accepted if it satisfies the + following condition + + .. math:: + p_{\text{original}}(x_{n+k} | x_1, x_2, \dots, x_{n+k-1}) > + \min \left( \epsilon, \delta * \exp \left( + -H(p_{\text{original}}( + \cdot | x_1, x_2, \ldots, x_{n+k-1})) \right) \right) + + where :math:`p_{\text{original}}` corresponds to target_probs + and :math:`\epsilon` and :math:`\delta` correspond to hyperparameters + specified using self._posterior_threshold and self._posterior_alpha + + This method computes the posterior probabilities for the given + draft token ids based on the provided target probabilities. It + calculates the entropy of the posterior distribution and determines + a dynamic threshold for each token position using the provided + posterior_threshold and posterior_alpha values. The method then + returns a boolean mask indicating which tokens can be accepted. + + Returns: + ------- + torch.Tensor + A boolean tensor of shape (batch_size, k) where each element + indicates whether the corresponding draft token has been accepted + or rejected. True indicates acceptance and false indicates + rejection. + + """ + device = target_probs.device + candidates_prob = torch.gather( + target_probs, dim=-1, + index=draft_token_ids.unsqueeze(-1)).squeeze(-1) + # A small constant added to prevent computing the logarithm of zero, + # which can lead to undefined values. + epsilon = 1e-5 + posterior_entropy = -torch.sum( + target_probs * torch.log(target_probs + epsilon), dim=-1) + threshold = torch.minimum( + torch.ones_like(posterior_entropy, device=device) * + self._posterior_threshold, + torch.exp(-posterior_entropy) * self._posterior_alpha, + ) + accepted_mask = candidates_prob > threshold + return accepted_mask + + def _get_recovered_token_ids(self, target_probs): + """ + The recovered token ids will fill the first unmatched token + by the target token. + + Parameters + ---------- + target_probs : torch.Tensor + A tensor of shape (batch_size, k, vocab_size) containing + the target probability distribution + + Returns + ------- + torch.Tensor + A tensor of shape (batch_size, k) with the recovered token + ids which are selected from target probs. + """ + max_indices = torch.argmax(target_probs, dim=-1) + + return max_indices diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py new file mode 100644 index 00000000..dff7a63c --- /dev/null +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -0,0 +1,471 @@ +from dataclasses import dataclass +from typing import List, Optional, Sequence, Tuple + +import torch +import torch.nn.functional as F +# import ixformer.inference.functions as IXF +import ixformer.functions as IXF +from torch.nn.parameter import Parameter, UninitializedParameter + +from vllm.distributed import (divide, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding) +from vllm.model_executor.parameter import BasevLLMParameter +from vllm.model_executor.utils import set_weight_attrs + +DEFAULT_VOCAB_PADDING_SIZE = 64 + + +class UnquantizedEmbeddingMethod(QuantizeMethodBase): + """Unquantized method for embeddings.""" + + def create_weights(self, layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], input_size: int, + output_size: int, params_dtype: torch.dtype, + **extra_weight_attrs): + """Create weights for embedding layer.""" + weight = Parameter(torch.empty(sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype), + requires_grad=False) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def apply(self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None) -> torch.Tensor: + return IXF.linear(x, layer.weight, bias) + + def embedding(self, layer: torch.nn.Module, + input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + +def pad_vocab_size(vocab_size: int, + pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: + """Pad the vocab size to the given value.""" + return ((vocab_size + pad_to - 1) // pad_to) * pad_to + + +def vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size: int, + rank: int, + offset: int = 0) -> Sequence[int]: + index_f = rank * per_partition_vocab_size + index_l = index_f + per_partition_vocab_size + return index_f + offset, index_l + offset + + +def vocab_range_from_global_vocab_size(global_vocab_size: int, + rank: int, + world_size: int, + offset: int = 0) -> Sequence[int]: + per_partition_vocab_size = divide(global_vocab_size, world_size) + return vocab_range_from_per_partition_vocab_size(per_partition_vocab_size, + rank, + offset=offset) + + +@dataclass +class VocabParallelEmbeddingShardIndices: + """Indices for a shard of a vocab parallel embedding.""" + padded_org_vocab_start_index: int + padded_org_vocab_end_index: int + padded_added_vocab_start_index: int + padded_added_vocab_end_index: int + + org_vocab_start_index: int + org_vocab_end_index: int + added_vocab_start_index: int + added_vocab_end_index: int + + @property + def num_org_elements(self) -> int: + return self.org_vocab_end_index - self.org_vocab_start_index + + @property + def num_added_elements(self) -> int: + return self.added_vocab_end_index - self.added_vocab_start_index + + @property + def num_org_elements_padded(self) -> int: + return (self.padded_org_vocab_end_index - + self.padded_org_vocab_start_index) + + @property + def num_added_elements_padded(self) -> int: + return (self.padded_added_vocab_end_index - + self.padded_added_vocab_start_index) + + @property + def num_org_vocab_padding(self) -> int: + return self.num_org_elements_padded - self.num_org_elements + + @property + def num_added_vocab_padding(self) -> int: + return self.num_added_elements_padded - self.num_added_elements + + @property + def num_elements_padded(self) -> int: + return self.num_org_elements_padded + self.num_added_elements_padded + + def __post_init__(self): + # sanity checks + assert (self.padded_org_vocab_start_index <= + self.padded_org_vocab_end_index) + assert (self.padded_added_vocab_start_index <= + self.padded_added_vocab_end_index) + + assert self.org_vocab_start_index <= self.org_vocab_end_index + assert self.added_vocab_start_index <= self.added_vocab_end_index + + assert self.org_vocab_start_index <= self.padded_org_vocab_start_index + assert (self.added_vocab_start_index <= + self.padded_added_vocab_start_index) + assert self.org_vocab_end_index <= self.padded_org_vocab_end_index + assert self.added_vocab_end_index <= self.padded_added_vocab_end_index + + assert self.num_org_elements <= self.num_org_elements_padded + assert self.num_added_elements <= self.num_added_elements_padded + + +# @torch.jit.script +def get_masked_input_and_mask( + input_: torch.Tensor, org_vocab_start_index: int, + org_vocab_end_index: int, num_org_vocab_padding: int, + added_vocab_start_index: int, + added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]: + # torch.jit.script will fuse all of the pointwise ops below + # into a single kernel, making it very fast + org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < + org_vocab_end_index) + added_vocab_mask = (input_ >= added_vocab_start_index) & ( + input_ < added_vocab_end_index) + added_offset = added_vocab_start_index - ( + org_vocab_end_index - org_vocab_start_index) - num_org_vocab_padding + valid_offset = (org_vocab_start_index * + org_vocab_mask) + (added_offset * added_vocab_mask) + vocab_mask = org_vocab_mask | added_vocab_mask + input_ = vocab_mask * (input_ - valid_offset) + return input_, ~vocab_mask + + +class VocabParallelEmbedding(torch.nn.Module): + """Embedding parallelized in the vocabulary dimension. + + Adapted from torch.nn.Embedding, note that we pad the vocabulary size to + make sure it is divisible by the number of model parallel GPUs. + + In order to support various loading methods, we ensure that LoRA-added + embeddings are always at the end of TP-sharded tensors. In other words, + we shard base embeddings and LoRA embeddings separately (both padded), + and place them in the same tensor. + In this example, we will have the original vocab size = 1010, + added vocab size = 16 and padding to 64. Therefore, the total + vocab size with padding will be 1088 (because we first pad 1010 to + 1024, add 16, and then pad to 1088). + Therefore, the tensor format looks like the following: + TP1, rank 0 (no sharding): + |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| + corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | + + TP2, rank 0: + |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| + corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 | + TP2, rank 1: + |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| + corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 | + + Args: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + params_dtype: type of the parameters. + org_num_embeddings: original vocabulary size (without LoRA). + padding_size: padding size for the vocabulary. + quant_config: quant config for the layer + prefix: full name of the layer in the state dict + """ # noqa: E501 + + def __init__(self, + num_embeddings: int, + embedding_dim: int, + params_dtype: Optional[torch.dtype] = None, + org_num_embeddings: Optional[int] = None, + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__() + + # Keep the input dimensions. + tp_rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + self.num_embeddings = num_embeddings + self.padding_size = padding_size + self.org_vocab_size = org_num_embeddings or num_embeddings + num_added_embeddings = num_embeddings - self.org_vocab_size + self.org_vocab_size_padded = pad_vocab_size(self.org_vocab_size, + self.padding_size) + self.num_embeddings_padded = pad_vocab_size( + self.org_vocab_size_padded + num_added_embeddings, + self.padding_size) + assert self.org_vocab_size_padded <= self.num_embeddings_padded + + self.shard_indices = self._get_indices(self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, tp_rank, + self.tp_size) + self.embedding_dim = embedding_dim + + linear_method = None + if quant_config is not None: + linear_method = quant_config.get_quant_method(self, prefix=prefix) + if linear_method is None: + linear_method = UnquantizedEmbeddingMethod() + + # If we are making an embedding layer, then our quantization linear + # method must implement the embedding operation. If we are another + # layer type like ParallelLMHead, this is not important. + is_embedding_layer = type(self.__class__) is VocabParallelEmbedding + linear_method_implements_embedding = method_has_implemented_embedding( + type(linear_method)) + if is_embedding_layer and not linear_method_implements_embedding: + raise NotImplementedError( + f"The class {type(linear_method).__name__} must implement " + "the 'embedding' method, see UnquantizedEmbeddingMethod.") + + self.linear_method: QuantizeMethodBase = linear_method + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + # Divide the weight matrix along the vocaburaly dimension. + self.num_added_embeddings = self.num_embeddings - self.org_vocab_size + self.num_embeddings_per_partition = divide(self.num_embeddings_padded, + self.tp_size) + assert (self.shard_indices.num_elements_padded == + self.num_embeddings_per_partition) + self.num_org_embeddings_per_partition = ( + self.shard_indices.org_vocab_end_index - + self.shard_indices.org_vocab_start_index) + self.num_added_embeddings_per_partition = ( + self.shard_indices.added_vocab_end_index - + self.shard_indices.added_vocab_start_index) + + self.linear_method.create_weights(self, + self.embedding_dim, + [self.num_embeddings_per_partition], + self.embedding_dim, + self.num_embeddings_padded, + params_dtype=params_dtype, + weight_loader=self.weight_loader) + + @classmethod + def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int, + vocab_size: int, org_vocab_size: int, tp_rank: int, + tp_size: int) -> VocabParallelEmbeddingShardIndices: + """Get start and end indices for vocab parallel embedding, following the + layout outlined in the class docstring, based on the given tp_rank and + tp_size.""" + num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded + padded_org_vocab_start_index, padded_org_vocab_end_index = ( + vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, + tp_size)) + padded_added_vocab_start_index, padded_added_vocab_end_index = ( + vocab_range_from_global_vocab_size(num_added_embeddings_padded, + tp_rank, + tp_size, + offset=org_vocab_size)) + # remove padding + org_vocab_start_index = min(padded_org_vocab_start_index, + org_vocab_size) + org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) + added_vocab_start_index = min(padded_added_vocab_start_index, + vocab_size) + added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) + return VocabParallelEmbeddingShardIndices( + padded_org_vocab_start_index, padded_org_vocab_end_index, + padded_added_vocab_start_index, padded_added_vocab_end_index, + org_vocab_start_index, org_vocab_end_index, + added_vocab_start_index, added_vocab_end_index) + + def get_sharded_to_full_mapping(self) -> Optional[List[int]]: + """Get a mapping that can be used to reindex the gathered + logits for sampling. + + During sampling, we gather logits from all ranks. The relationship + of index->token_id will follow the same format as outlined in the class + docstring. However, after the gather, we want to reindex the final + logits tensor to map index->token_id one-to-one (the index is always + equal the token_id it corresponds to). The indices returned by this + method allow us to do that. + """ + if self.tp_size < 2: + return None + + base_embeddings: List[int] = [] + added_embeddings: List[int] = [] + padding: List[int] = [] + for tp_rank in range(self.tp_size): + shard_indices = self._get_indices(self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, tp_rank, + self.tp_size) + range_start = self.num_embeddings_per_partition * tp_rank + range_end = self.num_embeddings_per_partition * (tp_rank + 1) + base_embeddings.extend( + range(range_start, + range_start + shard_indices.num_org_elements)) + padding.extend( + range(range_start + shard_indices.num_org_elements, + range_start + shard_indices.num_org_elements_padded)) + added_embeddings.extend( + range( + range_start + shard_indices.num_org_elements_padded, + range_start + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements)) + padding.extend( + range( + range_start + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements, + range_start + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded)) + assert (range_start + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded == range_end) + ret = base_embeddings + added_embeddings + padding + assert len(ret) == self.num_embeddings_padded + return ret + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + output_dim = getattr(param, "output_dim", None) + packed_dim = getattr(param, "packed_dim", None) + + # If the parameter is a gguf weight, then load it directly. + if getattr(param, "is_gguf_weight_type", None): + param.data.copy_(loaded_weight) + param.weight_type = loaded_weight.item() + return + elif isinstance(param, UninitializedParameter): + shape = list(loaded_weight.shape) + if output_dim is not None: + shape[output_dim] = shape[output_dim] // self.tp_size + param.materialize(tuple(shape), dtype=loaded_weight.dtype) + + # If parameter does not have output dim, then it should + # be copied onto all gpus (e.g. g_idx for act_order gptq). + if output_dim is None: + assert param.data.shape == loaded_weight.shape + param.data.copy_(loaded_weight) + return + + # Shard indexes for loading the weight + start_idx = self.shard_indices.org_vocab_start_index + shard_size = self.shard_indices.org_vocab_end_index - start_idx + + # If param packed on the same dim we are sharding on, then + # need to adjust offsets of loaded weight by pack_factor. + if packed_dim is not None and packed_dim == output_dim: + packed_factor = param.packed_factor if isinstance( + param, BasevLLMParameter) else param.pack_factor + assert loaded_weight.shape[output_dim] == (self.org_vocab_size // + param.packed_factor) + start_idx = start_idx // packed_factor + shard_size = shard_size // packed_factor + else: + assert loaded_weight.shape[output_dim] == self.org_vocab_size + + # Copy the data. + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + param[:loaded_weight.shape[0]].data.copy_(loaded_weight) + param[loaded_weight.shape[0]:].data.fill_(0) + + def forward(self, input_): + if self.tp_size > 1: + # Build the mask. + masked_input, input_mask = get_masked_input_and_mask( + input_, self.shard_indices.org_vocab_start_index, + self.shard_indices.org_vocab_end_index, + self.shard_indices.num_org_vocab_padding, + self.shard_indices.added_vocab_start_index, + self.shard_indices.added_vocab_end_index) + else: + masked_input = input_ + # Get the embeddings. + output_parallel = self.linear_method.embedding(self, + masked_input.long()) + # Mask the output embedding. + if self.tp_size > 1: + output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) + # Reduce across all the model parallel GPUs. + output = tensor_model_parallel_all_reduce(output_parallel) + return output + + def extra_repr(self) -> str: + s = f"num_embeddings={self.num_embeddings_per_partition}" + s += f", embedding_dim={self.embedding_dim}" + s += f", org_vocab_size={self.org_vocab_size}" + s += f', num_embeddings_padded={self.num_embeddings_padded}' + s += f', tp_size={self.tp_size}' + return s + + +class ParallelLMHead(VocabParallelEmbedding): + """Parallelized LM head. + + Output logits weight matrices used in the Sampler. The weight and bias + tensors are padded to make sure they are divisible by the number of + model parallel GPUs. + + Args: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + bias: whether to use bias. + params_dtype: type of the parameters. + org_num_embeddings: original vocabulary size (without LoRA). + padding_size: padding size for the vocabulary. + """ + + def __init__(self, + num_embeddings: int, + embedding_dim: int, + bias: bool = False, + params_dtype: Optional[torch.dtype] = None, + org_num_embeddings: Optional[int] = None, + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = ""): + super().__init__(num_embeddings, embedding_dim, params_dtype, + org_num_embeddings, padding_size, quant_config, + prefix) + self.quant_config = quant_config + if bias: + self.bias = Parameter( + torch.empty(self.num_embeddings_per_partition, + dtype=params_dtype)) + set_weight_attrs(self.bias, { + "output_dim": 0, + "weight_loader": self.weight_loader, + }) + else: + self.register_parameter("bias", None) + + def tie_weights(self, embed_tokens: VocabParallelEmbedding): + """Tie the weights with word embeddings.""" + # GGUF quantized embed_tokens. + if self.quant_config and self.quant_config.get_name() == "gguf": + return embed_tokens + else: + self.weight = embed_tokens.weight + return self + + def forward(self, input_): + del input_ + raise RuntimeError("LMHead's weights should be used in the sampler.") diff --git a/vllm/model_executor/model_loader/__init__.py b/vllm/model_executor/model_loader/__init__.py new file mode 100644 index 00000000..d1ec171c --- /dev/null +++ b/vllm/model_executor/model_loader/__init__.py @@ -0,0 +1,30 @@ +from typing import Optional + +from torch import nn + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, + ModelConfig, ParallelConfig, SchedulerConfig) +from vllm.model_executor.model_loader.loader import (BaseModelLoader, + get_model_loader) +from vllm.model_executor.model_loader.utils import ( + get_architecture_class_name, get_model_architecture) + + +def get_model(*, model_config: ModelConfig, load_config: LoadConfig, + device_config: DeviceConfig, parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + lora_config: Optional[LoRAConfig], + cache_config: CacheConfig) -> nn.Module: + loader = get_model_loader(load_config) + return loader.load_model(model_config=model_config, + device_config=device_config, + lora_config=lora_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + cache_config=cache_config) + + +__all__ = [ + "get_model", "get_model_loader", "BaseModelLoader", + "get_architecture_class_name", "get_model_architecture" +] diff --git a/vllm/model_executor/model_loader/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93f94a4a5f99f3a5ebb4dbee7bd5f6de26f8060d GIT binary patch literal 1118 zcmah|J#*AB5Vdu-?6ZA$zI-qQW~i7jE^;@iXcz_vB?%eQsqq*~xpPLcJ+d!wGp2%; zo|=w=zrYW1OT}LZ;bSG=C80tdJ?&fXu2#=-F&KCTjQy|Q<`d{^Y=2!T9-S$zVa z1~s`MOk$eKMjVMA>1n&iEnyQ|V1k819N`jI^hsZM#1lU8b&SOWF(3mmBty+@J`y1b z#h8qNW7ek*bEwNO?K6*h%%?sxnY&}qfZ24wdS@G_Asy1u2a9aZ3>tnm$rc@-7}3TL z=+21D)_qylS&=6kY?SAb)A|QVI%fw(ewxkN;5j?b(lzs56bW6u0sGa{mF*uEl<^fA z9w##4{EClGpd4LrCcA(LwWDWA#dMA;$YdMNSRD&hB<9)!L3foek?B0ESz0e7i&LId zRh%b+wYRPlZici$S0wGa_f@fYMFvzKWlcxrs?D_{tk!+5J&-PwH3?EB%bH=yprt%j zDyreYbXW5L*6JQWZ8YY4^PST;XJ%tGsIi+~V>OtXA2ECbbQEDZf;c%=u%%!_!KQ+3 zfC$L|7UJo1=%Yn$m!hs&mv^0_E2z8{cPf4{O_Kadk@88F80Nn zDbHR{%Vj+;^65F}V%o{a-m-L27ji|rV$(Rz@~n>INx5wMSNacNgGzPd+So)_5B@3d zfHSi7qK_eca7Yy|hX6;>c8lv}ndLKyfD;>bOy{zvY+rRV>AswV-^`G<0<1u3EtS}^ zuR#5d?&sF&^| o*c-0=V-r}l3lJCqx{pG5JPTC%qbEYYaS#1M7P_=7?=l$v1yM6C)c^nh literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/loader.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/loader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50280b702ca75fa6b87e462e56d73603722ca724 GIT binary patch literal 33745 zcmbt-3zQt$SzdQ_b@g+4dY+omvn8#h9j(;bhrAE1{Lp)4OSAIM?8C*{t)8x$QIEQN zMpZR>G(Be5+KtyS%aY&_65B8X#$gF?fPf(;7>5Kx2oEO&lT?y}59H(=Cj^WQ2iVu- z`~F+qT|F~eFJz>;U3Kfe?|=XMzwTcPeSPT&{_gw9FI`#vN+j~fdqGh_0ro4WkQ&A9v;rXjxxGl5^NmaM1DR6T8`qbw&@%ha=G zR_^s$Up;5$>iuSaJ#Xgg1Li<|&>XA}nM3tqbGSZYj?_oZQAr!EZ85hy}rZTA?c0Uw)#$Ur`#uMyXpnAP~UCtmgi(`PkpbsSMF1_yXyPQ zef9n3{`vv)K>cp>?)ro|Q9o!NtlwkaQ$J)LsvkBFOWJg;SU+MOso!hfE6FN6bg+$IN5(N6km;$IavNuD|x~`g_dx z)Zc5qSDy2=_tj6BC+a87ll4>PsrqU2bp4EZrhe8uTR&%>t3PHwRzGi^ufN}XfBggI z2PDrx?Lz&cd9glePSzhcAD6hn+NJsj%@4}`Q0}~Tq z=by4h@%!|wZf&u~UeIqx&1dYR_J{CmTI2RXYpZ?0+GY>g4~|D>4Qso#;|1OPj#Sio z)Y@r}*vLN~v3BA81bz$ny$8R$@p~A*+w6Y(SvxnYA#d8+W9@w*ZhknF?=E|{wGV0c zA?<$rqK>yI3w6!rtpnEGXu(IM1?~2=>)&VZ3)Ri^S!=>Nh+0eMBGx_Dp%)_7Av<+D zW?shgVXKJeqJ0_9Wjr6T?#1)H@;rm*`>gx%e7`(b@ce*v6wgQH*~0U?tOxP@pgh}n ze#m+l&kxJ@aKIR166w8;t0kDWMq%FmT&T+b<2yt?gOS+FLe zey+06@a&sjz1%3z+0H}?2^+V3{dBeB`Kfbu!*Uw_i@kOt? z&?wjZ`1H~uqo;2zBCM2M54FV~n?12u_4}{PmaZ(#l&pmtjoLz)6>5zJ-VdKx@)j;G zSa$8~f^(|8f36i%T`HikNaO zh{;_lJLOu(1|QEI2i* zXm8>bDWr{Kls6=a>J`O%N+Qi z4RJp|E%l?Ipk4l8)h%7G)#^dZ@W%3oIr^m=RqskS*71j3+o_goNTD)FAN*~qM=Y^) z!>-O<@!V3Cvky2xr$^Z-)!VXDxq>$pZ^^O!EiU3Md$x>$EFo#-O3Cs76B8N#Kzsg6 zZh6*L#iPuIW#25#R%>=?)>){RuFU#lorLX*+i&(D9okT`0BqGt*|U3!Vj-@-FGwR# zrN)9+YS^}ATcugdPqi^?J9eXD``g%oxw)m;65!q`w-_ z(0{+rU0PgRa4Wqg1!+3(l=l`FgopljUxpBmlkN=an6fjU~&hq7knIe@XoIsqjM zBQg=2^kbF9r3u5=i9UP-7`bEA{*8H;&Y*Ge;J0VH;~4A+|fBP?CMqu3&3%f=8k~+c~0sii@}WQ8g@B zGi9sVnByYn_V)nla-T(@J(rFdnjY1)d@LJHM{`l_IW3lq8hSpO#J_YjtKkX%H2gDO z&l<^SPSc*#ziF(!eG2WH&P7z(?RBg?T?Ox_%BVfo-0%d4fW#8)F%BhF#Zu(15*BcBFEtYUG8 z6bR#=P27%tEQ;6!a$$MRCtLCBQRlR0@E$4n%bbZ;{L_)o##QPmv;#Covbp`7?G?N$ zc0sUYutOD8r;3x!vFY`%6!}qTK3b@{1t97|fV^z00DL4x1vsj3?$VRR!bPCj`cl16 zDKD01ssX;VS}dGWh^!~xj~1H;H!ruRdF?g~7YkDglcIK6L+-J{M9R#DQ#(bB zyz?*ubEJy}f#BQiF%544=-D#|`MM5F+I{DzpDNdu>@$wD;Fx24-|21%w3yl^U0F{P zhBL{U9!KD-76ki{neP@@@28df?-r^0Aj?Mu7g*R>i4}+Y6ck*-TlZcBX+2F4 z2Y6=z+UY2k!CP9)cvD|X#?y#NMzJi`G{ov_x~@l^X_lJ_z`0g-T_4bl4K;w~f$m1s zj>@}j8Y+mX=qRK87sm~s1=KI&20MoC0pM>(v5#Ocu%gE!4cs}SUQ~ct14za`8zVS+ zF-dnH_<$J4xlS^K;h~Q>~O9Cp!BKG z+1rKUDmqW0Nv|0fN$3@tpD9=ib`S6tNLwB%%q)3wM-a3d+(WGngay!zH>!qu`Bq`Z zE|e``R#Yc|Kf6S7s#rKbi$|p7|2j8%&?E(Kp#bcDsDSA&yLMq=(QXvV4Wx0G@YE>G zfx=swDNZ_1qpi*}2qxmryLkTu0zX+Q5&xD-&V!6Qs@^}zdv>my%g-v>NR7#QSSkoC zsfq6$q>{>T3x&B`5$HxzO9P#1fGNO{kp+Aaf$Ew=Z0o1MAOzr&Rn(Gta;EUM100YG z2o5AJ6yUfEHv|mew|L9Atz-KJ-wzN00boTx6Kf?t7yTaMrr0D%CnG5HBgiEMOMfP! z(gf^U(*FUKKOcTMQTakI7YAgD30-e&D_nq&MorQ7$OC;_E zfaa>#0TrsxOQ71pEx1L1!JRsXR|n7pX{95>z=bWaFp=n@BLyK+2;oXb&5xGWkIDeB zV231@vH3hw%h+TSnuf6vRHA;3zj0>pwt}nYkJRM*aJd!&J3=uKL_kr9e>7X4)vTDM zgT;;8h84FHmSH8p_9m^Ah!c`QY}!s)89Qxfz~pBke(1CNtel;*47-0mKby1qt-Pck zusEgze*a0VsEb_O7ldeN?qEcrLc|tA3W}6)i-Mwic50Huykt@-@&Y_YN<#X+8l8MK zDmh-&K(^VnS$SALc37Sqyi)0xk=tci>1cCoUAv0yL~3d-A=TuDc7O<%c6^KY)B`G< zh&CTQ9dO|tkZ~6PFlDb$^{|6Bh>5Co5QbH-7qwdj>8VwG75wSsf_J_SoKpwaU|X`0 zy{b8LC|fmR-O9*y*f;@zFC(4wMvpajuJ286Qok>t&w?e(W$5>JxauHQ;E#}4VL$mt z%)mv}?zdCL1NwXq=d#n^w_hU8Jo6+%-D)60f&BvSI@PY9w={=JA%b5+LeC@*O(AGMQG(Q@0!%&1s;@AJF&Jl1XE4HmaOR9L*v4Qx z1L7ekkHF_v}x@y5wQLp1)l1otV`=9+{0=am#oiYbLD( zR%;TgHHFohQoj)YrL7Fsc?RpZ`?e2jIc4|RDLcp1c6Js#-)z#tibh(jYG8+yotG2? z^Me!yS_9UAq|&WHYY3@^tYLc?b&kxB-Z8CVjaZ{dyG7EbP(mQGgfN)7$L(aaW$4q0lAMN$pRcG^qk@zRbFQntpu`8l}fT_VW7I} zocQC9*_AAUEVx&Y^3mj)zSi8f$pp7|7(XucJ2#NY;YL&8aJ2tO&=BXI!sWgnffsT1 zS|H48(K>`X(TEqD*H_{kGItlqZtRBh5XX4Q`4q%2Bxwu?U}7)F$)`?XP}Q$R!aUym z`Oa=7ZBTG8BRd6YgIxfbTd2+!Hj$iil&75~yO>TFg1;w(uXcqzB2L^t$ zKUERQpjCCud~KluDT$IRe;Nfn=28Q8XD+n#SZr*4c&fp;zJ(b2$!0 zlKUGfb3TD&zP12)i!4IJz6qhdD`4(t+f6E=e#jM<<{+qCB z+0tWAo-93n_Uwi8lV|+?pix14Tu_ce1H6RV6p*l9Gp+i`nnTM{xX@iCuskz%XH=Br zx7(esBfrZ{OgFMRmnmc{B5OhTx}E~#3Ak}h;c75pM=b_z};?6TTV9a zleiS(?y>Y`2w>)-h}|!-X~aG(v2n&)D4{VfZ!&mu!b(Wmyu@S?^L~`c+zGGGf_SEl zb->d;)588a7kwzQoNVdt6)z`CO`q>?B^Bo|pSMz2W~l~Pq~+9HWI5eRLI%5iInzpe z11kg4tHD;Pm08tRSx(v;YNc;St@Ksh`TMPmrJ??p*w$#I5xv%E8O{%(rXTQzQ4?4% zj6;7*yRBo%kF?S#CviKroNnmeC`$>Ira3?3ZLz??gKy6&z8$I4En_x@Mc>!_5EMWl z_UuBin*{uFunt2D`uxV_i9Hhw=@5z88`WBk)NC8j!GWKBt?ytlBw~Lpey})GZA>@E z3GNHEDuj-<)sv*TH~cD`xhSfO;=H@icuhZ8T=beFPd2VLKwitnO??B9`LP0M{(k_7 z5~T3jU8iIh!(QG1McTO4X4{m81#Jdey8K%7^2C_)51Hwc489wIAFsMtsfuHGfw4mj zwjeO|6HiV*Zf2+>n?5r+_4pH0X5!-csp%(9Trjmu(|%&|(y0@t9y=rKzjKiJzJ~#u z>?v=^?1lPXtJ2JqF2_KtvEm+UYec zQ};S5*9{CHD`MRD2eGnaktU~42 z-qhnd7; zcV2d-Z#4o;0{m2i5rhVQshl%m20)QV+Y~uWPz&WQOi4&^u zy_A|!?i8p6d)H}a_0E8SwkVe3RvIF6if3g>O!~=o0UbsTqaaK)$K(NA?oI^HV`PA@ zQp=#Xs7-navV7r9@xPT zXCeNHE_33s#e1Fexb+RCO@%P)pW!jo#nkaRDDQ7B zI`5fhx45f8OA#@mqifAa|Ms2glm^a~pz3hNb@4z5IT0^8pJs3$13@>ON#}kB4j` zQ4|?T59w^67S2comphB#c@WQ7#h@;5#jGeK)*4soZAiLa)}h#mfRKi6@(jdDj{zt= z!%Hy7IAz-!x#-*Atw}&*ouoxW8gIqB>96f45KSZ|otvo5+_sMYf?D07HjHa@8spVv zt_EBiyP#EuPB<7ZHKs{t2?Z(U=_THAOYjp|8Pqa6Kh3B<2J9xZ8gsm#WWdyB|DsdH za_tuSGkhh}#3>8GabD(%>|#)Vozql6B7ie7zL1E-`o=1Ha|JOQcO(O00E zZzWzC0PsMyj-COa1j)yjlTV{pI{w{8{~F2rBJQ3W*cABhL%3SW73lv~KmfOrBoHMA z3Uz?m-N0EX5O^TsFh0l+k}_jOpq+dc+fN#Vp!P~1(q~pcYBJ}Bnw}2VbnR&PJ?hP_ z0&LNq`EaUsJecjajx4Ze>T}Ma_oB^q?&h87sAqUb4&Tobz}tZ2{0m0)ks}b{9Ry{r z^RrAVdfFX}&O*g@-FEU)d6ot!ECEZ!*X`>tun34E0qi?Xl_Az0@b#gHOr^&(Qj!nQwIx4tyqOhrG zB||N{2nt~JlIlFD6+Oz)+Asvqa z2RkIU+)QLiDW%^Rh%YKOmV8*d@FEM4#YD{W6?s>!XAH8dWcA5M1I)uNmXK zgTmphyq11Tdq3Ze@@_ZdbO<#^_4FHqLq=S3Z4h)0zAhM!7`o>#7d+iTH6i2@)e~9R zrcyx>Qcl1+l`&~M3B?42oT8Y>KrxYpVxkX9$5RV<0~NPfGwH@p&{UP$-a}~B8G19a=wefa}1tmz|F~#q3s5{bDTGSny1=R^sBA8B2)p2fA$wOdalsRDYXbL}D%rz5|dC!-?!# zb6oAJw8vGWu@1qKFzn$j*gsyUaEUOLWNb%w!FyN~5VLgf?nKP_RV4PuuEQXB7F0P6 zg@t+unrv?S87i+#JfYh(9%Qj6kY zvP178n+OOr{7MH0aicm2+qEdCkwOY36j0qVx3U*dB`v|&Dl{83_i^?kJ2)P8!0ccD zvbP^X*ZA$mry(k8?%&w>&4dFXGX5i|s6r;A{b%5Y#FTV~z|`4;aO9)AE4-5LA>4J! zd4Tz&J!Jc?9-JRnWcw=6lMvpWw5|9G$af*EA6}P3;wQB7L+B=jKl*ai4s+I zhpMeiIR#6-uF7RPuw9}`qtLKPNn=$OZd`$FN?5bfIN%ko!~Felr$DQ4w`?RdBsP_B)G%2tXCks5Ux9SxWy}cuc80&O+rOC*h8pIMyBjJ2MR5(iQ0Q+J z9wbDB+Y5xl5=HAevZ?d(i%2Z5j|wKwts0Ca6y4li(5);5-vlNW+3FK~=PuT1g=# zctj{M;;m!}N`m<;ezW)m-&u-!eXZmyQL)%|EvEqddI$Ii!6SYk49elRt<*}6;J#TL z^7Rr*L0V`af8ia%e+cN$++4(I2p5ZHa~LU1+0f%(dFQ|l(?o&(f?IK_i(M5Ytz_HN z#l4W#1zJI+gH(2IgrUia{!PL00N#lSr;^>#98YY%D0^GHCHq0hGVU(E7HCa}ko`Jn zeUgc$7>KkYN2la~lb@%3ei1e+I6?sHSR5i?9)U13*TL#e2{K4$FW>w-2ET>? z#+hBpq6=)jqA*ojFOn2l$~Xs?%H%BMf{L61O}>`aMY{u-g5&@O5F22pixL&|5{@@O z4(S&e@jPy5zS+V!zl;$lG=MyO6&h2#kzPVnxR3Y$fkAIW{j>`TRw47lZD4?@l@btc zY+E<=#Lms+I_nu7u?pkdC^1cjcOug{$2-~?s*PxbcOvHz$xO$DOmRS*3pFdkIB`s6 zg#9efB2_C&{3i6S^ILrRB?jG|zQ{+Cg3gx_{3h4=JZ==X1QTJ%Yqa>Ngp|(E!!#UF@*fzqy(#rpP-V|wE_ibPJ8pWpE}Srq+3~P?12gj}9vpfF^Iy-ERR7O&b4xGf=gKx&~Rn4(p@Lm=y_l`B$|GI{3~ND548{b&gyD#9Ai zkOj=z;Wif$>{}!aq4^KhB;^J4qiuo7gDA@eNm|7YOI^Fm9znW=4OJ0(LRi8?6>Trz zph4y8B2?QRtnUMI@^Xiklw~COA^Vk%fRel{ZLFXUk9%raRR9fUs4Qtyb3#VoFb{bi z{!WI=?2UK$I~gvAO$B6g?Li(fz#7pte_bN;Fc%792kO2h=uWDU1_&uyn}DqWzqSHC z6lgoX3d5pQIuX(}}M1n4?{_ zZzZcqfbVR!J<|JW(_MapT8vljgRK(A(jH{e;>>TxFuh zy{Y|2+}zn3-@*eGBOZ8PXp$owS;*zKLM{g#yE-!E?8R?Z4xXy>#sy?o_%yNtavhhY}@sHR6_A{jCg+am^r2u9Zeiqm^m(%Wq0@ zFG6*bX@O)$s_VFqBV`I?cJg@S+QT@@XaEhwn>Pi33-~)L2WksAD_DR&ELfB@kpm2H zWeA~KaG@P3s6(N&jl-H)nxT++K@4y@g|$=6!YW`vtw)i_bh#quCo%jRXNa@Kl|ZUd zPK=Ovv7opj1F;gtm_b^x8rh;Y)vvGN59)9Idey$+C#ki7^{9i>D#|uJI8d$X>TW+G zgvgLFw;MS2=s6|<4p(Grw0UeZ(1HyLl|ATe09#%4eyEsaH1YEYl)` ztm!y1nT-zO!Xn0R9^n|4G3YJq>M*W(PWz_b-`w|(M%?*6)F5i*9^n@0t^}@AxP)1t z8XN*GfQ$n=m1h!RCdZ>QM3*qo105^JXXask_i`MIAU5fIibelp1TZQ1B=5hQ!9Qhi zkO9$F*HCIgS5#=HLCz2JeJ^~8E^Lt_L)s||1P19Z`Y(p`2#wd9dpGO%I!hrDqV<{y z5l-9G=C@clmk1EPXr)&Jic{oW&d;%3H3sV%N3!jsDCk=(Y&hE7y-8En@fmj_w|jVx zttgIEqs+!Pu$aaEH&Tb*8I*7|;5d@xIC!8Nvk{#)R-rS7G=P$T=qtYmi9rgg4voD3 zD|h3GNBtt;{lR}g$OR<{(HoHE5RheNIScc~tOfrskRkwaiUQ?*K%4!oEX7>C>JU-O z^vDHTNgN>0dHMN)6&gH7D2;AGK5iK=!^m^F-y2yOY4u}+$hY$Df;Wm2ul?9ylYy~f zWMvE5Ge(ix@<1z(R(*JR5GIh2rd#QkwO1}K54DC?#-+XQTOP*!*48l6jXlrn(6jV+J2##>ujW8*xp`xC9vm7T3IZ2IG?vE|WgpJv+D zma8%6m%Lpo1sGBCdAPM@RmbR#uIz4&u>Y-rl|8HxBk>~S9lcupuY|eQYd2d1&L5$N zS#NJ^;5JP%`&2K(`Ow?D0#V_gg>$##CvuZ*o97hGD@){hoqrFI5EZu{J;)PJ;td0) z$d1?CeM;nX*d>%DWS6#*Qx0z7K=a`Pu*0U<5NqSGtH{<64F=kh^qo5Ti&UMHue*J1 z(W4>QtOK-2YPETclnseFg~qvUp+K0A@58Ce1_(+Lq2OLXKpxoV9tgj0_FcrWO6C72r37Bw^sLaNcbX3Rg#k~rro1fF(-2SGKAhyoI?k|n=&6-AAq8w~VDUE_9 zDb7c8Z=*{9yg^DKQQu6%Tul20Z8G#h$-SnnrT=U4VE(JggUuaN8`!QQfhaKV5x`(i z)#N7Y<(opoREY4~fBm@WY)@bN1QxKtE${Wsj1$wojoI$!0BZ!uW6=6(=y z5TN$Xlk+B1N-eU+3B+cS=naQEZu)5thn1AeF>fPGizkmHyF_!O^}F|P zq5LlrjJa;IiM9s{f**htWL7>pOt8d19J2a9(IF(gpZCMPGhZz@!h z!J}S583p||@5ooJ7bVH~k{(c&LF-iQB3TlFln9W9nT{M0|! z&8}BcG#54XIk4bA9EHV?ZW}!Z8xxS?Byq4Y2{sWF0;E8xR0f{FvWV@2&OQm5O%9rS z{wD3Lia~06FiXl2<+#}=sW+xle$EUWZX86p{g5FcCeX|0d-9Cp*z*=#tYH9G(jJ^2 z5(!qu%Ft549!A`V)XJLE)|fSpGDndEX5rRWYa5=n1W(E*+gR|VjyaA)val7t(6-HQ z$3g52^!oI^rkgt?o-w+0m$eTuJ0)hny~{d)eizX1yY1c9guTZ)XzxXtC;>7r`cBi$ zyF`Aq4>Wtx-*OUSpA!x1~fSeP&3f%I|kkSZ=M2Di!Lwa(qu&fWFS9ga6j z{&;b=a$m8@pZkl=M9~6&Y!{mY#cNA8>|jA-l!VnQHq*sxSKImfRk{+~b}f{SUMV)) z5l1Bk5$Pg0cvZ(nSrgZ25*Ima_oE2b%bJ{=oMF`9>}6;UM0yE0pGWZ^g`xrvj1$Oq z%1dHU<>B-_Y?FrIlf!^9sE%0J7+U}z<_i;zECWiC9^JCwaDCF;dQE(Uv`0LA9?Oq9 zelBn`h54gBCKLc4L$ejyxy`#67ND<;g4JxvM&RqDJ8_l;M{>Y<12Wn+$B!eE-`7(q z9AJqZLg#SZ73XypU1IPe10f?W@a_tNzaqBTi5ut;Aejeph0GZt_&c5e$%0PEM)$nc zEZhM^JCZ5~j8Gs0L6Mz5?(K10!bFi6=(4oS*wG=I!kIF#Dlsw&^oRn6mnOVW@L*o} zgt-Cx3)ho4oz>Ua<{9E8D-PKe_Su(lJ}tLSV$Z}s`g-CWlm^ck;`>S2X&Y;pXdDC*2RLhRVQ}{y zrbGQjX0A>ucuVLU6JT3!~Y;3Y*{6erGR^r5iL4t0wo1H&Fzy5fqo)5 zF95mr67p0V#|oF7e}W_vxpg4v2rnhDN3e7H2ug|B2BhJYD`2=p{edz(@2-U);I8y(J!S^$eM~rC1`2hwp$GSsD6mpKyAq}sl zDzLQa3i4#WK8N@42h7>%uF`cgAq4Fijy*+Qps;C-u$H8mCq}h>95DNK*kQe-Zxsdg zj4?{@I*=iC47_j9(1r$W7|oDjo8~-#_j07Ft!iN}mcxHS=WwMQb$f=r;Czxt zNNUXkcZ7)Ee0~Z&P#(nml<=?*&s4n)Wz>gh;h zK!=6ftwI!hYBbV};|c|HNG_^?nX z31G9-YOIwa3?U{Caa?Z|mBHg+Jamuf^8UErJxim{9piVZ8dy7Yp7VmAI>M0+Fk93tm4+b}!g~nX5a7j7(61!3! zY7VvLJ?~a!6r0s|kg9OKS}t6MBSm0Bu%mY<>oN&woZ;=MsNEymDNZE(Y`6`6rbOFL zHX0smGH?(rgCXcp8W?$9+*>+&4qYQ)YDYJuW)Xa;e5Ka_iaCpV0Rkn`Sa z`;ikkTXKP(XvIJ~o`Z=W4!2lF6~4(R@gY?pM^%x!doL7!VzQ|h-O9{s;FHwwzN)>J zD5_g+I#^sk2#doJcsChHHrt#>T}G(@#P*=8}sOO0)*UP3IwWzHJ}W#eKV& zDHCjCBr5&B;6cek&2;bxh6^qpLkZWRO=lZQ`g;U`{iq1CJxp>IheEX|&J%(5=MxWt zQl?14F{|l+(Y)`DQ``lz-eu4;xkQ~r%zJUU-$3wuN7VNTkan0V=(#x2%R10tTn>9u zeNE>R2BwKjz_x1x*kVcIAHbmxeU<0@;3E%HJ-~@lcNXluELdb(iI?DIT}~I!eOeZ_ zmz_EQOb_;!?Vi8(-j;#QJN;P*7q3U$r{yW5o}5J>SsofEVQH)+Y4swY26W9@yKyZz ze-dy(Y2AAPmq42UjwAg+I$@rojft}dQ_&m?t!98t7c0ZXErs~v+0cgxcYQIuW9Cpjz>_tl@K72aLbIL22>eW_?am(~sdJF;1Rash^am*ygzJP*x}PEJ zW$}_e38+`qp37;?i|+u!da-F&bG@;S=1zzo=<8Ypls%&gPm!}n-}e3N;!#<-&TjMt z?%214C#5*M)DW$Bx#rx%zH&7?`xzW(|IaWGLo@=sbAUwz=)HF_oBhEWZ6CCk5p|#_ zjKC>6g8LZU%Rq)x23A=2?TGt94&rIlpgjlkQ7l(QD~mhRSRM`{l#mc&`!zk&Uea_C zMavV0G#pGqx{Brf=kI8F_YQu~@+SA1!X?YwLhHKAds&C#QzO^z#VEDMG9-tmaQbJS z)jk1(Xd5GX1}Jo`46K}9gtj5aIIo{7S*U|SkYf+Y5_wp;m&sA|E$70rB~j`AV2>&? z*NVLqfoE8)m00e3Dbm6rC_yhnf>QcBdV@ZJb$Yw!Bz5%In2{cr6W^kon%nA!6LdK7 zQslb0Fc0}*5J-b4RvZuR% zJ5Ct7>+X)daA@4;Sg14HTBUh=S7)`zLSYMiwvfaHokU=dTXTMoS$_jT@$9r212UM< z%%RFc9ZUcYMH68_I8&{-W?G>T8n(bW#QiM$0S1(qDIKWnOp_b(h*PIgb5qlwl@l`vNrU6j|&aBFaxg%o5nbD3$6tNjM9(#Cza=C~SmK z?8PL~Dl_3>djJf`4fWCri>$HK40=T!4nIs_LQ!f3w*vtRf;mv2QZ@U?0D(AFZ_Ipf zSiBPx!bmdOGSK)=OysvY(9fVY!ARuMg_Ov~NK4Z8EepRByoX>BdL5XN@E$5gdKU*b zzh=NUBb?~Mh*|l^60Ogh+t<>tm_^S2+${d>z^m((p@$iz?uLTNQ@GquL0kz&6y)S> z&CS#KGMIWey_=7L8fZIU=6f8U0}}+dj-mI4V)a2Zf);@wfG%v_C8Z^C*zP{40Dhh; z*?=)$4pfK|ROe9dtku7)H-K*-Vpq>F(tqdSXutZo%E5R4X!MRO?F?xZ1sT>7lCx*)ea-Q3qx zq#!i-u3PWbyxG^&BiRy~=OL(*;(K%+9ZPQdGMm2C9`TNWPBD;EcsvNmG?y+Fox9lO z^rem+PIFAY2+)?=cI7ffvwx;B6DE;q1p*hMa9s$~8`-d$dqOel%L0(}P=F{iDTY4g z0E>a;h+~x&W@ll|>Y74-%;L|tiJ1OlQj5F#79uc`kI)8-{d>x;oSx}ZfALermssm-sBfBOfVqv2pg19ma zI=B3%sM4#mu)k<_1sObYmB0wOn&&@2% zol1e32W(?q`N2L3!h3-0yU<-z2|(r@7&&+?C2 zr#F(w$NWitfh3mYQBI_WxB;cAH#u-IOivk8c5M`fO*A{D`y&u?|F?IEU+g*%DfaZT zhd-uf61|p<-}&F1d5gi%F{m++vFZ$%_;hI?gY$pzvbjHuQ^FPgU93wWttkC0vJxaR|4}5PeUL%NZG(d&+BW?Sif-H1K3KkQ7<*^~l!Luc zZV3K^g11=Da`tRHmLG`k*R;FkgxZ*dM;IDM)slmqQC*1;ka?Y}ba)VATPkQq&tEQh za(@;#y|zUBxWRf~AF#iH4$X|xFo?Npy^6sat?)tz5x{jicLrc zDTzv88m@Idnxe{=15Cw&RG`QiDJq~pTxlAsnZXg20_v`{2^J|YFf>+ePzP5wTGS9b z#)9je`bv6<*zx1Y)Erwka9zs~>NGZ>VOAu#k3>P&BtV^W;0XuN;Zfy$94Ahj{cVB^ zXYW7+79oQCBr+;XTKHUKq_Um}zTLn|*Q0Z;;_o4rpRHZ*&^E|b`-r}x5+%x#U1UDi0onKnp{ay8b-HsWDy}l1N#Mp9@DX{bsXHC??L4$ zkxZo_2?iU5R*Wer6&If99U+5))45!WtRxN^@Z=SS-e~2}4oZigYGtr6lI10=cLBKq z{sblgJX5<`oGhL^ z87NOKquPv$^=mu1M4Me-$BVKfFuxARc!y&4%AYy@`hV+YFx7|J9xH?5% z@w`R%*pVZ__li*3kvSZPn#1X_>$t1ej-aVCN9y9l;|M;4;Uacvwpdpm2(#d3vRVsO zZQ>h0JU-fl_wqWvWojLECY$4Jr_9W%mU@XK_^|@WVw50^pE4ZoS0BE4FXj(!k&)cY z%cnR?C!RRSFd_%H`ZQ(md!t4)oP44`Wcfq)NxNs=t1+5%4?hcke{&xmqB z83>7CFpg!p_?#uarpnwLC0H=eF}+X+CElqf@k^UxHSEX5O3+NR2|?XS=46HVQignk zQc#$xVMxAS3FATN@;k{A!9(Ce)?J##R~gKdBxRy?I05c***WV7;}Y0O68{fC&a{l|M=r*Mh8 zUQ&gSMnNs3sG6vA1h3k55rX8TsCD$zPb1M6+L#Uw0YNk0p=Iq(x0#+EOxNBKq&Y-; zN~22Wo3PFYF{v7*Df&nVyE~E5|B9_)d2oxgbNv|oT?lBBX&nNZoE1Fw@sqxtc3!1K~^Z~9RqVw=&_X?CO#IC@(@JlEDh|)rI!@IN)-NdrM zu^}w`$a*Y`I{R3Yaw8f}-%5$i=o7s;MAO#H;bXQ}aPm9AeCCi>h5PRX@b5M-nMKmU zorYPLP@poxPZ4wuqr`eS@B66{~sPY<08!#l+rI(+r$Msw?W0O%6$6WcbW+t*R+G8z=) zIx-sE&qGcigh}~j#<9mC&zwTqZ$@?#wnL`+U*co8pD*%Jq}(6C*gwzCQn@NF9*i9R zX?wsyuYqqBH=J+lB9A5XC^qoiUGg8p1C~7OUZF9QFU-QX84l#p`Ui3N=m;o>5!fq3 z@Y;zpNKN3-VK|Q8YM6liV>pgqWq1Hb5W{i&`og>6en{eQ{7`=Jaj8S+XhozpmNwZt zb0?SQ&`JDj4$%t?h%1x>|M=-AFJAnR^DBrj)5?YadGpK@&X*a{H}%+wCr+O^U7DIc zF?~i(CYafi=clJmOrAdZq3JVI!eE#>A;saKDW}XfOYhDMCq;eNSoQ_s{B7r}oFUq> zWdU|PQL_F@YR)q((n6YvE6#smp})@HmlzE4mDo+fatj?|@JYPzW96BOLmVcbt5S*@ zI31u@KGaBu<+ntXD*RvkJ|=2gC8jC3p#Qm1Cr-vN(!1~8kYmiaW``u{u**Cz$adgL@bp zVjxJk$UAPz%C+cyyt|*l0|?BmcdW&Ah(pu|EFWaj@y(K|${*tUoV;&;*VW$QXfr8MiMT|Y^kNnMmj|JAC`ut8|M5v;_z7Z|AAo?u-BE2A=NH8e1Phj?3dy_q zEeZ(!D7Bzxng9R* literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/neuron.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/neuron.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d66454ab4e12be7b9d1b06726ea9592a9418deb GIT binary patch literal 7533 zcmaJ`NpBoUcFwJ`vQ{pVxM*v(dQn$PB3l|@+8(!irdy)A?Usr{lN!$i#!e(FqR66h zDMe;c5?y5*rhu^z8u;MbcmNU@7-+yg_zw)=lMg=nmRFzV7uaaei2hzg7FDD)Lm{JL zd9l5C%lDp=Mx$!r@8-Y$YWtV(8ODE8WAbknjUV7ke`OlRLxUU4;3l_1lbPz*VwU=~ znT=mNbfOX~MJ{urGAlg{FFkiyi(eKE^t_9nxA+oz zKICtM=8AB|wM7FX>bL9iclf&)z0ThgQ~Z4~&990X^nSow=)FO`U5kInuc5W9X!I`E z`3>}}@MXU8++ZI~&c@m}RoCMm@tdHz$*pyxb!+d|!>kwfvYtqnx`|v$MJPI1FWy=T zV!jk60jGK_1~Q44qJ)buUH&ymL%ZeV&YfN-%bk0@G|RoUe%4FkAk0e}gMKJlWGCGUuz+LW(EZ{*F;4GJJCqzw4 ztc%A35qHEFG8y#sq`Zu+`!d{BBW${8g~;pK6ZZGb2}KML3r;z z*17V=bs{_m*jlVKdtWyv*P7@%Ud@v8cvZQ8B46;^s}H|kYy0aDA3Rw5dSmsDe}C=H z>b><2z5L{#S2WUpL`4{|00y%vCZFe)u(>T9?(kCA{i)G4xy#GX8?3}V;qH`qMU_=m zI#fb3p^=bGk2h2=bdc*27dkqPkt#Hj=IW7}s6+c)=Jv$ZXg{#@1?-Yz0CLJ1#BhBDzoZt(g`;Gw6`bn>0%~?R4a$PNOW}+ zd0E{^NQk)uo$l%-X#5*9ulRl^4ARv1zcIdTuVhhwrQhq{$N~w~y76%|*p6;SPgcU- z=1Turww=T)yI~luDADxAQ_+F^%c4mpLdX>@#LN9>@-6If4PQ!rqH4RQYg(pves zQ+FAv`y`P3kI+czSzOn`CEb^fy*GF2dIf3T*zR#IVjp{FBF$%xB`zuq(g{JMOCDd0 z7&$Z!AT)HfLwjTo?RC6kyJgBHjI|tn_#3FpWh#jOrw`riz}gKm;p;oy1xZTVG%d&6 zyL9T*Cu!AsC(cKgrcW{cXQ&Y4Weuk^)fp{)#-Vv&9U3EN%NXH7Mo@ z26tXqH1>EdtK!*9FHAhS+m-+)P%+l%2U5hDpNhwLaaK>1NAR=ck%*ZGUXY1Tf32wD zK(yMVYzo~`KO<%-0w`)(Pfc4jt!Hmz1G4HtmSL$XZLme|BV}d1S(-b&OhmaG^!p;_ zat0IRG!;`+G*Gmv`svhz(bKV7ES_ficq!9TBDiiV;jQWuqW%oR!+@_P3t`BC%rh!tpox`%+Fo%6iz?Q>y!9^U6^bg zqYk}~`~w_=O?4sk6Pw5nskf3P1$C>TrS)Svv-=1pa0^?~o!A+Do~OmRl@PGGiB1v1 z=K$*#|mKbp3x`1OvO*4~Bo2?Vh`xmL@(+nOO)t;FBY ztNxe;eqQ$pr-ph&y2LeSGf6__Bl8lRC^+YCh#0_DUdOO(Qt=TLq%g`v$nT<-+rV`4 z4%Hfk^X`Z+1T=W%n%y--SEF&*fFr9!V;06-r}~?@OKL4i4=yjE=#ZTp|B-{DODQLip^`zX4=h#MLultK?7) zT)00|evEOf`S=D=waV?i8K4y6rIwB5&0f5*SJNPGc_&Tcy}2zB3pp7q=>Gb08gvEB zHy$9p+bT<9nEXD9ywpn}W4W^f-OAlw+U=1xhR`eO_F&J`C%tSNz=4><=1!XfpC+Vi z{`G#q{s2*vTChT;QkIE_RB!mlSoN>*rG(k$;G;eI%qGbv-l_?irC-OAM5%1aiN~?g zTgImz=W|rZ=yG$**tbXU(Tb(k(8|nRL(UE@3+4_i^)id&fnaj&%+AaM=g`leP4Fdzk6MdVm>DNemY3F#2T#oZ z-V&Hf;94r%Sp8uT4#cXI30{NvpF5Alv$SPv`?*epbji~U{6qBp13rZd>tG%IBPH;l z?Bv`eQRKFWck>1p-Cz)A{x0ox!g-N^A0x3JpaG$Tn}Cx7_qZ`2)C0r;z5%5?hZ{LS z+;eEc4cVCkH)f;ez;%Ixz&|AjtkXX6P^3}xSagw;!Y@3NF)))FJs8Cn`$(%i(l}Oo zAS<^+5hF^-WUmhhYd$@7_uTsg;o~L`K0TmC$rmpedzY0T+8lHqi7Z|I1b{jS(@#;+ zq~Ds?3kAqsI2~e(3+c=dh?JD&&Fv`Y=Qdo8e25iWuKY9n$RD9#6_9J@v9rYvz7$ac zKPdpCp0V1v7t>jN+@R+?efIj~F@KK5bs&MqBV5unflBa%jww4B%j?>&77^VPz@ADd z6e)pxIvr;eik=fFmZH}RK4O2mi$dk2j0nI4CWdlXRs?7Qm|z3BB9?uDHw4U}fUb;e zO^LfNtiLx@bX(@$a}N-v1T5?D3Mf2zV@QDkdcndw0Ix+L=z0ZEU_He`heEmy6jfW9HIpmsK`FuKYW6+K>ekP83t zqh0w3yRkDkO~H3xF&2zx4GkB3N|2#>_Eoyzl{;zG@;BqE;(A-O`sFtwFHj&DjI&e{ z9)TnWPyKihQKSM?#VNJ6xUu&2?Jxa18$W)q>Tj%#(=wmke~_2-J8EVYi7@#kEp%0J ztl-p}&OFYSd6I)(3M#r(&|%eK*!ef`qnO676iXKpH0f3S%|S0jI)I%y^x*lc6QQ3b#{V=+J|%a@H$Qm{gY47i!^|8 zmrqeYWlbf)vj4!RoIul5?Gyfucyqi5VbEh6Na2y$uk;My1p~Y?RRoUE3c->=!IJW2 z7K^0y~t zT+{hM7rx3bmJ$DnS;~^s$>GCq$iGC-8_mCBZy%$?-W?M$DU6+R;Dk<*asgy=GMIIY zQcMa5z|p%?-Xba&EkJ>yRUX+QAn#_EqY{ClhIFE+mPKYw`ptJRHlwonXjtbMiG_P@YajI>+?W_{!@x1wqOBKx$5m7LSG2)N-OH7CDtM*N`(GeRDT9LO2dEm+l7sTdLdw!Pa&&1?-9tX4 zFgH+G_gl}Ce}ieQTY1^{dD1~VHb7f`j|z$k%{{#g?t_G3I zavbMY9LvwBpG*R)QOb3>;4cArNp4XRTw&MLm}d1j9lH$2hW|}KY)b!Mf*^jJ6Q(pB zyZU=(Oxkhf%h{FRL#=Z@Qe~M{e9GQ?6ZNsZ9H%!`Xv-Rk79LLhZ;Q@@P1C(pZjA7T zq%^K1(4i${k!pX6BA@d8AVyH391e2VxqDybNOXuwu1|+T1ncD{6(p8gddajZ+o$AM zJ=%m+DET0>4B!l|BLDCyq4M-EP`I$p^lt$H7)23v^DKVr<}}qzMaXMWMKZQ2hS^ib(<_ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/openvino.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/openvino.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f256d7f1734bf7e055f35a72e328490d876bbe28 GIT binary patch literal 6194 zcmaJ_TW=gkcJA)!>FMdY@FGfbSJ)&TxC~vx*NZqTw~Xg>+HJD_u?DLO?ET! znXmKx_>JUE_NHlwtK!cNP4)vm$LC)f?1y|ow07R&i;zF!7sT6RhcEGqFKss9H~A(0 z+DnuD`9}u7%&)vO_!Tj?^Uh&|E%U455`SG>tlM1^?hb0kF2BxipzST*ST_d#SG1x7 zv$P+F`Szeu+7BX?m+qrU9%X45m(7joBo=GXXns7tGl`J6F$uQ>55hbbDQN|{3{y4E zWD-&q8jFvfJj}R=Kg{IaaH_)i!9$c&99U)dp4g2> z;%=6Xqph-|JE<~gITxuQL*v~j&4STnilR=GMtKltTU$b+`P_qSE6UYlnT>=}nJkxr ziJYb)*p4_CX`u2j7Yd54^)SJ-WZA0|kEZ!#nxn^gT@a;Pvr&y;T&GV1sTSV5c-0tU z%Q!F#}mE)9vKl)1GG=8d9xXmaamLtes;+If@mhvtC;UF_vSD{mdS_;>67cKz=a&QY5; zj!k*5aElhVt{6qT@V4xO&S}2IZ7M}sH@%|uo6!H|Y5jNg+vo=^w7&Vw#t1xoty5T1 z6rI#8y2sk`P~I(S8^r8-#|FU;tjk%GWaM9w*!P}_y=t|FUs+Vp)!Sbe4FHVT#b6T3 zFcG@~C-DXZYoAW0Y-$5$fE9RAHG`Q^({#$F9x(ILX%cM4+32Y%J^ddjxD9J> zi;&kz7+5tu*4~;WwM$gu2EJTg4q%|sxL)rwi~0^~)iOl4K~~*`g?I7qHptpN*!rbf zT9%ERebTkMro4p)=Zu;nB)w$l>UR(TCcq2~+TzC4l-&ZU*Nw`+j}3XFXmIm&gG{_@ zOs*q`+A6nhl+O z?(S`xFjdp|g~7db&BPn+Q4FMIgVi6unRdrQ-l?3_~X*1 z!Y@s>lu|&9VHU86>E<|#0c2LBw=3ZRoDwJ z$x1g=!6Z}B3+Ae5Ge&4pWlM@l93pbGS6_^TrUnCWsJa!GE9iu^PiM*6RSmwwob7QC zP9pgpsdTDxS|P=H*={|r38+W4GJgpzsa@mZ_s;!e)T_UMAU~jApM2xQX;|hWQXbMC zJi(wts`*W*VudOMwl#)Tv7I?oRK($@@EVR9-YW{2r-!?tb1x z$n5ScAYd*QE$SQJzd)(N&X*32!-fgWIl8Dz++*vYU9^ub9n)xD(?P7gwlOf-9JnAD zU;rc57{0#Cn1_cQGfYUtk|`+M0GOsvwID&eb%7)9e~G*hY=8zENRCqdG( zPpZ^SGEHuc;>fQ&9Yqz}oqbOM3;3+7;r_~*QTvhd!`(27!_8ReuS{N@csleqwxOtJ zP|XZq=2~5cNCA52hw4_Omi-V400IJmYH(Squry%~#<}N*K~BK>j!+GR^s3}3swj`*chofFE|^H8S1wbI7wD)!cs z2M-6k2uT(Vxh0u|`7rI8dwQYvafKo0_6dhq5tJ@86fiFho3 zO8vY~;%`X!B<_$PuFBj=Uia8W+sf-%M}vZ1aQ>xPwu4|4he}~7iIi)El^-I-@v7Sp zUW+1+3&bS+w1K70iH$PCQp6&M$}Igu$v2KmtOR8z{YKy1KRwdTKxsJJxw2g`sfeqx zO;LqTElAD?xhi`&r=$U>q&@0HgyGKo$#e#Q*7bb#d;M+JO^2(uH*yEnRPT92B7VwqtbpJ!fBc~AC0z|r)?2bbh5`hV%# zE{RAET=@`x@-c~@k@$qfIs}`ao#19WBCk>11_^B{r}k2|aM*XS(cUAVz9n8xG?X+;sZw-RN z((JL3I5h|wCkz6naA?sP949oLJ8lr2T{q;9p)j!IJpSZIl(ws7_PtF4#@LkCM5He%SBftr%I&1{T%WHd(B zp`%lfXHQ{T-`x?X!nl8jo}y25n;7A1uZ*zL8eAkmCAJy_r56xpOk+xSgWwscL{%ec zW0`#y_$|9g$__+nZ{T310YiRD_0EPq=SkWeNjoBmMMy^C0*TK^(2(RJiGP47do>7+ zL>$k6NL$w$sSxRtw;?w1D#9k$z~RQK%nJ!^UY67U+FGVk8%DQ?QcD}1_qBTsrg&oG z_-{G0R*t>+d)?<5^!xxlpW7eWT$;k);Zz}dNflRPeJO0W*=G;%27i7hiOi91(amj$4yVD zIfh96FvX|T6hS@pL6)|sn?u~`G=Lzb_k^x`R;A3OUbumCX6by)t8|_M1Gbdp$W)O@ z@1Yi&xa+gO9_@cZMbZ9u9j^`__KE8tsGP`c6c4PmwZX7#5&g$yP<=kZ86JyOnU=O7 z+8pL)H5JkZT3O-T_H+yAx;4hhB)}~6S%$qa(;Nobc#QLl_|E;L(t#PrkBV;%BLHr! zqq}U=r`mKIH6&^v$tIYu^QX6+!_9C6XXI))`;az%*3oK!Z=&1eHQImJ&8oNhR_nCQ z={;A)4XVZ!Kaq42Wbf#9dL2)7hjF-v8<$}PO<*B2#zXu_108$wPc)k`BH~BdFWC>j zPx}=Nu3S2xR`*HJy&G#jB<3f{TT6R0%i@aZ&>^dIPzP+OT#t783~+sGDDG{{4r=ey xwUhb)qH8Xhu0_`!^$(XC^Ey3m&LfSwMt{+Qk9*e_uUfw#kGE literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/tensorizer.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/tensorizer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8993fb11f841fad4f86863c079edfeb70d2851ab GIT binary patch literal 15181 zcmbVz+jAV(ncs9z&lQ8gg&=qp)e=QZBN0Q&vevdt)1r8>X_F9Fgsix2xz(6%00Z`1 zbh-x+W{10$DR1_LR`ybxR5?k38*f#Vs$8m4>xWHM{){~I>pr<^pLbJ^N#^(aPR|Xn zl+6Owr~8~fw?5~){J!ry)0v$uXYlvxfBv6afAw}I^LNZl{_*XbxS}tcnT((DjWFZr zyWtp;Hyu;%Stl!Z%gM<-@8oeeL!(u23I^L`!(yxCl;qh8%dHt_rd4q&tyyQbRduSZ zIcKgl@65}4BV2G6&?Xlyv=*I3Da(gPT1(DS`u=EZ*;$t7LU^oo+&SJl;hd0sF+ABi z<(!h|Qh2)cjPr~8MUsU zC@=1)r1*9>Zg$#UnB>-a-7rY3 zwY_eT6ph?UoD2maMgd!xBoGn1Ly9X|+lHmTn6loy7>JeriRw>MG$ zirVBfO1Ey^)&w%f4zO6WN|HMM;&!7KohJ>vmdBr zUYFe+m2i@sZqRn4SOs3ImQ7|Go%T+kVt0Kn4x(gsGl<<*&~kf`w;3c0?T#xS)lqIN z{WlD5NBRej&g5`4a7Cw)e4H6%-Y?xZ;!JD|z}*kc?`Mq6ho(4Uk3~s-9=R(}-Ug$QvE3v0Ap_Q+;dRwhmTe}y+=K6*1UcA+5U)Tx5 z)&=$!y1_?5qlZmTvth^c19d^qm+MgP?j@y5o1l%4y6W;-3=vh4m_M;HX4ZUc8m3Ww zqHGj#GHSh_-CcjBR#Hb%qL!E}GdaeDQ&uOKoMdte$#9uDl-_2p$JC#N+?%)}$eT=% zq2P1OcQZjYuzceZgq)KL@C{QPGqE?87W^v`BBMNB)^ROG0byedzq{*Z)WkvbdCoFj9$g_ zoIn3b#yJ7zKjJTerB9ZQmX=EkrJVm91i^8XoPrpjP_dlT{#pN-Anz|iT$uhD|5-5l zGky+?{(^cQE0k2n`{wpk^F}hqoldC_OYA2zpx5Tcp1b8mTdT=TyVr6-%-HfMSqw&O z;ob}OHkx4k`RFy*Yczr=Vv*bQlO>cy;3$y#_(d{<(xB~kJI!{S%xKCIX_J(;He9d? z#6yzvV~UX(w;8$7mZ$u{pW+B|2RHjFH~R%7F*st|h|LFPry6JXvE9r+g2wCN$%UrF zr<<$&+_{TC@_k6Nax-eSqu6UV0!30yvfx!vWsu~fz*CJaRYxgVDGI_3VHlk}%gVE^ z+wDZLW?J_x6h(O?6?5JwKJA~;@?cD;^{MJf9!t_w?W8)cM>#k3h2;4XF8RW*-~rzw z--x@3D~rp*mBW?CRq%~}TZBX?`erjTFn($LoKiizS}V%uDK3&a$K*>)US#qTlNytk znNTQz9l09+lb@ zrw|->frMlQe@$ZR|MW&j*`A#eqb+oM-k$huFcIE%dZBMO+l{d22do||C>?vFqgq~! z+O2Nr#ep3*?*;aY=PrK5f6K_tHB=RmUA8R4d}^AeVLdHA z>7PHe3184jQb~DQI9XND=bZL_UsTMje8Jq;i7uY zaw?N;wQR_1u#psYJ=G?PNEi4rrhjmOk58`6+Flt0;38^k}ai_^N=+pmFN=n z7Au}fimu!CT7l~(W!J?r_Ch{aTo=0&>Nn~lswQ);>$TgRSm0mex>|Hm8Yk9zrxT8a zUXq2%`JIyQ6%JoylN|63WIw+!$={Y?)p@?1VY0%6QdyLPqFC1J_)R`tVInI!UO0K8 z$`Gwfu8i1SIa7qHnEXGE+)1Ni6wPwZoCWJWF|*~z<(cCJt*;ki!G&b^p>*TAPU#G? z6{+ESxFSx3T2b#kcmBRPF!zl|U?{!FW->S4xhueJ(8vB5*VeM?7Wzmk!AHRE?mf`{ zW|Wl2^Ef~s$r2HvD}rTQsm-}?QuAifP`{6d2Ck_;F8rIgqW^_t8ntcb0&5$HEhq-1 zpu9a3R5pm-W>Yj*4d#M*$!j#XFv@E*w}^a2-fJ{>1o@)mHJV!*-)N z&6P6#j9&o)I~E-GXM+=dH8=soi>So|6CDP8P$L_<@*q?(;G2{9Dg@O6VxSt&m zh0GF4zXw5W;r~Hai|ki3(b_J=H~)WtYbeHR*6L^31Hc1>MqrYZ$EZ@T%UzV#7G=?# zB0;9xS>Kk%i50YWB4_^YYwne+SFhi_>wfS058RvA1kE^0D876B>O0rhCf*9hapv&q z`s%eiZ{J*9bKiaEmg1Uf5h0L>;3zpV!3An45y?-9MFcFi3MQmFEka$$Fu_21EkTnk z%HR?inUcc}#zH~l0fomtu80psqiW=dex6oMjf0Ahtpd?d@o_#6HH1Q9B++`?X%g_1who(WTVfY2V z=$HKRL+q{O!3O@UUmfM={0eZ>yua|!N^2~l#*v8{OUNIc$S)&*EX@;T9q%6>iz55# zt(!K4jTq-qe{n2K?2Z`ck@p9{WZLr(5jxo#zW{;(3J^Q32V34w6E9NyG||Q6-rBoc zLE9Fwr6JlkCn^FhMBSj#+-MS&u_8@$vH1mmwC#~o;=COIP6f)|X?ij~zd)L(UYmeH z3I&6_4Y$CRo9JpIP@wm~#vVc6Yt-V|-@W_x>JT+{;ex@?04FOe0wh+n$-N>1c}0J- zO;{R@m&@L4?qI5DG+s~*EA)iMooZ~NyM%2|8J~99M0+eEavHK@2rlyIbL;ePU>zH( zDf`i%Us)M1+zPg#zjntXjP3CAw3`HQi(Z5jGDmg$s@Ep+^|1a8FSNtUHK|wo(Hdqm zS{N>;XRnJFPM2bbVoY1eargp_S+N(1QDKke(8P+5yVQ%@4&vR8x~CV}ObULm;q}5; z&kc(W+kYjs{rkTLHN~nEM)p=`7oURhu(7pYy2O*Ys$F=O3(g+QA3sL8?vcm28r~2ej7&g`j;D=ncYr*1@ z&}*@{vJ$G2zeV&b6s0_E;B zRi{m_q`l*Tb3n)S{<4g!T`3cT_wR9JUHs3Vz#&EXIg7m9p1=h=3fH7`&r-mq&P5HE-)Uj3o0|edPu9YLt>_A5y>X> z`ujQ2=b1k>KQw{*i9jerX+FVo^9-_>UL31~>sG!LHlw)FX>~6@W`BeXi<$n#1K%kK zX!xP^OCrak%Xkl50f@~vL@)R=DXoIoqOJ!w>xZm6 zM#1tVP?)}f7?)5*Ia7tDSbjP`Wgx>_@|<~U$uPfl(hyUMV^00?li$P@y@h0GMIua> z^-X&FgB)x!z*_jm^M2khz>X7Gi+j;8`!l$g07_?ZFZ)&S*PObIj=Wz1X_jsLU9sf@ zMbxEQLV(@KEQhKcf*tLP7{W)byPy`NA3=eliQtVA-xYjRFAa}eR`HrGZ|Afz0E+wnYsmZ^fh30Nk*$$aYe4XCa|UjZ%qo!%ybaho@= zi&}|B=hI0}4cLsNGvX+!en1f5Dym+)8S8kV?Kf4>&?;Ay z++p>!JNy3`gwhS&Ufc{LgrTHe>b($eUW#lRl#s(e5)3c#mi!nGF|1TLA7^63S@+Et zqL`lG0fe$kI~P%fN3^YmS~4~P=XMMeEN0!4t$Y7ZD~o0RO#>;;<=cZWdo3fgdl$64-mZ@HQ*}!Fdot4 z`ybS8aR=LAK!zFLu>at`8sI}~GfC=57I%`kKHXNdnatd5bvr7SR*-YzwU%YnUaqg9 zt-|301WO=?r7MPL7>$+vOLB5c4PxDp<4Kzd&KJ1|2cNWb=fGhRqkKj9o1t$_>v29; zG_9w3NPr6RRb$dR6h{?oUETPQh8g)uK0=SQY_&@{DtPvag7vaNna7 ztP$rPWIjdoP-<8{>fk94j|sge1*+fo5I`9FI__rR>3IF1r-N}9AbicWvQAo)6!n^j zLy^8jXGYwOu0*@kD_kl@I&}*!bn0k!%9=k2B@V2k`WV%o;}X(Af$$Uj15ArpN_Y*P zSXt}w3s0vY3O#-GiEW!&Lvly~*Hv86&yjE?=rz2SdH-9`)zr~h=;~|m1X$ZR2LYNl zw)6Kg>RMcYuL;k%O7gZW>5OkawCH>EvyzuKKLfAqS0q16H$OOL)SsV~w^bbg&~Mz? z4|6_YkJW$wcE?9t(!SMs=ZYN+1N(O9?IAQdWR>+MPX`Q*O_2wX;e0Cwib!!-a$U$J z8*ohwmV02Gy24U2ti|m~@_w+>Y#@+6r+&(O0owpabfhjIr--tWycRu44k4Ou;GAZd zbTp~Rg{GkYCx+tyw%&KyB8MnYyywi0YPo*1rCrN^jQ38i(dq8FiABueRl?SLHIffW54ICVgfIUANB05hBp~kId%MyM-V29p&{y#orog}^d z#QKN+i(?6uGTQI%Z;ED2^h>%CnjvuyX=dLPx~T{Xg_jbMHWR^}oQ|=5SV-j}jw=Ax z_5WU*GY9+yU?SSM*jbTmu;jIo$0kr#+@JFhfxv9gy5ea;q%G>SpeoY_yTC{yYsJ^$ z+g*kDgbMdY61TAxH10tiF+>NehQT%TPHIZsN5Xw@7qI`aPe@F8-7Kv))c+tSX>#6< zdaz@m1DkPVPsWw%wY-p&1K))g2Oxn)oaA>H!iyBiK)FmVAxULGlGopJaL1o% z_UVJ3qil$wjnQ)sKIZJu$J{PL?p~ghyNBN1o_rh653KD47*_c~ZeSkI+%M>oLE#k5 zt)hHG38OAPg10(8vcKe8gS76T_y~SYDa#FTrXYRJ4@!K7mOsiixG(Gb_S^@@IQRYI zjJ4DX-nYRZBM~%#n1tFiZD}M(>eCxDQA?%H3hXKnH*mwKtCSQ+_)Q@9^$1Rg4PBp#5{~*fh)@stUv;xDLZqX<4)PGc zetL$fLtWfWf-w&cKkDnB3zD%9EgbvU38MO_uWOx#xFQ7Gr$Pv8B*G%H8`_z<(dog@ z1drhadqHQ9SNk0~`4PP_>Xurm-LBsj#q!3%Z%yUMcozL!qvv~XsGorWSNqRSkcn@f zgP3XC=PuU2ya9ZEF1ir)kKGPh9kqwu1pdRW0MY>lo{Rd+Q!l&?NIRC-PRMyHxV)OJ zAgR#K0d(%Q@fxvLI@{7CZ5qg&#VoYZOJSgoZ%K?)6rIJXI@Q$umkK2HE^0dS;&Vqh zwv)zoMHtF~FK01EFgHx)S$bYB*9bcxjBlZBV!?ObQGdz-A2Ppu;NX(m@)T@bgoq$c zYh}@#PFa6D&)h*(xGheOE9Pkn(1=saE4TQs=$0XDOlLt&bX2NyTE6BM{%ztc? z-^Pm^KY}Qym;qEMnXt->i^ZAZT%|0=v26X6ZXj|b0&n1o{shU#;?_!$95AkBcrS2#zTI$h@Bu37+53CmwMocZ|jV5IA1&_@zn7 zuS*ek^z!(eF~d-oAGT|UW*GxbEr?BDkRqv-_0xkS=c&=hl96pdkqrTn5=|n`yn5$d zIDQdAY2wJ-goI^3-h`wU^Nr>*y0AAqn9eHg11yLz$=yJVencBQoE{3I@qtTM2w%iK zY~Eo7TZd(&yP+)Qm=k`7x*!a=uev=P4@N&O=E5fg(oBTo1bKOY|36!~RQki*64L?rF;F&G*y+bMxJ6Y}>hC!raH9Iqh4e znU_~*x#9Jeg>(*MPwsF4hIVzM#IPuhMv+k6Fu?F z8c3Cs8|~;wAKU-gW>A9|Lx$;UEd3!v9HjoTv%Z&A4MM=D-2v@J&5)S`iqj z9q_y=4z&xnq`+xnCAx|+WLY6>Zez=9Zz3FXtbU-=RO8_6q#UXu%ZyJ}X@(NkB4Ll& z4MrW$*J%Cu3qFbT9f{y%5rx$04fyl3Cb*-wD~1F zq|rJRdYEBTT*4r~#RC}xSlrkKZijsax$9dGO&$FgbZhMAi0b_ePJ6(%w=(y|TcW-Z z7X~;K{lNURaz8i7?H5riKPim`p}#sQiR02B?-w8P+=#hdzF$CpGwK#k7KHe1$^9ZQ zbYW0@kXa@=uMP@YJ8aL1O>pdf35dM}FWU@GDU=6goI!zy4sFUBvmY9}Gc`65eI-6R zC_E4ZKOj5LuBv~7HK-Zt8@Q>T>t|YDH)@76tAY=Ea92hMKH<>EbNcPSU`^}-11V>C zieIx+>cFJHPVbeQ$!{Ji(*C@u}hF;|nqaBJrEsb7H=(vAR+NXEHvP@$cuB~F$0+15;^t5aOUdi(kKoeaY zrytLQAJl(AgSBKPkYAICvP$Op69pi;V0d^2TQf#{5V zMB%nAc{bH$kaA8~!+-<%a-81>V7Z@z!E50Sondr^i6iXODgaC#uCLs-IKDu(#*|U} z^(nHg?$43Tg<0m+ok3w6za^j}jckfzRXF4wACc0f?W5bv45R7NTCV>$@4_A4jO920 zB`&m+rZ~pJA(z`@p%4Frb|{ba$;f883H0CA9x|So5Q{4cNa=9^87cvV;W3v~rD6Mc zrUlUr1Pq$-Q2j8-j$glsGo-WIw)HQ11OV|IzE%;d>K!b)b2RdHaHM0X-gJB<1WQq7 z){Z$#x-s~KeFT{CN(B6BS~^X`$Dj4 z4h7Df27zb8SukG^m1pGxpZO*K(a0W-Eh4|f^{Cb8<1W!Bp2cYY} za$phv|BZvqPJhOK;@i)d5K}pM{q~@K82j)ixa4oiC3G7Vixp}J(FaVa8MKF8N)gvK zb5-N?Z^}iu0BM#OvP4K_tIAg1>673+SU`kwj%G&x9s|oVC>lX=y2nm~$#Pfz*xwj#^wvYAUBB0*puf z)Ee_Mk_WZ{(vg$pa>Cy^npPtv)PYBJ97t8!M6AQQUcfqk;gYVU?6K}oAMRH3tp1vG zHND4m?OLu+`R{n(WJqJx!DbAu=)l5X@JY!35Al>3fesSGScKYwv3QN| zMdm-xCx*u4%&tDYN}JHhYY`#fTL+B=YEaG!*-TLB7|IkcVM(D>E>m+x10+Q`L9Q*` z3Iw+eOP6>4ghuZ2T)}Bb(>#vSB@$`2`nYN>8O5KNPm9@N-ptM$`8O<>rDh(#M*XAe jf_abj^)lk>i%aH7z#;yK0akbUv3aKWgzrqV`sDutDA^bo literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/utils.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33127c29f7141c705e3c2672adb6039f368da4d4 GIT binary patch literal 1365 zcmZWp&5qkP5GJV~%kt0K2AW`#qVTDxP^{gY+CxzkU7)9WfhK7Wya=jD%QC2+kd(8# z@B%G1kI)B5ddnfN&Fgt=24s{Bby14Uu)C0dOyuzox^?9OK1T?U^FZxADLyS1y z4p@(S+u@q9fc0Ovx0sGt$OAUu4tKe~9iJk8vL>f!?Xn>oy>RK|uzmubF*^azl#wSW znp}VRi_S%^bFSv=O3oD*Jk@!b&6ASNMU^n47Zu||E#AX1&_zUA?`cyD-nvh(Y98U1 zlqJZ)%0!)4ZRqHVg!6`N=4j+Npfsw*cUx4@nY4j4mLN^#@vPZA5 zIr^^%8$a3O&W5(h^;b{0R&%}KbH>+6BlKKXGTqE=70~TcmhX7kxfU0Z(|R_BPBX6K z!?xINpi}i+u>90$d$I$KIeWZA zFuNIsYj9q=26lklyJ$xVAe?L9MW62k<6ZOWNP8>j+hQI16p?Eu@6l?+gU$ cGY~Ec@B{~Bh=(v-XNE~YF!?aS;fH(Ek2#fQUH||9 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/__pycache__/weight_utils.cpython-310.pyc b/vllm/model_executor/model_loader/__pycache__/weight_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2a2667be164d90be245841a983671584ed21604 GIT binary patch literal 20227 zcmdsfd5|2}d0%%=&$+V)79fBHD5!aWz=*&sK~g**h!lu}hrlHSfTY+GHJY8*yE}`y z@VXa^9nLDWfJ9VWw50O!hh^EjjvbqS*tu*cc3iexNu1M3m8w#SyDC;WPU0w+BP+5> z6iMXw`(Dqn07%jKGrKi!y5I4=<9qMB-?Yca3l{#4{L1evm6j~)PxvzWH;I>z;S(Nn zElXLxrEKLiY~Pk|$9LpA<)`G^^k_!)es8rf#f&)ID2HuBAaUucf`W6h#p zY>xZm%?W>^Iq6R}r~IkrHh)`l+MkxXPGh^jUEu8ScgXi`{!aPcBi3H z9)C}BufMl>yMMd1%``sH+~@CWe$fA5)AK#a%Qo(4-s#^d&$-53&Aa`(oBRF!@}6(p z(>&lGkmo|_f4^-7 zrQm_9w*O#|+4%6Liyfs%2YGT56k`zGkWE;LfWl{}EMH+k;2dj$kTy zbjDh6)op6$wX}a!?NYnZ>LY3frF-z%tDJ?5x?O$Xn(H4^`-1)IgTdWuZ*VViJ#`0i zKbm*)j+!5Zr|t|U30K`^;H$gwJsmu1au=K#3uSwQM;D4}zq$wgJ_ZP8L4J>ytmUrmG%M52EI&_fhl1>LJuTtsce*9tn!*_fhG!Kjw^j zR2@Z)CvQ>X*5k~mkEmm4@s#=~U_KT+tsckcWB43Tu&_6nzzm-bo-r7}>OY}QpvA}4 z$)JFBj2D!1-YTi?qt8e6Mm?$rp|{Z1o@%eQ8ttm8 zw-&u>OL_Gc%Bqceml@5r3L4&OP+we%!t%FBlIBWwJf?z$>PjP0^;%TQ$EoA3wK#X8 z+Gteg8$s-zM0T8iGH3<58ntztdpg3C6rHJuQJj0O6V=YLyWhDt!gJ+YDa@Cw8?Yyl?615^I>%%sI)sli;tHB z$;Eix=#pxdoVaiXRZp~A3-!f#>{-suyr9ay#UQFgK`U(Qip;vwscOt9XjF8yb&-?i z%2XQd#l=7a*7PS}%v}U>X(Yc=lg`k3N~)g)%;x#pQqZh2uePFf(26RZ zMl}L1n#h~(=sI#%tGWdQRNAdtP~)=q|7<+u@Hv4`cod1UBWu~dZ0o(N=rO8P}hrOsQf6=l};*r0p1^^)YQN?J;H4^~&Ab`w*mVXA9hwHDPcS0n5UtbBxd zhYNu&c=(HFo;qE5>iH)sr{<17apqJ7Zx_xTpF4m0)H%%|&0Q#EW2YU)*`Rf~uG_76 zYy<=_xw5#3`7Bgx0Z|&~u`pqT1q}7=XdaI}-R!h=bV}>CF4_8n$a}-mcj4>J3$szP zGux?m4o6ju1wQ<6b7iS{q`5lVsL#)K)}p0$YxZ)Z(VR7S3$6q;%t|M(+#P|QHG8J8SWD(Bh7)jEwRCdJCtJ6jF|zb0+Mfow1$k?ho{SWNyI%-mMf9{DquAV1XA~ zzViiZL3r}DoS*svR^E4U=CH3%)bccAT$N zYK>|bRw}Zp`fhZkOGqAJ#}1NxcK5c?VU`Cl(}druHiJqfE>w6tROTY@4?eU_=FH;u$6Z>nmO^b!JYua8SGS}CG3(a&Pqkqb&VtJ z|7G#OH9LV%sF3vRPAan2oycD1C$-`BoK0D~*B#sHSsQ7UQtqa+o{BOX*+#MrB`j)-$~fRz5d(0qd9xu7HG9+85un zXWz6BchluYyLPeu8^9?}uggi4#^W5wSBK-)AHeY9T(f#Hz*`u*GB03IbE0up2RQpR zJSlHGdr!r! z$gq=olc<+o&qV27<|+_*(bf0%T-42M;vihQy~kta%zcX|sg%*&9I@LVZ%Nm&^r4?q z)u>wOL^{q?JKzT@PBj87RGc+5KJ;A_hOvDywlDkHg(_HLYcaMfahiw;Z(NV53h<%> zRq0^8!Og;_(PgqOQ}zzqwWqK;1>1Smwe@)pXRH$m`w={kg$u)%j1Hm|-P*FH$I*l; zfUMk3Op8RlCt|)zy`EZk6t?tbOHcLiG{dd#^c<{gs+Yp5PH`Knw6gvxL=3!V*V8>` zBZV!@)!KJyCpYpn5R?snY@< zHwvt|Jhq;dXB-|p=X%**ZnxFT?X^@97`hl0)p*bTs--4+S)`LaOP+H`r@ofIYJc92 zP>$Zw14g-Py?~wyn9W!(FW<$;+L-8#^@^KL_!^*X>lMCgeGT*cynX4CkiY47?oiwXnwi;0i|j@ej?3*apg2}do`#MG8unnDK22hkMnK>|{1y`0C6!@6K+ zIvdue_2bSZh)c2!aIOnrS5bX&rM(h@!aC*q^mAyZ&oYriBZ-`x!ZG$*Yd1TORoMae z<{qniKOkHVbX1wIw<=M)LP;eNH+U4w$2gNl_*i+^Tz?o%;|v9ZY7{%22m{GAgQ&{A zz4*$GUw!hAyVoA?K6qg%@Cd^T>)pUp^#w3(NEKc^g#4xhFIuX$yd(RM3M2RUci4N= zIqG$bqdDEe@U450Fxx9pXC?BI0|P z)9&bxb8SfdG)aj5D3gyc*@gs63J8j`prjmhTs*ge^=$^i|227GoDRraO5^$h8$FN2 z&j>G9K4~P*qCb(G$prPW%d;ju`B@w|LiV#bhVqcppq&r}l%USfBj3!-&!et+yb+iC z-WzEgS@S|D*!W^=Y>V@d30lRRmpdZO3-6%g1rwah$?Cxs0h@v?nNFyshg1 z5*H1|aVC2!z-%385VTJde`dr>;5nUp=0v_K8Emt0#vPWZhP;Y*)s^!ZZHUQX*XD#LHc zqvu&iWmT>Y(t&!&RT&)ibcsiRP;4>~pWLvG=5i7j9YWL+^5P zX#2)?>2pUftMZ$W5Y}@duZ^kVwOlWEl7ulI-PQxCQsbD>uSYvU%CM$=B9xR(g4q2& z%BD;i$W$IRT#~6?UX5dpg=kl=pr$dCkFJkJyEkTfW9Y5O`ig7yP0X#Rvd65}J(9Qm zqVr{2Gj9iU?c4A{j{d%^a&gms(Q-y+jG63IJCnY5^~OM6cYiI--nXnv{)qKuJDeI_ znLa!)&zZT{U0hsQ@VDPWU-8GH;7XKiK7X3TOb1~IMW>%voUcbA)W!KVXbb+HOO3X! zRzR3K;EhI9Ok_BJ$E|W>tDE^yc{zmA2jNl&m+J%`Pe2Q4uTu5{m{uqPO*|!(QP^jg zP(u+wX~*KFeXs;!;Ztgi@>FF9MFkYH1Y zp-A-|vWFibl-#=gnzeylylKAxJ?aw9(|PL+drm~cmI_zv(b5}^{uJuR_7&vX3x=8q zQ^+4%gf@JoqaXWws1Q;?85bKd7v~Vi#x&#DAEWkDGvYubgJ4~d&8E#-mv|d zP8+D9E@LRiK+p*F74#!QcU(9&0QGk{f(axU3)BaaoCDgFx!%3|{ftADy{>nn+9EGl zsKa0ofmN8E4Hu$OL6~mWJKbJEg(fTob*TBHwT$gSe-^`tZSCiT^tC&YpX+F-7qviZ z*g>uChn{ai!=Y*zC~tJ94uDA)?6d2_cHzZ#xwJ>`Mq9r)T0b6J&~0oQLtL|%D>j+- z^U(iHoS+$qs_{2g3w1AmAyoXJLeUY}yqXW6vyLbb7C*kbI8VZ-IA(eo$-Z(L@@i%9&u zOcsYC^l3Ip-ek7^1eM(3ffi*U&ecPao*K{%Ae^=v(30cg$bdqBdK8K|fd00d^7I-9 znlUZSp8p(&B^!V0#!b~lQTS6x3Mu^CIjGw1^@$YuCU_z=3-@}?!MSjtdV_PK%pw2G zT+g^gyicG8^Z8+o+?%&w$4M&W95;s{F11}WXmz{8bl)4H& z*Gn(E6yd=$9fcfBn1(Od)(8r&uYj&gS|y9gc*{$*d<7J)NTtMHmW@=A=;Csw8CII~Yc~bkw8VD5V}tJse9ayybyt zIZ)`mgMB05r_l1{q^T4`gw;U}QstT#wY?AzlwlLN14xu(`-VyH=>>1D-3ktQtMx|1 zTM04r(_Uhv42QjMWiQ@AS|dlhmvRuAJ_8al7hT_?Q=xTX*`p>7Ih#T;8qULp)>`x0 z3o_sV7#Y?4Rxo|45vbw#`i$__i!&%qz{@%h8{e=~M_`C`mGeC04QBjuf5``fjM`hK z35^0wm`Z@m#Vqw-X4YGvB`e^WL1M~MdQKGKk&T5hnmxo_L+lgp!rhg@Z52?I z+b3%ZR-YgC4zhQ4luGE#WM2fQgAU4DB+;n%&wQ;;~&?xso8q-0u9R=m?-W&2or1yl{m*;Rk=I~)&_!CIT*P&nINQupwZ_vCq?GPq4Xn)k(DGyApyHR>0 z({o|#hT(6W{vN0YqemgiUWGo~OG65oSkFM4$qo8}nRUjZMhwm7KGBt6FNaQS zjD%-lesN{ zOnMjs0|p60HA?>#Ks`n@kRWBOX}kN7A;J{}KO&r1u=$N5Q2^aaohs#r!n{b^ai$)! zw*KpYtbddVMeH~g1`+fvrC?+KIpzvE=7#boN4AKZ1L?m0b+a}9IBEIv7 zkN88JB7H#UMu#`0%MKwZ@7vR>TlI7fcccHZk|Lf&ex%5rn|nXjrTT3HLr91m02=gzv{%pf#3UhfE?SmzjtTE_;CJ*VKQ`^|!F|>TauhBH0kKSu0ShD`w51lWClQ z{+bUB!~;!!XcW+|5cWC}9Haa^TuK*NYiIzFqjQ%1Qjp154ty5QJKRUzNB^Jumy-!^ ztsD%Q;@{*jWSj4QUr?Xmh^Rv4vW2YdYFGafUb~f7R1ntIk9x1by;*}bySf-0eV}~b z!uo#i1k6zl7|)g0^ImCH7Mjue>?=__sy5`=`_wBz17QMzT0iy*7!bA_z64p*#rG=6;)4;2X1_*61 zA*0oX4ytNRTK6?Rv8BaMt8+m-J>6|1Tq6|YX!p)DZ~{<~X>}?_HwIy=T@)vzNAHIU z-$)_a4`)Rm!0z=6^Hoj40rf$@hz$KnB(bwPKNs7phJQPHzO-9cnLma|f=JirS8$l( z2^fS1!9REx7cgUAZeW$hi$jSGPHH;CLrwp1JXU$)__@mI=gvKQ{DLq~;{pBcUX*Dr|3C1s_F5#>pkFR4vDmV|H6671)(I>vzIZ&^c8 zWcr>#u}X(=j{Y~2T7*u}M<`~-IL%<6reBaRi9HaRWGJZ^qBhe`sL91*?iiq6n*`P9@Q9@-(aQ%{c!Cm~i^Q9SjbT+a= zV~TJhBQk>0By3l1FP)s3yK%+_mL>7Fd}Q>DUD~A>9g*!m{Ql0=P(Dk}mN6;s@9*?s zdFb7=+4UQkvHlE_68PK#a2Bk#^~E{JbKKDSSC~*i)qjf#_oeSxcBLlo?8~iA@0jpsfz}^UbU|_E-yfu-b z1roXMh|Tg%{oAN`6LN`QK`spue@W!-19IV7Gw4+ru#h2o>ECv@Aa?)fXh4Laagz>> zT`CJS=DH8OKQ!VP&35PkfT&(JD7?LoLgc>-=17t*2u-5a_ueR@{4FFTc55rD>W?9$ zCR#IOa1?R+@3Z1RVDhd=+YWDULa8hLBl|Vt%0shT;flu;n8L^X3;jE&6zAzJGn8Yy zJi>_&)s7;*s6w!-4*rP4RPqT=;sI<3HWC~x#BD(Ff-nJBBdjG1pJrUfG6VoRrFaB; z?rx}Zi4x{OVu!+Z6%N#rJGW)CutRoo9v_KB5q&F7eQP=KZ1n{+u1lh8#hF{WxeRuJD`W^2D0PAlTSW>`W%=lTt$C__5L9fl3;9(-{teyk;DkLVHJNG z-ZvPP5y2yq5=t-`TtUR+NJhcTm4#*i|A=rnig@pPtjFKx=qSY&oFe4v96bN}_wl;L zZBL?2=RpmhkOYTkyT2>21Lz#Mo9A^I_AjW*hwPW+BPxwunCUKv zT6Pne5XVs8B)?UT;*uwV5krjODq<*%fPsOxDTZDKk@RCtq{L61e&GH%V~#DHd5Lrs zt#uI9k-U-cb2sRJ%!J4n8bJTGMDnt-evH{wCclRSA_l@|&~C6t^grpB4HS22$5Z1^ zjdUx!U;k6qz7=QKD?7{h$8IDxzsW6f2rZpg3pqNWU^ap?N?cM18Z$Xt^W5+0f6lIc z1_{E}IueI04-zu74dPK{sjgJesx+!=P_W}9;E#jf;-{i+!|izjpYXpR={`sU?p6Cs z_XtB^I37sR&$!T8Xm?^nMwSu8&;Sq?h6qSRN6?JR!z)?|BnF8|NcDA#7eR(#dM%Gi z1~33W1Osnw?L=bS&2JRo(-OS|#@~Y|LuAJqv|;{7EJeh}CbX8sG+bO;wh>DqCT5mq zpidPxQ`Bl0$Gl$nT%iTIhTp69Dj|$3hO_~~A}vaSzjWiuT|a1^yhkM_iRB%UQw z2JX|3zhEqp9_<7awh^Tar&D4GS%T2&%?CkC;2;j-avf1%2$l{5EphVDLm~V_lD=5> zo`=>+C(yuM#UeyQtsuZVkX%F~YD7oX@h$-0t@EtO0U-h+UimpzAw@*(L6Ye3uiqF@yNi*yG(DgDXYTLumt622mxNKPKWb1+erXo z_^4Ia@486F3G?wFpi+d0vxSwGjED;xuC;3L$kCI7fh6H%uq=b@5*Oj@Lg$fLA-SN! zVx(bcw)6|r?H^#H>VL)LUoiQXNJ@5`1@WLSd#-!OX~-V*Fb>^?2lj^t#A+o5=yLZC zu@u7*h)`Ni?3we=&2eCqT0B-RcMl$KdEg)b)2>N4zVcR=Sj^qQHmr6OM(u9vq`CCK zOf1X0A^%ALu_My?h9@oH01?6JvYktwM%Q{|q#zjD62{i{sR#_}OFUUPT~XTJ)@UU>g0m z>|hni7P$mKMPw2}cspK520?`2$qEsi1!>G?1ns&pcstdvqd~%~yt8ZP5wdw6R%UFv z`DMrw#$i694h;BM`GT+?V>P%%!z837aXkU+a9l6d!T)f86#UqrH_+jO{$Akmky8P> zgqX;LOUi++VILtB5&<)E#;Q$tULyLRoAA8Afj%E_PbHMj9A<%Jip$fnR z&@)H_sDhUa&qR+_)#WN|zAynNCmTDDmxlyQqkb{)K3l$A4^|JBKFh=S*>Z#@pbP2Q!XcS8cjIi1}_@-T&D`kD!a)O5bXPBU$28C@@ z;0@sdXFo5yYfRZLE9X%y4B!ye7K6dYG%jyJPTAD6^=dBF{lJ zxeY^!Q>xySm4X#&soDwR3|KfC$}~`ExIE;2h)|}f5AkJ9M7GPLENLDVEJT#o)s-7p zaSvd@jFntqOGH?+l>51q6M&jYK~=#*=0HCSj{eUaR|35Xv)&fM$t}zUP2m&XiNxGG zQ#3bV36;B$5rq{IzSoM>K(RpC?uip1Jdk^;{q18$07?Z1ld)b3)k=hAui9vnw?bA} ztX~dV9)~fq6fdCPxh*&*e)$nS^L{+^am=u&*KoT8xJ6U~@W`7bDQKU-rQi`_C{6Q- z@Ta>5IwjE}7}0;sr~k-=dkRHck)i*DMgIv2R8;nG34*K|0K1oNn-4fbE?AJx5eJMAu{~K$wF@j9C zUN#{X2iPRMTVeAHu$kjZ3Gs=mCW6nC?}bl9p+<)x+tv^$3_e0E4)D4_I>8x&8I3IG z%?QN=;J!`}N(bg^5?XQ<50fQZrrJYQfI;pFJ%y`16#s! zZaI$#9vXquxc7sw11M+C#+Z~K7#$(`&P7X~G4BX#M2n3Hw4J2d*vnE)#HpjfIfdIi zaCsq@b*2#V2w0ccVx$gpk|-UTc%JUxKkf$-_IHse5>RCnq7;oDllY-8V5=E_O}}b$ zC5#~YQ1L-+=xQT=4I`OT)+cF#Od;M9NH~dHR1H_)G=z4A;J)KZLDu0-<;L0}+0tA4 zNfa{5OtmVb4(F-b#tAY8;P!aqn&CxZ-hv^D$!Yv}4$Ye3Okq&tw zH9gQ}IAEGeYM&}$pjoB#$=$pJ{RE6sUK zlKN)DAXL^1_ZB=_fION>jED%e5)p(Ppv%;?=bm4OoS zgr6q~x(AJ?rV)n?P6Z+a!xV*r3A{6E8p;;puW{e=Rrk;ITkLEJ39dHMTNcXox|{g% z7;0>xKnmnq-xKMk+AhIXVCl5sf5i`y$X!9)T7*QKp&SbW4}Sz158UDT zMcx@rfr=dhC%147L#P;G;lfg(=kkKi2*k612zmur(CUyHxZ{#D!JCMbg$;ZMPv@;m z2PMR5vijRtlylHyjfTr|w57&h)h7a7fQ!s9r4NXvDQ+anN z5-FVoDL;+#hof1;9yK21L=uvL$YvKRScQtVfO8`onwMoDtfRv=6g~834i>_8P55lz z@NFWph1%h6;E0>M%><6bJvwFZ-KNmj$HZ~i5ze1I({Vp8m@3y7>b9^?!n2H!I^XG;csLsdIxFCLL~Xg zfl-XK3z5ML0FfCAO(bmqVo8RG^3|i2S31A&A-O=~L321y0UpX>+vn1Vo83q-xrFH-OD~BKp0aAqx0XLs%TWbW>Fu*faI$d&s8Ov?bf88fi%%*~kSgH8c-campl1SsUq71H6oOz7BvenOh8-$vq(_qVBd zD*PRTn_;pcjPs6`I)uX+Uxo;)l8CT6)2yz|2Ni)*>94=kTqCnMiarY4#Kv~Lm?ewA z$H}~d>ot&Kb5L~(aD6c!7t7?TE6XxzUTI!QMa5qSY`B|@%?`@`8JZ$i*dGs z-*u_sXCr=`nXfT{@#j&CHQ0f$r5&g%f5O$tL-45Dajs*%Mr~qsckM9=$)*1 zKq?AB!(}z|lP6_kxi5L*o57ZW#IT6!Q}wDFhptQ1A0Bs5AL8lb>etvrK-G$uBXHunp?L5(6W# z65@y#&#HLF;uI`Vaax2WWAGJo9JO|_X803l#CX&vGDah&j)kDBR-AcOe#b=Q8aM^P zGt3Sho{8bX$Q{E%Bc7Dk2ynpwsf!=P82%USL%3|wf8PNi3YW}aTj5o+qI~0YWOAuY zaVoVtlN)!&aEcfl@{W@#yq&p6-f$Vzc?n|I5BQcrSq6r!jO)BLopyfUWD0LOkGOC3 z+x5$wbm0eu1A|;#E_ZfQY=bZHx;q2Ym^@R%G%$CfKj$!O>}1{P!P9oufUR9x@@kxU zlUg3%Mn4?g1lsR7n42{EYvFqCw*J%3iRqLx$@&ag#?^Ps6n!yl-ge(&k7nl1+YV;O J8SJ~~{{kCdpTz(G literal 0 HcmV?d00001 diff --git a/vllm/model_executor/model_loader/loader.py b/vllm/model_executor/model_loader/loader.py new file mode 100644 index 00000000..813f5833 --- /dev/null +++ b/vllm/model_executor/model_loader/loader.py @@ -0,0 +1,1259 @@ +# ruff: noqa: SIM117 +import collections +import copy +import dataclasses +import fnmatch +import glob +import json +import math +import os +from abc import ABC, abstractmethod +from contextlib import contextmanager +from typing import (Any, Dict, Generator, Iterable, List, Optional, Tuple, + Type, cast) + +import gguf +import huggingface_hub +import numpy as np +import torch +from huggingface_hub import HfApi, hf_hub_download +from torch import nn +from transformers import AutoModelForCausalLM, PretrainedConfig +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat, + LoRAConfig, ModelConfig, MultiModalConfig, + ParallelConfig, SchedulerConfig) +from vllm.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.envs import VLLM_USE_MODELSCOPE +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.model_loader.tensorizer import ( + TensorizerConfig, is_vllm_tensorized, load_with_tensorizer, + serialize_vllm_model, tensorizer_weights_iterator) +from vllm.model_executor.model_loader.utils import (get_model_architecture, + set_default_torch_dtype) +from vllm.model_executor.model_loader.weight_utils import ( + download_safetensors_index_file_from_hf, download_weights_from_hf, + filter_duplicate_safetensors_files, filter_files_not_needed_for_inference, + get_gguf_extra_tensor_names, get_quant_config, gguf_quant_weights_iterator, + initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator, + safetensors_weights_iterator) +from vllm.model_executor.models import (has_inner_state, supports_lora, + supports_multimodal) +from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform +from vllm.utils import is_pin_memory_available + + +@contextmanager +def device_loading_context(module: torch.nn.Module, + target_device: torch.device): + if target_device.type == "cpu": + # If target is CPU, no need to move anything + yield module + return + + original_device_states: Dict[str, torch.device] = {} + + # Store original device states and move parameters to GPU if they're on CPU + for name, p in module.named_parameters(): + if p.device.type == "cpu": + original_device_states[name] = p.device + p.data = p.data.to(target_device) + # Parameters already on target device are not touched + + try: + yield module + + finally: + # Restore parameters to their original devices, ignoring new parameters + pin_memory = is_pin_memory_available() + for name, p in module.named_parameters(): + if name in original_device_states: + original_device: torch.device = original_device_states[name] + if original_device.type == "cpu": + # `torch.empty_like` does not support `pin_memory` argument + cpu_data = torch.empty_strided(size=p.data.size(), + stride=p.data.stride(), + dtype=p.data.dtype, + layout=p.data.layout, + device="cpu", + pin_memory=pin_memory) + cpu_data.copy_(p.data) + p.data = cpu_data + else: + p.data = p.data.to(original_device) + # New parameters or parameters already on target device are untouched + + +logger = init_logger(__name__) + + +def _get_quantization_config( + model_config: ModelConfig, + load_config: LoadConfig) -> Optional[QuantizationConfig]: + """Get the quantization config.""" + if model_config.quantization is not None: + quant_config = get_quant_config(model_config, load_config) + capability_tuple = current_platform.get_device_capability() + + if capability_tuple is not None: + capability = capability_tuple.to_int() + if capability < quant_config.get_min_capability(): + raise ValueError( + f"The quantization method {model_config.quantization} " + "is not supported for the current GPU. " + f"Minimum capability: {quant_config.get_min_capability()}. " + f"Current capability: {capability}.") + supported_dtypes = quant_config.get_supported_act_dtypes() + if model_config.dtype not in supported_dtypes: + raise ValueError( + f"{model_config.dtype} is not supported for quantization " + f"method {model_config.quantization}. Supported dtypes: " + f"{supported_dtypes}") + return quant_config + return None + + +def _get_model_initialization_kwargs( + model_class: Type[nn.Module], + lora_config: Optional[LoRAConfig], + multimodal_config: Optional[MultiModalConfig], + scheduler_config: Optional[SchedulerConfig] = None) -> Dict[str, Any]: + """Get extra kwargs for model initialization.""" + extra_kwargs: Dict[str, Any] = {} + + if supports_lora(model_class): + # lora_config=None is used to disable LoRA + extra_kwargs["lora_config"] = lora_config + elif lora_config: + raise ValueError( + f"Model {model_class.__name__} does not support LoRA, " + "but LoRA is enabled. Support for this model may " + "be added in the future. If this is important to you, " + "please open an issue on github.") + + if supports_multimodal(model_class): + assert multimodal_config is not None + + extra_kwargs["multimodal_config"] = multimodal_config + + if has_inner_state(model_class) and scheduler_config: + extra_kwargs["scheduler_config"] = scheduler_config + + return extra_kwargs + + +def build_model(model_class: Type[nn.Module], hf_config: PretrainedConfig, + cache_config: Optional[CacheConfig], + quant_config: Optional[QuantizationConfig], *, + lora_config: Optional[LoRAConfig], + multimodal_config: Optional[MultiModalConfig], + scheduler_config: Optional[SchedulerConfig]) -> nn.Module: + extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config, + multimodal_config, + scheduler_config) + + return model_class(config=hf_config, + cache_config=cache_config, + quant_config=quant_config, + **extra_kwargs) + + +def _initialize_model( + model_config: ModelConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + cache_config: CacheConfig, + scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module: + """Initialize a model with the given configurations.""" + model_class, _ = get_model_architecture(model_config) + + return build_model( + model_class, + model_config.hf_config, + cache_config=cache_config, + quant_config=_get_quantization_config(model_config, load_config), + lora_config=lora_config, + multimodal_config=model_config.multimodal_config, + scheduler_config=scheduler_config, + ) + + +class BaseModelLoader(ABC): + """Base class for model loaders.""" + + def __init__(self, load_config: LoadConfig): + self.load_config = load_config + + @abstractmethod + def download_model(self, model_config: ModelConfig) -> None: + """Download a model so that it can be immediately loaded.""" + raise NotImplementedError + + @abstractmethod + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + """Load a model with the given configurations.""" + raise NotImplementedError + + +class DefaultModelLoader(BaseModelLoader): + """Model loader that can load different file types from disk.""" + + @dataclasses.dataclass + class Source: + """A source for weights.""" + + model_or_path: str + """The model ID or path.""" + + revision: Optional[str] + """The optional model revision.""" + + prefix: str = "" + """A prefix to prepend to all weights.""" + + fall_back_to_pt: bool = True + """Whether .pt weights can be used.""" + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + if load_config.model_loader_extra_config: + raise ValueError(f"Model loader extra config is not supported for " + f"load format {load_config.load_format}") + + def _maybe_download_from_modelscope( + self, model: str, revision: Optional[str]) -> Optional[str]: + """Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True. + + Returns the path to the downloaded model, or None if the model is not + downloaded from ModelScope.""" + if VLLM_USE_MODELSCOPE: + # download model from ModelScope hub, + # lazy import so that modelscope is not required for normal use. + # pylint: disable=C. + from modelscope.hub.snapshot_download import snapshot_download + + if not os.path.exists(model): + model_path = snapshot_download( + model_id=model, + cache_dir=self.load_config.download_dir, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + revision=revision, + ignore_file_pattern=self.load_config.ignore_patterns, + ) + else: + model_path = model + return model_path + return None + + def _prepare_weights(self, model_name_or_path: str, + revision: Optional[str], + fall_back_to_pt: bool) -> Tuple[str, List[str], bool]: + """Prepare weights for the model. + + If the model is not local, it will be downloaded.""" + model_name_or_path = self._maybe_download_from_modelscope( + model_name_or_path, revision) or model_name_or_path + + is_local = os.path.isdir(model_name_or_path) + load_format = self.load_config.load_format + use_safetensors = False + index_file = SAFE_WEIGHTS_INDEX_NAME + # Some quantized models use .pt files for storing the weights. + if load_format == LoadFormat.AUTO: + allow_patterns = ["*.safetensors", "*.bin"] + elif load_format == LoadFormat.SAFETENSORS: + use_safetensors = True + allow_patterns = ["*.safetensors"] + elif load_format == LoadFormat.MISTRAL: + use_safetensors = True + allow_patterns = ["consolidated*.safetensors"] + index_file = "consolidated.safetensors.index.json" + elif load_format == LoadFormat.PT: + allow_patterns = ["*.pt"] + elif load_format == LoadFormat.NPCACHE: + allow_patterns = ["*.bin"] + else: + raise ValueError(f"Unknown load_format: {load_format}") + + if fall_back_to_pt: + allow_patterns += ["*.pt"] + + if not is_local: + hf_folder = download_weights_from_hf( + model_name_or_path, + self.load_config.download_dir, + allow_patterns, + revision, + ignore_patterns=self.load_config.ignore_patterns, + ) + else: + hf_folder = model_name_or_path + + hf_weights_files: List[str] = [] + for pattern in allow_patterns: + hf_weights_files += glob.glob(os.path.join(hf_folder, pattern)) + if len(hf_weights_files) > 0: + if pattern == "*.safetensors": + use_safetensors = True + break + + if use_safetensors: + # For models like Mistral-7B-Instruct-v0.3 + # there are both sharded safetensors files and a consolidated + # safetensors file. Using both breaks. + # Here, we download the `model.safetensors.index.json` and filter + # any files not found in the index. + if not is_local: + download_safetensors_index_file_from_hf( + model_name_or_path, index_file, + self.load_config.download_dir, revision) + hf_weights_files = filter_duplicate_safetensors_files( + hf_weights_files, hf_folder, index_file) + else: + hf_weights_files = filter_files_not_needed_for_inference( + hf_weights_files) + + if len(hf_weights_files) == 0: + raise RuntimeError( + f"Cannot find any model weights with `{model_name_or_path}`") + + return hf_folder, hf_weights_files, use_safetensors + + def _get_weights_iterator( + self, source: "Source" + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + """Get an iterator for the model weights based on the load format.""" + hf_folder, hf_weights_files, use_safetensors = self._prepare_weights( + source.model_or_path, source.revision, source.fall_back_to_pt) + if self.load_config.load_format == LoadFormat.NPCACHE: + # Currently np_cache only support *.bin checkpoints + assert use_safetensors is False + weights_iterator = np_cache_weights_iterator( + source.model_or_path, self.load_config.download_dir, hf_folder, + hf_weights_files) + elif use_safetensors: + weights_iterator = safetensors_weights_iterator(hf_weights_files) + else: + weights_iterator = pt_weights_iterator(hf_weights_files) + + if current_platform.is_tpu(): + # In PyTorch XLA, we should call `xm.mark_step` frequently so that + # not too many ops are accumulated in the XLA program. + import torch_xla.core.xla_model as xm + + def _xla_weights_iterator(iterator: Generator): + for weights in iterator: + yield weights + xm.mark_step() + + weights_iterator = _xla_weights_iterator(weights_iterator) + + # Apply the prefix. + return ((source.prefix + name, tensor) + for (name, tensor) in weights_iterator) + + def _get_all_weights( + self, + model_config: ModelConfig, + model: nn.Module, + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + + primary_weights = DefaultModelLoader.Source( + model_config.model, + model_config.revision, + prefix="", + fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load", + True)) + yield from self._get_weights_iterator(primary_weights) + + secondary_weights = cast(Iterable[DefaultModelLoader.Source], + getattr(model, "secondary_weights", ())) + for source in secondary_weights: + yield from self._get_weights_iterator(source) + + def download_model(self, model_config: ModelConfig) -> None: + self._prepare_weights(model_config.model, + model_config.revision, + fall_back_to_pt=True) + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + target_device = torch.device(device_config.device) + with set_default_torch_dtype(model_config.dtype): + with target_device: + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config, + scheduler_config) + + model.load_weights(self._get_all_weights(model_config, model)) + + for _, module in model.named_modules(): + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + # When quant methods need to process weights after loading + # (for repacking, quantizing, etc), they expect parameters + # to be on the global target device. This scope is for the + # case where cpu offloading is used, where we will move the + # parameters onto device for processing and back off after. + with device_loading_context(module, target_device): + quant_method.process_weights_after_loading(module) + return model.eval() + + +class DummyModelLoader(BaseModelLoader): + """Model loader that will set model weights to random values.""" + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + if load_config.model_loader_extra_config: + raise ValueError(f"Model loader extra config is not supported for " + f"load format {load_config.load_format}") + + def download_model(self, model_config: ModelConfig) -> None: + pass # Nothing to download + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config, + scheduler_config) + # NOTE(woosuk): For accurate performance evaluation, we assign + # random values to the weights. + initialize_dummy_weights(model) + + for _, module in model.named_modules(): + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + # When quant methods need to process weights after loading + # (for repacking, quantizing, etc), they expect parameters + # to be on the global target device. This scope is for the + # case where cpu offloading is used, where we will move the + # parameters onto device for processing and back off after. + with device_loading_context( + module, torch.device(device_config.device)): + quant_method.process_weights_after_loading(module) + return model.eval() + + +class TensorizerLoader(BaseModelLoader): + """Model loader using CoreWeave's tensorizer library.""" + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + if isinstance(load_config.model_loader_extra_config, TensorizerConfig): + self.tensorizer_config = load_config.model_loader_extra_config + else: + self.tensorizer_config = TensorizerConfig( + **load_config.model_loader_extra_config) + + def _verify_config(self, model_config: ModelConfig, + parallel_config: ParallelConfig): + self.tensorizer_config.verify_with_model_config(model_config) + self.tensorizer_config.verify_with_parallel_config(parallel_config) + + def _get_weights_iterator( + self) -> Generator[Tuple[str, torch.Tensor], None, None]: + tensorizer_args = self.tensorizer_config._construct_tensorizer_args() + return tensorizer_weights_iterator(tensorizer_args) + + def _load_model_serialized_cpu( + self, + model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + cache_config: CacheConfig, + ) -> nn.Module: + """Load a serialized model with tensorizer to the CPU. + + This is only necessary when the model isn't vLLM-tensorized (see + examples/tensorize_vllm_model.py) This should still be faster than + default HuggingFace loading, but will be slower than loading a + vLLM-tensorized model. + """ + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config) + + model.load_weights(self._get_weights_iterator()) + return model.eval() + + def _load_model_serialized( + self, + model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + cache_config: CacheConfig, + ) -> nn.Module: + """Load a serialized model with tensorizer. + + Expects a vLLM-tensorized model. See the + examples/tensorize_vllm_model.py example script + for serializing vLLM models.""" + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model_class = get_model_architecture(model_config)[0] + quant_config = _get_quantization_config( + model_config, self.load_config) + extra_kwargs = _get_model_initialization_kwargs( + model_class, lora_config, model_config.multimodal_config) + extra_kwargs["quant_config"] = quant_config + extra_kwargs["cache_config"] = cache_config + + tensorizer_config = copy.copy(self.tensorizer_config) + tensorizer_config.model_class = model_class + tensorizer_config.hf_config = model_config.hf_config + tensorizer_config.dtype = model_config.dtype + + model = load_with_tensorizer(tensorizer_config, **extra_kwargs) + return model.eval() + + def download_model(self, model_config: ModelConfig) -> None: + self.tensorizer_config.verify_with_model_config(model_config) + + with self.tensorizer_config.open_stream(): + pass + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + self._verify_config(model_config, parallel_config) + + if parallel_config.tensor_parallel_size > 1: + from vllm.distributed import get_tensor_model_parallel_rank + self.tensorizer_config.tensorizer_uri = \ + self.tensorizer_config.tensorizer_uri \ + % get_tensor_model_parallel_rank() + + if is_vllm_tensorized(self.tensorizer_config): + return self._load_model_serialized(model_config, device_config, + lora_config, cache_config) + return self._load_model_serialized_cpu(model_config, device_config, + lora_config, cache_config) + + @staticmethod + def save_model( + model: torch.nn.Module, + tensorizer_config: TensorizerConfig, + ) -> None: + serialize_vllm_model( + model=model, + tensorizer_config=tensorizer_config, + ) + + +class ShardedStateLoader(BaseModelLoader): + """ + Model loader that directly loads each worker's model state dict, which + enables a fast load path for large tensor-parallel models where each worker + only needs to read its own shard rather than the entire checkpoint. See + `examples/save_sharded_state.py` for creating a sharded checkpoint. + """ + + DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors" + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + extra_config = ({} if load_config.model_loader_extra_config is None + else load_config.model_loader_extra_config.copy()) + self.pattern = extra_config.pop("pattern", self.DEFAULT_PATTERN) + if extra_config: + raise ValueError(f"Unexpected extra config keys for load format " + f"{load_config.load_format}: " + f"{load_config.model_loader_extra_config.keys()}") + + @staticmethod + def _filter_subtensors( + tensors: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Filter out all tensors that share the same memory or a subset of the + memory of another tensor. + """ + same_storage_groups: Dict[Any, List[Tuple[ + str, torch.Tensor]]] = collections.defaultdict(list) + for key, tensor in tensors.items(): + if tensor.numel(): + ptr = tensor.untyped_storage().data_ptr() + same_storage_groups[tensor.device, ptr].append((key, tensor)) + + def get_end_ptr(tensor: torch.Tensor) -> int: + return tensor.view(-1)[-1].data_ptr() + tensor.element_size() + + result: Dict[str, torch.Tensor] = {} + for group in same_storage_groups.values(): + for k, t in group: + a, b = t.data_ptr(), get_end_ptr(t) + for k2, t2 in group: + if not t2.is_contiguous(): + continue + a2, b2 = t2.data_ptr(), get_end_ptr(t2) + if a < a2 or b2 < b: + continue + if a2 < a or b < b2 or not t.is_contiguous(): + break # t2 covers strictly more memory than t. + if k2 < k: + # Same tensors, keep the one with the smaller key. + break + else: + result[k] = t + return result + + def _prepare_weights(self, model_name_or_path: str, + revision: Optional[str]): + if os.path.isdir(model_name_or_path): + return model_name_or_path + else: + allow_patterns = ["*.safetensors"] + return download_weights_from_hf( + model_name_or_path, + self.load_config.download_dir, + allow_patterns, + revision, + ignore_patterns=self.load_config.ignore_patterns, + ) + + def download_model(self, model_config: ModelConfig) -> None: + self._prepare_weights(model_config.model, model_config.revision) + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + from safetensors.torch import safe_open + + from vllm.distributed import get_tensor_model_parallel_rank + + local_model_path = self._prepare_weights(model_config.model, + model_config.revision) + + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config) + for _, module in model.named_modules(): + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + quant_method.process_weights_after_loading(module) + rank = get_tensor_model_parallel_rank() + pattern = os.path.join( + local_model_path, + self.pattern.format(rank=rank, part="*"), + ) + filepaths = glob.glob(pattern) + if not filepaths: + # TODO: support un-sharded checkpoints too + raise ValueError( + f"Could not find checkpoint files '{pattern}', only " + f"pre-sharded checkpoints are currently supported!") + state_dict = self._filter_subtensors(model.state_dict()) + for path in filepaths: + with safe_open(path, framework="pt") as f: + for key in f.keys(): # noqa: SIM118 + tensor = f.get_tensor(key) + # If loading with LoRA enabled, additional padding may + # be added to certain parameters. We only load into a + # narrowed view of the parameter data. + param_data = state_dict[key].data + param_shape = state_dict[key].shape + for dim, size in enumerate(tensor.shape): + if size < param_shape[dim]: + param_data = param_data.narrow(dim, 0, size) + if tensor.shape != param_shape: + logger.warning( + "loading tensor of shape %s into " + "parameter '%s' of shape %s", tensor.shape, + key, param_shape) + param_data.copy_(tensor) + state_dict.pop(key) + if state_dict: + raise ValueError( + f"Missing keys {tuple(state_dict)} in loaded state!") + return model.eval() + + @staticmethod + def save_model( + model: torch.nn.Module, + path: str, + pattern: Optional[str] = None, + max_size: Optional[int] = None, + ) -> None: + from safetensors.torch import save_file + + from vllm.distributed import get_tensor_model_parallel_rank + if pattern is None: + pattern = ShardedStateLoader.DEFAULT_PATTERN + rank = get_tensor_model_parallel_rank() + part_idx = 0 + total_size = 0 + state_dict = ShardedStateLoader._filter_subtensors(model.state_dict()) + state_dict_part: Dict[str, torch.Tensor] = {} + for key, tensor in state_dict.items(): + param_size = tensor.nelement() * tensor.element_size() + if max_size is not None and total_size + param_size > max_size: + filename = pattern.format(rank=rank, part=part_idx) + save_file( + state_dict_part, + os.path.join(path, filename), + ) + part_idx += 1 + total_size = 0 + state_dict_part = {} + state_dict_part[key] = tensor + total_size += param_size + if len(state_dict_part) > 0: + filename = pattern.format(rank=rank, part=part_idx) + save_file( + state_dict_part, + os.path.join(path, filename), + ) + + +class BitsAndBytesModelLoader(BaseModelLoader): + """Model loader to load model weights with BitAndBytes quantization.""" + + possible_config_file_names = ["adapter_config.json"] + + default_target_modules = [ + ".gate_proj.", + ".down_proj.", + ".up_proj.", + ".q_proj.", + ".k_proj.", + ".v_proj.", + ".o_proj.", + '.fc1.', + '.fc2.', + '.dense.', + '.query_key_value.', + '.qkv_proj.', + '.dense_h_to_4h.', + '.dense_4h_to_h.', + '.out_proj.', + ] + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + + # we don't need to quantize the whole model, only the target modules + # that are specified in the adapter config file. If the adapter config + # file is not provided, we will quantize the default modules. + if (not load_config.model_loader_extra_config + or "qlora_adapter_name_or_path" + not in load_config.model_loader_extra_config): + self.target_modules = [] + return + + qlora_adapter = load_config.model_loader_extra_config[ + "qlora_adapter_name_or_path"] + + config_file_path = self._get_config_file(qlora_adapter) + + with open(config_file_path, "r") as f: + config = json.load(f) + self.target_modules = config["target_modules"] + + def _get_config_file(self, qlora_adapter: str) -> str: + is_local = os.path.isdir(qlora_adapter) + config_file_path = None + if is_local: + for file in self.possible_config_file_names: + config_file_path = os.path.join(qlora_adapter, file) + if os.path.exists(config_file_path): + break + else: + hf_api = HfApi() + repo_files = hf_api.list_repo_files(repo_id=qlora_adapter) + for file in self.possible_config_file_names: + if file in repo_files: + config_file_path = hf_hub_download(repo_id=qlora_adapter, + filename=file) + break + + if not config_file_path: + raise ValueError( + f"Cannot find adapter config file in {qlora_adapter}") + + return config_file_path + + def _get_weight_files( + self, + model_name_or_path: str, + allowed_patterns: List[str], + revision: Optional[str] = None) -> Tuple[List[str], str]: + """Retrieve weight files. Download the files if necessary. + + Return the weight files and the file pattern.""" + is_local = os.path.isdir(model_name_or_path) + + if is_local: + for pattern in allowed_patterns: + weight_files = glob.glob( + os.path.join(model_name_or_path, pattern)) + if weight_files: + return weight_files, pattern + else: + hf_api = HfApi() + repo_files = hf_api.list_repo_files(repo_id=model_name_or_path) + for pattern in allowed_patterns: + matching_files = fnmatch.filter(repo_files, pattern) + if matching_files: + hf_folder = download_weights_from_hf( + model_name_or_path, + self.load_config.download_dir, + [pattern], + revision, + ignore_patterns=self.load_config.ignore_patterns, + ) + return glob.glob(os.path.join(hf_folder, pattern)), pattern + + raise RuntimeError( + f"No model weights found in: `{model_name_or_path}`") + + def _prepare_weights(self, model_name_or_path: str, + revision: Optional[str]) -> Tuple[List[str], bool]: + """Prepare weight files for the model.""" + + allowed_patterns = ["*.safetensors", "*.bin", "*.pt"] + + hf_weights_files, matched_pattern = self._get_weight_files( + model_name_or_path, allowed_patterns, revision) + + if matched_pattern != "*.safetensors": + hf_weights_files = filter_files_not_needed_for_inference( + hf_weights_files) + + if len(hf_weights_files) == 0: + raise RuntimeError( + f"Cannot find any model weights with `{model_name_or_path}`") + + return hf_weights_files, matched_pattern == "*.safetensors" + + def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool): + if use_safetensors: + return safetensors_weights_iterator(hf_weights_files) + else: + return pt_weights_iterator(hf_weights_files) + + def _get_quantized_weights_iterator( + self, + model_name_or_path: str, + revision: Optional[str], + pre_quant: bool, + load_8bit: bool, + ) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str, + Any]]: + """Get an iterator to the model weights with bitsandbytes quantization, + as well as the quantization state dictionary.""" + + # only load the bitsandbytes module when needed + try: + import bitsandbytes + if bitsandbytes.__version__ < "0.44.0": + raise ImportError("bitsandbytes version is wrong. Please " + "install bitsandbytes>=0.44.0.") + except ImportError as err: + raise ImportError("Please install bitsandbytes>=0.44.0 via " + "`pip install bitsandbytes>=0.44.0` to use " + "bitsandbytes quantizer.") from err + + hf_weights_files, use_safetensors = self._prepare_weights( + model_name_or_path, revision) + + quant_state_dict: Dict[str, Any] = {} + + if pre_quant: + if load_8bit: + return self._quantized_8bit_generator( + hf_weights_files, use_safetensors, + quant_state_dict), quant_state_dict + else: + return self._quantized_4bit_generator( + hf_weights_files, use_safetensors, + quant_state_dict), quant_state_dict + + return self._unquantized_generator(hf_weights_files, use_safetensors, + quant_state_dict), quant_state_dict + + def _quantized_8bit_generator(self, hf_weights_files, use_safetensors, + quant_state_dict) -> Generator: + for weight_name, weight_tensor in self._hf_weight_iter( + hf_weights_files, use_safetensors): + if not weight_name.lower().endswith(".scb"): + continue + + weight_key = weight_name.lower().replace(".scb", ".qweight") + quant_state_dict[weight_key] = weight_tensor + + for weight_name, weight_tensor in self._hf_weight_iter( + hf_weights_files, use_safetensors): + + if not weight_name.endswith((".weight", ".bias")): + continue + + qweight_name = weight_name.replace(".weight", ".qweight") + + if qweight_name in quant_state_dict: + set_weight_attrs(weight_tensor, {"load_in_8bit": True}) + yield qweight_name, weight_tensor + else: + yield weight_name, weight_tensor + + def _quantized_4bit_generator(self, hf_weights_files, use_safetensors, + quant_state_dict) -> Generator: + from bitsandbytes.functional import QuantState + + # First iterate over all quant state weights + weight_iterator = self._hf_weight_iter(hf_weights_files, + use_safetensors) + temp_state_dict = {} + for weight_name, weight_tensor in weight_iterator: + if weight_name.endswith((".weight", ".bias")): + continue + # bitsandbytes library requires + # weight.quant_state.bitsandbytes__* in CPU + if "quant_state.bitsandbytes" in weight_name: + temp_state_dict[weight_name] = weight_tensor.cpu().data + else: + temp_state_dict[weight_name] = weight_tensor + + # Closure to parse quant_state for each prequant weight + def _parse_quant_state(param_name: str, + temp_state_dict: Dict) -> QuantState: + quant_state = {} + for k in temp_state_dict: + if param_name + "." in k: + quant_state[k] = temp_state_dict[k] + + return QuantState.from_dict(quant_state, device="cuda") + + # Second iterate over all prequant and normal weights + # pre quantized weights would have a quant_state + for weight_name, weight_tensor in self._hf_weight_iter( + hf_weights_files, use_safetensors): + + if not weight_name.endswith((".weight", ".bias")): + continue + + if (f"{weight_name}.quant_state.bitsandbytes__nf4" \ + in temp_state_dict) or \ + (f"{weight_name}.quant_state.bitsandbytes__fp4" \ + in temp_state_dict): + quant_state = _parse_quant_state(weight_name, temp_state_dict) + weight_name = weight_name.replace(".weight", ".qweight") + quant_state_dict[weight_name] = quant_state + yield weight_name.replace(".weight", ".qweight"), weight_tensor + else: + yield weight_name, weight_tensor + + def _unquantized_generator(self, hf_weights_files, use_safetensors, + quant_state_dict) -> Generator: + from bitsandbytes.functional import quantize_4bit + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + + for weight_name, weight_tensor in self._hf_weight_iter( + hf_weights_files, use_safetensors): + + if any(target_module in weight_name for target_module in + self.target_modules) and weight_name.endswith(".weight"): + weight_name = weight_name.replace(".weight", ".qweight") + + if any(module in weight_name + for module in self.column_parallel_weights_modules): + + total_size = weight_tensor.size(-1) + start_index = total_size // tp_size * tp_rank + end_index = total_size // tp_size * (tp_rank + 1) + weight_sub_tensor = weight_tensor[..., + start_index:end_index] + + else: + total_size = weight_tensor.size(0) + start_index = total_size // tp_size * tp_rank + end_index = total_size // tp_size * (tp_rank + 1) + weight_sub_tensor = weight_tensor[start_index:end_index, + ...] + + # bitsandbytes requires data in GPU + if weight_sub_tensor.is_cuda: + loaded_weight = weight_sub_tensor + else: + loaded_weight = weight_sub_tensor.cuda() + + # remove the following after the issue is fixed: + # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1342 + if loaded_weight.is_contiguous() is False: + loaded_weight = loaded_weight.contiguous() + + with set_default_torch_dtype(torch.float32): + processed_weight, quant_state = quantize_4bit( + loaded_weight, + compress_statistics=True, + quant_type="nf4") + + quant_state_dict[weight_name] = quant_state + else: + processed_weight = weight_tensor + + yield weight_name, processed_weight + + def _load_weights(self, model_config: ModelConfig, + model: nn.Module) -> None: + if not hasattr(model, 'load_weights'): + raise AttributeError( + "The required method 'load_weights' is not defined in class" + f" {type(model).__name__}.") + + if not hasattr(model, 'bitsandbytes_stacked_params_mapping'): + raise AttributeError( + f"Model {type(model).__name__} does not support BitsAndBytes " + "quantization yet.") + + if len(self.target_modules) == 0: + if hasattr(model, 'default_bitsandbytes_target_modules'): + self.target_modules = model.default_bitsandbytes_target_modules + else: + self.target_modules = self.default_target_modules + + if hasattr(model, 'column_parallel_weights_modules'): + self.column_parallel_weights_modules = \ + model.column_parallel_weights_modules + else: + self.column_parallel_weights_modules = [] + + self.model_type = type(model).__name__ + + logger.info("Loading weights with BitsAndBytes quantization. " + " May take a while ...") + + quant_config = getattr(model_config.hf_config, "quantization_config", + None) + + pre_quant = False + if quant_config is not None: + quant_method = quant_config.get('quant_method') + if quant_method == "bitsandbytes": + pre_quant = True + else: + raise ValueError( + f"BitsAndBytes loader does not support {quant_method} " + "quantization") + + # The quant_states in pre_quantized models cannot work with a split + # weight tensor. So TP does not work with pre_quantized bnb models. + if pre_quant and get_tensor_model_parallel_world_size() > 1: + raise ValueError( + "Prequant BitsAndBytes models with TP is not supported." + "Please try with PP.") + + load_8bit = False + if pre_quant: + load_8bit = quant_config.get('load_in_8bit', False) + + qweight_iterator, quant_state_dict = \ + self._get_quantized_weights_iterator( + model_config.model, model_config.revision, pre_quant, load_8bit) + + model.load_weights(qweight_iterator) + + torch.cuda.empty_cache() + + param_dict = dict(model.named_parameters()) + stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {} + for quant_param_name in quant_state_dict: + non_stacked_param_name = quant_param_name + + shard_index = 0 + for shard_name, ( + weight_name, index + ) in model.bitsandbytes_stacked_params_mapping.items(): + if shard_name in quant_param_name: + shard_index = index + quant_param_name = quant_param_name.replace( + shard_name, weight_name) + break + + if quant_param_name not in param_dict: + raise ValueError( + f"Parameter {quant_param_name} not found in the model.") + + if quant_param_name not in stacked_quant_state_dict: + stacked_quant_state_dict[quant_param_name] = {} + + stacked_quant_state_dict[quant_param_name][shard_index] = ( + quant_state_dict[non_stacked_param_name]) + + # save quant_states and offsets as the attributes of the parameters + for param_name, param in param_dict.items(): + if param_name in stacked_quant_state_dict: + quant_states = stacked_quant_state_dict[param_name] + set_weight_attrs(param, {"bnb_quant_state": quant_states}) + + pack_ratio = getattr(param, "pack_factor", -1) + if pack_ratio == -1: + raise ValueError( + f"pack_factor not set for parameter {param_name}.") + + num_elements = [0] * len(quant_states) + for seq, quant_state in quant_states.items(): + num_elements[seq] = math.prod( + quant_state.shape) // pack_ratio + + offsets = np.concatenate(([0], np.cumsum(num_elements))) + set_weight_attrs(param, {"bnb_shard_offsets": offsets}) + + if load_8bit: + set_weight_attrs( + param, {"matmul_state": [None] * len(quant_states)}) + + def download_model(self, model_config: ModelConfig) -> None: + self._prepare_weights(model_config.model, model_config.revision) + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config) + + self._load_weights(model_config, model) + + return model.eval() + + +class GGUFModelLoader(BaseModelLoader): + """ + Model loader that can load GGUF files. This is useful for loading models + that are quantized with GGUF and saved in the GGUF format. This loader + supports loading both full models and sharded models. + """ + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + if load_config.model_loader_extra_config: + raise ValueError(f"Model loader extra config is not supported for " + f"load format {load_config.load_format}") + + def _prepare_weights(self, model_name_or_path: str): + if os.path.isfile(model_name_or_path): + return model_name_or_path + else: + raise ValueError(f"{model_name_or_path} is not a file.") + + def _get_gguf_weights_map(self, model_config: ModelConfig): + """ + GGUF uses this naming convention for their tensors from HF checkpoint: + `blk.N.BB.weight` and `blk.N.BB.bias` + where N signifies the block number of a layer, and BB signifies the + attention/mlp layer components. + See "Standardized tensor names" in + https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details. + """ + config = model_config.hf_config + model_type = config.model_type + # hack: ggufs have a different name than transformers + if model_type == "cohere": + model_type = "command-r" + arch = None + for key, value in gguf.MODEL_ARCH_NAMES.items(): + if value == model_type: + arch = key + break + if arch is None: + raise RuntimeError(f"Unknown gguf model_type: {model_type}") + num_layers = config.num_hidden_layers + name_map = gguf.get_tensor_name_map(arch, num_layers) + with torch.device("meta"): + dummy_model = AutoModelForCausalLM.from_config(config) + state_dict = dummy_model.state_dict() + + gguf_to_hf_name_map = {} + for hf_name in state_dict: + name, suffix = hf_name.rsplit(".", 1) + gguf_name = name_map.get_name(name) + gguf_to_hf_name_map[f"{gguf_name}.{suffix}"] = hf_name + return gguf_to_hf_name_map + + def _get_weights_iterator( + self, model_name_or_path: str, gguf_to_hf_name_map: Dict[str, str] + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + return gguf_quant_weights_iterator(model_name_or_path, + gguf_to_hf_name_map) + + def download_model(self, model_config: ModelConfig) -> None: + self._prepare_weights(model_config.model) + + def load_model(self, *, model_config: ModelConfig, + device_config: DeviceConfig, + lora_config: Optional[LoRAConfig], + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + cache_config: CacheConfig) -> nn.Module: + + local_model_path = self._prepare_weights(model_config.model) + gguf_weights_map = self._get_gguf_weights_map(model_config) + # we can only know if tie word embeddings after mapping weights + if "lm_head.weight" in get_gguf_extra_tensor_names( + local_model_path, gguf_weights_map): + model_config.hf_config.update({"tie_word_embeddings": True}) + + with set_default_torch_dtype(model_config.dtype): + with torch.device(device_config.device): + model = _initialize_model(model_config, self.load_config, + lora_config, cache_config) + model.load_weights( + self._get_weights_iterator(local_model_path, gguf_weights_map)) + return model + + +def get_model_loader(load_config: LoadConfig) -> BaseModelLoader: + """Get a model loader based on the load format.""" + + if isinstance(load_config.load_format, type): + return load_config.load_format(load_config) + + if load_config.load_format == LoadFormat.DUMMY: + return DummyModelLoader(load_config) + + if load_config.load_format == LoadFormat.TENSORIZER: + return TensorizerLoader(load_config) + + if load_config.load_format == LoadFormat.SHARDED_STATE: + return ShardedStateLoader(load_config) + + if load_config.load_format == LoadFormat.BITSANDBYTES: + return BitsAndBytesModelLoader(load_config) + + if load_config.load_format == LoadFormat.GGUF: + return GGUFModelLoader(load_config) + + return DefaultModelLoader(load_config) diff --git a/vllm/model_executor/model_loader/neuron.py b/vllm/model_executor/model_loader/neuron.py new file mode 100644 index 00000000..00c82fb7 --- /dev/null +++ b/vllm/model_executor/model_loader/neuron.py @@ -0,0 +1,240 @@ +"""Utilities for selecting and loading neuron models.""" +import copy +import importlib +import os +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import transformers +from transformers import PretrainedConfig + +from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import get_quantization_config +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob, + SequenceOutput) + +TORCH_DTYPE_TO_NEURON_AMP = { + "auto": "f32", + "half": "f16", + "float16": "f16", + "bfloat16": "bf16", + "float": "f32", + "float32": "f32", + torch.float16: "f16", + torch.bfloat16: "bf16", + torch.float32: "f32", +} + +# Models supported by Neuron. +_NEURON_SUPPORTED_MODELS: Dict[str, Tuple[str, str, str]] = { + "LlamaForCausalLM": ("transformers_neuronx.llama.model", + "LlamaForSampling", "LlamaForCausalLM"), + "MistralForCausalLM": ("transformers_neuronx.mistral.model", + "MistralForSampling", "MistralForCausalLM") +} + + +class NeuronCasualLM(nn.Module): + + def __init__(self, + config: PretrainedConfig, + on_device_sampling_disabled: bool = False) -> None: + super().__init__() + self.config = config + self.logits_processor = LogitsProcessor(config.vocab_size, + logits_as_input=True) + + self.on_device_sampling_disabled = on_device_sampling_disabled + if self.on_device_sampling_disabled: + # Use default sampler + self.sampler = Sampler() + + # Lazy initialized + self.model: nn.Module + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + input_block_ids: torch.Tensor, + ) -> torch.Tensor: + logits = self.model(input_ids, + cache_ids=positions, + start_ids=input_block_ids) + return logits + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(None, hidden_states, sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + + if self.on_device_sampling_disabled: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + # On-device sampling outputs the token ids directly. + sampled_token_ids = logits.flatten() + next_tokens = [] + sample_idx = 0 + for seq_group in sampling_metadata.seq_groups: + samples = [] + for seq_id in seq_group.seq_ids: + token_id = sampled_token_ids[sample_idx].item() + samples.append( + SequenceOutput(parent_seq_id=seq_id, + output_token=token_id, + logprobs={token_id: Logprob(token_id)})) + sample_idx += 1 + next_tokens.append( + CompletionSequenceGroupOutput(samples=samples, + prompt_logprobs=None)) + + return SamplerOutput(outputs=next_tokens) + + def load_weights(self, model_name_or_path: str, **kwargs): + arch = _get_model_architecture(self.config) + neuronx_module_path, neuronx_model_cls_name, hf_model_cls_name = ( + _NEURON_SUPPORTED_MODELS[arch]) + neuronx_module = importlib.import_module(neuronx_module_path) + neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls_name) + + split_model_dir = f"{model_name_or_path}-split" + if _is_pretrained_neuron_checkpoint(model_name_or_path): + split_model_dir = model_name_or_path + elif not os.path.exists(f"{model_name_or_path}-split"): + hf_model_cls = getattr(transformers, hf_model_cls_name) + from transformers_neuronx.module import save_pretrained_split + + hf_model = hf_model_cls.from_pretrained(model_name_or_path, + low_cpu_mem_usage=True) + save_pretrained_split(hf_model, f"{model_name_or_path}-split") + + self.model = neuronx_model_cls.from_pretrained(split_model_dir, + **kwargs) + self.model.to_neuron() + + +def _is_pretrained_neuron_checkpoint(model_name_or_path: str) -> bool: + # Checking if the neuron checkpoint is saved in the old format. + if os.path.isdir(os.path.join(model_name_or_path, "pytorch_model.bin")): + return True + # Checking if the neuron checkpoint is saved in the new format. + pretrained_split_files = ["config.json", "generation_config.json"] + pretrained_split_format = ".safetensors" + for file in pretrained_split_files: + file_path = os.path.join(model_name_or_path, file) + if not os.path.isfile(file_path): + return False + for file in os.listdir(model_name_or_path): + if file.endswith(pretrained_split_format): + return True + return False + + +def _get_model_architecture(config: PretrainedConfig) -> str: + architectures = getattr(config, "architectures", []) + for arch in architectures: + if arch in _NEURON_SUPPORTED_MODELS: + return arch + raise ValueError( + f"Model architectures {architectures} are not supported on Neuron " + f"for now. Supported architectures: " + f"{list(_NEURON_SUPPORTED_MODELS.keys())}") + + +def _get_buckets(env: str, default_value: List[int]) -> List[int]: + env_value = os.getenv(env) + if env_value is None: + return default_value + buckets_remove_empty = filter( + lambda x: x is not None and len(x.strip()) > 0, env_value.split(",")) + buckets_int = map(int, buckets_remove_empty) + buckets_list = list(buckets_int) + return buckets_list + + +def _get_default_neuron_config(model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig): + from transformers_neuronx.config import ContinuousBatchingConfig + from transformers_neuronx.constants import LAYOUT_BSH + + continuous_batching_config = ContinuousBatchingConfig( + batch_size_for_shared_caches=scheduler_config.max_num_seqs) + quant_config = dict( + dequant_dtype=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype], + quantize_method="vector_dynamic") + neuron_quantization_config_builder = lambda quant: get_quantization_config( + quant).from_config(quant_config).get_quant_method(None, "") + # TODO: Add Paged attention config to the default neuron arguments. + default_neuron_args = dict( + collectives_layout=LAYOUT_BSH, + attention_layout=LAYOUT_BSH, + fuse_qkv=True, + quant=neuron_quantization_config_builder(model_config.quantization) + if model_config.quantization else None, + continuous_batching=continuous_batching_config, + weight_tiling=bool(model_config.quantization), + on_device_generation=_get_neuron_on_device_generation_config( + model_config)) + return default_neuron_args + + +def _get_neuron_on_device_generation_config(model_config: ModelConfig): + if not _is_neuron_on_device_sampling_disabled(model_config): + return copy.deepcopy(model_config.neuron_sampling_params) + return None + + +def _is_neuron_on_device_sampling_disabled(model_config: ModelConfig) -> bool: + return not getattr(model_config, "neuron_sampling_params", None) + + +def _get_neuron_config_after_override(default_neuron_config, + overridden_neuron_config): + from transformers_neuronx.config import NeuronConfig + overridden_neuron_config = overridden_neuron_config or {} + default_neuron_config.update(overridden_neuron_config) + return NeuronConfig(**default_neuron_config) + + +def get_neuron_model(model_config: ModelConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig) -> nn.Module: + + # Create a model instance. + model = NeuronCasualLM( + model_config.hf_config, + _is_neuron_on_device_sampling_disabled(model_config)) + + default_neuron_config_args = _get_default_neuron_config( + model_config, parallel_config, scheduler_config) + + neuron_config = _get_neuron_config_after_override( + default_neuron_config_args, model_config.override_neuron_config) + + context_length_estimates = _get_buckets("NEURON_CONTEXT_LENGTH_BUCKETS", + [scheduler_config.max_model_len]) + n_positions = _get_buckets("NEURON_TOKEN_GEN_BUCKETS", + [scheduler_config.max_model_len]) + + # Load the weights from the cached or downloaded files. + model.load_weights(model_config.model, + tp_degree=parallel_config.tensor_parallel_size, + amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype], + neuron_config=neuron_config, + context_length_estimate=context_length_estimates, + n_positions=n_positions, + batch_size=scheduler_config.max_num_seqs) + + return model.eval() diff --git a/vllm/model_executor/model_loader/openvino.py b/vllm/model_executor/model_loader/openvino.py new file mode 100644 index 00000000..88b7ac46 --- /dev/null +++ b/vllm/model_executor/model_loader/openvino.py @@ -0,0 +1,203 @@ +# ruff: noqa: SIM117 +from pathlib import Path +from typing import List, Optional, Tuple + +import openvino as ov +import torch +from huggingface_hub import HfApi +from openvino._offline_transformations import paged_attention_transformation +from optimum.intel import OVModelForCausalLM +from torch import nn + +import vllm.envs as envs +from vllm.attention.backends.openvino import OpenVINOAttentionMetadata +from vllm.config import DeviceConfig, ModelConfig +from vllm.executor.openvino_executor import is_openvino_cpu +from vllm.logger import init_logger +from vllm.model_executor.layers.logits_processor import (LogitsProcessor, + _prune_hidden_states) +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput +from vllm.model_executor.sampling_metadata import SamplingMetadata + +logger = init_logger(__name__) + + +def _flattenize_inputs(inputs): + """ + Helper function for making nested inputs flattens + """ + flatten_inputs = [] + for input_data in inputs: + if input_data is None: + continue + if isinstance(input_data, (list, tuple)): + flatten_inputs.extend(_flattenize_inputs(input_data)) + elif isinstance(input_data, dict): + flatten_inputs.extend(_flattenize_inputs(list( + input_data.values()))) + else: + flatten_inputs.append(input_data) + return flatten_inputs + + +def _modify_cache_parameters(model: ov.Model, kv_cache_dtype: ov.Type, + is_cpu: bool): + # Apply hardware dependent modifications to KV tensors + for parameter in model.get_parameters(): + input = parameter.get_output_tensor(0) + input_names = input.get_names() + if len(input_names) != 1: + continue + input_name = next(iter(input_names)) + shape = parameter.get_partial_shape() + # use real block size if available, just a placeholder + # to provide the expected rank + num_blocks = ov.Dimension() + block_size = ov.Dimension() + head_size = ov.Dimension() + if input_name.startswith("key_cache."): + cpu_shape = [num_blocks, shape[1], block_size, head_size] + gpu_shape = [num_blocks, shape[1], shape[2], block_size] + elif input_name.startswith("value_cache."): + cpu_shape = [num_blocks, shape[1], block_size, head_size] + gpu_shape = [num_blocks, shape[1], block_size, shape[2]] + else: + continue + parameter.set_partial_shape( + ov.PartialShape(cpu_shape if is_cpu else gpu_shape)) + parameter.set_element_type(kv_cache_dtype) + model.validate_nodes_and_infer_types() + + +def _require_model_export(model_id, revision=None, subfolder=None): + model_dir = Path(model_id) + if subfolder is not None: + model_dir = model_dir / subfolder + if model_dir.is_dir(): + return (not (model_dir / "openvino_model.xml").exists() + or not (model_dir / "openvino_model.bin").exists()) + + hf_api = HfApi() + try: + model_info = hf_api.model_info(model_id, revision=revision or "main") + normalized_subfolder = (None if subfolder is None else + Path(subfolder).as_posix()) + model_files = [ + file.rfilename for file in model_info.siblings + if normalized_subfolder is None + or file.rfilename.startswith(normalized_subfolder) + ] + ov_model_path = ("openvino_model.xml" if normalized_subfolder is None + else f"{normalized_subfolder}/openvino_model.xml") + return (ov_model_path not in model_files + or ov_model_path.replace(".xml", ".bin") not in model_files) + except Exception: + return True + + +class OpenVINOCasualLM(nn.Module): + + def __init__( + self, + ov_core: ov.Core, + model_config: ModelConfig, + device_config: DeviceConfig, + kv_cache_dtype: ov.Type, + ) -> None: + super().__init__() + self.logits_processor = LogitsProcessor( + model_config.hf_config.vocab_size, logits_as_input=True) + self.sampler = Sampler() + + export = _require_model_export(model_config.model) + if export: + logger.warning( + f"Provided model id {model_config.model} does not " # noqa: G004 + "contain OpenVINO IR, the model will be converted to IR with " + "default options. If you need to use specific options for " + "model conversion, use optimum-cli export openvino with " + "desired options.") + else: + logger.warning( + "OpenVINO IR is available for provided model id " # noqa: G004 + f"{model_config.model}. This IR will be used for inference " + "as-is, all possible options that may affect model conversion " + "are ignored.") + + load_in_8bit = envs.VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS + pt_model = OVModelForCausalLM.from_pretrained( + model_config.model, + export=export, + compile=False, + load_in_8bit=load_in_8bit, + trust_remote_code=model_config.trust_remote_code, + ) + + ov_device = envs.VLLM_OPENVINO_DEVICE + paged_attention_transformation(pt_model.model) + _modify_cache_parameters(pt_model.model, kv_cache_dtype, + is_openvino_cpu()) + + ov_compiled = ov_core.compile_model(pt_model.model, ov_device) + self.ov_request = ov_compiled.create_infer_request() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + kv_caches: List[Tuple[ov.Tensor, ov.Tensor]], + attn_metadata: OpenVINOAttentionMetadata, + ) -> torch.Tensor: + flatten_kv_cache = _flattenize_inputs(kv_caches) + + inputs = [ + input_ids, + positions, + *flatten_kv_cache, + attn_metadata.past_lens, + attn_metadata.subsequence_begins, + attn_metadata.block_indices, + attn_metadata.block_indices_begins, + attn_metadata.max_context_len, + ] + + self.ov_request.start_async(inputs, share_inputs=True) + self.ov_request.wait() + + logits = torch.from_numpy(self.ov_request.get_tensor("logits").data) + + # TODO: remove 'view' once OpenVINO PA will drop 'seq_len' dimension + return logits.view(-1, logits.shape[-1]) + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + hidden_states = _prune_hidden_states(hidden_states, sampling_metadata) + logits = self.logits_processor(None, hidden_states, sampling_metadata) + return logits + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[SamplerOutput]: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + +def get_model( + model_config: ModelConfig, + device_config: DeviceConfig, + kv_cache_dtype: ov.Type, + **kwargs, +) -> torch.nn.Module: + lora_config = kwargs.get("lora_config", None) + ov_core = kwargs.get("ov_core") + if lora_config: + raise ValueError( + "OpenVINO modeling does not support LoRA, " + "but LoRA is enabled. Support for this model may " + "be added in the future. If this is important to you, " + "please open an issue on github.") + + return OpenVINOCasualLM(ov_core, model_config, device_config, + kv_cache_dtype) diff --git a/vllm/model_executor/model_loader/tensorizer.py b/vllm/model_executor/model_loader/tensorizer.py new file mode 100644 index 00000000..da224ecf --- /dev/null +++ b/vllm/model_executor/model_loader/tensorizer.py @@ -0,0 +1,480 @@ +import argparse +import dataclasses +import io +import os +import regex as re +import time +from dataclasses import dataclass +from functools import partial +from typing import BinaryIO, Generator, Optional, Tuple, Type, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +import vllm.envs as envs +from vllm.config import ModelConfig, ParallelConfig +from vllm.engine.arg_utils import EngineArgs +from vllm.engine.llm_engine import LLMEngine +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig) +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding) +from vllm.utils import FlexibleArgumentParser + +tensorizer_error_msg = None + +try: + from tensorizer import (DecryptionParams, EncryptionParams, + TensorDeserializer, TensorSerializer) + from tensorizer.stream_io import open_stream + from tensorizer.utils import (convert_bytes, get_mem_usage, + no_init_or_tensor) + + _read_stream, _write_stream = (partial( + open_stream, + mode=mode, + ) for mode in ("rb", "wb+")) +except ImportError as e: + tensorizer_error_msg = str(e) + +__all__ = [ + 'EncryptionParams', 'DecryptionParams', 'TensorDeserializer', + 'TensorSerializer', 'open_stream', 'convert_bytes', 'get_mem_usage', + 'no_init_or_tensor', 'TensorizerConfig' +] + +logger = init_logger(__name__) + + +@dataclass +class TensorizerConfig: + tensorizer_uri: str + vllm_tensorized: Optional[bool] = False + verify_hash: Optional[bool] = False + num_readers: Optional[int] = None + encryption_keyfile: Optional[str] = None + s3_access_key_id: Optional[str] = None + s3_secret_access_key: Optional[str] = None + s3_endpoint: Optional[str] = None + model_class: Optional[Type[torch.nn.Module]] = None + hf_config: Optional[PretrainedConfig] = None + dtype: Optional[Union[str, torch.dtype]] = None + _is_sharded: bool = False + + def __post_init__(self): + # check if the configuration is for a sharded vLLM model + self._is_sharded = isinstance(self.tensorizer_uri, str) \ + and re.search(r'%0\dd', self.tensorizer_uri) is not None + + def _construct_tensorizer_args(self) -> "TensorizerArgs": + tensorizer_args = { + "tensorizer_uri": self.tensorizer_uri, + "vllm_tensorized": self.vllm_tensorized, + "verify_hash": self.verify_hash, + "num_readers": self.num_readers, + "encryption_keyfile": self.encryption_keyfile, + "s3_access_key_id": self.s3_access_key_id, + "s3_secret_access_key": self.s3_secret_access_key, + "s3_endpoint": self.s3_endpoint, + } + return TensorizerArgs(**tensorizer_args) # type: ignore + + def verify_with_parallel_config( + self, + parallel_config: "ParallelConfig", + ) -> None: + if parallel_config.tensor_parallel_size > 1 \ + and not self._is_sharded: + raise ValueError( + "For a sharded model, tensorizer_uri should include a" + " string format template like '%04d' to be formatted" + " with the rank of the shard") + + def verify_with_model_config(self, model_config: "ModelConfig") -> None: + if (model_config.quantization is not None + and self.tensorizer_uri is not None): + logger.warning( + "Loading a model using Tensorizer with quantization on vLLM" + " is unstable and may lead to errors.") + + def open_stream(self, tensorizer_args: Optional["TensorizerArgs"] = None): + if tensorizer_args is None: + tensorizer_args = self._construct_tensorizer_args() + + return open_stream(self.tensorizer_uri, + **tensorizer_args.stream_params) + + +def load_with_tensorizer(tensorizer_config: TensorizerConfig, + **extra_kwargs) -> nn.Module: + tensorizer = TensorizerAgent(tensorizer_config, **extra_kwargs) + return tensorizer.deserialize() + + +@dataclass +class TensorizerArgs: + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str, + bytes, os.PathLike, int] + vllm_tensorized: Optional[bool] = False + verify_hash: Optional[bool] = False + num_readers: Optional[int] = None + encryption_keyfile: Optional[str] = None + s3_access_key_id: Optional[str] = None + s3_secret_access_key: Optional[str] = None + s3_endpoint: Optional[str] = None + """ + Args for the TensorizerAgent class. These are used to configure the behavior + of the TensorDeserializer when loading tensors from a serialized model. + + Args: + tensorizer_uri: Path to serialized model tensors. Can be a local file + path or a S3 URI. + vllm_tensorized: If True, indicates that the serialized model is a + vLLM model. This is used to determine the behavior of the + TensorDeserializer when loading tensors from a serialized model. + It is far faster to deserialize a vLLM model as it utilizes + tensorizer's optimized GPU loading. Note that this is now + deprecated, as serialized vLLM models are now automatically + inferred as vLLM models. + verify_hash: If True, the hashes of each tensor will be verified against + the hashes stored in the metadata. A `HashMismatchError` will be + raised if any of the hashes do not match. + num_readers: Controls how many threads are allowed to read concurrently + from the source file. Default is `None`, which will dynamically set + the number of readers based on the number of available + resources and model size. This greatly increases performance. + encryption_keyfile: File path to a binary file containing a + binary key to use for decryption. `None` (the default) means + no decryption. See the example script in + examples/tensorize_vllm_model.py. + s3_access_key_id: The access key for the S3 bucket. Can also be set via + the S3_ACCESS_KEY_ID environment variable. + s3_secret_access_key: The secret access key for the S3 bucket. Can also + be set via the S3_SECRET_ACCESS_KEY environment variable. + s3_endpoint: The endpoint for the S3 bucket. Can also be set via the + S3_ENDPOINT_URL environment variable. + """ + + def __post_init__(self): + self.file_obj = self.tensorizer_uri + self.s3_access_key_id = self.s3_access_key_id or envs.S3_ACCESS_KEY_ID + self.s3_secret_access_key = (self.s3_secret_access_key + or envs.S3_SECRET_ACCESS_KEY) + self.s3_endpoint = self.s3_endpoint or envs.S3_ENDPOINT_URL + self.stream_params = { + "s3_access_key_id": self.s3_access_key_id, + "s3_secret_access_key": self.s3_secret_access_key, + "s3_endpoint": self.s3_endpoint, + } + + self.deserializer_params = { + "verify_hash": self.verify_hash, + "encryption": self.encryption_keyfile, + "num_readers": self.num_readers + } + + if self.encryption_keyfile: + with open_stream( + self.encryption_keyfile, + **self.stream_params, + ) as stream: + key = stream.read() + decryption_params = DecryptionParams.from_key(key) + self.deserializer_params['encryption'] = decryption_params + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + """Tensorizer CLI arguments""" + + # Tensorizer options arg group + group = parser.add_argument_group( + 'tensorizer options', + description=('Options for configuring the behavior of the' + ' tensorizer deserializer when ' + 'load_format=tensorizer is specified when ' + 'initializing an LLMEngine, either via the CLI ' + 'when running the vLLM OpenAI inference server ' + 'with a JSON string passed to ' + '--model-loader-extra-config or as arguments given ' + 'to TensorizerConfig when passed to ' + 'model_loader_extra_config in the constructor ' + 'for LLMEngine.')) + + group.add_argument( + "--tensorizer-uri", + help="Path to serialized model tensors. Can be a local file path," + " or an HTTP(S) or S3 URI.", + ) + group.add_argument( + "--verify-hash", + action="store_true", + help="If enabled, the hashes of each tensor will be verified" + " against the hashes stored in the file metadata. An exception" + " will be raised if any of the hashes do not match.", + ) + group.add_argument( + "--encryption-keyfile", + default=None, + help="The file path to a binary file containing a binary key to " + "use for decryption. Can be a file path or S3 network URI.") + group.add_argument( + "--num-readers", + default=None, + type=int, + help="Controls how many threads are allowed to read concurrently " + "from the source file. Default is `None`, which will dynamically " + "set the number of readers based on the available resources " + "and model size. This greatly increases performance.") + group.add_argument( + "--s3-access-key-id", + default=None, + help="The access key for the S3 bucket. Can also be set via the " + "S3_ACCESS_KEY_ID environment variable.", + ) + group.add_argument( + "--s3-secret-access-key", + default=None, + help="The secret access key for the S3 bucket. Can also be set via " + "the S3_SECRET_ACCESS_KEY environment variable.", + ) + group.add_argument( + "--s3-endpoint", + default=None, + help="The endpoint for the S3 bucket. Can also be set via the " + "S3_ENDPOINT_URL environment variable.", + ) + + return parser + + @classmethod + def from_cli_args(cls, args: argparse.Namespace) -> "TensorizerArgs": + attrs = [attr.name for attr in dataclasses.fields(cls)] + tensorizer_args = cls(**{ + attr: getattr(args, attr) + for attr in attrs if hasattr(args, attr) + }) + return tensorizer_args + + +class TensorizerAgent: + """ + A class for performing tensorizer deserializations specifically for + vLLM models using plaid_mode. Uses TensorizerArgs to configure the + behavior of the TensorDeserializer when loading tensors from a serialized + model. For deserializations of HuggingFace models, TensorDeserializer is + instead used as an iterator directly in the func hf_model_weights_iterator + in vllm/model_executor/model_loader/weight_utils.py + """ + + def __init__(self, tensorizer_config: TensorizerConfig, + quant_config: QuantizationConfig, **extra_kwargs): + if tensorizer_error_msg is not None: + raise ImportError( + "Tensorizer is not installed. Please install tensorizer " + "to use this feature with `pip install vllm[tensorizer]`. " + "Error message: {}".format(tensorizer_error_msg)) + + self.tensorizer_config = tensorizer_config + self.tensorizer_args = ( + self.tensorizer_config._construct_tensorizer_args()) + self.extra_kwargs = extra_kwargs + if extra_kwargs.get("quant_config", None) is not None: + self.quant_config = extra_kwargs["quant_config"] + else: + self.quant_config = quant_config + self.model = self._init_model() + + def _init_model(self): + assert self.tensorizer_config.hf_config is not None + model_args = self.tensorizer_config.hf_config + model_args.torch_dtype = self.tensorizer_config.dtype + assert self.tensorizer_config.model_class is not None + with no_init_or_tensor(): + return self.tensorizer_config.model_class( + config=model_args, + quant_config=self.quant_config, + **self.extra_kwargs) + + def _resize_lora_embeddings(self): + """Modify LoRA embedding layers to use bigger tensors + to allow for adapter added tokens.""" + for child in self.model.modules(): + if (isinstance(child, VocabParallelEmbedding) + and child.weight.shape[0] < + child.num_embeddings_per_partition): + new_weight = torch.empty(child.num_embeddings_per_partition, + child.embedding_dim, + dtype=child.weight.dtype, + device=child.weight.device) + new_weight[:child.weight.shape[0]].copy_(child.weight.data) + new_weight[child.weight.shape[0]:].fill_(0) + child.weight.data = new_weight + + def _check_tensors_on_meta_device(self): + for tensor in self.model.state_dict().values(): + if tensor.device.type == 'meta': + raise ValueError( + "The serialized model contains tensors on the meta device," + " indicating that some tensors were not loaded properly." + " Please check that the parameters of the model being" + " specified match that of the serialized model, such as" + " its quantization.") + + def deserialize(self): + """ + Deserialize the model using the TensorDeserializer. This method is + specifically for vLLM models using tensorizer's plaid_mode. + + The deserializer makes use of tensorizer_args.stream_params + to configure the behavior of the stream when loading tensors from a + serialized model. The deserializer_params are used to configure the + behavior of the TensorDeserializer when loading tensors themselves. + Documentation on these params can be found in TensorizerArgs + + Returns: + nn.Module: The deserialized model. + """ + before_mem = get_mem_usage() + start = time.perf_counter() + with _read_stream( + self.tensorizer_config.tensorizer_uri, + **self.tensorizer_args.stream_params + ) as stream, TensorDeserializer( + stream, + dtype=self.tensorizer_config.dtype, + device=f'cuda:{torch.cuda.current_device()}', + **self.tensorizer_args.deserializer_params) as deserializer: + deserializer.load_into_module(self.model) + end = time.perf_counter() + + total_bytes_str = convert_bytes(deserializer.total_tensor_bytes) + duration = end - start + per_second = convert_bytes(deserializer.total_tensor_bytes / duration) + after_mem = get_mem_usage() + deserializer.close() + logger.info("Deserialized %s in %0.2fs, %s/s", total_bytes_str, + end - start, per_second) + logger.info("Memory usage before: %s", before_mem) + logger.info("Memory usage after: %s", after_mem) + + self._check_tensors_on_meta_device() + self._resize_lora_embeddings() + del self.model.vllm_tensorized_marker + return self.model.eval() + + +def tensorizer_weights_iterator( + tensorizer_args: "TensorizerArgs" +) -> Generator[Tuple[str, torch.Tensor], None, None]: + logger.warning( + "Deserializing HuggingFace models is not optimized for " + "loading on vLLM, as tensorizer is forced to load to CPU. " + "Consider deserializing a vLLM model instead for faster " + "load times. See the examples/tensorize_vllm_model.py example " + "script for serializing vLLM models.") + + deserializer_args = tensorizer_args.deserializer_params + stream_params = tensorizer_args.stream_params + stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params) + with TensorDeserializer(stream, **deserializer_args, + device="cpu") as state: + for name, param in state.items(): + yield name, param + del state + + +def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool: + """ + Infer if the model is a vLLM model by checking the weights for + a vLLM tensorized marker. + + Args: + tensorizer_config: The TensorizerConfig object containing the + tensorizer_uri to the serialized model. + + Returns: + bool: True if the model is a vLLM model, False otherwise. + """ + tensorizer_args = tensorizer_config._construct_tensorizer_args() + deserializer = TensorDeserializer(open_stream( + tensorizer_args.tensorizer_uri, **tensorizer_args.stream_params), + **tensorizer_args.deserializer_params, + lazy_load=True) + if tensorizer_config.vllm_tensorized: + logger.warning( + "Please note that newly serialized vLLM models are automatically " + "inferred as vLLM models, so setting vllm_tensorized=True is " + "only necessary for models serialized prior to this change.") + return True + return ".vllm_tensorized_marker" in deserializer + + +def serialize_vllm_model( + model: nn.Module, + tensorizer_config: TensorizerConfig, +) -> nn.Module: + model.register_parameter( + "vllm_tensorized_marker", + nn.Parameter(torch.tensor((1, ), device="meta"), requires_grad=False)) + tensorizer_args = tensorizer_config._construct_tensorizer_args() + + encryption_params = None + if (keyfile := tensorizer_config.encryption_keyfile) is not None: + with open(keyfile, "rb") as f: + key = f.read() + encryption_params = EncryptionParams(key=key) + + output_file = tensorizer_args.tensorizer_uri + if tensorizer_config._is_sharded: + from vllm.distributed import get_tensor_model_parallel_rank + output_file = output_file % get_tensor_model_parallel_rank() + + with _write_stream(output_file, **tensorizer_args.stream_params) as stream: + serializer = TensorSerializer(stream, encryption=encryption_params) + serializer.write_module(model) + serializer.close() + logger.info("Successfully serialized model to %s", str(output_file)) + return model + + +def tensorize_vllm_model(engine_args: EngineArgs, + tensorizer_config: TensorizerConfig, + generate_keyfile: bool = True): + """Utility to load a model and then serialize it with Tensorizer + + Intended to be used separately from running a vLLM server since it + creates its own Engine instance. + """ + engine_config = engine_args.create_engine_config() + tensorizer_config.verify_with_model_config(engine_config.model_config) + tensorizer_config.verify_with_parallel_config( + engine_config.parallel_config) + + # generate the encryption key before creating the engine to support sharding + if generate_keyfile and (keyfile := + tensorizer_config.encryption_keyfile) is not None: + encryption_params = EncryptionParams.random() + with _write_stream( + keyfile, + s3_access_key_id=tensorizer_config.s3_access_key_id, + s3_secret_access_key=tensorizer_config.s3_secret_access_key, + s3_endpoint=tensorizer_config.s3_endpoint, + ) as stream: + stream.write(encryption_params.key) + + engine = LLMEngine.from_engine_args(engine_args) + if tensorizer_config._is_sharded: + # if the engine is a distributed engine (for tensor parallel) then each + # worker shard needs to serialize its part of the model. + engine.model_executor._run_workers( + "save_tensorized_model", + tensorizer_config=tensorizer_config, + ) + else: + # with a single worker, we can get to the underlying model directly + serialize_vllm_model( + engine.model_executor.driver_worker.model_runner.model, + tensorizer_config, + ) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py new file mode 100644 index 00000000..b95c0b7c --- /dev/null +++ b/vllm/model_executor/model_loader/utils.py @@ -0,0 +1,39 @@ +"""Utilities for selecting and loading models.""" +import contextlib +from typing import Tuple, Type + +import torch +from torch import nn + +from vllm.config import ModelConfig +from vllm.model_executor.models import ModelRegistry + + +@contextlib.contextmanager +def set_default_torch_dtype(dtype: torch.dtype): + """Sets the default torch dtype to the given dtype.""" + old_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + yield + torch.set_default_dtype(old_dtype) + + +def get_model_architecture( + model_config: ModelConfig) -> Tuple[Type[nn.Module], str]: + architectures = getattr(model_config.hf_config, "architectures", []) + # Special handling for quantized Mixtral. + # FIXME(woosuk): This is a temporary hack. + mixtral_supported = [ + "fp8", "compressed-tensors", "gptq_marlin", "awq_marlin" + ] + + if (model_config.quantization is not None + and model_config.quantization not in mixtral_supported + and "MixtralForCausalLM" in architectures): + architectures = ["QuantMixtralForCausalLM"] + + return ModelRegistry.resolve_model_cls(architectures) + + +def get_architecture_class_name(model_config: ModelConfig) -> str: + return get_model_architecture(model_config)[1] diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py new file mode 100644 index 00000000..746b0be0 --- /dev/null +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -0,0 +1,682 @@ +"""Utilities for downloading and initializing model weights.""" +import fnmatch +import glob +import hashlib +import json +import os +import tempfile +from collections import defaultdict +from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional, + Tuple, Union) + +import filelock +import gguf +import huggingface_hub.constants +import numpy as np +import torch +from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download +from safetensors.torch import load_file, safe_open, save_file +from tqdm.auto import tqdm + +from vllm.config import LoadConfig, ModelConfig +from vllm.distributed import get_tensor_model_parallel_rank +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization import (QuantizationConfig, + get_quantization_config) +from vllm.model_executor.layers.quantization.schema import QuantParamSchema +from vllm.platforms import current_platform +from vllm.utils import print_warning_once + +logger = init_logger(__name__) + +# use system-level temp directory for file locks, so that multiple users +# can share the same lock without error. +# lock files in the temp directory will be automatically deleted when the +# system reboots, so users will not complain about annoying lock files +temp_dir = tempfile.gettempdir() + + +def enable_hf_transfer(): + """automatically activates hf_transfer + """ + if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True + except ImportError: + pass + + +enable_hf_transfer() + + +class DisabledTqdm(tqdm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, disable=True) + + +def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None): + lock_dir = cache_dir or temp_dir + os.makedirs(os.path.dirname(lock_dir), exist_ok=True) + model_name = model_name_or_path.replace("/", "-") + hash_name = hashlib.sha256(model_name.encode()).hexdigest() + # add hash to avoid conflict with old users' lock files + lock_file_name = hash_name + model_name + ".lock" + # mode 0o666 is required for the filelock to be shared across users + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), + mode=0o666) + return lock + + +def _shared_pointers(tensors): + ptrs = defaultdict(list) + for k, v in tensors.items(): + ptrs[v.data_ptr()].append(k) + failing = [] + for _, names in ptrs.items(): + if len(names) > 1: + failing.append(names) + return failing + + +def convert_bin_to_safetensor_file( + pt_filename: str, + sf_filename: str, +) -> None: + loaded = torch.load(pt_filename, map_location="cpu", weights_only=True) + if "state_dict" in loaded: + loaded = loaded["state_dict"] + shared = _shared_pointers(loaded) + for shared_weights in shared: + for name in shared_weights[1:]: + loaded.pop(name) + + # For tensors to be contiguous + loaded = {k: v.contiguous() for k, v in loaded.items()} + + dirname = os.path.dirname(sf_filename) + os.makedirs(dirname, exist_ok=True) + save_file(loaded, sf_filename, metadata={"format": "pt"}) + + # check file size + sf_size = os.stat(sf_filename).st_size + pt_size = os.stat(pt_filename).st_size + if (sf_size - pt_size) / pt_size > 0.01: + raise RuntimeError(f"""The file size different is more than 1%: + - {sf_filename}: {sf_size} + - {pt_filename}: {pt_size} + """) + + # check if the tensors are the same + reloaded = load_file(sf_filename) + for k in loaded: + pt_tensor = loaded[k] + sf_tensor = reloaded[k] + if not torch.equal(pt_tensor, sf_tensor): + raise RuntimeError(f"The output tensors do not match for key {k}") + + +# TODO(woosuk): Move this to other place. +def get_quant_config(model_config: ModelConfig, + load_config: LoadConfig) -> QuantizationConfig: + + quant_cls = get_quantization_config(model_config.quantization) + + # GGUF doesn't have config file + if model_config.quantization == "gguf": + return quant_cls.from_config({}) + + # Read the quantization config from the HF model config, if available. + hf_quant_config = getattr(model_config.hf_config, "quantization_config", + None) + # some vision model may keep quantization_config in their text_config + hf_text_config = getattr(model_config.hf_config, "text_config", None) + if hf_quant_config is None and hf_text_config is not None: + hf_quant_config = getattr(hf_text_config, "quantization_config", None) + if hf_quant_config is None: + # compressed-tensors uses a compressions_config + hf_quant_config = getattr(model_config.hf_config, "compression_config", + None) + if hf_quant_config is not None: + return quant_cls.from_config(hf_quant_config) + # In case of bitsandbytes/QLoRA, get quant config from the adapter model. + if model_config.quantization == "bitsandbytes": + if (not load_config.model_loader_extra_config + or "qlora_adapter_name_or_path" + not in load_config.model_loader_extra_config): + return quant_cls.from_config({"adapter_name_or_path": ""}) + model_name_or_path = load_config.model_loader_extra_config[ + "qlora_adapter_name_or_path"] + + else: + model_name_or_path = model_config.model + is_local = os.path.isdir(model_name_or_path) + if not is_local: + # Download the config files. + with get_lock(model_name_or_path, load_config.download_dir): + hf_folder = snapshot_download( + model_name_or_path, + revision=model_config.revision, + allow_patterns="*.json", + cache_dir=load_config.download_dir, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + tqdm_class=DisabledTqdm, + ) + else: + hf_folder = model_name_or_path + + possible_config_filenames = quant_cls.get_config_filenames() + + # If the quantization config is not found, use the default config. + if not possible_config_filenames: + return quant_cls() + + config_files = glob.glob(os.path.join(hf_folder, "*.json")) + + quant_config_files = [ + f for f in config_files if any( + f.endswith(x) for x in possible_config_filenames) + ] + if len(quant_config_files) == 0: + raise ValueError( + f"Cannot find the config file for {model_config.quantization}") + if len(quant_config_files) > 1: + raise ValueError( + f"Found multiple config files for {model_config.quantization}: " + f"{quant_config_files}") + + quant_config_file = quant_config_files[0] + with open(quant_config_file, "r") as f: + config = json.load(f) + + if model_config.quantization == "bitsandbytes": + config["adapter_name_or_path"] = model_name_or_path + elif model_config.quantization == "modelopt": + if config["producer"]["name"] == "modelopt": + return quant_cls.from_config(config) + else: + raise ValueError( + f"Unsupported quantization config" + f" found for {model_config.quantization} in {f}.") + + return quant_cls.from_config(config) + + +def download_weights_from_hf( + model_name_or_path: str, + cache_dir: Optional[str], + allow_patterns: List[str], + revision: Optional[str] = None, + ignore_patterns: Optional[Union[str, List[str]]] = None, +) -> str: + """Download model weights from Hugging Face Hub. + + Args: + model_name_or_path (str): The model name or path. + cache_dir (Optional[str]): The cache directory to store the model + weights. If None, will use HF defaults. + allow_patterns (List[str]): The allowed patterns for the + weight files. Files matched by any of the patterns will be + downloaded. + revision (Optional[str]): The revision of the model. + ignore_patterns (Optional[Union[str, List[str]]]): The patterns to + filter out the weight files. Files matched by any of the patterns + will be ignored. + + Returns: + str: The path to the downloaded model weights. + """ + if not huggingface_hub.constants.HF_HUB_OFFLINE: + # Before we download we look at that is available: + fs = HfFileSystem() + file_list = fs.ls(model_name_or_path, detail=False, revision=revision) + + # depending on what is available we download different things + for pattern in allow_patterns: + matching = fnmatch.filter(file_list, pattern) + if len(matching) > 0: + allow_patterns = [pattern] + break + + logger.info("Using model weights format %s", allow_patterns) + # Use file lock to prevent multiple processes from + # downloading the same model weights at the same time. + with get_lock(model_name_or_path, cache_dir): + hf_folder = snapshot_download( + model_name_or_path, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + cache_dir=cache_dir, + tqdm_class=DisabledTqdm, + revision=revision, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + return hf_folder + + +def download_safetensors_index_file_from_hf( + model_name_or_path: str, + index_file: str, + cache_dir: Optional[str], + revision: Optional[str] = None, +) -> None: + """Download hf safetensors index file from Hugging Face Hub. + + Args: + model_name_or_path (str): The model name or path. + cache_dir (Optional[str]): The cache directory to store the model + weights. If None, will use HF defaults. + revision (Optional[str]): The revision of the model. + """ + # Use file lock to prevent multiple processes from + # downloading the same model weights at the same time. + with get_lock(model_name_or_path, cache_dir): + try: + # Download the safetensors index file. + hf_hub_download( + repo_id=model_name_or_path, + filename=index_file, + cache_dir=cache_dir, + revision=revision, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + # If file not found on remote or locally, we should not fail since + # only some models will have index_file. + except huggingface_hub.utils.EntryNotFoundError: + logger.info("No %s found in remote.", index_file) + except huggingface_hub.utils.LocalEntryNotFoundError: + logger.info("No %s found in local cache.", index_file) + + +# For models like Mistral-7B-v0.3, there are both sharded +# safetensors files and a consolidated safetensors file. +# Passing both of these to the weight loader functionality breaks. +# So, we use the index_file to +# look up which safetensors files should be used. +def filter_duplicate_safetensors_files(hf_weights_files: List[str], + hf_folder: str, + index_file: str) -> List[str]: + # model.safetensors.index.json is a mapping from keys in the + # torch state_dict to safetensors file holding that weight. + index_file_name = os.path.join(hf_folder, index_file) + if not os.path.isfile(index_file_name): + return hf_weights_files + + # Iterate through the weight_map (weight_name: safetensors files) + # to identify weights that we should use. + with open(index_file_name, "r") as f: + weight_map = json.load(f)["weight_map"] + weight_files_in_index = set() + for weight_name in weight_map: + weight_files_in_index.add( + os.path.join(hf_folder, weight_map[weight_name])) + # Filter out any fields that are not found in the index file. + hf_weights_files = [ + f for f in hf_weights_files if f in weight_files_in_index + ] + return hf_weights_files + + +def filter_files_not_needed_for_inference( + hf_weights_files: List[str]) -> List[str]: + """ + Exclude files that are not needed for inference. + + See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233 + """ + blacklist = [ + "training_args.bin", + "optimizer.bin", + "optimizer.pt", + "scheduler.pt", + "scaler.pt", + ] + hf_weights_files = [ + f for f in hf_weights_files + if not any(f.endswith(x) for x in blacklist) + ] + return hf_weights_files + + +# explicitly use pure text format, with a newline at the end +# this makes it impossible to see the animation in the progress bar +# but will avoid messing up with ray or multiprocessing, which wraps +# each line of output with some prefix. +_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501 + + +def np_cache_weights_iterator( + model_name_or_path: str, cache_dir: Optional[str], hf_folder: str, + hf_weights_files: List[str] +) -> Generator[Tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model np files. + + Will dump the model weights to numpy files if they are not already dumped. + """ + enable_tqdm = not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0 + # Convert the model weights from torch tensors to numpy arrays for + # faster loading. + np_folder = os.path.join(hf_folder, "np") + os.makedirs(np_folder, exist_ok=True) + weight_names_file = os.path.join(np_folder, "weight_names.json") + # Use file lock to prevent multiple processes from + # dumping the same model weights to numpy at the same time. + with get_lock(model_name_or_path, cache_dir): + if not os.path.exists(weight_names_file): + weight_names: List[str] = [] + for bin_file in tqdm( + hf_weights_files, + desc="Loading np_cache checkpoint shards", + disable=not enable_tqdm, + bar_format=_BAR_FORMAT, + ): + state = torch.load(bin_file, map_location="cpu", weights_only=True) + for name, param in state.items(): + param_path = os.path.join(np_folder, name) + with open(param_path, "wb") as f: + np.save(f, param.cpu().detach().numpy()) + weight_names.append(name) + with open(weight_names_file, "w") as f: + json.dump(weight_names, f) + + with open(weight_names_file, "r") as f: + weight_names = json.load(f) + + for name in weight_names: + param_path = os.path.join(np_folder, name) + with open(param_path, "rb") as f: + param = np.load(f) + yield name, torch.from_numpy(param) + + +def safetensors_weights_iterator( + hf_weights_files: List[str] +) -> Generator[Tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model safetensor files.""" + enable_tqdm = not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0 + for st_file in tqdm( + hf_weights_files, + desc="Loading safetensors checkpoint shards", + disable=not enable_tqdm, + bar_format=_BAR_FORMAT, + ): + with safe_open(st_file, framework="pt") as f: + for name in f.keys(): # noqa: SIM118 + param = f.get_tensor(name) + yield name, param + + +def pt_weights_iterator( + hf_weights_files: List[str] +) -> Generator[Tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model bin/pt files.""" + enable_tqdm = not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0 + for bin_file in tqdm( + hf_weights_files, + desc="Loading pt checkpoint shards", + disable=not enable_tqdm, + bar_format=_BAR_FORMAT, + ): + state = torch.load(bin_file, map_location="cpu", weights_only=True) + for name, param in state.items(): + yield name, param + del state + torch.cuda.empty_cache() + + +def get_gguf_extra_tensor_names( + gguf_file: str, gguf_to_hf_name_map: Dict[str, str]) -> List[str]: + reader = gguf.GGUFReader(gguf_file) + expected_gguf_keys = set(gguf_to_hf_name_map.keys()) + exact_gguf_keys = set([tensor.name for tensor in reader.tensors]) + extra_keys = expected_gguf_keys - exact_gguf_keys + return [gguf_to_hf_name_map[key] for key in extra_keys] + + +def gguf_quant_weights_iterator( + gguf_file: str, gguf_to_hf_name_map: Dict[str, str] +) -> Generator[Tuple[str, torch.Tensor], None, None]: + """ + Iterate over the quant weights in the model gguf files and convert + them to torch tensors + """ + + reader = gguf.GGUFReader(gguf_file) + + for tensor in reader.tensors: + if tensor.name in gguf_to_hf_name_map: + weight_type = tensor.tensor_type + name = gguf_to_hf_name_map[tensor.name] + + if weight_type.name != "F32": + weight_type_name = name.replace("weight", "qweight_type") + weight_type = torch.tensor(weight_type) + yield weight_type_name, weight_type + + for tensor in reader.tensors: + if tensor.name in gguf_to_hf_name_map: + weight = tensor.data + weight_type = tensor.tensor_type + name = gguf_to_hf_name_map[tensor.name] + + if weight_type.name != "F32": + name = name.replace("weight", "qweight") + param = torch.tensor(weight) + yield name, param + + +def kv_cache_scales_loader( + filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int, + model_type: Optional[str]) -> Iterable[Tuple[int, float]]: + """ + A simple utility to read in KV cache scaling factors that have been + previously serialized to disk. Used by the model to populate the appropriate + KV cache scaling factors. The serialization should represent a dictionary + whose keys are the TP ranks and values are another dictionary mapping layers + to their KV cache scaling factors. + Keep this function in sync with the output of examples/fp8/extract_scales.py + """ + try: + with open(filename) as f: + context = { + "model_type": model_type, + "num_hidden_layers": num_hidden_layers, + "tp_rank": tp_rank, + "tp_size": tp_size, + } + schema_dct = json.load(f) + schema = QuantParamSchema.model_validate(schema_dct, + context=context) + layer_scales_map = schema.kv_cache.scaling_factor[tp_rank] + return layer_scales_map.items() + + except FileNotFoundError: + logger.error("File or directory '%s' not found.", filename) + except json.JSONDecodeError: + logger.error("Error decoding JSON in file '%s'.", filename) + except Exception as e: + logger.error("An error occurred while reading '%s': %s", filename, e) + # This section is reached if and only if any of the excepts are hit + # Return an empty iterable (list) => no KV cache scales are loaded + # which ultimately defaults to 1.0 scales + logger.warning( + "Defaulting to KV cache scaling factors = 1.0 for all " + "layers in TP rank %d as an error occurred during loading.", tp_rank) + return [] + + +def convert_pyslice_to_tensor(x: Any) -> torch.Tensor: + """convert PySafeSlice object from safetensors to torch.Tensor + + PySafeSlice object supports indexing, which is done before loading the + actual tensor and can reduce the amount of memory being read into the + memory. However, it does not support more advanced functionalities + like `.view()` or `.t()`. Therefore, if we need to modify the loaded + tensor with these more complicated operators, we need to convert to + tensor first. + """ + if not isinstance(x, torch.Tensor): + x = x[:] + return x + + +def default_weight_loader(param: torch.Tensor, + loaded_weight: torch.Tensor) -> None: + """Default weight loader.""" + try: + if param.numel() == 1 and loaded_weight.numel() == 1: + # Sometimes scalar values aren't considered tensors with shapes + # so if both param and loaded_weight are a scalar, + # "broadcast" instead of copy + param.data.fill_(loaded_weight.item()) + else: + assert param.size() == loaded_weight.size(), ( + f"Attempted to load weight ({loaded_weight.size()}) " + f"into parameter ({param.size()})") + + param.data.copy_(loaded_weight) + except Exception: + # NOTE: This exception is added for the purpose of setting breakpoint to + # debug weight loading issues. + raise + + +def row_parallel_weight_loader(param: torch.Tensor, + loaded_weight: torch.Tensor) -> None: + """Load weights that are row-parallelized.""" + tp_rank = get_tensor_model_parallel_rank() + shard_dim = 0 if param.dim() != 1 else None + + if shard_dim is not None: + shard_size = param.data.shape[shard_dim] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size) + + return default_weight_loader(param, loaded_weight) + + +LoaderFunction = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] + + +def sharded_weight_loader(shard_axis: int) -> LoaderFunction: + """Create a weight loader that shards the weights along the given axis""" + + def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + tp_rank = get_tensor_model_parallel_rank() + + shard_size = param.data.shape[shard_axis] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size) + + return default_weight_loader(param, loaded_weight) + + return loader + + +def composed_weight_loader( + loader: LoaderFunction, fn: Callable[[torch.Tensor], + torch.Tensor]) -> LoaderFunction: + """Create a weight loader that post-processes the weights after loading""" + + def composed_loader(param: torch.Tensor, + loaded_weight: torch.Tensor) -> None: + loader(param, loaded_weight) + param.data.copy_(fn(param)) + return + + return composed_loader + + +def initialize_dummy_weights( + model: torch.nn.Module, + low: float = -1e-3, + high: float = 1e-3, + seed: int = 1234, +) -> None: + """Initialize model weights with random values. + + The model weights must be randomly initialized for accurate performance + measurements. Additionally, the model weights should not cause NaNs in the + forward pass. We empirically found that initializing the weights with + values between -1e-3 and 1e-3 works well for most models. + + We use per-parameter random seed, so that dummy weights are consistent, + even if the model is partitioned across multiple devices. When the seed + is fixed, the random values generated by this function only depends on + the parameter's number of elements and its data type. + """ + for param in model.state_dict().values(): + if torch.is_floating_point(param): + if current_platform.is_tpu(): + # XLA device does not support torch.Generator() + param.uniform_(low, high) + continue + + generator = torch.Generator(device=param.data.device) + generator.manual_seed(seed) + if torch.finfo(param.data.dtype).bits < 16: + # uniform_ doesn't support < 16-bit datatypes (FP8) + dtype = param.data.dtype + tmp_param = param.data.to(torch.float16) + tmp_param = tmp_param.uniform_(low, high, + generator=generator).to(dtype) + param.data.copy_(tmp_param) + else: + param.uniform_(low, high, generator=generator) + + +def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]: + """Remap the name of FP8 k/v_scale parameters. + + This function handles the remapping of FP8 k/v_scale parameter names. + It detects if the given name ends with a suffix and attempts to remap + it to the expected name format in the model. If the remapped name is not + found in the params_dict, a warning is printed and None is returned. + + Args: + name (str): The original loaded checkpoint parameter name. + params_dict (dict): Dictionary containing the model's named parameters. + + Returns: + str: The remapped parameter name if successful, or the original name + if no remapping is needed. + None: If the remapped name is not found in params_dict. + """ + if name.endswith(".kv_scale"): + print_warning_once( + "DEPRECATED. Found kv_scale in the checkpoint. " + "This format is deprecated in favor of separate k_scale and " + "v_scale tensors and will be removed in a future release. " + "Functionally, we will remap kv_scale to k_scale and duplicate " + "k_scale to v_scale") + # NOTE: we remap the deprecated kv_scale to k_scale + remapped_name = name.replace(".kv_scale", ".attn.k_scale") + if remapped_name not in params_dict: + print_warning_once( + f"Found kv_scale in the checkpoint (e.g. {name}), " + "but not found the expected name in the model " + f"(e.g. {remapped_name}). kv_scale is " + "not loaded.") + return None + return remapped_name + + possible_scale_names = [".k_scale", ".v_scale"] + for scale_name in possible_scale_names: + if name.endswith(scale_name): + remapped_name = name.replace(scale_name, f".attn{scale_name}") + if remapped_name not in params_dict: + print_warning_once( + f"Found {scale_name} in the checkpoint (e.g. {name}), " + "but not found the expected name in the model " + f"(e.g. {remapped_name}). {scale_name} is " + "not loaded.") + return None + return remapped_name + + # If there were no matches, return the untouched param name + return name diff --git a/vllm/model_executor/models/__init__.py b/vllm/model_executor/models/__init__.py new file mode 100644 index 00000000..d6637351 --- /dev/null +++ b/vllm/model_executor/models/__init__.py @@ -0,0 +1,23 @@ +from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal, + SupportsPP, has_inner_state, supports_lora, + supports_multimodal, supports_pp) +from .interfaces_base import (VllmModelForEmbedding, + VllmModelForTextGeneration, is_embedding_model, + is_text_generation_model) +from .registry import ModelRegistry + +__all__ = [ + "ModelRegistry", + "VllmModelForEmbedding", + "is_embedding_model", + "VllmModelForTextGeneration", + "is_text_generation_model", + "HasInnerState", + "has_inner_state", + "SupportsLoRA", + "supports_lora", + "SupportsMultiModal", + "supports_multimodal", + "SupportsPP", + "supports_pp", +] \ No newline at end of file diff --git a/vllm/model_executor/models/__pycache__/__init__.cpython-310.pyc b/vllm/model_executor/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d17d8c800ccd8a5607c0faddce4393cc54246270 GIT binary patch literal 684 zcmYjP%Wl*#6m=dmlV*|*EkdgJ0L7%?u{@2*Sn^FG+kwvSuw%h5 z@FCu^V$X^dzMjrdOF7a#Ip^G)>-1%r6MxQr{9ONu3HgP^-Qzj(%s=LnhlVtfA`sCC zVj4q26G&+a6FPw@ox+UHAfs7?x>y}RPIH*kITW;jl9qu_)FCYBA|fqm(r*#H*G!hA zn*NS_e3iNKH8*d%PUxlOR=8q04nwc4dDmY(cgMTxVpLXM^bJ=o-*K0hZn5TuNpv$4 z?4`N2Gu3PE?(a*$00;r|Jvj_j>K=Sl3VgIsFMIt0R-$QS*Sd#y=?~(@z7oC-x3cft zu{2EVl+33J^sp%*dO8V+uFiV4{|uQT(`oWK7nx!XYjRFa#ar-yY9RCi(hOVw!_4YxE^h< ix236>zcpzYdpvAh_S9=eCC>4WFgWZZa*`BD5&r}JV!wO< literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/arctic.cpython-310.pyc b/vllm/model_executor/models/__pycache__/arctic.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3dd3e7cba6d2c1da99dc0c987a5d381ed043fc13 GIT binary patch literal 14378 zcmbtbZIB$tS)T8io&C7Iz0>K_(pr{fuWg+yD~avcapc&RtwcIImh2=$lFe#o_Oy31 zyL&xz`ncLlf=`J}fgpehAz(O#01-u?pd3D{po%JfQ3d>f6g5RrQ2g>={>7GZJn!4H zA9p$_sba6TxBH!*o}QVvpZDpf`?|%VtKf6&`+s}kKfb6a|3iiGkBPzweB=LA6~$LP z#aDeTR3pvPBHhy?!!sh&Gb78hR63@IwrAtl2<^!69JZNZE^<9L%6oY(TVWw8dPUCL zVJVvMCZe)ejwZdysNz-F?u1j(v^UN9T(~2e@n$&hhC8EO-mYl3x0}oPa8IT3?T-$42coK1WqUEaH#+DYB9h_KGy=Pxk z{Js9ZtBSuba27tXrhCu9a*w|smi?LKc~}nkRamN-HS%Rn54DTghKdaOpSUG&E(+Wh;X$D8So$yNaY^Qsv6*d-w>a(Jmw42qa z>j&ZNd*mTCGj&cSfoPl$gVZ?Pj+4}RxtFxNokp0N=azdgo3C|Ho~x;;*6Gx=H1}+h z1RbiAP7bqYf~4U$k_Kw!pKCNP1kZIlt@eD)NDK2pQt$QZ^P;=lOZQU=r^j7U=YG_C z4bce0AgqfLT!$n*nvB-DxNFusc5=2o&A3?Qr?o zj(=u3#61*S%W>e>gUh`@BysA>V$$s`qN@GsnX_|U5ux*Y&IDpU@Sp33%Tec*tf!~j zouDDo$xnak^)06It3fYpHycUd4=TOdy|lR!E--PrJKs*?S46iN#CQ@gRX)Akz%yNG z&~we6ot92M9|XO#y&&*Uz9Nn3^q7enjsmeR`&nh;OM8s%d>Qo0kivqum$8nC|IGGF53FY0q2Ru?>bk8pLdR@GUS6-><>CSdc z4imNG7?)uO;k!&9h~UNbAnGMo>g}=CGlz=ntwuBHiYsZJD~63Lfry*r=i{GZR)|j{ z6DXbvMAUL>PRr>z!=LcYK=-Y{@NImZ!1Qy071+BK-wn)G;Z3Eb`gy-_wdgs1(Jx(9 zyxg0Lp?EIFUKwM9huE3Zue`T`&&j!Ssnz6>kUHF*dfQKP((Ncafm;={pq`ymlHN>J@G9b_J$4_Ir>~B0{0$_DlBj)UO-(1wiCz7IX zQRz1o-+o75Q{PaN(wc%g6TX7S;mp0Kpe#r4s~x6|b>o=75UpksZ zy`#N$??}=B&Ll@3kCrb)$D&I|!}j^3y(`IuZs+L5FpQ2$tPCy(&1HDFEXGG0>}|8X zE8+=Uf^I&32uaCyR11HuW~j!7rMs$zKj)TZ7@R}?rfXKzg0@l_I_&J=sWs`P4L$}k z=yoGqv%OFExb!5(=?{q@=oPNj21!$x+v{I%xe7 z)WE2#<;e>=Ku36iTc=R798$heuVaLT@`$*fuAsvk;Go@ZDClm*Zc6BRiWli{F?(Un z0MTW}IiKA#HJ#I}k-bcEjY1)r9UAv_j+D*&UKG3Yv`{jH1oF=Mo z`V67zn=lWc$yYrc;6#5qh=&1KG672#V95k5*#u7u4ss51IsE1VHv=ly&jY9cS3zN+ z=oeA1Do#jK34*3WMAfM)mF?$aP;> zdD^YwbA|}TkzN)at8EHC5hwzW;A*!uARcGBFH}zxD9`d~bLU2*CAE{DjO_(#CyMKS zas?xLICO5qz8oHrj&IyY5ZQu#dM}^oj6??TiBQWV+{3TUNa&!HL&?Q&9tdabA=JwE zHDBFH$XS>H5-t*E7L!unOeWSSUj_;)^~t{BE6ecdq_RGBU0F5ygqG82r5L}Q?C4`W z^v%ANq38@DVX_lP(TfG!N8N>KV5!Mix(lVUxSH%nTYLC8uCS1wMsF6(mVRAD+oo?{ zBMjd=szpbs-j*8sI){^c*7qlsg#$Qh1Dl6!`GoV+e(suyHmR+#dN{(ZZK<;_sp809 zpQ6CMZFNmk6}Idfw=mA<=L9*vpkNMoCx{)rx0ApC2K-ls=HtlKG%+-!MswM3fSp)= zaIxJ4&VvZ6Xo-|s-%alBcB%SOmN1!Q9MA8AT%abW9Na&v$v(({V z^`yI)njn7lMe$K;Wh&|hbs4H#B3N1mkG_%?5T=Lr)NO;8#7P4TTkN58931Aj%^*+B ziRJ}wk{H>5MM-bwTjm(0BiM@>I@uuX6MH#(mdLGkDMkQ#EEW{q(;SgANq-KG*TfeQoh5B_s47c2OhHZ0stZnLd8~wKXI0N%b00<8=TMo08+@)i)@`3P(L;3ii zOd|{B$Fg$G5d~Pp7m%byAD9Y<=4l2ddo){M*px@0P182d<@Z*nHa$1KfOySd?wOq564TjvzlHPTDiTYZhQG}qsvo4t zt{EHfuMJl_2rJ_J#>&nt&cjbO0fKk|XQ!43l4a4^L|W*ki7K#1{1_R$eI+9->N|F=+u+VKQT|#~6TpW7UF(5cI-ZB#`4=8y=!8SmHB)aE$@lUR1>y*qrrT-&uFB zE1*FLDhv6svYSxgHF3OjUFkau6TWp#UoRs^wVe)utFu0dFe1;SoSRf|b-LCU`#GlS z^2$Nq_;_KoJIZ|D%kB#FKl2IJp*2Sq0xi4TxC~<%*lefsQNFm&d_DWZ)+X zFp)?*yna>d!#f7VU6#c2sD_aJGx!mLlDaeyZRZgumMF&oWfWD2DgO(;F$s}Cch%6{ zjWLdnu{-4Gb93P5)8Zv+<`gBLM3UO|y5DVrU=t@7pQ41>IC+YofF*f3&Dmx2r|2kr zB&j9Ug;b9d!Pp`@PJI$*C>eG31QqFVFpVImia&(JQ4pi}@s6{03#{E4W9_Bqna?A3 z&6IaN$SpK+@}>_9v9QDhk^=@0aRGmj7Bol;dSEbb2agXduyqr^kQZ7GOAD57v$Ws@ zxdj(Yo>G_7d_X)N<(A9BgW*ApfbdXc;bB60#88i$V^)yn2BO2|w9q73E>jkmpKXew z6hR3y$zS5nGNoRFpGYYIlo7sy6le)dgrJa^9AgKhnuG!qq*-0m;sc*l-XQ#fg=zJO zY%2?H-&{1s6F7=oR{wFT-^n2+!U*HqsE8l(U9-3ifn?iGG3_!c0*7$Dx%_Fj(T?i}MSqcO!(eE#e2 zP73nFbK)aNYli2FE~4`lQY5@W)VN&lb>lWmuEzOB9C$@A&Q8$1T#u70VF1mnKuEZF zv3>!nUMw!76@JCFeUdaYUuJ3I5*;GGEQmgP%!e76sdag}H<94#VV#=f_PpFE%O@-b zSLzo@2+59DMkfsVkm~5RSfOh;)Ck>5YAwk*o{d^8yX+jlHz<3V{ut8(=mquU0%T>+ zj=|}n1d1JKEp?V~a!+&@q(EbJWnmxP62V3~cy?$_kVmHm?ypm`#>g^1gyBPywD|-z zFhVqXVky2c0g!8xoC019ZVcH3Iai%lD_VsCy2P9r+?nJShIlYz%=0DO6KJF4$YTzd ztgJ-G3LmWO+A?@Yd^k6F@3CY%4(^@DaayFCs$*invb%`T%T42%I1oW3V?V(pa9$y) z23*Dj6E+u^1;Ilxxm?iaby^d;rhQ%~rbBFG9aCpw8rSs!`&kFC?K}M(dGvL2mEwri zce%wpTFe88bjdUz>FRjjn4o+aR|biIDxL_Mafnq)zDmi@Qt}lfgOL6P87QucHA=1` z@pQax4&$l1l&XuVdeO`Cog>ae(oQY;I|lJ|!>ruw0Eor(7)o{(Z;jp+B2vaCpB=1W z4C9iHxKt}`2}?2>aUkN5MDRW)5OHuy$VQV8zUrML&P7!DcYNbVkqlHK;{(!%DAHIQ zX-XexPZ_X)4%{+eftK`vVO8E2baoTpO`m*Liiw<$N1IWsfnH>0amHjB+w#?x%drL& zl4H&0C_@pafLH@9=?<|b(+r;vn$Ud&3pz5~h%MvlHm9T@Lth==_+2D4Rm*}50X$FB z@T(7%7(*&CAm`9=y1OYz)XfB{Gm%@Gn6Xw(IyPAd)XjndrJ8~iDcGcfL*oIsa{#%` zLNIQM&)#M)B?uwt1>!NFomwjoCt{$}htYSbcVNCq`A;K(x~m7h_#8`@fM1m&g!m?C~pR!T|lfyQIHwP&q4Q1kO|6CXKwrP=^bM^08SiMoR{is&P=gRa zLkCSqUEnHd=hP0qBzw?R%E_dUl}55Mp|3C@cCfd=t8~RYGtLnSY~oc8Ka{5-O~@A$ z*-MnSR9f&{*L=)neLS)?^)2NsZOvdFlB%;TdH{>*waOXg*O9xS3POY%x@5^riW$O( zBOrC24ja*Vzwyi}ITLxLr4+D~0}8jXa(L_=3=wJeX`YA2&mbdDU9o2wTx0QDbopzP z5VxsKjY5HkwGLnE7g!AT@+i^jJMx!P_ad##GfNg#I+-=@0zaLBoPM`tt z4ke5$#Hb^BpcFh^R*FegrauRlB}9neiCl@7`Y10s4>Y zIX2XvJy@T07IM&ZD0Ksueu4T@*D-6|iFZYjp_c`Xak zr^4&EQv=(u#w6m0m96Af{Fz`18`5^rdDJeBY5q=s7tWpG*s>Guw- z9F8m)R21!MJ4To*VwYA}Z3jt0uayIskhY?KN2`DzcZwer=?~$!VC(O8bypRChGT=CQY(Ub>g7AZWo);U3nW{*Yf0z(C+a3$qJUN2ff6;|MOiJirO(Hk z78s~U^oF>5U)B_59un|(Faw%bR&{(SP~!Qnnh9oDYbpZfML3AQ&O6G$9`$zyIRiP~ z@u1S;>cBcWZsE91`yFr`Vk=Yub5RqY8>^cew&iZy*64VC)YeiNJ)=Gv+`GdF=LW7` z@QZyL{VTnLV4hu->=T*2yv07KKRL25)zLQ1Ij!p;rmIe3uRDH)`%ll|dl1!fkkE)k z%tCa_uNGE|{X#!?Q5D}^`gT7Da!}~!@tSkUyZ#gwkKcv#=HU3dOW#iN$U_tte~r43 zFq(UP&%^o_Mn}l*f04zu+LB* z+$-%&S$doCeg-?vK=?^?^q@7fO8h1YL))UXy)|yj)a0LoZN-)hd3I&j>~NwCDP$J= z0kg&Dm7QCU3>M!pkd)hw4o;Jb(Q&F$o!u(u%x<~FfjiY0^x;kwhF3dJ7jm+lFS9Ey zZeY;OCLMxdut|XOgY#B?a4M-@YKLJpq5a}tKK5ueYIlHXag}yt09mUo(T%Fm<*Ua& za^!p)wRwARJX?J>#KLO_TfQy^;gv_KP_?)krlf)i3ex0mM_xXOiGcvS*DZcH&{jdJ+u!G`nKXp2z-DtY9C-&B?tn6Ee_W zC&M2i!A8wq4^tsQK^wat+Rel((}og%w81!LXvco>)M$5F-4u(_ybdlxYK0;#x1HM9 z*#r4RhY*;P5lU6jHS4$#o8=0+f#vJXR%@e-m_fTW3~i8F8`=j2kZKBf1w)dO0#-_}U2-~@BSJS>~{PW8G+equ|_|1u5r`vxX zNp06pGZA@$s*gl{or*M76o)7o%|Q-Q@y{vQPsv|Ug6O9Rq8EavGk;IHGn9;;{s}5l ztQ6lz^4s*}UquFU6)2;|EvTPRMCEFLW|VGf2aNZ(pQpG#(h78hbS=klZ)!O_IWAc! z9dfw++9?sE_0+;fO8B7cAm0*&d)ZaKCB9)BHm^i`jc4K7QZnY{GkySGo{z|F>4Ii% zUYRXE{$)h8&jTWSp18^lM@6~-31``PU_=;If~Ha4{d^u%Z*0wz`SZ` zhwf5W{>ui5;gfW=3blQ=`aGmq*dPhXyZ0_lW3dtc;K5tLdCieG>hW@;_we0LW8ODL z@`?L<58UO93>kT!EH5j0$A8M&vp)VaB&PoiY4vy$BJUu)YxlAx<>6#T(A3^Lc$W(f zrYAI6VNbtn_vEC9aEkw(XLKE&TUF^|N4C3|bOqk=C$b<_UryR#%<#IszuZRj1r14W z_jc{ew(g_jMLr(OZJhy+YVR=H5@;(iMIQXeh`jo&boO{w#wZppJeQRv{F_M-iwDqR zYRc9*Hq>Ac4^i?GC4WQ5EoimadCto#a_j0vI>wUZwN9`| zB__L}?1Ms9P|Nu4QVS4Z%D_tG49)?AQO2?C5BusIy`onRD_^O&mD80McOCyfw<+{{ literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/baichuan.cpython-310.pyc b/vllm/model_executor/models/__pycache__/baichuan.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b43274eb98f896016456dfc377611de768f2b3b0 GIT binary patch literal 11579 zcmbVSTXP)8b)Ngq&R(!sfB-@9vZN@IkWA2|WJ{JMnHNcxWD=r8OO7Y9C(E5#V1b$4 zMb9iR@FH;;w8bQ)%C&N{T}9*}36nULN>yHyR4P@e$`8mx=fzcdV&&0QDT}1YeCN#U zE&zgbl10ry_nGdVp6=7%`TF#9N+n0Z@8nPZVX^vEMfp$4Z2nouoW>RZo2n?j;wrxC zYoY3@_^*drq`P`#xJG2UW@Ncmlyh@xTE=#5lrcg(%DZ{inV}OE+ybYquo#uxQdD-! zoX>@0QN^u99O!|^o;vVbi_Tv`AT>+I_4gWX5CrN zkB84j&$-WWx*C2iI_@6l^h7upop4VeJ?ZZXKOdcRPe!NQQ=Fd)pO0Q}Ux>coej)mz z`$g!cgBShXfxYxn)9`2fJvR*ZOJ7y|r~JJ)6n}4!UwV06cfSnHK7T(n`!mfe&>Zk< z(9|-`tI!%1YyWD^8^SZZ~Ol+VxPHU+;CHHa}=1KdVZu zO|o)P1c|7(+JS$r({8pFW;K~VlO#c#s>$(T@7xgd)P)>>?`t1!SH0i4x>bed#?E&Z zT1k9SbQ(d7o!GC#i>CyBB$jc)M6yFrS`iH>fLvpN&Kg z*EwAAzeD(HqTHfO>nj`ThB~3FsfoU10F_cLbiTG*DJTKUX(oB9(^q#Z>)M(%cmnEJlgc#p zm7ZgMVyxwma{2~8&Ftqkv<1z#{2cma^bM?Bd;X4wx4WZ5VA@PSx!w)L0m>QDjFY-Z zq#m?=X~ap;6|=Y@c*<0%`>klUAdRS=EK0K(b~-{DVQ1l#G%@InMb}D~jhn;0dh+}! z*MfDo7XsJBWIcaM>fO#&sYk80)Eo6=))vp8b+_6GJ8=+W33jdq!s|4>Q_{Fxj|10Y zP3&562n&u~C%A@X@A16}nobc!KmoE5o>;e-7D%hzR>T+4{@r&@BvJQ7x79t4m&0;6 z{$kWyj7~;ZPlT<@C%V^@#ZLRgl`xD>q{}_H7BqTzp)?zxxLj{F2-)Vk*X6iJ>#2^4 zEidiV3wT8Qd5B5XP2-BtG?rFf$r{ zT|o`oSJ405yf}*X-@L84#^qK$7Ox`b8ZmHnb@vDDIE9SBuc5=OD~x+D*H#bC)!rqC zM8#^YxHc-*?1);sb9HVu4@2+4>ZR=gMp}vIi7J|gBLGf$y{^|4oh30zs=VL1+UC4; zKCFkm;0+-;hHnWK1-#A%3k zam9NfoSdeTb!e&$>!AOXvSBAhZFO?f6wVF0+n^;p_!&shWSqklKMT=UZUHGa)Q^Bg zYdUh6lnr$a_yH_S^(yr0C3J6oRu^YbN4!qrEX1rK-oQVpUUPHVct|zt`|t&14ug5M zE_@$FW5UidUZD&FU!B<8;jEN+iyC%lyxa9C@I2{wUzl78UT=8xQThVgHJx$XdXu9^9W@z^*$VB1KVOFmHCk`tadwb zizox!YUbvpLxjOg7Qvq6bX30vWZqsTmc<4Usth66*c)UAgI^g5$}>#73=cAb3G!qf zyyUvFMjlb|01`_^-=2aWTh`*~ca-m;1ZnmV7?@Ok$>6lrhdq0_KEaCJHzD(h!}l$& zfpUNu^0>%*g`}8i;UyV>=nmB_L7xKzY2rIc8U4uj^U#=+%9_*9`{p#B%eg`_MlGzX zYsG%C@AM1b*JhMkl??@X1zz3)D<~!7RGRYr5=!OZZK-Ef0j~JqG#B?%{`bT zJ=I6aU7K1^mUi_mw2>dRG3(5;3w~3OQ;_DF^KYMhn_c!oI@&+|M-aBr$s6@D1GB&a8dJ%H3SjWfJc?{4&gTvKIs85x-0=Unaq5!nneaavHieu6Pl` zaV!;oS`}!g!Ic*^&Z__zXy}gt{H|uFpbjv<@8m4TM@Oxw4$7xxG_`nNv(h>SN);`- zlin-dU)}Qv4h$WD%nz1iEK38op1^y>(q6ve@zRtf8d(p_nfE~A2>QOolNX^FQeJv( zb{udOa2)2E)i;-Do33HWXdmfoxiuThL|M|jCz;YoKM;s>L(z^vNKu{aK$a3N+b@h$w5Cf^Vrk-kpiS4eyt zVz!tX1cA38_ag!lO$b+CS-v9PK}xDCQeBqn6}P~H(BYkqw9@~nBOXmJd6vJ$6_bgU zbCZ~=D*o7=%{)2@nNx$KyoB=N>k#wvvlVfka^!i$JcP`7p5JM}Ns*%$7fD5=xt^qs ztw7o-5~oR=gpgLcNfN{Y+24uxD8)TxG4gWnf)7u3=U03To{UZ%Cs-zb)P{uY!$MGB%1$PRc}?k><8B zQa~L73l2OaNrz=2DELLxfd>Av6ci~%R};Y6Ec#`C4BoKB-mnaN8J9x>^m@=hXhEE( zP2D^!hpB0b215+kV&7!j1TpM_et^Vg7qmo<2TllX0XDv_ubG?%FGbn{V}+&nRlvFi zH$!d*toxH3>R1UR-?MQ)552{J?j!}|sZNo$TPRt2L{ny7YtN_f#n!{1NhUO#`(I!j zY;I?ksR!M7c5EAj%N%wquz$hx>++Tf*)&X&?S=D1T|HVPOo>MP6&tm%E5^|?Srk$1 zwGqAW@Dw@LY9ngk@#dP8ayR({Y9ecRcn7aUR8j~MACeeO)FuFDn`Y}CV(mYpY)mT0 zwACWe7H0iW!O{%r&h|2&j;~Y-miqi4tH_4-Hw5ip@E!0u+2p z37{yEbJjK~w5H8tl6UAKwAb-$afQSxiK`I6OCI7KDv0k;As*P#=n~ZZ6jyu*!p8U& zRRd#YV?&z42Rzz+!zPWR+3M=v9Y=D2fEsXbY#Z(gVtovEkHtCBr6LS+CZ(uTk4L%{ z-2Eq9@hJ%Kc@xHPADx9kXb&tPnPrskYx}q3nhn$|=c=Cys!zkSoWl?t#<)_3a206io) z3R*a`6Pv)4I^h<(;J-;{84;A0?-r1wm?+ZMq;rMhr5sohPf;@j(z3`qs1)V(O&>8fxP&1zhWaW*DHYwpP7gwKR%jJ+vRLPVGF(GM{>fmLD^S$^3wh#!S)V zhS7#Jru0GfVg+hVA z`x%gaS^L9-Ck+zP2 z37(#(JR`gXhHGI38}pmYM%$EjLt#6pOP zrn;I1ps*uh66xHtGdgKMs9^e6cWs^SfdkOz=fxV@njQcDdI5s2)D`gS(mMVH{(uns zGhFdY5UD?aN4U4uBM`SO-_!OCyn>yvT?<~JNM3mE-0>*=Rdy&?|>L=0M8KQ17f%E4M73_;T3`+*lp@L(x^}= zZ{awu9F&2i@DPxdphEvqemodQbSNKGk)8-9NE=i$?|~!38>TzyPjQ{;U>8c{(|S$I z-|f%9w@mqaw%nb@vn@U{G~C_(zF;QU6WCOO+QdE0-|rtly{FhG?S(}jlGB6vK8ry7 zn;mhk-izzu`3vdpEYE>KijvA)7A{3Bfi`X&OaxWo^tT5J4lK^^!~>#9{#hGdL z2R}-a%OC@Z3I9gEjRHO%@p8nLITU0h6eMDvwNBw3;^^wqfS;?MkPpLdbV*w(i023c zmWoRh0&>WsQOR{`aV)8D^qImr)>0LkD)V(PGBu5nZ1)M`chN7t#r7mkK0y$BNc%k! z><4mTdhp~_R13!I=sU_b6(Tp6-d=q|U|Z@tq(?|0ICt zPQ6KwAMw7^$eyQ_PrT0oH=+mzARh4x;>=*s4{W`R)o+vj;p*@_cia3A-Klsi7#+bA zwy+e!@gbH^A;Ua5n=i0siZuX72PEM4P-3uic~&Ed&q7NUxK(Uy~L5ny~MVPe~gDb(sAlMcf5kcVC-=WxvA5t z_(R+q^p{!@)L*yI#);a1!E7CFX{Gk~9qKGSqG-ZT?0;NG)50>B9sX}Q!Jt4CEqCtZ ze=ld8wRm#)KMaTWuSDr;|8E~?$ ziFZ&I&jfLB`p!~et$-}Aul9jq?M7C+u~Hn?ZlRx9ZA7FUKbMpcWXRfB z*HmStm<0v~r!l{cK}e5Xv*VfHAqAb%u1;hjU69MUR{M(A6v4`BIfIzFba-3Jf2qo< zId>VSX0o*PsJ<)>ztu?G3Z1w3qr>$$g=dDXlk_xNvaZwYwQ;(}A7kW+esNU^jy-5&hQAd1%It7Z0 zPPrxis=&{_F<@47IXi?sB^@r$=SXfLdo`|Q$BQ_S0w0$q7fCy_&1j3%bk=aZPNw@7 zI>({g&x$6&AD~bz=CA=q%6M4T>3>nJVtY*CT)-o3nbj(Sy@NcJb2RO{#(%6Hew_Jl ze(Y_qruh`@_HCn0?i9t%F-Ym**l?6EdV|wWU<4dycHbQzp4tpmGb7btkCK&s{MMXS z;3y86?#~JTc4TJz=mv3eF+<{qBs3C)kmAosyhma*^z4jyqw!m~L8n0+Nk`Zcdw3AH z2i61IHW0xnxuJ&iCBQR@QIliPtBK{l|ZWp#K0?~#xwN>&fg7$)GqfK1rA(G0 z&)cR&n-_mc&($`?9{vi7od*10X~4;Ama$XI26}r>w+^)UYieqzRrxo#@h@mk{DewS zZuV?C3v)>0yzV+db-KtI7IM$pZO2%QV4mh1EWt5VkIZ75Ti47G-)8?aFpNa4hCm(IP&poNeb`nqA-aYz+8vMyk_5mN;`xDzA(SY>h zJU&8o4?Us%!Ty0vkfGas{t2bikQRAT{`PU`RMOoIMY-JLJ5^7og*~{)=!n-#T4BtT zVCT7egnSF~F>Z<5kAsySeP=2tK;SwYKf-r@K`d%eh=UN)OrO9U)*6Uc)5`4LclR~qpxA1DDf*0#)PfzMPRH5N>I@% k>Lj8mMUpye5HzDa-J4V=jp|HwfAxdvt5dI5u1!||2M9UqdH?_b literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/bart.cpython-310.pyc b/vllm/model_executor/models/__pycache__/bart.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..363f805070215d477f8ba0b47d44006e6245eb95 GIT binary patch literal 21136 zcmdsfdz2*Cc~{q?tE+puduDoO-+HvPt4HfGtCeiamSx%UYHiD2jbv$&Ny(zsJ5@8= zv;CO8RkPY{dcb%UO9Yg2AUJ`<4!a?QEMN!$1L2VnULimToWtQ!e~=s!4mgKDI7f#Q zvf2l~-*>C3yXPe-22M=(>D#w%-MV$_-tT_j?|a$mGp4GKIyPNm&UB`0_eZJroaNh0|x~}I+S-w;3 zmb{YWoz7IZ?3E>7=v2B@uiBmVrn@uVjFh>Z*>26Nb?3Y}IWBhQyL-Gnk}q}kcK3Pv zBtO;J-#y?R=pOVAb`N=nq^#Vzt$Vw7yW}gKJGyszcS^q6sdw-4?&{v{-7Uw{ox|ND z-jVJ--aT?W)48{MpLd_+XFK-qFwI z{5}5OH*)^o;GRpC_Xv*n`TKFaUyhIA_<(;9#|MKWIR7Y)5Bay@__n}ZeGDZZLdos^ z9VodYDfuu;?)2*@sV5~LLCIbI-6**`DR~?vhy5ccIg*ro6eaig_oC$9q~v2LxzE2J zCHE&KPoSjXKY)@4l9G?3K<{z8`cJ zzC)r|Y}m2;Y!s;0=}r*aC);5ZyU(vh?S8M-iSwt{*E&JNh|OLPY3b7~6&>&QmfI^h zDs=iQEA8G&!-|VfM^Vt@Me%HweJ+Svek*Flm8|I0#kHVe##6^zOJ@SzP+U6MfAQ(8 z(IYEC)I?ihUp1wh&9#@H~SSpDF2_85dq^buk_) zt|Xb~*Q2%d2v1ba8h!3_LCcT#zT97GolXl+bWaDq&!IBnJ$|s7*Ms)TnW)+6 zxBP&&oRKzBDIGVI%{|-0)N}*C-HL)!GAiK`R&DmNkmK(-e&GU=VQw|IVMNBzxMXaY zL(@0TnrhZJhPlh;mE1C@VQBa{-~4{_Kn|2)Eq;g1eaA$Camrr3n!|>>7P}IWO#dwm=p{X(+B@xSOL3;!2ipVz`&A zv4>`!Oq~DW9Y{F6&0J($!XyuoU(M~y4b3Yi*c~U?TvWHC7N*kaFE58dgn3?H0~x!` zX1mvpnoXRt!=SUQ?n6CwKa#6PTx>R%I;}8lHf5^S5uSSx$tUm&*#RfN-x%Dt`#>$E z_lS$>$b@mR6-B*f1a1&JDu~uqZ|4WB<3>X>%B>n_t+4R<+*ggr#DiM$p!UAp6$6iG z3?6$eBh&lpLdJDWO>ki?Sc)*>82r;g{S{DH{X!iy1`>O9p^;ZpXg!`jv3UIXXHLA> zeCCAws|Q$GSnjLWTgq1r95pQU5dIYDQa!@tVJ3%}$iO^?+zI?-V5)_hIk@*d1}149 zw1#=;wYovG8JC*mdY~rc%gyGw^;Rc2Q)o8*{t}98bQtx_d9`eU#muN8;veW3O$xwrSopE%SzL@4c>AV+p^BPa!#a=Oi-Y ztF37IWv({g#xEb_lXd0zg*PgmgSA#zb^RjKWIdMrsfiU=iT825URr8(0{=^W<&T!y z*)2=0e?CyE?FV6fdp>(#SeI*(A+DdkSnq=8w%0lr!C>pJx1%%Ti!_-yiK$+mZdhrb{EK8G1@T#$R)e@?bnNy!c-K(~y zsBFne>1MLxdixTa=Y6Vn$r#jk7^ZBYsk=~B9YeB2ww`{NXGi%9Ye>@0ULnUY=h02| zG>X%Xl9F^sP2Evxb{ney1E}~8Tdx^|J9p|_G80z~^%U!FoIVWQ}B3PNV9SK!^3Xdu?tv=1%aVDFll~lLN@@J3? z4s9Kj?~>RKrNpMlGSz38Z0Z6h{2a?ai)3(cn=Yj6nSx(E%Vay!_ySIx#&1L#r7fhv zI$KDiib8(g=^#m2UC}6-{Q9$Cb{3eO1!iXhROAEaKn}1X9~AKK;@>@xTP|ndqT&_( zsi3qv<(DO`1ZBU<+_LLW`!jFYUPZu16~e3*&+~Cl^p=3r)d)N<#zoDE00`nD@xmEk z%`mo4w_9QCED11=%S!~;&GZN&q!%SeODVbYx!mMu;s2dyfC4o?xRNt+LwnUE>!OHl zT0`b-@Cf{xc`~aCt% ze=V!SGPAL;!Wqlue*G7&n(75))Jse-X*qQp65zKoFFAW&lZ1LPt#ftnNbJM~U+YwA z5vRTBs2{aD&FofQZB&pp%k=Z|?xUyjD$&}Q zKDe28kbS+5-#fn_NvY@>4!*8YLONsG2EGssrJH5T#8HXI284y-Uc<_k9P-zlLJew1 z$(*-pcFm~b>*#BxM)}>r?b|KVj4{TQGi~1wdd)C`?iR)_9#vLmToLfwO|)Yzs43mk z&lECToMg=M1kl}}wOuj3Y;M>ac`QznJeLpf#j3Sp(SGx``DJUv*(hwdD6@wSNZrN4 zwW2(ho%Ob}0dl=u7#5|K5?X$4^;9Ts;uMp&H@rQXF9r{ z_U~v^V%{)q^SYBa-*p`Gnp3F$PW36&xMdr)TZ-52D3TP>H^jt#KISulh>4e=zKw|r zS>ijcpB)8(I)#?i?_fo-O`jD*zH970M6TDhx%Vc*6jj1jR zwxzICTO)<3K9j>)l95UQOj4akxdzVLjn+bnB*-(=g{%!}O&(ra`-3>~OFTJX!lNB0 zC}fo_n4$K;F$0js+%~u&PviicF)aw9%(V$o2x6SP zdqxhlfCM{%IVHG0OagGtBSEC<|bfz%RbF zo>O(-+HmnJ_V&s3cETc^VQI(foeh@^l}H3^wwN(m8;o{J810m_QpR1%Ky-O^2HjI_ zCdAc?ilFknC|o_t`lRkgMf1-;!~&lQ`KS8BNW20CFNRdDvFIEV#U#v5VmMXAA~K`x z*2wo^;==oJvN~0RbSvL-A?X~WYU(4$+QMK3k8hgbiBqf?U%D*R-A5HN}>;EJG3nKOuS8*fKQy3ao+_?G!;C%Lld!!uG^g0ox#K z6=EA)Eds*3NQ*(qFEIz04(JY;J|!@nlUj{u#>C9Y*2O?A3Q8K`lnEt_@`OK&J^N84 zV}RQbz->i@!LV!r3Ik>jA=%%~1BN_E4&( zS!)jF{`PsX$TyeQdpLl*p)y|Bi>K3DE-&{4wa)NLYkBFIXQ8IXrZ{{wo>^|g!!dS~ zs|CHrCZE-tW=koj0#x<(umCo=%q}ko)+fWkz1yt;t`=RFHsw(6}OZ5V1g@SU;I zFT7kApkD{k!<(dSLafUViN2X)|E^YiHFWEhn0_Pb0kdK{|$Tzo-C`|?x|OCsxhs;${KGld5g){nEX*DKY#?j z6dLfwM&}j8Xm{)*aUObZZ#lMFy^GBET0Jk1l?K_sHsd@z3f(mbzHQWj*i}K;_F?jd z>3!)~6Wte?=1s{(&GS?}w-aAK!Ikw0nzt)lM_5TWU06zOvkK*|YPx0#-lH;*br}+* zmaH-7rfm-dKqrGHGnqoct0pvr%4Q9L+*|7g-;G-E6|^#7m+wFZ!Y^+c_I0WRkh3=& zyL!EPFCIdx0j$e;n)pz^;wL0lo)B-<_ZDwO1NJ7kzcz_Dnc)C`ZC> zMhm+HKFe6We_{7C=;sF!c3=f=BHmyt0;5PpU%8lzDm>C65^66+T6Llhtv6aA&OjhS zDTcsH1)?niu@=o`B2bsiPqZMGoJM|8{SoxLVTerEGOsaaO&ODR)CIhKDXiTj?er?5 z1XJJ7L{<^Sp1QTuzvg2*!9^c=pD`lzNj_QLO1jNo&h!QK~=2L(y3dGbf4* z6}Dd?!F>*yt=6t1PpJ!ZKHHl4>rV6J%}e9!Ehpm zk<0mELC)naqa7&+k{`RakX{Q>W3|4HG0Pa{R#B7A_XiX?$z{KLk+Vs@WIg;pDqh%F zN{=7#0h7yid%$glS3*ngLwKn_4swD2Y#Ta*Xcj16Ysh!i z4>4I|BI{|isAz_|$wfuujypKhH6iR%s?`fII^95}^jL>$BLvo>Ta6e_dpZ_UY8C7gJ|I_9nIPoS+J~mB zaG`zz9+HI25F^m4zw+AHjYYP-_NoSeldE3eiFVR73@TfE+K3K;e-Nt zx%Ou6P2){-87ic2zu`u6I6r^McqVrV03UbSgF1V859IeH3Z-`Z8rnGt`prHja(uu! z{KSjCg2#B(MHEx+E#s}+TjnL}PnhQrxg^U0^7t#Lc-4%}g-2v%0?NLK60Lf_#GEV} zioUq?BvJWNzq|I-E4a@)ypNMJjhS16`^V-y6I}~Wb`ZGL37*G<6^WNRE}duenD&f( zgLiOru$aVTK*uF%jT+QCbm=*X8<7CMJQmGL;8iD=MC|CLA-E%+lLKnkqn@O=d{PBg z@}%sC*6kU*pdrQ;<}!%!DGac1!x9h$JudXQ%qQk~S0FjnUcz(RuHu}1%eggU55}FH z*+l3dhNFD=VybQxXE;A;|SOtY2$Xzi-ofQR^*a6C`8QH6O zlmi6-3FKEbD$p1L{y-R8SW<{+<13M_U#$Q`(B?-UqBR8L6XaoS6wWn<1wjkfK+_pJ znB*H2rdf1SR)?khJd1cb1Riduk0MZM9g`S9hZDxq?n!8XkjHwbr7ljYsch|qmP8sN z#--Dt>#@CpIYs~!9FIxlA-p~P^^RY^*k9k$7kKOtuz;(g$(B>XS#eo467`|1B= zo7uE(RfB~G|DMsRQ(&^2KCNxL#3hL5vd_1J*A>+}CClb7GDodV^^nA7)Q=rMIFDIZ ze}x6b08Uth;KHW#EgUyW8kyY1E4T#IUu8m*Z(I>N3iX;rAYH}_1g;=B%)j7OCf5BJ zh>)d9;PM)mY8{!{JoRKKpoV?3Y`f5uYXV)UN4qy{XD~O;B@>i#)rS9U@oH_edi*#V z)Lt{2IU)<~I;$|p73vjFq+)TiKaJu{aV5WpQ;mkx$Ju=<{1WO@(pbxWms&HNfr%bX zBXRZJY8}Ql2?Mgt+D*l)m++hTWDw|UDBczV(R1_NLZEyGfl@G31sIwZFjN)~j_r-W z(3}QC3F!@fRaWXa81l6fWd|^noYi_`_&|z$q?PT_PjaUO{Yb@0CiZ_7{ETGG1lFO! zg*u~Pou5MHy|51O2GLGI9g5t`MdV5e&T%x(VF@BOOtBlJNavAFNGH!)9}Mg0J8ecg z>YM2DM7%Z#H^VyW$8nHi8e((xXPJml%@BnLF84X&Jsmf&K2L&xZiIy}r z(J|X&P~y+C2OMbi7nq1a{}aspG!tQFSDCw)iLf0iF=`3M*gS6WY}(mO zIFE`li6>sR5eL?W3_f-GA!8?@9>*^{jRXJ!5F{})z>gD|@Z7)zF)Yv=0pI~>LBI{e z2D~}|ATCPWh=EA1fQ>4^MscGwEWx>G4`G&riGsc$d&50fAG&}WW~+q}$;9{MN~Kud z@?=?Ep5`0#?n~W__@xlcB*!?*s;YTVYC#)M=p~%QtkX*1uo14T$ z|9_#HKLc`xp=1+^8KPAEH6}mHgezP9btZydl31U=!-Budgl_7w1@t#~_-!UjO#UVk zuaeHRXeHZ0mT!{-)KRyz4^H55z+zMWYs0^L-7aXz@{AUbsXbr)E#B&HGZEG*$;hVi zb2#w>yunA25l}J3Msb^KBg4Fo)p4EbHTQ0?8#u|$n{MKdw(Z(=#a8%86Q62w6n+Jn z5xDTp6%#fV`69$4iJF%nQd(9RM9N@N29dgBb`|VXk?_&UP*R4K0#sx{Ar*f{0z;|@ zQ?0>gT@@S4H2B7RTqK}_t{8N#9ZD#AVcgcRpjFLlhmslD?&N0Eg)s05m5lr`6WnKT zFcRGG>PcuRqt_5tYOB47Xk%2Hvo`M=^EsF$OsHuP;=??)u`tAx`CQY5I;Oqi@CV|! z6v2ZiUUGe07DI!*C=rno8wpzhRdqy9Jb*uqIH}uu+$&O*=c!Ocp;zszm9g^zEaO5lph5|Vw$wD(aJF+S%b0m3xtrW)JFER~ zktQi?OR&d--XQX*HiI(Z%R)}Dy342t&}#YSK*HB$ax-u+rby4GEN@KQr9~Ns9b2fP zXj8JKADZklN%sT}Xh0fTmvdJT?0eQ$43%ZbuDDwmeIVhw3?nw7HbcGk?YHxgd6Z!J z)-UcZ!Lm+n;rZVSk}Dg}ExrHlCD!{JnPg^vmoj||U2K(%lMk}{?IuO<_l%5`rIZT% z@!)}MRu?s+u}8~nw15!ywUZERSgMZoqy7PtUqq66BSt#^Kjf)@#3a-Hb->qjDl4@; zA{>)wnSxJ^N&L(9{<4|IZ#g&Yf);B`op&KCXESM#O3JGibo<84Cf(kvHKR6+j#@Z` z&9lM9aA#GY4#%;*JLt>%Qtkg9>irjTybmD*DQtq%N&E}k;{^GcY7ZCrUP@p_gi#91 zo;9m>L$QS|5_lGc{DdDh#wBp-;qV{cO}S5`Z^3ur)hNOnvcQ-y$lFv7r&6~ZRN&pn z2USRJ+1R0jGZRUS08L2Zyf^L7$<_1049+BBoXd{vI=8)9e_vYWr!{JfPxJTt2O!7i z{Da(Y{tXlVwI3>}P`>%~lM= z%6Ew=-XAg`Iv`Pq+jkrpteRm-wsa69Fd9zeJ4~(6eNaMFAdU>6h9g64y#p?=SKY^R zrbY=x^{_bd;m{DZA2=v;Q|a1Agah58kdMOY6*S-Y7mR}Q|O^+@Qt|< ze(>h74!HHeADPWV?Rp^bsej6Z%Ca}N-XpHSq8Y&m^$R?`os#_poWPEHy`@LZ@n{D7 z>#;4JyD0R&bhSq+g?bbEF0f)ujXum93R~c!JcdyZA4R6*l*at~?hO~7{~5R`x2n{R z4P_c=V{i{y#ZJ?|4cr*9t$2PSTrLTVQX>W@K!ZfV4;Vm>K!biRj!HO>M61+*5!~1S zfN{kENgAyWqjJWIw<^Iq3(9d`E{6Anr>4?Uuwflr-X{gu&Qg8n2GeWc0T4U?WQ(bs z{QzCvBx!f(YT9sm`$;X827oPH&KgdyNxPg4F)#a`5#GAyfCCId+%cx?+gG_Z~LR@`G)0T zqZ&7{dCuA7KiT8<0$ZUa)GKCwRK7&vG<})_ za|b0t)?AE@g@zN)o^FLf`U({l;Jp)Z30u)r6ms{wx)XPftMdF|v)fu*0}UczGYYz4 zTu{MUr?nISzdnR&>ff@iv!)xdLbZggU`N5c;|EKb_^i~j^@(D z^9zO4C)_bj>}-80cMc%JmlvVnDskYMumVtAI$(z8yg$VNr|N7jj1Cs|j& z#sV3XLNaoj$6`-9786bWI7fxfrkd?S+RhlX9}OPem1?(bC@v&8d}5GlXgQ-SMw>?* zCJe_Fgk@~t4H{T}I&Vb?phTJN=MW;v@H!hb^$G@3hsj}Ei`b}hI$%!9i*TqH&ga6% zv8`ACTgg9aW8j~Wu#sH&$&{Z-z@Kq2vH=Wo!~>Jd65I?8AR%^6!zv zZqW0EAH)T`e+Pk5N*zFXqox=BZy@i@3b0QX1#iKf!)p@-wNx*R_`awnZ{_XNPd|dm ziSpYB1Ti|k!m=H8`$IhY7?Xoc{uPtGOn#NguQT}%NMZ|QsOgZ*N)sb7zxpjEA7^qn zOGYO8e}b|Fj_lWv5fi=G{kNJ1%p>+S8V0yk3~#i#j`(`nV2kL6|791=-?pbp|I@6q z2F(W8C~(~@rbYyZM~Ek|Z%R~tk~+U_&WOe22DalA=@6(|r5i^|ihXK)LchD@r;v{w z?1;b`R`NXPY%+xMpdaHQg_u`{Z8;5U)K(O1y&o%DSdZGBP!WlFRlYlGAq!!Yjq*B7 zVv47ywpVRk~bs8K39DEGytBj=!Wp_kz&1`(-(0q2!06{M*FB_x0QF-E_Jc% zAMd)-LFlxu$hIo)!QHOdtjbcu^&Z>h3VClu@&dL6!7|BHQXTZ}-sQ>!=!v#06u9es zv~o`NA}^WCxUbsB2iwn^J-pq`^e76eQVAb#>Ukpv-F~RIfPGmiUPanZovw-t$&>fn z0hX`eMMob(fk1d3+=Bz{eiy$yHRRZNPGbLL)3|I*mTi&hE3Ep%On#Kf&oL1r1?hHd zk;!s9g0k3s5|4$qPdx?nkPAPJ#GZAH{oq`90UN&a=8XO~%2pZgQoxa%DS=fDnd7pb{XAb~N5IjY#NQ%-*A}G?f4$4*}QIt)KglLhzCX$EC?E$chncZd2 zEJ&gjwqwvG<)D*PD$Zd!1te9e;)DGM-}m)i02nK~ zH9g&5-(P=sPp4G!H2lu~>K~VX_PVD1I~68>MHJq^lPnsVCNy6Yx-cR=Hhd#CeKWRv zE4F=GC%73ozQgZa?D{UltSBFQzQ=hxD#S&<7?=DKmz}5_Px(`v&qdSmj6cJ9H>$+@ z{C)APKg;EOv_GEn=Q!_02jYYNLCzPVL-Aq%aD2o+5+C)CGOQRq5g+r9alRBi8CU%( z=gZMk@p1one8NA$<*Ddo{IvhHuB~}k@pN=5e#Ul7Ztzu zeXXtE=322Yd|u4r*&jZ?=!iLS;GX5bunT%n90K&TI2_txDJ(7;hcs?)i6i1DYUUqP z^Mp8tnlr)_Pky5BsIOwSr|=xdb0Rdu=MQO%o;WFz9L=%ru5CccU;4+i5**cG}g>V)d=4+5Jv4LFr0IglN%D!?p-THSJVyhSg3xx?AlfD0{b? z>2kAOtyfpMc{QrHmwNT3uo_dx`A><=YgXo7N<&${8HJhqUYBO6N11)Shj}ym+ijE= zYIOUr2xb?PFN z*#2yz-dGMrz{~Eegl#NOW>3km8`iO=w!r%FMD<3v+=Zen!s)ljwyJx!1?&-8Hv6<5^$GzqFY<&An)Vz77 zdpBL~w9l+YQG7<7N_Z!1^wN%0#pKLQ9PfPhZg!XuK?UramcAsLB8bBFQo3BlWQm1j zT6cB%Br-epVgf+5yq8&IQr?Zw?;GnzAOG3GKAu2bO$|^p$D8pB6LXDsCXbDXgy8l)%w@%w$ZoN?Nr}1`exszBVPn^ z`?|^>L}`lk$eFlk_Kwz+f9)!kCC-sL6?>27~)W# zMjttkq(Qi$AH4}bHXg>&`^bR6KzHlFOWy#tbXn*d>sH_7{7l~@d=Uxut-cLx8N$48 zt~%v|!-ygyv$FD&WC(O2bP)b z@OCYy(8-t_Ej+V%r9fXBT@re;qLMI* z1v3vaCup*y-$6@crHyDQLZNSLLhx`f>m6U|hw^wyJ)d$U_%%~gsx-+1;CsMPo zqnu-!%cWM|;StO&9eB15X`sNIu9^t1u1sZY+AhsMA;Nc&G2XU=mD^aUmPe4CR||tHEc$nr{q;6H77H{wvr@bmS=7y z6{CoMKeZ}RD^I>m`QwzlLdl5Sc@+6If{|eHJsfAC7U!2J&tqaC$?l^&$Y`gwjW8jF zJmdl&<0Qm=o;!`B^9&b!U>W$~C>;n>L-#(n^7M8dNm$WiNsm1LNRrYop~Z%pufvoE z5r^iY@sYL&lUkVfioPYR&~7=xrgYH~j>y3z%zXrg@Le2(o6VA4p{$RK@y$?(W_u~g zoPqovG8WA=WG)g&$G61gPU9JlI3u9p zYW9LWht+*b>#SLRq1g@^%k_3Uj1s@t>80aRVI`CRvw7eC)!nQIf@*yEC5Y2e3tumwC&hlgw#UZS~%!Z0@Ec>>ECDYEP8c{t- z0`5nM_8k~VtLH>ErIh`sf_0!@n9#K#HB!8tq)GN)l2wM;@o12rRuV>wywnFU@q$3j z7bwCcMJ`D!{t!<>TFTQM!v)E?`ay_d1EffQ_cu-s>2?vYY2Djid*X34GC#yjR_r$K z0E?^js23*kFvcATWaKr{uOmV8GZa7<5HyE0SS8T^Eu&?wgKjn&5|Yl$EZtQ?}D!Y0N*{~W|oPQbzZDTIFC*(?fc zOQ$?R7R^K20S)aP7DXdDF4DW*@KYjmMMB&70g)z-xK;v5EZ0F3-$ZrhC_7eOrGk-m zvfN@+C+i#{24|h%bV5l|I&;E1T^zpr1{KUkJ(cfL&SE>cnLFJIM9%uo>rS730`JNw5Z<9Y4~r^h-#lj^&_l<_$3=%#bBu=)l3XSS~^V z-7EPyk%NPn7jEc@eCWaL_Cx`au^@^BhZ#^3Wt597DNCS|>1;n?W$5mVJh8Q}T_H7g zdO3tcBdSaT>{*gURefgEu6ik{z8OW;?{|9Dg)kJ)RJ%k>3remOfw?v$ld_sgat(i! zC+?wl@()P9K}N_Qur4d%U%a>^OyF9PQFenC#-$c zFArYy@Ghi9>PIzr96(FpKPYw7l~a<+RA&JOwGP;HIRP1=MylQzwIP^Mn_mJ} z@cXxadSLk0&1OB3A&T%D=vCV<-@;3g;v1BEo05x2vVAnt=)}~#S$;Hp=F&`#w^z}_L16j_49*iw((}WcOCuGywB3*4NT`j+b#x*;$R-55%*+fCMDEU5;%mY`X z&81$am((nU2T7H*2ANPY79fO-%n4Xf|1BvVrNR34T7`aIt!T5@ zcUvW>$UIr()7rY%_lDzamT8SRAQMn+bE@z0o(kAgLF5+U2h`lP6C5|HGLp|%-y*G5 zy;Hs3j6=LGC{s(m2^>jszvMMa$RguYW?j2bo0cCCBt`=FX}Q;4ktqSXl#I?^l3(KI zzzcCNMf7YDVWX54il0k5iz!@j`CaNkYS_=oP-*BkO0}XQmD^NJRO08-CB&Veq4mJ` zh&K~HW*@RQlyFs5$y*6W`68P%!x{?=-$sxLj9#E@fE?myyQq&f+`nMHx^};K*itV;eeP+!5!Or2;bz9W0oHtJZ@N) zQF$b(6%DcDH`K=e`8L2qZJgJkU5yZ7}ebUd~YB!5IoIFNSdo0_e3C7PNANajXy&nWf_N}3Av(t z2XnD;R;dK!&dFQU_AxU3zX4CkqV`P3C>jpSYLeEDMN-?)ix1a0S`7S*q_kw3r(Qcm z7~zMW3wV;}kbuD<#+bo%vc;7MTL+#HJ{rRj@KAJT%HKmT`D03cf&{KMp<7M2M!7LH zu~s57%OM=XF~TxzO@w-kwW+~*&QB8cgTgMN{_V8<1r+=vO#EMX5>mvj28w+IWNkY} z`O&c%Nwp0;JAOmTl|KX+RL5Ao*lr-QFE7{c4lT942ImvvOI!#h$28TkDRuhf+(3pofHNrNRddgKpF(~#k4fhy8M_HQ%tOu3Q|k5 zh{$gVy9p%im~NK=S9Ztrx366czW2_1?_at2{Pyp4z^H1Xl&T z|Bj5AoU?L7<)95ffElvFxO|q8o0)Ocodln&*}DxGR`MO1<1u89TRSZw5(3$qj!rQW zL`1edn}c23WjL*tUffbRj*0bPrTign>|;vqQ}Ra&I^q`Jg{o=_cuVsC;+s>1 z0+Qks9-@Fhg-Xd=0|y=4v+YS%k{dMTm=(u@{T?cCIr><(*e|Je4_lB%p&tv@p+FZp zxmR$oK%Zj=i-JsH4&vQmM~gyDCEtaM1vecIRbF`HV6k8?0$s(dFtLrbUwc*ju9NSK z&BURs;%0}KuL|=oX(lPcC3IVNL-|PaY$>Z?43l4Sqe`?}yuyB5s*f zq1sHU_3H25n3()BHM;TPyvLp2luOBZ&e4k0bTD&3X&2Z%hj~nr zf*zCb`3)pvN%1i`O=NjdG=KVI4IvQ=aaE)#!XO`MAL+m#EXsQ>6c|+5+qkXZAR0m( zz@@pc<9fru*d)*P(L{7vwS?h6bl3G%`&j!Jk;41NpBlHwnA35a^#&R!S4n3YKp4rG ze^C+UpHLl>5^-bZeKo>KG!UPC{oSElcQnKBww>*@XD2W^;=}o`Mjg0I$?M1ryVl$Z zkwmj+GkY+|lE?S1aB$%=dS(Ro>xwbW-I9RA=P^pB3>ju z5bq#$u*UJvfe0C`IxDGwxr(jr^aOT@5JHU-4lV6%?L!A51408)apM+l8-|Kgmw%3S zL+9n^v?!7jerZ7B0q<#oWqI8{K$ELOEJF(558mH5iRA&?<8pGLR@$vV`!o;nuOxy~ z7U4SOE>J@B$7bOds8R~peT7y*bi~G>hdALwrA`MXVMS7_2A&O6)gpNlLWyO+TOR;;(BLHKKYf-XQIA#pv9)^5B zokF-hHO499mk|Lukl8=yNMR;IR3~u?=8YpZh zKcs}HB+CtmJ~QEqF=HsMU7;3aZ^+-L~L%3W0-G>zi)&>5~wzXYBh=gnl&sJ)7f_g^cVQ-9^ z$c3H@c#?S}L)%)#xZ!EQrrce-!d6e+I>1l_<>jj> zZU8s)0-<;TN04q05w);ZZ22l$85rj%;w88(MD!ugcaIluy#|SDZ&KOa0;jJR`h`|u zz1T1I3y67m8zy~G)t;hOREvORK{#n?!yLBRFm&zK0oIebg_?;ETsSgEkvQMEMYnUU zOuq%h{|&b7J1Yao$_TPLfUNR?td$1@Jg+oSrq6R=BP;Fvw52Tn3gcy#Xf{%Rnxf+L zDa+tqD3QNFrQ}m4-H>?*KgcwpFN}B=Ns=0&#%|P1GXysH;#2+|0du&`gn=U^sdtEd zCXZht@6Tb7#>#M>1Yat2yD*!w0?(Tul8L1ZtaZLD%`94%U*fNEhFt6Cd1JWHyhGp5 zV8tBm_X~slaTizdc~fLIH{qr2(qLJU#vRCldCyM zmbLS;k5}g4?g@t_c`A}6Jf9?kQANC5zDk^Lx0P*VUcGc#lB&XG(b#HEBylmTegD)X zd>;2SxB;EN38|p+n2bNkU{B94(6>qR3Q>|oi9gK{0ozM9p%X&EF=}=n2d$Nn#P{d^ z)7+XT35MS`4LD?e92b#g8u_W;(l+9+%~bGYnzll5bN`v&(u1!~2H%^pmE)g&Tx+#7 zrgS+!Amv`#j1uM}f4}NOcR6TTkLFb!eKjEID*?a64e&t=e{!Zin{n?bUnP+TsEKt4 zU!W;HShy5-J2K@P1LkG^UQ2!2bdttAMG2`S{$_*k_%r7Uo4b6`#r`;Z(`=Np?qqtI zAUP&JBDu`{Di)1vII?rKgviR$4#Lo-_~JC4B3wwy&*~I&L>}S1A~=p7W7{(4j5*tI zz=_-BXHVn`&Swlo^v^Jjhs3vz<9%ivz?TW14{IFD_>E(Os}DGzkzoja-O;*oyz*RS Mp>m{hYVO4U0xr!$$~XQua8MCOZl!k;iR89(D2 z^-RO?41;O2ZZ<5>YS^CL$a>j^<2j9-muuv`e52qM8m{L`JF7nA4WYeNFE)m~VJWlg zrN%mMU1P)>X^eWKjWKUb%Cq(H#)LN^X{WxvG3iY0@9D-KZ%<>dw^zzW>NAaf-oD0uZ@=V6>jxSKy@QQI-l4`b-ZLm0 z^A7vt-jSCw{)E5&cE(>H6mD7GQRFB64ajeh{Ih`@91WhGH>Wb*=TNuNpF-VKFuZsS ztv`>FP5x$-Y)(p!qhyP}6(w7flIKwJn7<7r+me#!QBw9FN6F(!$rn(v-G2flPb4K@ zM9B{SNt8U9l$=1xPJb6lb|ob*_`8F3{&aA1-u9pJpT2E-r~Ey^rg_uf>(3zfqHp>8 ze%6py`-7AI0Xzrs913#5$*Ih|>p$ZkzU_FY`IgTXM*=5!3H@QU^NxSie-^!bNoqeA zY{n?Z{LiE0OtACLm+xi(Uk+ZpYXnnw&IVtSzvqH8cMZ(*xc?mH`5fmN>*xx zs!L0?=H)p5%2HHoHBpqku)KtZg$p;90{>KPE{d~nG*LA>ZO20|)N4xzPqv!#waf9C zWL`VnQjI_*WusE|W-Y`ENzoKWsNbl}Ei5;$V5Caas?4|4wW{(l*U=ME6f`;Vxge_g z)u=jc#zQB$faLX&bIbLpcCO`D>$+sxic4o^Uwz|3<@Fa|I&=QQ>u<-!GtH&t2-Q(= zJ&Frw&z_Tl5CgcEgQ!xSiz@R?NUy z?VHClL3YtW$~vC$ZJ=s4UN3lZhAV!xc0H);^NNd0%v7#c>&rnH+YwMOwuu$9_i}N* zQfXEjL8TJA70!CO&UCR-d3U*5*EK4ODa7k4m1?uuisY0-w8=&-HMbBu`dIE|)Il^8 z@bOFgqQ=s`rP|WosEQ4U_8x64FEkD`uI;PWF6~>o5iPWu`?zKMWS=U*^Q8Y$!~LN$%{>M4L( z^)!>6W$4r)mK_FQzv`JZ_rUl=sNpZ12S^XhH+&O(&BC$SfgNP0GJY0+9sG5IoS*aa z^X_|@dBZRG?(LG7_lNxA?Tl9_}x8})Cz0lqFp{w5pvB}FNLl8aulpQ`9?F;Tq^L(4{3g^ zJiR{7q8Zo|=vx?6MIFN!@sOT$g_DlQF|RICRAff6`zHCsi%PXrJl>hERO+{etMz(M zHW$uS>lAiovl%;QYt5jl;w%`ny5Wt3OM?-rO7E!FmEcC46>b#gz#K4!QeWWY#%Ej6 z848UCq*vhUIpj`3;Wx49}=vEykAU1oY zD_47`0^%rCDs%NJXrt0Y?JuA$2|eVNxp~JNF-_q-&Z@nd&)VQY&T7dl8e^ty&AULxhuHq*c`ysnkTLC^K)q+3TS8KsF>=h6} zonqx_fH()7X#mfrP4znS#`PY4NLCn6M5@{hfq7c0hI1_ygzumxB#+)AYacOJ4t>Vj z*Xku6PVodt0a3Mn5PhV0jtfYCv>DOI=&pTd=x&B(i$$F9(4Ap9VY73>rQ~eU z0_T~ZRc~O0AKu5I0(C9F){wJRZ=zJrR=tJP^g8vYSa6o$9Kp*3`vGF7QH{Wyyh3OD zyXX5gnFv`9Yob!q7q-i z97BqA(5_Idy=t4F+tm?k#Bfd5bWG?Kt3^<-v+8Dx=7c#Zcx+oMJ3j-DlOgUIv)K1N z+>E^hGXz9VMeJ0Q1S0^keW}%|D{>fpN<0>IfHgy>>#b^}wJGJZWH@0?a{)MR!k|Pu zInS3wF*%A9i^AOi`HY!&J~mDBzGIu8I9c<1SlP#BuJqmfw3)X*1&g|$x5hqId}|I* z-;Yd(zmGV%t^xzq1x#h$7BvNkd%Nh_P+p*W`Z<9nA&juRUx3M#6Xhfibams&-n}~y zrmRf>&6=OXC|$LJ1CvGBcu3zNC(%_aB9n9or2vnJ{cL0|I!Q^+H$I>$;Gk6=WuQ6a zTng-a<}4{CSPG|y)CqK{cm51hUnU^mh_evmA&j%LMBZ6gUU5nAxY9(rf|u2+Y&M>7 zr(U@u1anU_OT8{R8#yI8Ag<$w@q9#Na3L}Bf>~qc%C@z4ry~`FzT#qyR#*l09BiLH z*2it)c5w+G?F0}OMP`NNgXyIyXkKk|Lq^Jt^BEPO={-v|@kf(;OBcM(0-@=MDOo~y z9Yn|<$D=jztZS68j+H+3vwOlbDOD8KBnj?`@FPKd!F9FJKV}!h7g6y(2Oa~+XC_Tx zGVs{8%uj85>=VWMIXr#8jxx?}ySV&EQ^ui$uk(k3!lLUJp@bI}hoH9Sf+9=A?J(?@ z`qc4}!~#py@e`@KeOghDuceWwek$ORX7P{|nGT-<=%oT4W!q}oz<8?7Rs@Q;MQqQC zdJ1f}26Ucjrw8vz_i>@aOl9w42g(%<^Q`9T)OiN_P;21yU*Yt~(cC;34m2PB%YRbJ z_2Wcm5Bkws#6}-BbJ;02hmRoxVM=SlBymHW6Qe1LO^9(SHzaK71)9oDG*^EXAYFrC zF(s2XL|w3=R~Yw58eOs$l7Abog(Oxxs};JUZJR4wdx^EXs07c2roJ!!ujbLSWF_U2^D>djB{BoV&%0Euw3`3tEYcS+|Qf^n^tQd zqc6l8dv~YTW%GTpdP>b+B=4|#C``iEb71OuLGuF9T<^BqgVzK6e6-f%H4>vmbG!8z z$a51e4);|&-q%n&w3I@twJ`oSQ67>q$;C&EmB$~p=zdgxK)EMjpJZPf0oLT%DVGLn z3zxoyhKWM18T3IEXxtSUL#+92Jjv|nBUvdwbapAu-sp59-E}$c52E5X((cG{A9EDT z9zY>(27jss0D0f%yZrwl>r(_UpvNFx1>{|k_3-}#Ev|4D#Wm65hd4`a zy53MqJhenX!LHsV5Y6{tXa8^5A_Q33+`F%Ojs54XC_VdFn}vduwW($HL=mj65?mu# z<2Zkh<<|jLHmxz>XCC2gy!kuqVugUY2M*6gA%Edl0Q%`a@l;ufr^<$>3ie9ksUr0k z;i)Q!U~~lz1vw~yMSmE4fgY>EViA5Wz{F=|dk~d1w5qc@Yeu3tklNg8dq%f8uTQ1Ouh<_K-(f@ zdfuAK+@y2RP-Q@~orQ{FLdD2NKnHO^AAHbTcM$`F0GyZFQ|0@9Aq~7)pGNl?+QInuX z6K}LP2NUIbv=%F~K>*NO zKzRHp$_`_2^^-`fJoWtyn~?j+nzq`2`@+H=^{B6+x%!;O0RpaEaRv|9VZVyn-^U}q zRp_w%J>ae|x1}bYX}h18PXk9*ZNptP9i19mRqW>hnMR(#U-%}#qh*?(ozII)18xk+ zGf1@{zgU1&BXk4!=#Uzk*Ece@R}@J#3|bwI$9p&RMeX4pkW~W$%EWYtj9EMu9`WmT z1=RZ#OlD-=fs^Gu<2~>wdKPczqwM>}qI1hQmAM6D9&K_WWUSda%9QlS&bjb$JmD4q zxDqq(xHpX`^S<%E`97>DKl6e8b@N>l(>JDjEkrW&X+t&>_n+p>e#H_Pku(Z=)+9)F z2xYN*ypB80xmIK8`7kjQfwy6eS! zfX+{&x=ZIcexY;mW73+z4T5+sbD;em&t)`}Ac|H|iwV`MI(k#@0Zb3CMJ`Ir8X}_D zHi^DWALgK$!|jZ7LBhJ|%2`XJo97`-pptZwr{f%#UN{;E?Be}?{Z00!L*c2`mRzt? zk?S*vNH1iCLNi1WYz3}OKqK1EL(e7X=8oC6b&3>}>g-V7vN#+|Yuf4#UI*3u82+fg zO7I%N&#^S2nxAI@=b%Vo-bg0|BT;I+bD4~5woitjFI?UyC&J?a z*Xnp$5V1q1t5w1zSjJF)4Xp-tD&E5AnoA7cNaJpTD#8Z>F>-!{6DEZ?hWZ8kiLJTl zdfzN;lv;QMFP=q$I$Nau*x;x)AtKvmWZyE?Xau$W7Wk)cL=IEDv%hIYxwgfddvMi{ zH}_tYUzOOYvW|1W^>{uW=2W{oLWglYted6b5H4P7hEWxfMrs)Ks!T9V@CpF1?jF;! z5q}G**lL8A2e)Js$4LW@(rwY3_+#9VB3^V22iH(}ql8EOGP^e7%}L-=&jwanR|BPL zeh}vp3o&+iFw!!NOZpAzyHfMjub`t2*~z}6fa;uV?-3+uA1!B8fg6#p$!Npb2}U1CjA50e?T)ka)5B#fm$|fLi1*hcHKPo z!MCD(+fGmqfhldcD1mzRY@C(lL9db({|$^c?W(_tKZz|AgQl0Ziv&3sl>4VuI)YfW3@9aAJxQayqAv=GW3b^wInsWWC{LaIHJ4=Q68D z9z2*Sebw z1*2g2dwdh#et7v{>trP+EsMW7#-{m3kGF5w%LlNG5TOPx;Xz9Z5U+qMU_kiywAYXR zy%Mj5gLR!qHQO8UM+12JdfGewxIY0cYYb7$oi8?chUfwP`;uFBPnONTpoo+E$*O%a}YW=04 zi9lcma?Ogu7>^20YLwJMNLyIqB@}5)!byKjlb9?#;rjrCDkOBFNbF5i^B{W5^v~uG{F+KmFT$j&;!;l zPGW>fQ9|LU+KTzW3nAJsr+;DE@a#*qYB;@NP$~UQRIA4bL_8J~@rrGLB25D?F1%V* zh(3keMf_4;7QslZQ2lk(d8K!uMInGTRa(frQ_WHfP$hazE@>wh`reR)gXleu$Lp{e zml*&InpzCzcp_>C`@sc&QX{kRwl%6(0z};{MF<)0aRVh6?_r-aRMyqmfVEWT?~q<+ zFp%pw;HO24V#|Ha~Q}$xsqq`+%tE2xK3FQQxH1Y-eslj{_D%1(~-X5+v*c5h!r6PKlBN zc5fqEbH8t0Hli%pKlH(GAp{PhAO|nKgV<7vaRiD`H2Y2(XNuLsJO36JMilnkOVwy@ zK{V5qoebv;t}g*Y5LGF@QwD2F+;myTO*6Q2W!+5q?bdRc0hsj~TE@lDI6cH{7nhUzZb<|O^)&^m>oNSUFIMmTUp^jNpda2mRo zrPcd15Vvh2){Yv0i6aKPgRKm`0TLftnDl0-H_(HX^zdc$K+O$0MW+WV>A{K$&_OKy zZsZ~em^vFG>vm)2ic0ayQ)gfoD+OFAGxl~65oXFSA;yuumHqD{A>WUYc1FP@};1>+*AQR*5IclGt)z{wbWYDO5uJ^QtU(Gk!Xg9 z--BTmfN$<%(Z3})3=o%2gdsju0mqX#T39QBbid06oy87f)3(Y|RUxvo+Vs09ppja1 zlS!i0L3B}sTYH2q0da3bV_bEih7iS^`}r(g-rX|jkSJSu?lVyC(IcyWo=&_^vnTdp zPFl04Ek(}>bjhw`UzV7zzSBu+EzN5aBplp`sl8VXV9uDf<3`(J>u!jHm~hEUWtuOk4V}e-CMW z*gP}6Mlk`_Qy@~w>|5qZE{$v%x<2&%B!>5)+Eroi5`1KJVuZ+&%^h-5 zRt;v@&lz7cZsywv%|MH%Y1Xy#VAcqLUIa_OgS&L(a$v{?!lnx_r*T2bewr}4VNOdc zk}b7PNcS8}Z|A1lc5#k&^#?!@uP-;t&;|K*TX_k=`plGRi0wT%C@LXh#n z1-?coM=h{Xe0o6tjZfjt?=UWQHtJ%zS_hN$ZncyHkNFFozFkg8Gd;^f`s_*{E{0Ox3iE1B8}&tTW8T?XTgpEb{4N<+m29 z(JnnzyoQ+1C>>h{`$G9dGrSh4W7{wGtos5DT1N4gH6n&RegDaWd&^4( z_mqzuKD4KN;MqN8qz@c=7MagU`ryH%&!FVM5i~n+% zP_8x0FTYt9^x9LthHqi2sKTD>-6=2C(P*Jn=eo77-dXRZ0H=qx3plDO5^^NmPrS`D z5`BD6In+6+<(_Ga7^z?>XyU_}PJiMjN(L$KMkH^GomIo~#l!m#9yxq*dJm`0FYtgB zwPx?;NP|7)nVA{v0vqos_wSd~ti2;|apuH?@}Yz2=x`TuYa*OE^ap$+!HFizf}<|^ z1W5d-o#}V=g5>x(`%A>)p#0ii@LpIzWkF%a0r%`vYQX3X>%|9Z6xXUiJ@#?2gT_F4 zP{PGNR)8`bfuJZpTx>?2X>k>DBb?ipZh%^YA zczk>xC85k>E8UGDDFdmi#g!s|3pADD$Z(y za)N$ez2NQ7!!X;q*FU zrB5%jFn+4ad_-?8#RY@XwbGKtGRTgVc$c3^pjlkRCnSC)=`Ee? zhpfSp`ML5S!u)U4{R-#H!-7g;Bq1XYopIj)=IpHAr5%I!T;k}a>rsFIGQ!j(!3zZ3 z9PMxDS~Rp=5Q~0^Zkl*xkQz0Wi7$x?t~Pc=Z4=7l?A1Dg%>*t1)hrfo>nPyX5 za$ZqB4Fj!1@Pzupf_9a@kRT;{C4RX*BRpOlYTS2x^2I{%d;BK`@}-mF=_kdSZ>ll& zGfqICx%Z^JBJ8sB1^A4P>6n27q;^$2sA@gi?s`O9`Gz_9?*|0QyzP%@o%*Ec=@J}S zdmaz4H^Cxro$k5wLBHC>*$qo+dX21ca0@H!OSlZxE>OA4DYMWO^?M!~Qm7tBf!Yob zXP2WI&XHs+E=nycew|qg42|)JnfgBfv2#vBW8_mV`Hn+JivA!Q+$Q)c!A}!>i{PgS zeu01&jB@>?1pRrXBR2=$=JfwBNQ^=)z2LgSH=*8Cr;?YM+Q!qGoj2a8(sy>ebMZEI z3UxOf6;5!shYg7Y6QeYL9Jql0e}M5hnQ#%HnAvUQt)j6J9@G(I0(TwoIwWucU*%31 zHev|z6`xZqj9FuEj;$YiVyp!C_So*R-6h)`#piGdt&(~)mAN0g<1Q$4Y|@#eODu0} GH~t@#_+-EU literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/bloom.cpython-310.pyc b/vllm/model_executor/models/__pycache__/bloom.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2bdd4b197f3e05418e7be44d6a6411ddccee410 GIT binary patch literal 9104 zcma)COLH98b?*1{^gJ*afCNB7Bt%hk0 z;q4xfKn96p&>89i7F=YXRq_Y2PNljkl~kp&;5Ny|B5CG3r{@I_q&%!X zxcA&g-@f;p?>sI$m5QU__v%0V)7oEtOHuxdDw97GmG9w6ZmNpnD?;&AUyD?s;;$ZQ zu`cx35JqeYGq!{k7eqnL``E%pA0x8kq9`)YjGVY6N?f+0a$FIWxGJh#FGN#uP1Lw- zN7Hd#)Z-a36VHlS<`ttO@lkP<%T6>G&x`qZK`g|_#4+ZTqT}(RSmbg!dMZ94PH?#r zos5^nQv9@dn(Ni*RQ!y1CO$1rbA2j$HhxY#$K_h|eEfoVfy>iTBYsi5h;rSZiC&6d z7B6#oHaZh8i{<#NI2*qrUIFh&@Tz|_u-DJE41dm_zix=vK2ZDx|JZfKKNb|%U*FZm zx4=2>FM_i;;CvgLr~DJ(oEUK40OzE?1kTcc^Cmb?`=`J;MV#Q}T|<1w*DfiIXSTol zQM(n$pxq44blTC@(mNME{`lfj-0_2Gso9CUUK(DGf~Biry0-LTZ*?_nufFd!!3e_D zwKQ42ONYGDFf;q3G>{(pX2yjuNi+N7ZW?ylUX+=i_qt%4KWd}iP&2JfwDeBY>BR4L z+O2T4p=HHCNYkKABeLmH@nVpAzL$E`{avrQ77V)US$Q=`-EP-im7QKUJ3%!JPdd`& zrMq2EdQlWauJqa)+3Cl(U+u`qcav~Cz)KwJ=QUHe)o$q7?7N+)7q>qdOm!h_2cFEP zKmFks2U(wWuI^`HuBi*1)i6yyk)39cU}0d@Kka$=#BJZE{D16*Y zyS)^9QXNft@x#FLv!h>hn%?Ch^ZW7T!1r+;nE8kww7g!Fy7{rVQOENGdS#90M5l?Z zocXAYqlyDR^wQvS-nHZ|v{FHT=TfiR?MSTq6YRiTn9wH2VUl19-qbk2*zEb(+X&ns z?xtIAI5FGc5E8eAUF^uMti&B7Z!3^VHkEc#FLK+x*j>Z-B$*|Hv?tq5din5Yqfo%} zE}rCHL3}k;ZquXnl|6M&ol$nwR9`m$3ON@%Uz=A-$_zhfGkyJ*wxgkB`G#-w)j0*_ z0=W8)p6a*lzSh_Gv>mkcl?`2nlk@p%U!BF94xQ6Xi!`RM&MUjxjx~G(##pn;9Ich! zW8N}$3Me^!gWqQM3wzqC=39ON>oWQVRL@?ytKsYJ>LA#7(@(d$fn21Tk(o*A$u!f0 zwx1bE8g%8eXpn2cW3^rn|KZ22^17xiY;NrD7=)wvo-x6^XZWyWPM353O*L|94G z=>~~#f@`VtTyC488}%K3GxCJLJReDDoWU|N4WoY1}>A4ZCOXanQ6gZ^XT| z_|^F8*(khxw!4+Cb=qe)qbNR`>u_)_X!h_yc{MqEnE+$CyOmA5tfv?iy7`J;!Tbr4 zS=CVI@mRk-YLr!5Gt@<{8HS}A4=v46HL5=_jGAgaG&K1-1~u{Q`>`nl=YM|+5-2cI ziqMK`QT4077U;en7=gK<_y+!33rfqnp|n)r^sVa^QSb`@@uF`tEi!ci$1hQ-<@ja4 z0%$MYPz*(sAqP`gm631+dMlaLWK_(`O~O2PSi3^#F{m|-i3QVgNo`1!lK%lom0hUJ zo}xlf*P%A9hQB)g8hwT8>&7t!H4`Ozj9JvywGAyf`2*$WYHIB&5K{2_#!-kg_%9JZ z=kUlvYSRcjJl=;oLAg+Ni>Whc(FR=10L>ir^m(HpH>7Ur(`3bwJZ!rJk9Lleg3&`acZ zun(+Gt0>_2pZ^R%D|-_{Bi{s@`!%MaaXqMCX1l<%Fm>Imv=;h)&}P#htTqdFt|vCh zxC;sv=hNj$|6+^FHPGG z2f7|aE%^p@8ZAy#(ylu;Q=)(>SvBGCIv6QLz9zthd<%m`A!*`E+pBlftmwMU$V(E} zWi6Di(d2J~{18v_9*ASvs;ySlqpGFV)v~HVYv`YbfA&4CFe=}-YzO@uwWc~6^oe^= zukyX^x&1=CJle-{x08fq@Z1oHUm5Od3fq%p_Qs~mLY`G<2V7VeHu&-s*3~4Rari5v zFft1tqVgw@$JPc1)6}b@C-Og8nb{ox+a2Q$!D~Ux2g}`+DElGj|}HxAnfm zV@eoPg5(%0@-*hV`w%}EU8}j)Yi}g70baw9-^HKIWS)EnW%)hgmnE4e`96^kh|r4U zM<7DKvay+|Zl+$z)QwEt6ea$$4jc8%%Kzp^L8!>*kri4UdDWBtHE{Y}<=OC*EH?7maaXpjOu;ujDebg znqg@7(SFac>R(G5_XyxxVA7A}aSXV=GYPjXgJIV2O~9<>TR~yn2H1kaW48SwV717w z>flsMna#U)@xmu7xk3>jGzlVyKiUrxF^0_*6!ya=c?tj-wXXvn0cN)tKvVb@s4;h! zI96I1^dWxU%VtP2R_@X(8Y*LqFjj-8*C@*0125nF-=|UyB&#qwxNC0OabI7PQ^YG$ ze>a40m?Oxb&+FW24Wh{+$XLNx$FuSIjO2=6^Gi&Y90PGoi#3+0{h*Em%j@L_+tb6n z8tbikEmuNvQiea;DSGJ5eD17*@V}sf>JH4oZTL=m>JF(UOwNd}UQ+I;D-B(KiGlJb zM98v&!QbE#<027W`FQ!Cf%i9f5|Up69#)6iXt4uDkYAy>QIbExANd85KO{01`u*}Q z|A<;?Zuw&(5}E+koESa8!^J$iWGP1!MuVA898W3<2Gxz9oz+<8PTo zQd@yV(1UK$s2r3-0@ti~fjsq@exK1>l5AS>)>}r-RuRg+l{v1HzuJEm?E#H9W>cY@aWMi8s>zYk2X*~PT2pR zHgOTeu`HM~L$%@bLD1pqStyq^L$e=l&y0^|B=CoU`J-m2h^YyzSOdO7fi(a|2+obN zuF6s$rl_s;5gVWm9e~RN(}ROUL;zD{&B5Kl7#$J8aLluVF$EfP+vuBnh$hev&Jci1 zTY;({VrZ~2Excc@6Ims)MkE9Q$go}=GD_4&JL~4ANZ7+e5Vi4{4b0T1H04-7*moes z`A0kn*=WbMba(<9V1D2YSodwqQFCrphxMvYZ1zxLFVVc86B$oFRz436e~0Jsre)?( z(?a8#c=r9sN26bI4P`(bMn4>8+M>U*ceE*O-);1&m9;iM(h$~Tl7>J6=>lY?~K?i+uN z=;d!UM5_9iWX0fsyHz|>hs;n*r+sTxrFSx<)Q1lDWvppm>`w71LD>u*c4JcIrg~Gk zsqN~QlpLj1NvCv2l`;L%b2oQv#8@0i%rg7!$cr!g-ueHd$;b+omF;I)mUt6Kio5(a zAl8eL^C;vawzcIo*~SR=%^EWYpiUG6N}aF=KTof6HYu;do_;n3PuoXkV~|@(B;lg; zi3HSl;XwNYx-FqyOmj4lhYqZ2NmgPPJ@1$m;iC_V>KdZTo z9CO0vEg)V@DIO(3w|2Gnl%LEHh$0#+pp9Zi4N)T?mHNyp3rgSEHFm5kOZ*-?IiErq z`c|m!!V$qco%~IGi{UjFl9D91TU3zoYK0Q^i$W|Z!IKF{W^w{Wro`wG*#r?~bVqPC z=zW~{S3!mVyhRK~;cY5?MC6Z&d;pTEVP>+3On@MtA0g}mJdCh5Vy?1U)^J{MfO3>Y zjHVLKmcbbzY<_kFgl*_6D>A|yjp@g`MWa|C(-DY{1A^Q^(lI&@3HZC{K)=Ch+CZ0o z$CJDcl7|Ec3hs@A0ty4F_p}p3^tK0q0mSg@a-sekNi#fx;N%U{FZmacNd!l7{2Q|| z$k`k_kdbjH69de4f)cVaMo?xfuMjsV5AeIQ8b0rWQ;A?Tvg~%Yxmg1Kn%> z%)ZuX$^`kd{t>j*k;9F#zD?Y4s~CUH4cvtD9Ux6?9QMQ4}h zsif6iD2hud7jwOa5+_-51fQOO(=l+<$z>&E1=|TkFb<@cNyC8h7W>eY6e=r3c>-9@ zp(d{oVSjFh8f!>oB5&??hlz7}gvTU#mRyn<-xvJ6N#i*JVi+7*5SHiTFiCmvL;fJc z9;F3QepuFRc!+?$1vV-ZXD~7?k`y>A4l@{uJfOzp0W~L)j!ju@6jH1&&r#iiB|^BY z!jicNopoeekw4wgqePbP2u)cxo4kRS{7G+iYNKgBA3-eN@)i2`8LLvSfEriW)8fI-7y(l3B&x44WeeG^OF*))FrL9aI2h$`MN#+RD zYOwq8-C*Nww*2tjw6aOLTFSl70?f3B&>X;+@euh~gt-OSKSODOKAv)Mhh`Ly z8($ypIyk@4%G9DE+h`lkeXTgGS*V@1CC%M zub15&%8qgXASOjO;mne#(pA;+NU%7O&;F7qaZ7$}!wFYWV~$Vx{_rwD&}VNmzYU-O zRXocbVu}vk;l0;#9zQJODmC2XJ7Z1^il?8@7fF9yQgc#~ZzDK$C|-Kfh=b;qtgv|g z2_yI(YH*9qsbq2L3B8AQo2)oXhIs7>z4J(n6p3%|M{!v0D$LW;&Kx#8-#OMdIe;6t zmxfWoihTI?K4K0fiLj{f{7Hao9Q>~cNzSiuIMBhRc96)Yz{t#ea)KKU-{cZ;b#%*} zXYznE3obdb@th^&oFT`)?BudQPH4f%YeC$LZ^5YZ(1hHRHl}TLk#rw;Py?MsG=N7( X4gbo_C(f)b)Oe2Ah}6!m}}AVtX%Ei)1zlAp}*gEXJ&UnkaD`<-o3B6b06RR-uJRSI9SZ!^YBl6b>ZrdXEI-7qW8nnDSU(P z&t@`S#xok3rr{a}<7UHbX5DPla;;|0%{6V;Zsy&5({Y_|5z&AZ*Z zC0=UW)4bQcSK@<>`oLHD4fw>Azn54(q(&$!P>dRya2^I7*<#J795HI6o) zbDwKI?>;Z-9gSnne1j62hO!F>UFw_6$ag!`hm(>>|k;hyp;?rHB%_a$$ad*(>S zyUW}CddA!B4_wZ=ANB6`cg>mJJ>I>qTkgwV*1PYEhBx7V%)1}o2k?E+FZdtZmYH+B zhrG$xZP!JetheWjrqtN0OTCBVTDG^(djz#U&UU4pM}5bC#sAoxi8gHSL*8R}&#O}U zxPRB29CDuVjv(hqoHLJ{XT77yIU46IAm=&ndE`7F z=hTsN%sY;p<8jU+a%Q|2kn=*Ea}GHtycd!4;$_oqAa&9^h199Iyy;I}X?|J_dR}IuZ?Y)HhFR{M=od5jg++{=mxys9)%1oSji+gp#h=yisvkSgg zT~h5OUxk+@icxmDbt$qA)n~)VIU4$^cDCV1*0FjJM)_lPW;CMw@!HZ-y)_>>FD!-i zb_-d#)5}Y!SU7!Y$@k>x+)FJK&EPG$qs`j9kBHf7O_4E+{g5N%4M$NjM8)xsLe2M)6LaU;XWmm6XIc-yNr(gsTNepp>v zs?MwS@=~;$DKr?gRkhjn{6=-DhS4+-x!6_>uNu@>{E2K-K00&arPI}uhmRaRb^7GT zqSDdU(sGDKL;tlfDjYj@TrvU-sCcU0Se|ZqcsEk{lgCfZv{e&rY(MU+dEYzOZY(!j zC*s!}tGE1`iZ;FY{F(JxC)*eMv(VhovG#mD3{I%_tRJA$$QpTZxrPZ@sc}-`VH%Oc zfpG;fC;3ygCKf_PgK^}A>2JAL%{3x}qU zrBjS|j>V8DYt%mHx3Cr}x>NZ}el4ulT7c0SzKz z{%Kj?AS#{m&oBF}S^p4W1v9s#>MeAprdsG@wT+Uu3~mB58acJRwA5B%(1jo@Y@B!E z1XgoPJs=o0>p{SWm$05qAMcFrZPw2DRlm6uUaF$`v==d$1=YFQY}i(pq5-Mcs9o|^ zP|ui|Og&>E$nQdst01r#*htQzQ+*#0P)$mABjRUV!^^nlOBp}w8QU_x<(UUEes0l5 zEPEj1S=ipWXj{6YWb2)%zveeI&_<;tCaM=|jb%TGtPuM(vbdpUCi0O}t+p_U@OBO0n=OlS`M> zV`z3CzJY_Fm?;@W!!jf{%Fc$b&2mkX599nE#5cfnXTr>>%r%23Y5T!vr;i^#cKC&v z>e1uVM-Ep{9e%NT?C{Ky)6Y&A>S@&N4nR#K8SNxwHfyg{)9zI3L~Ah|?&mv91nA9S z{NNq8>GC-M@Urp+yHwj3e5LB1ALRr8e6``XqGD^gsap!7a!-rZx~HB&L$hp@AO7a> z<^0QtI1Qbwms!rJe3)6aT4pD^YK6uXvtxFwjxm-2Nj5qe9p8%7RTI!*&L~dGL{=Tb zkD5f_Zkki=rXERkw!Pd6gNd9v!s2Hc9A&`HxFa`?Y5hd)@Rpm+OI41KU{#$1x=rWo z02&F(2%NlaSf*tbjk00Aubx8|v4C#=V%-ZDx^uNz=PqcBlT4v{9?z&_2+~F86xsN4 znq-PjAcCpMhQ=x;1XE?GV#ip`c1(#6cT7x}xrnXQ$#yJEm+58S%&z7Zp6T z`q86@PCwfVdflRD4<9}9>}fTKr=x7kzc^v*nNPaXLmfg!W)bkjDOltrSfR!3h9|761|TlCEfeNKJIsfU zXTO<$#azv;+MOKQG1R^erfjkBcm`8vhsAjVF*_XSpd4??ze!HbS$Vry!6#+{Kz%CG z=L$GBP@PKHMmhlNR#BjLb>(}=B3-Ot+f@V-Dm_GVvM8;c^8Ly}7%l}*?AZerV`2I1 z9p1pIPU)%rK-p9Si{k;$SXZQM#?62Y1 zKl}Kc@B5GJt?k=4w|91KPoQS|9~ZRVgU9!v^JHmAOH<4MyIZ+)~Uz2v&2;WFlmOfP3dccc`0IWOV?muVBg5 z#~HW?CQ4Bjh(9W6(ljd!t9k$>Q9;@vo1z|M^^Y_&FN!+bU~XL!INuRAVYnGfgdd13a7005p95byA6zq{LbW>Es0HAt^h%NSPzwmYfUjV)?M4X;0phvF z&{QVpJ*@{E_g-%2KdVZ#bE5-7g1G) zhk?bgWnUl8S;ort4Tcv#1e);#U-uCPW}9IOLcZ5pkj; zTlruxTeBQS@?o4`ZoGi>j+^6(aTWY9XcacdJhlYZ7!(Mr4sNHK1?6H1bFz|?19NGe z%c^@q+e7?^GM@b<(74YSVg53pqw2_3$@rGZpk^3z{enmtPgGX!oUFXiYFw(YTBROTy46VSTJ4LI z69qMo7id7(fml?U2i#Scmx$vQ)lOy=y!OSGq@&^)%0Y*fYAdy!Wj0<|oofNK2)(i- zYK5`yWWX`#{t6gPF_~`BYe)rrLDA7zQo`$2j=#W~?>W}6Q8HIH_b&U!Ta|c?OLve> za;tVSSAfN?8ekwyu&-FkjtQ0oTNT{C&L>-ot*ZD8=-8+6qh4q58D>~|HyW?G`FKJi zBgQdPtR$o0&iN&LMYt*htAjT;R=VmLfazqs_pX^U&rA%g+ka6Oz#C0&p$QI0kg*|&4_G%V;HH&?k^&ITm0{5+-_Z_Miv44xmA3$CK z@seNk1`!`XyzH5CW3s1*ykYF=lDEkpTr7Jd5|;fT{_-2EDM!6AC?$uz%~C!t;T8$E z`kOpmv*>N}wxiaF;Du4(g&jSVFV%XzH4I8J^Hyd!lZo*}fpX&l6gkj!FnaDnoGMZi zR1FuPCyPoF4`xB2psm|N@pY*k)Tup!)*{yU1YNL>O0!g{CFo@pb2wYaQ=H2d4JpY9Y8(kiQ6le}GcR-GW0E89i*q)I zA9N}AcYlbOjt0Lp3;Gc0HsTR=-)+$~*`k;uz9H2uadD(4X0 ze7Gc?)BM2{s|iXZ@AVt_2IQ`bB^#`-ZH$ly19}|Gj#!MB@y8sn#gH^D`4g1?j_GKY z8R-A|h&=)_z|xo?ylXq%x}|q}IJtD+r{p0r&PT?%S#G@K!+5Ic;Bd603VyRX@$_OIT%h^afdsz{?%)T&LVuq8pDCs0hNZ$w zBtpNBKD_ll_~r>G$}3;9!?FN>7bO~=eweYpk07!x)cuQKsL3<~+GNST%+jwgm`VdK zLsYK9;?THMJ+GG^hC!`H8p)1e9g4Yp~slKM4;DG9#3?39ttI}Rpu+%)%>cn3LJn*&v%@w zCh8*x=^XG0bFyiU%``9p6j4`mz@eEd`A$*ZFaVXofH2{TQ%nXN`)a~bUqlKsxYVeJ z>hB@0eu%-3F}T9uDuXW}m?-J_C5s}%R{w~xe~iG*V%MqfW-^`q$hZ&<$ZTjnEwc6R zx>XTQdrPtu<$T2$Hb%@5%{6rS_U=-2`S$O_`)1Ue44;{q7+c3js5ej$<>p{`2sI}` zhDd#m0ij+P3-vvWjWXaSRZlW_iUBE>o`n&{$kp7O>-b)j{yM(GC2OujZ%=I1Z}f5> zCb*BXa3AKkY{&ejSup?7E}Gvk2g-lpJdI7g299aX9I(K#KpK9}0>{E0Qi*> z>LVXdf^T7rU*SngkXXv#HXxZm&K?9z4M$ye^pHObvzXH2c4WsgMoIycumfC) z9g@rf&n()Kn`i7YHVGsZ8xo2GU0LBxY_TeIY;ny3>J*`%WGh9_xeRtrNbf*cLJ8X% z^h$5Cf8`r;hDd;~4bEudL^u_C7**QP5Y=Jo6eB=_M0&X>A!Gz;cT^ZM|B|+{Xuj+j zh$ccdzg>#tG#g9Rv(y~qZZ3u>yzbiYI?5d7p|ZmERqAKieZrpZ67sY?HD(o|YHG|? zxzUB0?g~1fvR?GzKb7{kA$+f&jIX}ZLbYW;6 z6<%WL6a%7p*~s?;p}c(zsUmc>CHxgZSFJTGYeVCKO}S=fNi{`Lu$E|ch1!0@EG6`` zXgKfMdFZ+EjAfM8R(7V8p%=TJQ?`DWIHDr4E$430+t< zTg39vn0Io5^`(s3DdYfnJ>yMFh%s$7JvFXCwX9XmT9*zn3!>WwokQ+dIb%Q1;1?PE z0)lI?*t)?3>X%t*Dh*`5i5gE4fs4phz)aC7gINO-i@6m0x&;P+*^CRC8E~TfNXSqt*fQEfJ#nXJ}9Ta|Zu{!7nk`0Nw)( zNc}4muIx%@cm11h38%8MHwV!ZXj+5jB88ZK@IyKQH|Y8Ri~_>=mE24Uj^Gq^xdu!1 zuhA$#P@qrXkkF^$@ZT^tl?Ky{aaq5JaRxgOu>faX=ql!6i^`IRa|}ROmxH@u4q_-% zzrq%OmBCaR$h^v{NQTbiRqEGR$kYuho{%sBqlE5yhM%1?g}TBB_=~WUe%9H#p#(HXqY{2#NXcC{_dJ3`+zb@bZ<0g zDTGI*UWIYrERx+ImN+{R)bse*Y;sqTyDArF$IFm z`>qWuMYG;gKg#M>)33E6+XMK`E(q|zB-P;Kzs1MLdzY~~*AN!L_uPU9SF#%1ZvuA= zZB^l;(ffRiSm&<(E*|F0`RZW1b>R>c+JQKJMFW`A*#$V~_>F$Uwuwf#g9Hcd zMx={_=mG(F6ud(S6lNJrIp8igqy7`>D#D$3JY37V1Lu5&=Cv}%hPGE6T3|{Q|FK*b z0|_>XZ1!-U7g+$k3kmM{uWWAv-0^iz43QAIzJyAYuCF}!ZHzs|AtK`Tp$-n7(=Di@ zf{3<#VZ#aX05gW|s#WPkAu@2#H*_Z6XlcxKFbqO)7`xF#S&SUJ0c#QE#H|Y24C)Bf z|6uUn87woHN`q;}2qb@!jFy}&|LpgQ7F6yJd^@APDM-G-=Ko3{`P+En_t}>eBpt=P z8$dEqiUbExnvhO0g%x8{m^Eh!5hc+abaPO$1D%410=a^SLVShbS-|`bM3oH(dD0JB ze)x2wJ$nu&F=4m}D}Cx1_96x4-$kMe%0%E*@HpZYRqmKAdiKD&B`RMKS0Z<4#KhAF zfKElQBe_KVFynzpkW5ffTTE+b6DUFM0CR!tAwv?yR|n&#v+JHNYeK!4g$1pRxTR`0 zd5=h(J!R~H6Zc`5n@~1lVdkw0M*8(P|M+LF{N=Cy)RU*LL6J)WFV3xjD@K51oei zsE*B12z*%mH)g{Ts&(OEowd)+AA;=xZAN)7j!5~wXpq&?W?{Sl!b)BOQCBEPy6o!f zXj%OUgAI_yb%N~6C@jL8tc2}Dqd44sWGm6$mLL%4x|3SrhAl_RS>{|X z72;Isy7kSKi4TI&Ti_fbS~7itbGRqrc?Rf&P^k5{$a8$|cVLB|&JfGr0wHih4U`UuKkei7~+%*hr&4{`?jbF_)Dq{u>P(w`6N^UXFq4*xf^ z{)oXJGx$FY1XasA{2`OF7=MabqO_&<>Za1={lL7M>j#HVlOoA8x&0En7r-A{#rMJW z-}iyjoG|~N<%^3O5@Am9Uk&Yj4Nts77@`0Qc@k@Hy{mP8<@dHuDwfaU+xJPNhlDD3 zH4DPm##DJ4N)dJD10B1RB| z(A6iapJa(c(@qsALH!ON*?~wF(0$ zs1}74V1v#8F#|kVHHRb2oviZ@a%FxT!Nj0?oAEC*_zDBAp>Y1{A2Rlz8E~!K?Ac~h zH1@*vxcQ(x7t##)e^Hni9#S0P!hywy?GYRl+=1NUIcQ1acuAP2I29Go*238Z&0{(# zT|3RUp5j|e5GY#l;etT@6!WLjV45*9Sf3z7?MK8Cq(j70&Wiuq8Vqx56ikPKmYghA zjAfuCVE?sPwFvnDCZ%;^>-z1T_(Q1QOXx&-93I!?Z7q=0n2N8#POX6j1OqZv2uMTwvu8f77f#(K0Wk7uPDjoRBc*0m5 z7n{MSVJ1WTLPmWWzgs*Te%hbzl-Z|VE=nPl4cS;Cv6L$iL z*IP57mtPEbur~5}fM&gZC6@OdU5#if7M5Yp&Zya5s zp9oGR#2Hn%0neTbqCq+dbXmn>x2_O`XwYjfa+i|o3{ z*6l<&8I3lPQq+lxY%~_4v(35}<)udTVb?hdf6cHSUc$SGbRJHM_GyymkmQlDMU)LM zFb4j*%gDZq=GAouLNMQB?0p8`L;!hmSt(Aq0^nw%d>yXI7yO2@k&^DYj_ydkf2!DIsl0)@8p3o`yI&91}bt&`X=y)I3M2?By zgZKtK)o1wS%D`fkJOhV8fkBbM00X*P=xzIFtUQQhqU+;2X^(<3 zE8+ZgtpD2tH|DU~tIOMnKK~klbRXJ)XJnRcG-ZI7wKfJJfxuM4FLMA=pK>e$C(MDU zN4;K;F{+n7rUd!FuxC?gu*BFjgKr@C&OU_o+RcsNBz--bl~UHSy~adVU6petIs{oG^_uy`UmET>^DADWX`taZ-i) zj)626=bhjppmX7DMs+jKf&+eV5KM?jxl0vf9-_SHWMRjkRuW!W`pl`n4smM~_U~Dx zm^vk2z>x|Xgu5A$<%6$N{}J^zV)_4&3o(X5jbq68+fd;$gsbe9{<-Q$585Arvz2_v zVR6<3M@%Au@oH(Ca(80uBIgbX8O}NINx*)lZYW13d*V8a3k5Q2GlMY%i7Rjl>26Ld zr5)H%a5gpwfU~vkx(D0{X_2ib^3OR&2N^}!xrPC}Jubikx|6!zP2^e;wa97rc(-h2 z?_OixR2p<4d<3O`z?V{8Lk+W8q?Q}LIqxf$-2my7Y1vTlI)W~w<6MSG9~2)}FhH}Y zN@)r69kGc4ZMyI+@!*H@I^*y_A$ucn_?>8vcjT84r{4+d3?feNS;ouopB-8phW{)= z>=o@VyUE)O;2e+cNZ0--&soZ8Kb@~98k3Z9zDvCjR&6Y zg7n-YnvjF8Nk8A_h9c+wIw_?1{U5@FqnFv4zK8e#*Pas9+Z-Iu%w%5WF*P~v)-k?d zp2vy4E|6WzM&{(+9)WNQ4|PWq6%W9O6XG!0(vx{K{x)ZCJhO7(1J-g9C}JPa7f&7l zk6CL3Pfm6lf_aP6i4&Hg-^3vq%~`E`v6HlrOI|zHt5&+nOePQYO$y%C)!S`FBJ9p3 zOnz50oXeZdW)Af^o54;5aH`C_IuOq$*cA^}K?UUV12U^PB5WT$egyvzD?$jy5%h~V zjR)}A$?;m8kSI&+IfcghX1M_)n#}S5hAXyKJ0ptB_hCa=TbcTx`4&Lc$HIc1IZvC# zp?4cs4&Vfa2HHE(A+?Lax~`OZTPmit`z`;qCH3THP7jA>e#**#My(asR*wFar%CyEN6)M!tt;HslMxOX z2g|L+Htg2qcOg3C0-lU|l))4-geinD$!5k900!G>I`6MM_W#$LK2YYQl9QCRsURY- zR-&0P%)F6#!=P6fMUVv2q*s{HH&9pIg>ELZn*||5d_Q|G(&ZFuOfw*~ZDZ%jX4sg_ zku`e_h$PFppJR;&7)y6Biax}QZd=o+1i4lN;c&7tBj&hS%2}rUeveZ1CYw2awr}uw zM@l`_ZAjgU0;wJB(iD3u!R`IYX%=vx74%M35c%Lu1yJ0Fe)WJyx~VXePw)jdMQ!=` z;hlWlZoaOoSH8&<#82EX1>Bnl@eS@n5P~2Bv%o{;igi`?Voy#^=g_5IZ!lZfZr6q` z1qfNuJ5u@`Sx}wg=qShoF7oKEY!6J2$18%n7;3njBe|4h7Z;a3!qSphe7iutu9?0; zl8N9~21#C(_VwODdO)bi!^wB(u)#rL@HQ#ws}9H`kF-G-*)b1f&X2|CO|@)dDBSc# zVUumTIUKfb2kP%2u0FzvIKV*m30DyYdv1#;pCFyDu%hU#rx=r98nI|RUZ?~;#Zz-H zXNEgyC%o82#{9s!*|?#ofc&axuy2%7`sS*Fw?RUq?;U1LoR@%{dC5X2tK}5oEkSmc zmPwR7{{p;Z)D#Ck+#R!;X2wATZjrCnEN{*^aZ$cn_1d#Iam!I^%p>cwz$8(jU^I=5Wc$fF;wm#N4}-3pb4c>|#8et^ z6bg5^Wv-*Cpn{YG&HlJ$TF~Z0xH6%tG6&;0^0wPjdlnV@K2(|W6VUN3k$2$6z%#vD z5=RE{y~nv;@x90SK*%}9Tv4bhof&whx9ArZi*Rv32>FF4d@XkX=LLIn%ia)jO9jK* ziuxAI;F&)xb#Q(#PLKFyZxp3NC>u*_<$bB=4~hI$_BMOtujkxN;zTh5z}VKKO@Ln0 zZ_KEDoHo%4e*t5Yqg%W-!Jn(Yra_T>Px4`$m;i8)34RJeSJm(a#T*mViRuIkzhXQH zskF)=mDi{5!UEz@mr><_5_7Q>><$N8Mp*uw84jUr__G4t;E*)N%J zzR149wsFR;I)d+LIL3DI-AqFla>m0gNNttYw}sp3m@&^D>38228!Y?q5C$)doe*0} zzi;3H5ZNpouIWYxcj{PYp3aVd7o>_Kc=np5UDY{%xN}gShkzm*_Ydepl2a(q41;r^ zQEQ&{YEK?ZkOp`dD6K##yG{eLCaJvELxtjJSMrn6c%Yu;vMIKHfx#~$nAp*yn2{cp zy4O|IJ$E4XxWY|AuSG>Iz4CmWx|=QVtU*-L9-lEb;Kp&wQc(nHV!vplKTY`=S5Uz7 zS)a4}X}0z23_gUw-9%eJtklyzj|N0?(USysQQXue1(6+(jHd|Hzhv+>1L_~#!iDqk z8Ph0#;e3;js~$yZbjJr=sMY5n(p^HXoz^ZLWo()+BF1@)YeV%S9Ym<#hEfrK_WP!V z5E>?|n4BjVh6!-w-4O>)B)sHe4Z=%T_td|N!0o&>2D5≠xPHL41X)O_WawB{$SD zbPjBX@NRUwUx(SEG2StyTTlISgO@o5Voo6Ft_Ps*(?<0b6;*vI4DRrhIPXV-1V?W2 z5u&3F*`f}ELQF_AF0p}vnm(!GJqx2W;@Cdfm7f)5Nd=$N@hxqE*0SKI1+_}5deJaf zp+FleU9il!Pi(SsP!pjbZK-`Wtieyo>sQxEb6qV>1$V$zE~?s6pxfMawq4&PnY(<% z(#C=%f3{w0^yzlkU4{9M#Q|HJv0vm0h$8JcV?u9!1eN*}et*HIz8eAQD%4PQCZ`KnG(LK#pK73AaY+c6zmCNN5xG;tN9ktdhE%7)1RwOtE^9E1>5An zHoQ>4i{z09!EqJ7CIAm@TY+T#>Nt-!;>a|Pn$c@PY`R(6gcq%?I7kn@s&hF9{tibT z&)+9+&j-)plok#;df6}H;Iw!r!Pg-@8adF(1I%FA=wd;$Ctg&WF%EYS_wmiu74*z7 z0iIK*xqMj$GYoKdX-1tu;FeHN%n(xTAv`KLC~-=VEqIN&JjyP!tUk)v%M8*Us69wV zIYDRopyU*vkzks!iwNeq&U+DY3beIRgH5ZOgR{cGC-O%JHC&kUu7isW_~*Po3@o%V zo-$n;8e&f;2ud+ACqkJRUB_stA-I3S_{HNrqsv4_%IQb=-pT1l8)t7Y;BS?j zk#8!tH;bnFJ{$-?V$|#9#k@*#3X&%=^a)7(=D{JsNFH-y)IF0ED+oVjGX$pr6T5I? zCBG!MvJ8qXMxT8wgA=>c8hnG4>{c+Qv^QTYYNHF8t&5XQ{ z`Zz(D02JjgWbg~mBkqnax5x$I@}(aBXE*B$|C7fZZ@h;f8V&0{Z+Z24&cg^f))y90 z3{ZC;bt(9GL`DXXta;qCrtf86a;4wc%s&v!|8wR4+KxL z3>AT}W>fAq;SDq4W#^5MhmXw5a(Na!(rnyI25S4ob0c(=PGjJXhkb3cx}(h&}8Lf?Pa;X2QDXGB{C)<;=hO{)CmR| zPU4Ru#``3eG+MiiWWxP%$gm{EFX9QNz#YKFJvgeV6WuZ@8ue04;UUeskMmq^9XAI0 z)!xfce}YRyeLoLBTCkQm&;kHW2j!1axPB zgp}c_B{&Q4R?iZ*NC>qE&u3gVgWE#6w+u^!rT7v;EQATyKe|7dxETdf7!1KMZ9=w! z{)lHc$P2FAe-tN0l)|c1=nt@dNhT{t+PdV4iq^V7$B5?yvL*^R-_Tyd@dnF&VA>z-c)01+}it z>*l~MAs=VkpF%#oQy33!6K)D=_=;kow$aC%m=N9^SUh^3L9B%@9&GrX!Z~_*I=rVt z+w=hvumj%?WSZ%w%T4qu17UQ91sGvFFk!8$N$xEY6 zg3Ed2EWPW4yD|9?ee!z{-(U)X$Z=R8PPsVmW@UMIV$rZK7cHR5931kqi}|Z?r{vKS zqE}ugdjK~OCemL)eUfwg2?nw%GU*R6sdbw;s)24&-3hE$ZzZ@+*w6*$+L|=!d1c4C zGZ&R<48-|9_#^Pn@IT@tQvFfHu?HM|%Rv=}lie$OKX}9RH>4Kv=Gorqp&-exOwTYP zGCiCc(c&h zX)mwuT^#rAny}*PUYsvw&b_E(uR<#&smD-Cw|#E?4|&e2-D-9yBu%Ebz(DtfYh#~2 zzrSMvULXcx%FyGSc^=g~{Hh?`^7=lLFLCGnmTTwvMy_l3C6Ovrk94GJn zL8RmN;zHV@2Nz#l#Sy=qw`O*1^;LOeMtwDX>L)+wDa;pLB{t1WEUxTud8?3c zv@3eTLl-e&7NC=y)vxa=TFX0R#qOHc(8JI zIm8WzA^cqU4C^KQPlbL0$-}}MC(mu4m_+Lj#jOYOyjfc3wCR%XE!8gp3mPq?S zVswe1z{L9kuxzuTx|cNvb?;)ges?=hp>6P=5de2bMR{24;qE|uO)E!%E29^1p(xaI zQBJBz_uO%@@8bWa;6-^dA=;Ea#coS5jaXD-2a;0-7LZWTK{otip&2B{?tloH*1h23ILM#zlp ziKEBlws*JGd(*bo>I}0+ng1gK5r)38NA9;1?TtGiC9%Xm$}Vt8-C@b*>3UTMvViB# zW|ob!Oh2Mkg=!14Dd)&N((c{zoZyXW@?S5+R*A{(?W=M-s=NCZbwxiWhrg)tckjGa zRr!xl6g^Dj=0CZX)qUVrPY7GnBb8HFckjK$6S3E-C`?p^wQtpa7xnvcie25wCwASU z{X`)^^?-oA`^YV7YwZN_k^Fx%sdiuO=HqwsWe;uixLy{)iYlfwRUFFzz!J;K_0k(J z;0-*k;wE*sZS)e|-~`KubUANd$EyLp_cM3%y4*?q!~^WYgRC$p6>wL-{Ff_Qzc|T) zq7>-rq!7@eQ;W?j<^%OGpQm*u%IO+HO&(?LhZuAf0*9FVD1%Qjs4*a_(8%d9V{b5c zj=@U|1nIoY*en9Ka4fk)T&{RlIUpQ;=hgSJ&QCM=bq2q|;I|okjlrKY_yz;ot;Cot zN+|a><t@)NTNCfIFHNb^qT2cWaudjXz!G6A@z%#++hL zrV&KW0jv+qDYTqtf*(du%Iu{P3uK39+J+%F4I5*ULYR$nkzWFt(KQg7V}=7RJNX;V z;)c6|{Vq|goP&iH*UlB+H8;bsd0h~%+_Y4@UfcwS?X{8e$nKHxkvm5|IdT+#4~%Rc ZIXrS;oJ;x3RY}#ZDT>VZ zz3#aHO1o77s(3PNO;IOB*PQIqv4D*Bg2!y;qWo%vG9m;Gv48k6z{NK+|Zr#Y;!^L9`lYM_k!%21AN8I` z-_QGF%MN<~F{HfUy@-?-vy^WmJJ9XDQ!7$_eizQch+mu77m% zegEur6~7Jt0)F>x&in7k-=_cWbrm&Ed9R?xE3Dza=D)s?+fdVg$MbS;Dbug6J$Jgj z;0wRq^dIfCgKPEUOO5!IGqd%usMOwQvLj^7f#I9ulTLSr8t`T zoCj`NO$u|3(2rIcO&^h5Lx{$;>3m|GXf@-+Ivx9>fd&%eOe>0${A^=orPW?ctT$HT zR;S$vlHB>#733GrUt94#R+?ior^Ch~+tb=@c5$Y0%@=bW5l(AK;aD8|ZPrf4`@va1 zZg`EjF|8-X`Kv41tj_|DAAga}O* zTkTft2A#!4Um$9~)e2URwLNr>SpMAXTk<{hWY4TG7Jct{Cs+;JXR}_QOhAg9vrVD?qN;UzoY?lc?oy~LB@yzhBfJ@m4}^A{SJPB+~Rc-@94U$=bn z?PHH0KXdvl=Go^PS7j%tHMB$%vlq^sKRx@#iDPF5Q(7A=&k3xg+&@E0)}P}};T}78?%2ET`FGErOeRia;Y8?rtw!vh zm&+d|rMLV`t2_(r9?h1s(~wT$=PI|mhEl65D;*I>+*wJj7aI(!$%4Hvr7Oei>v##~~zxa;MXb8MWv=8_D5uA$xy9<}o2R)ke`n{soDZz5fi zo2L)Q;mYBa*2<%C1Dh&-^yzSQDLfKhIUKa+53gK{mpbjkmxCZYoNjXes^46VJ0gun zhnqO}i$OTEaxK~0Yy{2K0Q2Ie!&wP%w3j*o*7q>qf}@#I#WVOx^k#fDF?jG2Q~2?! zXg8VN`!f-+@W{8nia=LcR@Svx?P|*!S@Ngay57}2WmOTiSoib|^+R=SysIx8p3zmk z+-KUNigIAQc=Dg-mUHXIJ4(A^D6x4B?2G?#e$(=-u6iA;{X^x7(lxxoZe=|e7d*S0 zYXR~wyT!TlH`HW27ON4?fgg78dQI%mwPSOg`m3vpi`by2@SgRR-chQzqI%T#!+Q;(PYx}2GMXMWiS$Qi2F-(bG|Nndq1z&qx*_K)g%V@8Exn|PS+-k9 z8*O<}QkKEU^$A~5GM=3w86}!w&gMHUS5AuC@{*E26l>YjlupO1}hu4`^+CzSVU-`3Wx^+MO`7A`8$Bd;kR zV!L!QR_rOS<-pF0m=uo{sYlaC4l4t9FZ zi@{F?C-Cz-_?w1g+wQc1i=8BggWp_oY(^qCXk7$%p6_&mL|<#IB*x{IeUM*eifUcw`h)>j-AqQ7=Q@qs6P&Qp&9bundn zG65OqG_m$94@=5fz;s)RF5By`CR%>be?Z(Sd== zhQ6+KHHnzxe4AMBYO%4Qi9K(f?JCMR85=9-+n}?>Nf%WMv6)0hL3+&yq;xc=a zOKiuLu1&@{UC5T07$69s;z^R1Kob4jD=z~ma?MTv)LuZ+^q5#=Hdj|Hk#N4mA(9yq z&bU}6A+H}CeZjd*^tOLRTx3Q7GF?nrzzP#Ck%S~%XD8)1fY!*dxJr^4aCpg-WLoZ! zgux|TjF+H>hiSPso7uuNCcM=!ye9Me{NggoMI5%RmsCru5+~#qNvU?n)XdwaUX`3= zQaA2chTJ@?mzb&4G$OlDj^TvxiFSf~tSs;ZY9CoKHqVB70uRw-ZsZ8FIFIlItL+0> zMB(>rpuEb1`-yT<6FAC0rCq{04qgnPq|H1A;i2vMSCgD%-#1io1r4Nys4&fNCzZmH`0I91S442hj@%^r~R_3x0bQkPE&(t)u#qiejMS%hA^3O?r3D%_}+byPs0HF=S;&nqtHhP>j{11gyjy;0mN|nKE!E+sHX!CO&Kx}%Etu=mG*K`(vp`0f3?=8y4G?T=LbR~ zbae{?w@RVVvp>^uE{pgt;VDC)tgIIxP*x#Oj$ND21j_2>xK~MQT<-~$1#b-XtGkuW z+I0o}=Os>Di6`R8cn5g7)U4O@;L$tRZO~oo#dr$m)dqT$WT=Qx8D&cwDrk8_y{JV$ zMSj1itXJ@idDZnQpLln-gnUW|-@?8bU$1q?yERA$e}SdDmCZe9x7r=|#wi~_9#B=K zJGQxpW9g-DfC0$#w6rX{TLB&!ll}5(3qxtO_3EXc@3rsUrS0rZv$hed$afi&@ACb0 zbeK0CvdI{JjZ8+VWw=DtajL1*+n2W}vSv~QAwnEQA@PDN8B$*-3vL7{7`%k!#7rBQ z*2Rk~D%0{{UNj1v7sr_PGRWHg?ecO~s!oYH-To3PA4Bo!y<5?63;gw=F44iG#3Ss7 zvTl--61I4r39ph+Viu=K$il_zAV8Q7&tYOnniGxoqMzshyCGQRxf1Wn(l9FcZEt!a z1=T;mj6R-$tjXEBz*}@e4EktK%6qLIuo;r!c|6xC*7ICiN_>o$`4fO zUali$<3Vg#${V0pCd;QXch(5x_?|5Pr!a=-0}!xoU^7{_Em^m%*^t{!{zIatY;QCC zrIyR2`|oQ7^ZO;cDy93XPF!bc)}Mq5U_cBoR7!?y$A(;3RgF7UY6y2GH4{08%9Kge zq{twC0)uGc8TqrbSmYqklNt+ZLG{X>=Ig$JgRkQ^2ScDCxq#v2J}Ns9#{K-V<>e){ zBrW*RIvHB92{JGN+8=`-I7M)ZN>YN{**96uC8Z|q7MYZllptTW<1E%B59$5o5g?{Y zRirg|O9Z&zv-t`_SX$~si$WuvRO&_R}SU1KNo5@5`3c`jwj9-y%cR9w#E zH_u-&4QjQ)kz`F`X#x66P8>}`SsTx>jW%+anx&m&T5NmJY*=RxWEiwsL`zJ`$-YVm znRrT*?755(aXO7v7^3k5Ib(4ZG1g-Zj=z9*##gR@)*zi3pQ06)_?G6f1@?n7GM8_? zBi;$jfu02Zl9S=QXCZ9HyRvpmUIC*rX%?7NxICYtCwB^Xn~f5`V|a>PN`z(E=N^y7 znaZ40tfcTvp$UJ#{cATg$Cz(5BJnAriB0R8D>eJ9#P7c7*qKxck>+!yj^jvytW|wb zQBF-`YX~R}0)k8{bV?L@d*=-LSR?YCaw~G%e&?zi#n%E_Axq%i?#0XQ5{z?^_#zoh zY+@K=zhW|yyeK-WE3EF+1}*hQ2noM&Rs3;wZn1T~XkuQHV@mQUMdPZIk07W(4;On; zFd3tbgtH38l%`Op#Dq&I%*13Y9nD4mT594D6tSS~!i49iLhJ!1^kx=HzNj$4!1&uV zOGkdNkeHb{q(6gBK3++u>W`wdQyyN72MM5?lLTd=-^LTogV?48V6vd1 z0^(FdqmpXew)IK9X4KS*{D+YugDgUzrdl^Oi;5tCPql89EE+Kk)Fvb%P0vy4x z^u^kaVf2{kuXMS$(ooO*NV0>(!hA_^Ny>b2H_XiA;yDa;#HLX~LLSd?Jdp=Nc@4%i z6_Q5|a!zhp#fjj_)cGt@Q-ay<+xmK5+AWk#1E&S{it9RRAPx0%sr{m>ca7^gNJGjd zPka+aJ{9LgG#=R zb>yn_{*5mxSc|pEkrl|Q-O%RdFti58!kr+NQ%bryH~mn?dX!q8*iH(|RkWDf7vPmt+%J8SDleWmuF@ z&KH>Bunx0T3I+8TVpGI4X=yoBZ?wIBYv@$ABRi?jH<}kQ2DIPrz2EYTEH@{SbHqfT zV^&N&{Eg0oY-6)j?NzWRb=jm{I%7gm&_5*uf(c-Q%&ZXGERmI@(v#*FB`d+!Cle4q z34(3&Cm5EyXhaL-PUAEl#}hpaGK4BL`9pf?YS7+*E1Q@gi46wQjkkaaa~S7eMIG_iNd7v= zbZ&6+ogG-$zVD{*t`j#QKvNy=ivG^tSRM>Kmlf3JC zohFW^Jkf*0%+o9WGKri&Nd{;3uOQ+YTfpKA85MG-TiRBx3|pFtQmiv3kE=FECgrb0)78OQO&&@;CBlzd z9<0I?+S|Z1>D1}K!zAag4Wxo*werN{@HlNNnB+wmv544>Hxp$N9`|ItU)e>stpI-R zu~f6_XGnNYDwJggf|*mvFv5c*iuq2aAWp`E{K25)4uo(PtlY>B)+T%JFM=~BdvZBC zSAvP$*rPbHz}-#h%QhkD+_ChEX5v&?cPa)X7@cPNsFUc zU56V5|bA2eVGZ`H-`I5COXYzMsh_<}K4QDRXMJZt4sH-?X)Af9`t- z_m?>6@)m>>f0KRvi1Nx;Bi`RaD=tU(5CRhM3L|pLzPrjPto7AcNO9`_6ANu(AT57B ztbQYSSdw^HmUu{3Fobh!wV_B|O8zr*gY&TsqWVTb&>YVTwm@K zQ-P`Gj(KP8k{;o6I|JL9_!tRr zKU1EM#NR<=3v!CT%K~GmDXV|{#NTJh>m?glTv z&NA$_6Z!OH(p&KZ-~Qw8=+B&F<{5q7#C{75LXiGdGL2HQrV z0gxgP9=K4z6^S>)hCZm}F7C*{thP-Uic8ySN-@ZTml@`>)FrBl*W)oyZ_(^l{RQb_L|!kBp>*s)n2@Wcn$xywzgw< zk^3l_m=|GqSd3EGK$HDH9U+RobRI;@AaHoWoa^E8N9O-m#bX2#xS5;o7o2ZU{OMYQ8%=|m6)DrXx%8?I)zr1c z;rm4O5Z9;b^M1Vn)g5<8X3`rFbt&T05HS4L5hzA?hm`tb0hZ{@!N@qp< zTZB^w^g9fBAc@hI_Cu^pvk{9w#3XrZR9-R=?<3w*aOvMq-(ggQN)Aw8jC|bJz;=Vx z5w7UoH8&}DeZgg&MD3DJLmm`GRr`S62Ji-b?a=5VetlEYzmybuCf!K%K{%Nl(O_Ya z#m&vANhs|7QEMdEkjuY@Kwtc&fB-8u4x-6p2q70b4lst>;PEt!BbmodBkfb(7@(kA zfCvmL+;VQ&T&Mj#x15Jh5^w-P+3ps*6e}P+Bd36rRIoN6SeLt$j?8R3oX0S5Xy)*9 zw+lfd(sTb^({pc&f5lq=9wgDQb1pC@mw1z*H%ZQsxFr7?1Y&~ZVY0XXz|=vL|B+F- z#BU(v6w^xtsocyx)&O?(H$X814ST2dUpVxru1@&dB=u0%^PPuAsd|V z3&ZNg%v*-JHaW5%hIwIXOBmnE5GgusF$%(DF;8=t+`n>$hRlQm?)WHcQUUmC7Wprb z`y1T9&myqROt>E-ze69Hw$r$|m01GAZfbe$_YAA{1HpEhct-xpSuFZp1bR% zp8lEU=!>|@^5;~?Xemx^QE|*dj{n|)jQ!z`CN9+5*sAc6`DBqceoHWAz(ck`r+f*f zj52D}!8|L=V_=`8V}6ytC|~vA#zvhQ!ZoJhivNT+33fW+?T}@6`jf~hq~&off0s7} zHoe2!J(9Zk+KIQlIcwCb<%?YV(=yUk41%4`cwd6(V?6!{hV;I!*bzBmL)V}=b^g;E=+i_*7n@xoqAx$mXdKfv9Kcc z{Y*{n)!rZXE??$29?4uKGouxUMqqgvsP__@_^&7={tF2BiCiT)*8heCdGi>{x&^_L z3hzU$f{&-GzQlLn{MpnHIR1O@GXBvz;=iN(wlQ^jE znTCskTB>Avbeu!ggkhcP<{3EXjP;xcmpCpIYu6Q|?aZ8ZM%TKmh$Fam%-asQ0-0V# z2C^Up_h#V=ytKD#if>*T$F(X0WuHYTFGDA~#->h{2y+Ac*)%*j`@|fONWYBVOK{P% z)%60dcNM&1Ob5LU!J~jnUASI_{N88|Vl+jFG)2@@#7C$HK7o3;PXYPV+O)63PC|1w zE_s!>H(eM`r+8XhJN!c`EmB|{s%8}HLW^TeuQw@cRk$NZb6>)&#&#ut^+`1B;?Js+Lzr0;a_sbwxHlzb!JOS zqO9d+=IJ0Wk9?4m-dgzc*etl9Ni_m@e_(sZTjP5pIE7(J!JR|g^^ahV_T@dy#KO&^ zh~Imq+S7fgmEL#O`(n45eO^#bkLdqm!xPzjX1f)>eta5Hl>J0$#9-G>S$-M!59mIJJ=`(?)W#Me)DcC49W$JZ=ei^1`+=){mxR*a{RS_MCMV zUE}eibgAi->5SXBHme+n}m9(qn|x}MqYnTt&{k2hgxvEE!{UxgP(Vj~33>JouQGUe`&hNUbIkldQ6XgvPn7x%5LEv5P79%SNy<@a<3U zdiw^oZ$f@`S zJA3qQoux~ZelD4@19vrU!I(v$b!NV@PBeqBWS;DQCX?B`?q{XDF8(%|Oz`|VV`&Gx zS&N}G0`VYH#X}_1%%9wnFK1mm%&Z+-v(j-t!o0G~>*)pQM+A?u(HZ6*khw#gGD_ch zCOr|6#3sj=lIWFqIe-TFzU!V<{6S9xeoZbCQsxN7Q^s5wSqMtSRD8ZTP zC(83Xq=*oZhWJun$w;iDSPq~*zNH61($s0-Lxz%iAG8bpmhcJGBq$&L>6epS5U9I*E$t7oY zS7&A^iC7jzsMc;9wN3AUla@@cMicb%P!uSTqW?uPPX+qW`k_yKDdI@%@B7Z|g`{j3 z=q%>!nQzXV>-YV>%Q>A=$e`l|LcOL{X0F3UlVxv67Kjn4NVi8uL)fkp&l8& z5t+UjS-ut7z8yKfqtiPx%=s?==lwkYdr`qJ@H;CkMkT+*c{?me6aEC}op3Um@~1eT z3oFqce-G!~a5~!S?~V5P`}jE@?vG~t8P0p*f#{%rkn@G`P;}To96jbg79H`AFs&Fq z9##D+=S$%e(NX^>=gZ-f(J}v6^pyV;KTm{DN5}o+y0+?}x0B(CsOCR|H&gzz-_qIz zOADU6yRK{W6BYm2?`UoPHop~nf|Ftz_uk;-yd(CB{r4Yn4jfO4jI`UeJ=q*BLK z32`DT>57_o1|>hYt>m+hEcu)`iIShs+MW&eaIL3;nPJ;ba56Bt2gU)7`|-Rujc;E3 ziY8tVpS!1t&jp30FRYvXOQ3vSya>vR8RcbAz93!#<)w`B3Mem&S3r3sqnrWdj5rI* z*^F`)lyl-$P+rX_=RkQ)d=Zo{W|UWjc}=T*Y4zsiR=X87!fGq(hCvjxlSb0&w5y%@ z>iJ6-ue{xg@pQQ(0+eVcL0bf(nsll+gKDQ8-mUiHKzO%X$zrQrZB$pdbTw?W7kZ6_ zpc>J4vmcQ()a=x~m;|zMGYnGeQY%hU_ZwYmsu8C4^&T2e?Qgd6JXh0G`(lLek(nHI z;(Vt)-&z1|ifGqFxAA&{4p9{#7PyaD86$N>Yo#TE^w>g>)KMz#$U66?-fhT67zSbe zc1MPy9=BGho&2RsmoK)vy(GqiM;|qsNqxRuGt=qwov;_RuVmj{YPExgOsC%Z`rA86 zS39@2lTcUXtzH9tT;*9E4q>kd<{P~*soxG-3yVoT>@-9mYeqWVY%~`GQRi-VmV-9B zC(|cn&SnP!Ork+qK>UHs2aH|)zo5356q=9F%(~YmcnR-~=g@DJI zdfi6STvT~S21!r0o3uD|0i1^SdF106Bz>)~?*k|;8?pN}?YlZXAJndC9~yH~$NLYF zlT=V1pyco%`PrMNlBj#C+v=W78WMv!`BKzdj7~?lPlc_Ur@D8O#ZLRwN*G3`)T#t` zf@Ux2Nc9+>YKE=uZ1-+@kS2m2FmCGlf^3O;7_=9X#iM8}wvbHft}Y)(X4_ay5a=zB zWg3};Me2ji$v7>K z)mU!{N%P&LWu}WNcW~!$bHBfUOy3e(Pm_7o|24a9^sO~J(Kn2~*|&)W<}ulQUFB!+ zv}s(^4=a!_&*U^-w+s5}|{t`F2Yri=)ZftEh_#N(OT%X{qHb zk|xa!UGz`$W8>yIx{3@K4t%(VdFdONEnOD+#+ubPIX~4mX}*Xh`&Qq^Y#GA5Z>~AQ z>g&Q@Hsq6irv=XFI}4b{X^rwVXD&6be&x%I5Ljuu*$F!`wYi(}DB2vz8FeN#+rjNx zPE9Az-k**5!-T9<$WEY(zo4 zZDxqkiGz4%&LKlwgK>jbgmK?mvzTuzVII&nb)FUQ5RF<`!sa<~XfD=r{T${XH#ZpR z_A#kJusG|!)wt7&eVa5@r)H_{8SQRrH>7ObO##sa`!!ch4l&1QRsDRo71ntizT56* zxv3~p9My$t;ClUBM*4)A<3d9Q^wCC4XldjOpcws}pO@_l&!Ien7im!;wt}?O1BTaI ztU0&IVhKfuviM^>^o>_tAos6KLTVymeLY`7yxz*{5 z?UhE@60n!Z!b$FSgEQ6C=>j0SNjjlnnt7239*x8qo5RW|uK4wV^nhi>Y&+?VQw(7aQFm&0j~~E=UQ{ zv#6VLFMUAkQY*=`lxO8ZoIV0CZe6|(nx7dCgirEWDnyK&78YAV1Z}nPBB*GVu?% zT(a>G23~UT?9r*K=F^mY$&)51m@dx^fOZ|oD!rjGi+;&8>tx^U;-tl55{)vh-e z8|`)w#(uHWgTXa;Dl7*QWVPB(oUn4XZnhdRn@tRHX|dahTV#gQ!q*HYezb)P*zbYw zVi^0d(`);|nLRwGZXJlxO6qm_b<6z3qv)B{B{GN4FWh0zznDb*DOKRFPUrL*bOK)a3QHa0&S~rZP>yD7eV#opG*X8 z2|Nrih5FkhGaiac%R*F$+ztk24OCOO4G0(_3 zm;exkSc)d!Mv^+pOOo%>gOPO7+cDo_P-=cO_o*wYa#bUWz z6=Tw<=s-4O_3$H?E&H^VC#h(I(d+fptCQO4g_JMW>$iH1P`#04qcHdIAmtnMxPT-r zWo*_A!f?Q9>=uwhXSYCp4Y}XL9g~scY8B(iL8k3OQF>2TOy7=B)Hm~nm=I>b_!&3| zWmt@=ws7v1{G7-I?owX3fhY2Thk%YJ3c%HZC=wl}V@Z_pTx6UrffFXvy)-2QmwD*T zufBSjRPV{f08X)}@=9||f~@fB@nN~@#kl%<7*@a2=~d@~Kpd}jq4(q*i(Xp9w6y_c z%JU+89$b{CG17nh4@f=(vk8Y8j}H>k=7{fcJJV!3Tg#?AnIK$8`42RhGCf60Z6Sh-7rq&tT8NG9J zt7n-h2{o%xFHR^dLxGRDg(yq)=G`hm8%<+eJ+aV9syBwkZ=9*FR;s|O>WyI;q8YVW z;wN1H_Rp|L37Y*p)u2?sMAAuM4h=t%5en2y*a~C;sxk#=kimm`KP|7InBv3+P{>Xf?SczcMov=g?ycn&vN@LI z6hD$~%c z!WIj_e9JPNht^in0wQGa4rKVbkt3i-?t$w}>#Ii|DGDQ0cT^Bdo}j*(H{;djz}NwS z0|`Ulr3yk{-yjQ@AsuXEY+4&8nG&1EhP9am7RVEmzla2}IcudA+?E}pbSV)?QV+tD zv=(}uUJPDf0KG#msSollB|HnmsYsnVo1XuUR~%$0(g?STxE1>C67^a2L0TGEm54r$ zxn-zFqopEK_dM?Se~{47HZ>Uh#+pf5h&0uv4l~}ylTEIwAScy831=<0=JuTpnh3}f zOP#%GK@;V6(QAwT@`<-p z=;u|BHi~_BsRV7B@8|pOq_*buy+J)2W#wo>lL93+Ci)(asemyRL~b4~Qq5g`l0%eL zp5!ytFOwRp-l<-1MFIZKE2Hfrf;Cp9vQD|-DziI22h_DF<|$*mgloRoYcI=HD!4+) zaPg(Yqo0E)M7;!|*?EL06J>$r;?8`6Fps=RUyxGvb28vg$plZeqT-b`dQEKN=aL1) zIH9lgy6=%p!iB*)`@rZxA`GIQyyak+FS7YEcw<4(+X!evzzdYED~Q>_eyj=q1MRU{ zRkq8W6tsMbMpl-KG^iFOOGwgO zy)HUUXk3>_B$Bd130Zg}9i5tpoy$Ja2wmkeB_r7x>1gtXeu|reuPy}=ZR-I%poflG zelT;?uq>nUP*SNT?rqmVYybBfAP=;5UWXPo0&I9-YXeSirm?}%(AbV}0=Ul5+eqOD zxgrn!oM-*)f!_*gjzH=1r7KFMGMEpYd}^FnZKz2v;bEsHg_{m5a-F;zJ8_WD;U;Z~ zKoV;`$OlpA%^0=7`pulX4sU|0RcfTTQ;n4I%J(wpK)gOJA!)5}s2#C;Njq;!g-y}k zg^U8x0^c#{r{;X~v{DPmy&!K>*+(FI8OCaS5HCH`0mo3-lsr)Pfn%v$@xdx@Qec{f zXf2ub>8ov~H%%Aa=W)jrFNB~2&REcOGQSm+tzo(lp&Zc>Q&a3`%3s4*@_kBv9SPzF zjLO{HcZoV;D%MTJZI96ih&{`&N)b&oRwuG$ogL@x4+}eZ`_ozaGZ^_jv{$5tNe{ak z822Hj4L(fy;gJbRZ=1NcT>}P|zX~##k7xBlyNNv{d8u)CVA$>ELJDV+G1n2`Oe}c_TEJ5K8S)SXlb8?5qH@}H0^Vd!brRUC;7H=kLgs51lhRn}@*_%2 zFwa_QUj$KdG8M>~3A%A@a*T7y<&s4t>Et)BU8sNi&2L=2eBtV~`ZumzW5G`IQ68Sy6z`pq-1*0PA z?JmI}*`+HcC>$WW(8(WzM+OcZd1MsR3Ua`ChdncjY?W}BzzyJ=!vo6;k32F)_#&pO zm=?x{vHBCQit9SL)EG<@$SSrIN&KoX1SyJxI16nI-%w7~EL+Sf9LVe!->4GraxFvC z9J|5b)Ofaf0Xv)Oq1uY8jp{Gl7;F0y6}s{6tj8a}E*Ii6oTCn@*3R&L^m%sY_|=$F zN)PVp%5ZyRo1IKhK{rFiSCJV(#rx!FQLLYKj3(dL5C*ZZUxYM4=;J-@JslGW)AF7R zMFy4j26n_aY?sTF)wV4(19cOi?ZM5;d;!(J>#pgE_P+K$;*$4`?;E$ss?)K9I`l2T zy~F|tA{n#KE57_AJS$Ej{z<)aAr_*EX!C3D5*rb7IGSO2Th7+%iLn_Sis9_Jumg7~ zehrzyr`S@sLiMtlLanD$yJnDq$M-ICB!V~WDfC8>F14b&@#GJw$`2@Ee&?Pu(^E57 z2Tbo-aFGZ-AX20cRyo?5A;_@TX-RD=sLgiK+>lJpXW?+}V zxP=|9f#%fZ4^eI)|9^};!-QYTIK0kd8k1P=H>uB?gIQL>57FNuZUhL4~#&TzW4;uy-M{z!W4*z`N&=+(RHv#Q?bYfrl0$u zV#6%J&)KrJDtK?{#VyZP@r+%(F%)l(#O@AG6E4%ulKnC=0}DB`jtyvhifQ2Z23ZK4 zSBSlzWF4g;`OO~u5ppC3+4qoT z+J&$>721VRR{p^69K(x&y5?|*+(bp&lp!s*WE=grkV}gQYeV#+@?6K}(V%TJ zwc5d*gif4gyLZ?>6~RiYiPLtfLp%EN*Qn|52Yo3UtNV8LMSY#-)Mn6ji=c#$bGR8d z{uLQugeG$f*x34!-qshexkV=oHVo<|IfN7pT*CG*p1?S;8{XmoqSLoGa)^@Z@&uP~ z*%~}?Yli9A{*q^tJo28vrY)T!!1p-s(YKauc^0J%)N~kuM;E6NJOugZe8X36y$XxO z#zq)|ew%crq0lcZ71z{>1E=q;V>?#Y+7ncY-on)^2q!78n}agzhOXV3VLLN7S2F=# z6utAETO7?!^;<*AazOGKa7)+zneviYOwVm5_f& z#2gMlz>AQQY-re4%*8)O-rt8xn#+U6VjMZ^cHx?&1#UdXb_n{DdBl9a0?dfc`6WJP zIgn;Q&!fYp^_`4OdH>BXWP{{STqXZ4k=k52cWsB6m=>wt3?t%yBpxQ!WjWr*hHZRY zbjo@W4kW_hx@Xc!8bc@V+0Z8;{S+d1@lPl5Jj3{9>p#ag3K<(`$E%mDU#1_WU$9}x z-jUjH_9vnSHulxV42|XgV1uSEc1IBMP3}q(M@g_C33O8vd;jvE=y92n!^4E}jWs4& zb05&)tH{7ig|^(SrS?r2rl%jXC6%td#ujqwU>_Lkr{wqp0!ddl> z&LQ#nA-}{0aBz{28mr@x?j02_h~<77mvsjxk0q%$U08(0%Nnzed_BCfm=_ z4rgbJDDsaV+)w(S)@d&Qd1&Dxgp+P#%QE+w`|OGX>(+CA2Iag(QFy~N9*{^oj`uTT qKaR$IJb2?+#?Kua0>Af>)S2CNR8U9dMCF;vT;)LJSmo%x=l>U_`x5v7 literal 0 HcmV?d00001 diff --git a/vllm/model_executor/models/__pycache__/commandr.cpython-310.pyc b/vllm/model_executor/models/__pycache__/commandr.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91e8bc66795ba9d153ca3f477a06edf9f1827055 GIT binary patch literal 11345 zcmb7KTXP)8b)Ngo&R(!s00aTvDN6F%5~zzM`6A1bY0H*F!n7$n$wba%xjg_DoY`IU z%<_UP;>4gWRV8%d%B2$9sUlsKs_-Fs%Zn?OJmfDVd6<`^%1Nd4n5sO)6e;98r)Mvq zAj+Ao>Fqv!nV!?9&-u>j=~b(~hU?kC`G<|a|DvY-oC?#IgTkx0)1T>@CbU2kx-eor zF#;no12eG#E3pGRaRMiC16QYdGxh?H|4aOD#a`kEKG)fCIjIB{&O326sRgxUCYa%} z8`qQBU^Zz44K91}TrwZbCkw#>mrL=XWHDIeydNJ@#D!8!4sUX$4@373qF=KgJ$ya;Nwi2 zjX#k*6+FfHM*MWL60C53EtWJ2de`=oFyhSl-g+lXFUnp!O7VD*8ebo@@KW0? zda+`{dhXE_Wv?IMz1&N!1pJYCwa9#Jko5-{`X2gLuid&lddioQ%aIVB?m8yBAfmO_ zAkM;TQD=Q43*%l(V1%Wlb>ngr$|z~|!z)+AwB3rMu-i(ad|CFcP4Qv68=h3>M`P?U zzd=he$71|emdO!)NbpUv1#7XcZgz<>JjEGW5*h5L+h}1+ZNUxeTr@?dS={qhW5}{ z)b8l>8v5#^afTX~G42qfVq4g&@9yDxv#AH}l_=_qPV%mS2{hfn7TJw{lp9$uw=w

I80}bA=Q>u!}7&r=_??%Y{cZZL)s2Y>g1UH6dzP$-xsWZ43S~LCCYg6d&W*z^6 z$-hK0YeqqjQ(^RDV{UHt>2)I@e=m#u851s7xNYNO8z51Iy5aY!ntb&vlP~c7hy+&C zw5r<3$7LqhncR)UDg|ByhfmDSAAN3KZRH!7(FWgqJI||~OicE^!7Cv~Ws70Z0sRfR z5ndhVTN?a}AFqzDOoL&qQ`*QEo0xnLpNS*GU+{{9spwv%TD+Q>t6A)LtfIOzvQ^ zpUIyvfw(+~+??SR7kOVS=x^rjy-Y;sl;_m}CXcgbawqB}6Oy57nlDJ%DcTY!5=)AE zg(3l>HuH)nx*{*5cvz|}yqaagvq|<)#obGBACP4yb6Uy7=}i~8607$=MX@sW3ilUL zppY<=)Lm{n!++!VI-=cPHy&UPQ3bxmT#W3DCvWrfcO}G=HnPPx zjUCX4q=P>$kL+a1$l!w*QfuLzLbfnISt{K;S)iAL@pLb0^jD8hT^NCG2A|@{T}Pbi zv9Bdgb)gWI{1Oy6WJdm%<5f>0UbS(zU?pjOxrZ$sA)9z$Ty1dX=PEV~tR*JYEa_Ds(^Mw5EJ1QN z`YViF?0OZ*qTm{_@;a_k>!NF!i2UVRehxJt8yw4l`0e7+8DHW7H&)x7!OwQ)e>}&dxo`n;GLMh(nkseqT0syoBH4PjAFIQ zcE?Z+w5zT!fNX#%x+^=7wN=*;#9WKwXhE3@Tg)Q|9mtmez5%YHt zZrn$#qGZ(cw5u%#Zo_lg(t;rwR5!_17$8M!j6);y2-XTKo|1|~Q~N-hsrN$esMe@{ zPb*q{!AL&8WbKk?AlX_j6l9>Q@LeJfbtTl0E5(gHs9WMdRdh6{M*WOFSPD{wMmp&S zy&2!ftNWR}j|r(QYv(iI#;fn*&2}^Q)UPnVh(OGFsLl0ZGslM?XL5o`n~9u|a&G+r zZ@E1C4(tbc%N^LyJ&5sJX*jgcKukjF7rR`@n!YkH@Y&W_GC%KX z_}%#1KX<;otZDzIm+7C0m)j`mpHOj)vq)eB2YO}<^&xwxgH}EGJ+>O1=CzYJ z6ZbQ(7sSD~kWJSc{XR}bXApT|?6rebq~89Hh`obk;Oz%-=4FW&^!w3)SMC!f+hN-a zWHN}kw-qJr-SJLI2d8l1CGyyAL3$9k!=$+%gqh!=;na(h%u5Gh7HmbL>HQd;;(pNU zN8;vqzgxF%d1|S*A7(ox1y`;&uV25>m`rOnn~IgGsX>_RWLdwtEwU@GQ7gTmhZ3CkLE+%P2hOKz%uhl_wmVJseNav9xU z-CoIh{gr;$zm^3O+gy96H`wXj=sHj6{z!q%{z1Mt zqN41hZBrYSQ@y}kCeNVRMw$7MS>8rTpG74!a!fcSgG1^qlN&9In=PALFg!cA{f#Wh zL~gFFuWjbm>iXvDJs9}f&sIPDcr!OwKUv#cZ&Y%}_v4@^d_Q-6znAbqMD3dI|7s9K z#fW?cr^$0vou}#oRYdPpRj1dpsGdhjNpy~8GWt7A^`GFSiaEDw@>}3)mKR|P3(#N) zyIbKVw>mbrc?AIB@Ye}2(5iA5vs^KEQNzqRw9(=n7x2?i+l;K z#s72NGOuIyl32!G^BtX^;b*@vAF@_m^(gw#`wTy)de87T_<4Q-E6$>IkzYdV8Gczn zAE4gk&+_NcbB^odp6B@s=sAzs9)A(73uwK>Uq`9DtA%Rw@?AcG+)bD zm!3d<)aDRIH_+r6^=5Hx1HDJ~kUe4)3pTXJ3}z*t!)m4X!g$CkvOF&taQD zY%>*6XNstaS&>DNb%rqj%YPSdDLJOCJB*6U7WBiV%>u8R<+K^#CU3D&6gzZorgZ_( zna*`?d}%Nu+<2tX6CPu(AHF?7%G};QW>F31|DK7B^f`WX^D22~9XlEt3JQ*Xfzi>) zisdQIjAoq5vr;Ts_E768SKFn;fgQfGmX3p%xZe+@08|VlEDBLtvebJ7l%yk8xx!n%pwunw6 zM=FFQS{zg^@X=-FHxo6NoKR?P6Ak5$G(c{uXd(|IN*?G#IBhxc8DQK&gH@+VzTQS$#K8l@NP>e9^&&I+qn z*kGdXEc!+o=~~=??L3baR^b&x_=jH^?cnSh@*4c5s;tU@J)tzrAcSjgh-|_;T`;!N zBpPI*$gOAfF_sZh-9ZmH>-Cc`&X9J;UIvwtI4K*ac8XlmYtxm)OApda^kx)+jGyx9 zCat*Am_22!19RA)5Cu(-_Us2KE*8Y0sKNsfPKqrt&S9*}%6N-X|DWx_*t9i9M&zPc=gDu7S&Qkwz{T31rtivT&tFn%T7#9(vR|s*vSo4q5O61GcU2 zGWjM(%p(`|+|YPLi78f45{lj$*}v*H%KidB=clBlE;7ouRgy~SMvjl4owj$v*CpSa&Q_kndsm4=5>xpCt&B z@)K7lpmr#!#ov8u8%S-R9-b-jVoGf@Ql{u^peWS%4jSOQ$Q}&2NT@;YU9*guh^Q7t zm2vch#2Wg#{4Q3W42a5<-oa3D=_A5V4LKAo{)(bPPL0*U5dz>O?lIC6t3*fL;qD_U8>;FC)pk?RK{DfK|~xkydR+NA1F#qbRma{=~E^) zF=^~Vx5DCHS+w!RL~$nX&;X6DH=dOj>E#kt_?D>AH6@LbEcJa!?w?zoC<(GcH_Lan z(hT3}vei9!)CYin`7$khohkx~eEx^P*46uMp}yNm3WmA4l_Zh8L37Bu<(t&$^kvc) zGCPoTIW2<<-PLisdaA<% z-8sGL>iRdfp?__g`hRW9{f}GK&EtJ~b&2WbS>_t9Q@84K?isg!$zE_9%ZmHl6!D;! z8;FJSRUDFAbT3N_9&NM{S|s_gl3O8;+MbXpl6%~I@tux7lcZvPKLto#80 literal 0 HcmV?d00001 diff --git a/vllm/engine/__pycache__/llm_engine.cpython-310.pyc b/vllm/engine/__pycache__/llm_engine.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c106623e84b6ee1d465076ef21a3585c38e0c226 GIT binary patch literal 48342 zcmbuo349#ac_-LiUEOFj8jX`6!Lvz<2P6{IebCfFksw7&1Sx`)tdiws6ICF)*$4Ql z!2@pCvMF1168eY}=dcrlxs^C^?$eW*cxQjRn~5jRIho2GPbT9`CQjnm&d3tE|Nr-@ zs=FG1WbJ~)L)EKyeDD75SMTl3#PD<1Pygn*FCLD^zQmXIpA=pm!>_wN5sTR|J6?%d z{A*O=Rl_pkd`?snRns!%*{me1DJvz<$x2r>ZKdToRq3u~tW34X>ZxX}Y_-?wt>&y; zwa@CS_FMhc0c)T-Xbo0}tfA_#H7t3$kY|-OBF||&ueMfK*H~++qt69fSzp~?ZSdnZRySFjTT9-)os?c>UL{;b%(X1y3^WO z9ka%&w_CSY@38Ku?y`1CzMjhN>bNyt-DB;k-f7(_aoNgU)w`{`<+-{>t9!+pM?A^FU=^^rnNub-4P3^+ffEbws|6RNh{F(t1*!S67~@9<`3j z^P0+J^_X=`o<}Rkt4~``%k$dGiRzR!CC}?BC#$EdQ`KjzXR6Oy&r00-$~&shS!p}|*Hr^0eGu<=+xOu89%l$CUdH>q_I-H2@2X*4!`uD#19*D?Zy&PbQ?aoJ zzswahITrVlwsT?5!9#AkUTZj)8`Wa1c-B#f>9&iF;&i3xx_IfDEviPTSQ+c_dQU!g ze4=pRiHQSG9+^DsCHB{@cv+jiHKg#; z^^1W>!^#xGo?zYamA0!9WN@hy_M*=s#j+l`|aW^TI?t4 zn*vz&TqTScn3{E_=PJcU>7oz9Si&0#QrSV<53Bmztk)gHE3ZF%JT}*uoojGtGP))E z)meAU@N!d)V&kd$S&ZT1MHd~$cP3jODO6_%tkaI`);ZigQ;quUfpgCEc^Ump)5<9} zoI+jdDV(`daQy)(oF!DqvFfbTC{*faOAWU$8)|=#d&* z?e)sv6V6%8C3VH?Zhgt<{^y)PRRfalxN<`H5ZS}}3Amo;Jsj^7Y@FM`*V}cgri+!r z*}0PK*ah34M@!m@)V)*1DyFW|{mDU`sGf0bPCLIy{l)8zz79uT4;81;TUS`#@p`?2 z%$!wK_T{i{k3XLRFb!`YQoxg?nvJ0+EUnBbZ+!^NGK238=`nF+j{C)_0c;Ftl|R;c z0bw6PNRO_sbShMZ5I2^$?)Cq207is2o3l;8R;-r zVFqi{FVMwUWly=->##<-@nKrJg*iqPTrQb{Q{#SRd%ZfciiLuYXl2)d`W&(-FQs=B zuiwV}a;CXl&R;5G9eUlDoZ|Vy8H@@6FBGs_xok|~%SrZcw#seAhr~-Mr!l8$({XOf z{*QsbOnLyn?oI@aSTk0R(UaV2 z$7ip2>#>H4_R`^%Dj!1yZa)GeZpy#Z>*}E z$ENW$@&iOrql72$;KVF+lVzL&YG@^F!!n(ulXAL7W1}%AZ722tRkXShmyvf9?@6o2 z$>Li|zV+f;PQG>V9bKIEhCB9Qt^jBLkQ_g5zH!dUyTOj1*E^`2pQ)>SF@Le78gs=; z-tUwADc7;((^0ghw0o-I%uaf%UFU+{W(!q+wG}EnHV49D{f*f@Q@>Owx>st` z!FJ8<&g;QR-0)+@`n+_ZP{UbUD0rDd0b`Hj2+!F<;R4|CzjPG}c73`~P-oF|iesUA z8B8GX`U-_&tyV`%4j7?k+kJcGSUmHEDx0VhmbaGI|(2%g5Tn?;a`nZ|Iu zKc0;b$K}h)Bl!%Hw?@)QI3`9UX~*})@SKuo1J7NE!-zU*{KZJh-z5HaJ84OS5k-2A zbkgQ%rZJk`UPkZH2f3+r`NO4qfGzaW*chaDy)2%jQ*mC+dc$~|DJk?ZU;U2uO!s_g zc5iGtr1gKkQ#f1={l7e zd1>p_(<}*VL5_w32g%JLNXHGsyk#WJTPY)B%#Vf(+;8M~kV7e?ZmY}XYlFcYg6S>< z!4IE%@GI@zf(LBfvBh}9Kjh&D2@!OV_(@#7QLR?YrxZt56qF(pW;h8-KhM z&}gS-%N!W%QVWQCliaOUd71_E>H_>uWE0$B1gWGMH`@N0ucmL!Z(PzTk<6ec4lSMc zGQHn}BHLC#Yx2t$^CG{t;A+kL%lN>p#a)Xa9qTtTnN-}kWf;ay`ArzFr_9`Iis_cD z0B%f=;nzKcaR^pG^c288(8c|tn>$6fjrH(S>}nh&RffC2(}VSpb$YQb(oXM8%I>x^ zFPZq7!`B`VZ9TGLf)xEsiBws;S5gf)gU*oDki)JsTwZ1OIU}|K60hG`Z4WqW>_KPL z9&*;&!_GQ;m9yR+L5>Y5b2a{M#NRdey9s|s@i&jZYw>q8{;mU5TkvmQgH+8rdjLn;*@Mza zswS;FyZh{4w(ka{cgdH#@#P--UVOO+>4s3_efIr;?q0z8fc+qz?(6RBe#rh6d#`hw z{Wj-*=K<$Iw$I*&=ZBoV^8B##wy?}cQ1(82e-wWoMsM8a_r_z+BhnLt_I~?u)b}WA zIA9+{4Uakd?LPnCL-xd{l2X?p=kbm@2A#K|+yM5@@`HL?^&GaJcqxV6IpFL^8}z@2 z>|d4AkNp2G{p~27eSg3?Xdkklw4b_egtf4)hwNWNn~r`e?$`ny0Ow`<3ceJOZ&2rC9)Wk>emB0HmKL;sZ8}AJ!QV|X_IvF2BGnm5MHmJNg zDSAYc*mlP=nLK_@o-4U|{3pd!EtYC|;Wr9B#Hfp{vd8mBmMK~aAytozC8yR{wywM+ zV0kD`$|=>rbFu+TF&{DqcIAswTBTS!I|o8PFXVB4=WN||OJ^!q@-}%*!q~9$#c74; zyr8{c|K!2oa>q~t8j8n!6?C_{BE`A0;CJLnqLDwKNj;$Xjsn-SgqS>;4S-~4@q97= z3oC8zu>^U&k!jW`wsyNF*!^pJ; ztUfaX=97~@Q*4m-&6jF3jv@zFDq`1g2!b;AlUzlSdM>?j!8IwDyjOtQ2A^h4E5LKA zwzF*ZVtAaqu6|Ar2WC6ixu_St>`yhFWjriFhSq2LdU!b%t%MIO0$LY{;dJV&eQ2z`73o*^mFHThC+fj zfytjQdXerXlebl%+hV`J^>SNz!=dGv1Z?6~ExKRp3LI|8L^Qa1R|&qfEguD6Vny9jqE&)SZ|Hy`ugN>;Uc6I;B^C-SMb$MBQRb}C4@z=CEUHnSIXq2 zG{?g0@i`X_h$R#`InxOQ!Hr`V9DcP1@lx#fep-l zP^{R?fHPdfNrc_n>!$2vLaDm)OlBT4)m0|^0D})Qb&p*SdIqzO(ILt1_4!>ENRECU zpM`CeN~H}jS#D*FR6KplOw5nA`mZC8a4;f0L?FpcksdmUhe!`?KO0*>e;kaxux25N z-bywN^w!2mZ-HV8d&^u*Hk0JITv&CUVrCq?)64?WN?bfmDf8xU&7ti8(g|mdVRr87vvbOw+lvt*j3B! zPF^n9`}D5pb%z^EAZhV;4KGX7$d}W(UamyZA4D3y%x@5zM?uqrCZJ@e{ERW0$*TTuT+UT#F&xdM%FdwrdH5+pZ-MZf`Q&(M%)U*-RlE zYj)3md*k-2On*nS2jQ+}X7;-pyCr74*^8Jx%>jgWUQ3m?l#}kyUP~?Bb%W{6#@&d+ z#)Id35PR>9*oV4mX$W4AGv1S;P*$HS$vP7HCVLQFk;OV;VqBbsp}bhe@x)p@BS+w zdHe=?qH*AA?4xnD&F-?()K6I$LfoP9;pPx)Y~~tIpx=5?H%1CQdIZ03N6awdF*;Y{ zjVG^RYi&H)>_HmzIKHhy+9A99Q$2R(dIEh7Xi>vB|1LhoI)z0ZYaB()w({UrY(>(x z$(K`Tc`w-K=_TKeq1-%5DK?HH*V9+yA2MGyk=_84e!27~YKgP#;SDYgmA6tvz@hy^oR4*+bV8_VA}TI;RkG%3d`JCbm6-=dCwHzY1TT$J==M zov7n;jTh|I`b|dk`Nq4-1-x&^FJ?=bP(9cjY@B9XIoBM#YS?S8b50iJj2@jVrm$L% z1UdqmvT3I>cV67)j^!_v8s~fze6Pl|AKs>R<(KYP5*_ZCIuS%Pg)mA)0S|9;ceRK? z%Eugu3Z$m;Lah@8oq$5knNy)`TBobeLU0VN0l+JDC`JHHY80aYUW|Z`Uu?WV5zQO3IZNh!(5GhAz;lL7$@#9sw?>J;@1~twTC$ znNPqtDe5jvLuL#CIj9H#Y0ph-5Fj3`HA?e>1xXbS&@=@hz5~J%K%n2wU(`D>*h&H5 z?YX4ND#FG&Ngg?T{1~mO~`V{-mbYqpE}4)_cd<>t>g`N?|C*#(O7Ja`dvAfPF~>{uLR(HtsPTn9q6 zL9HJH6d^3q%L>Zi6M#UlKMiPrVE2V7v@p93O%&as)@Mv8PIG7*##~~CFcePNU zK=?Mkk9?Gh`^Y$8S`|*LcJj_^MPr_Z^UdpxtVeHj#nn6(hn@q~a0=%yx)BPB)Dx%D6nIP{0+b$P>@c)5`1*?B8RK_)@j@r^pV>z(g=s+j$cH$Tq?b}L3RY9M z=C{AmaJDio0W66^WR6n^D1dbnFdWVCGGxZr;~K!RIG_{3Z$jjOdgvLUeg?rSaV;DT zDHZi|i2X33q$&cSOr&l?@dS`Afq>M9K(G-H0zr?4;949iIdO2OW7mxUidRTjHZDVd zhJXWs4tE?MuXSHB)MxR@GA|ZNHdYi7Wvb!%8<|f@EC=rQw1aEjOfL^^cL45q0Pc{0 zYXXZR)EX}IE-W%kCgGv6xD!?>@7&5UrXIWR}DM`tvzhA)VJXy zl(0lWj{Jx@w-t-^oRggbXFE6}AipsAGp&OH;`4vUCZ?dY>Qtal$a7YI#z~8cxQKMi;Msi|})AneP*hXiK*VrGr|j&lWJ)qCj~DMQiv>ay7i{GnoB_XC_We9XU3s zUSL5VMqmvsv)fv^*2YGdw073W^2mpk6-2|wJgiI$?XU)0#RNEq*7o{QR{!r2c-eM* z!qQ}*MkdscG10Fw_-O{@f2hwOuzEDHDf^+dx_wt`JLj#GKIEx`;n>!|lB3!hUhbT> zhL$;`z11t6&|0!r$XyM;0dZe!9b1%E_HoAd_SLl5CIgR zqWXmDDa2Ymc1thO8;YJwa@LAYr8>%DxWRfoMChiq4xv_UTeC#p5sF1l#dg*G%rK}) zU*H)bj-Y;kRgd^ey1{o}o7d~>jL7M^9gyB2!UHW;nf2Z(X_>E+8yropqNbLWm$@D- zVBB9TU>g7<{t+ZoCzv)15bLU;^(egr1Saao`SB+hP!q&5Me94MAZ2m9H9jWP+9QL_ z@0Xb283f))fEaabkH5~3`C#kp&;VKhLw7qX>enp|8V42Z5xwKfCb=8}^Kq8k!eV5q3IZeZ_51i$rs^m8#J0*D zk(sN0n6ENSVN>;io?do){uL${)JgW}AL8q22GryEulRLg#}G>mV_&{y3>yDv4jKQ; z7&iXFTxI-EbHw<2bG7khbB*yo%u(a-%(cdUH`f_|Ypyr`o4LXGlDX0N8*`KK*Jj@M zD|55)m*y7ZFU+mRpPRQCe`ano{?yzK#jPF2e>Ha+e`1aq|HZi7_+#S^IvN=~ zfzO6#Gd^OblhklZffs2aOd;f(kzj1}A2@VhPxnIQE|JA~1Qn!s+B6OMXQqvmnbOaO z`L|G6-JB8raQbE@nMtIf!irjrOd|7oDwQ&alj&6Y)pRBUo?m~;00dMzGlctRzMlSO zHiHunuw(=RwBn}Og&6rK&@%p}Xk!g7#it^e3-M^8AO|>JTuNH|>_+UJ#zK67YQ$iH z+3~r!TC1PIaRz4;?DBXsaU%hyfT$SrTOkE~HU8~F7+XwJf9(cIe^M8T`5PgzE`r5S zj0uLm$ignw6^sgYx?{fVj^c!q5cBAG7o>cp0`Uojq7)$FEYIT@I0Fd-_}7A3Lr7f9 zpQ(f5=SPZIW#M43sLXRn)>tLgZlF~6y8=Csum z?)xMusIl!+;}IQB>Pm8E$*lTiRMw1_fr&2|;JX`*#8pFWfo#HPB*B3nX`}*q#SbJz zPGOY0)W!zpPZuvS&Ph z7#svIK5nHnKP(`nbCbdz3({H%m?~KbnEDVcKT*5_3109GF+FXsAf0CC7>6|+B!O%N zDy|Uy*YTnM7@nUwGI=cgnz$@ESnfMonLML9J8wVE9RpGXMRf?6CfLxkfI`Nln&?UZ z(LFzwKUciyfWP6+Ir(}mk0SD=nf#Ud9CMYb5q(64$k5?R6nB*zGL{qRKLF>C{ zj}zLqit$QNq83kUMY-R{0I&nKeVu=YRwB_PAa|#>QOP@gcLz{_{|8+WGH%?tSz=mw zp({>JO%zV-f37ff{OG~Lv&T+6IdS6QyA;X4v8_UIdF$IsT{6~M&P5Ry1%bhEtKEnY z>6ev8UqG}~yrS-89Qi3kdudKrAji+zps^|++Hs#f8bGi9ni)1m)1b5h=0ymM`Wq%% z8%-2R@TFxEaMKK{`CBH~6fGf`JFQgzeFb1Jk6MZT4vA#L_L4ww>#7s5Na8QAP$JKm z5V*f*qT8atg+njLQJD09BB^I$&b!u{mB!cW=&49udr-hwPK#~-63^-%7zm2Up^&at z>`uilQ|u8XEoMs;AytGyv7C-W#v2d=tpHXq$=t8<$axV-Qzpc_KI$Tfl^`-eUhd5P z9uTLvxjXxZX`njtyff>`-`lc(E`K*?|4ROD%zi0MGn)P%z_Wl5nZwzCLjJz=zxcU( z(qD%SgdyJ`GR@v3iZs8j?O-Dz)($0EY;vek#b=sFSn$(@2Xd<9NZX4C*iyxAP^1hw zN7K9!Ur07#Lteshv}oLj{Rkv?IIt2~9A8Z!P@gh?B>v&}LaLbpGGxx*c|Z&&sfpGu zvGQC`+l*Z`djdvxrK;9UP(P0nI$?)d zlNMVP3QrJf{KnFW`Qfl|*l+|1BCs)Gh|}cpLSs9dY_WtHTZpS1CPZAseBhA!RvLm3 zl2L?tR)obPIQ8e|8f2${^C!3(F~&|1Xm?MtFuZ;enZSxY zM%bQX(E3EKjAkn=v!>MlgRdpROC~vbQ~ssnsn^=ocRJW&OJEPX(Ly0YK|(#N4Dmkj zQ5dtLOIvTI+JrZ1=wMp=Da^$b-o!=_C zFEKay7wS;A=0{pHp*^GE)LtgwZ!m2|t-4;e2Ms|2hM&F{d&kCcvg zd>kcZz+A}s#&KQH_2_M;ZS#8fLaw%2()A$SDWuD`rPHaB*V9O4HhOIs-+=6c{>Mw4 zU^!}IEcEfI*|*r&=x_F2#Tp)sE%ZarWT2U2t7Kd6o}>bxxEvs3j=GhypFvLJLUGU< zk|R#kpP=}rQ(#$v6B1Lh?m@H0R00994P6eHy$XbB44Ygl9g`@Kn=$z`|KJ2>DcBuR z1O7Y=h9m9(LYfRUU!-e-@yJr+j>Th)Oy;k09~4nligx1Mx!UA-uf(kl+>*$Kn1gwJ zt>i*1E{X=SYX>4K^MFmKy}*Z@9H_N*^=ZPduEM7%l|}3zJS2$C*)4u4Tnba*X6+2&kVN}e3U7W zo%^QLvK6?#j|BAylWu*Zq_f%;gZd_`W>0uY81y@ry#&})RxevgPQLWm*nr;sZ!s+t zf*@9A&@tjNJvbSJD{iHj9VCK9XmT8g6+4Cp z9lC7yzndG-X+W3h@%f-Q9)Vc5afmd}GxchwJK!1V7#*g+kpXYAJ%-Qee}lCwH!|Sq zshh)QF5ZUBjV}*MWYVIYExb@4v$T;7u&v;jB{Jo077?58)Ic*2eH0%Gx=<9X&n|}- zD)?uL{RW%Q1p)x?Xie2OEikfb!__UT)=TNB8e$EkBe;jMN!&NT%EcHSOfpVjBnpxs z(_}Rgc21LM6`f<-!=d%p({BbXGJEG&wOZ7c0_TG4VUdl!9YD2)5D<}B3LKnQy*1#} zC~|xu1S&g!yU#Eo3Q4Z*4h_wYyqs3#_O|g*tU&+{o+rKkKzl;F+K4N5Q3yVsV?;wT z_?2_%b9iWW3ky31HXCtZva@rf%I4y3Kbe0`I0--;k&2kR3D_~zgNY0K3`v|V=M(BK zd?hnHdERjELYh=FjWk_;8WSbN)I(s7o0=*tC(C4rlMcigbKT?#xQKI)=U+3kn6|rb z#G5^K_dcFJ-ALEdcAll7Lr9NfWS#dEe-@KwzYT+;z*>rEID1QzXFEbGDyYoYD;@lE ze(ISrPQ%qoSbqUK>-i|x!zVu`A#3nljAb(`WR~)lh(^6El%@JgDZ=USklMY<8K{AUeO5tkW$5f;F0<1L{c8$Y! zg1H2&V>MXv!D|6#EL9%0;3p_6n5CH1ZPWB12z<t%{PXi2rpvD&b4RC!oGt2paR64Ry0iJ*_BczHPadf;=#_ArY8#%3 z%i@76Iy{d&@<@IwmS!MD^S9FoS?+5E#T2Gn>b7i*_ZBh0X|B?WBFyO;;Th6_qh zTOuw=hfR}y!hD(m6>fRU(I~z_FhDPwprJRTF9KUlZ+^wA!neXhZpz|Gli33JJ3z{0vQ}R@oq}P2mj!Xf% zvU5oqOg!j^>})|YwrjaG(Vkm2$(uBOY3sLAalBEg;cQ`d2W`S)q#hoY{}MpP$6M2P za(;kpz| z%eG;y-vVMttj!u~6&G?UG-cGmx+C8Br^FlgqSEv}6V9M=Mt^)b(GPtKu$sW+*Z)$l zWvFT))?-0j@Hg{s3T7Tq(dt+lLP$*$V+?+XL57?S>a%FR9z@ugW&(PVy$qX~^QOBB zPYJbwA>Y-81~9>9hURxZW*7q#DgnNSh_M$_zz!kK1||shRMRMT-vIg7AV0%63poN) zf}ha^3zly1J-UG__K3t_m)?AVW8hHH+TODJ>BA{HxyR>64`|KpVA2Yf0>k@Sem+BU zOibc4?l=ey&KRoSU2ruS3U{-eT#6P80vgUjSGRM&;HC~*kwKAOI(V}B{8KB45DM8_ zQ#dnMs@TjB^$6)Drz<%AtPOLsu%&dg7rrGLZ|ZjduNowf2k{h{a~xVF@V5x+7Nr#O zQPdWu8)nck$8v~!EP*-pAd1fPrXYj@Eel&jQnVS-05Wf-lP2^dppQe5lRxKz`IY*H z*)vZQ_zCUH?v3~c!4&jzKBDDs(;*=P7=(xk37PGPz{CNQhE|{9gyH0p2?Nefg-n`Q6`IkNW)>Qrhf5l-&~Ati=l{JeYuXkq*>cdKh?u5@e?pZvveC zXCN@=D)W~P6q583;~8)kP6KM7G}XdI9Ap{(p&cwpoQxnARNzQ~_|*z1*N6f4B2?z-BfMt*rU>)qp_A`1(IRU1@gdwicD!g^}!!4#T@vOd% zUJGPPa>6dd@Pv!B_QGH0q^2|`Ga#b7HVIB7-4ZSrW}s#ZF*7GCCT4to!(j(=k~2`Y z-l$UvD3@q)%`)-@wHXC|62N?wY^_wpVwfD#_MLo1Skz7w0x0q!6r^40&Uh&xB1l>W z5UukZ%=-_g2^|87(3-F%` zL29$t??|%*8syPaol*!2`fWobF>y*_u4<*x9U+8}LioC#Em=gcKe$Up94y?$?^k)U zybmi0TfQ&ipu2|)5QO`(Kn3-?n1W_qCqCeH?<4+kc~(90b0|(E8ODny1f2c1<{yl9 zeTR(1-&))@aYj%Sv;h+CTvZEAmg`$FF${s*opb^ z6XIJ~OY6x-z)l0M79HihrOsA+*o$NpLltA_ToJC4BXT2mxo%dtl_2)ZXqdVS5A7nN zjso7KQ;W4LD|hPZ70UWW)_X7O-Q4QR71DY+-z~4N`4T}ixV`fC0Di?)DEDp}PEkrB zd-${Asq~h5TXXZU^LO|J4omY-EOj4RfoG8%Jc#EG?AE(|g=Q7Nsw_lX_G{=BEeA<{ z3*)g&0fOvB`UfSp6eYIfr3p=hQT7qsu7J2gF99t}Q9~GnluDSpQn*tFe zOA`}XUkvD*nEVI!1VxzHB-Hbv);~YinkR3P*UO8ZG&27_yRXCBDILkjzI0+tbuAQn`JmNhD0K-m6++IMD2ns z7qSPf3?U_OH%%5F!3+)Ojd)d?nX8ExoxG;Hz>|Qmm!_HU%x-Ijqh?EMVh(&P-AgSl z{-txkh(&W%+)|S%;qx@w(8RDK;N&MlyaEl@H4AdHLgnN65a_TVae2Kre-!;nH}e5{ ztr7G{9W9p3a-NvUOx6OW=joXG1@LUc%7~Q1ixKpI0~P*VXx>t9&R`+J=7(H3fA1vk z7~0WJ6rt5q3Ya`Jr`b(J=lnRII1oS4yD3pYk-keRzO|Y_koM;x45uQpX0Er@E}r%L znX6r!0ZRkk?7$COsTOHhqh$mF-N3 zbzC^3y(e4U@GPW@>0Hit`|+Z)&@qi=n`*%dIK`0vceuhy8DqVk#8DmJr=pa zz4D!13+6&{A+^x806KpmQ-g9#*;q`#R;S|&xC7AiLP&=xD9gQvn*-kQhRwnxQ%N1e zmI4u_dk}xS+=G%L-86MxDAO2F%7Fq;&47JH7I4hLe^PIwn`+2(_nM3+`BN;99qoXfSDx}a{Tg~jBuZl(HloH2dPp=MvRn^sG;6v~2%Pg&bonChEN6Hwg0u&OZ#_a;M7A9}SpR35es zh=t_|o|*alSBy1duvz4W>MBwwpc$eZe>}b0>(0L6CDlCWTt%)X}1Bwu7%aj)r;Ht4Ygfv zrcwS6Am0x8%aosND;3Wxmdd$vcQAL5`)4tW`Y)nXc^cr8`gzpx>9^P;U+n0SPYaCy zpkY-1&<5S#zX5cq@=m~aJ7;crY++4vjr5a&5nofjy>Ul*7kZX&GOs^sxLttsE40t) z7$E}(e{_Vt!gu7|6G9@-%k@*U3+WyOl)J$vG@JcUK>%A6@8iuB$4PFhNX>s`KLnm4 z@b{Sv;dvz|6;58kkHpB1M^`|W+C&Pp9&ZaPpsAg{umpeHS=zDxY*yqFssHfxX`Ac2EPQZH|`5ycdz`4;b)t!yPw7EVPQ9I>@Y#GdRFNl-4Ns z@VY4RhApx;s2Mjv-Eox+5H+fD`f=Kv0VX$;*@znN7g@_bmh`81vZ(SYo+R{19q{gP zl~=tWTxmym_mad%Caks9;*~RST?DjSUuL0?YN_L4LYA5qg9dF#++>A+%xEz69mv1Q zFtgUoCGYA&ep)fY*UwWg_~~$ZV}G|%~FrCq~yVgr%vvtUjhQJZ5g}_ z-lz(IF&`V%cmq#fE#XMi>X+W+%7cq6d6-0!Hn@QtHfa5*|FHCeE3Zp%KipyzhpQ!Y zvDX(i@Qj1uaja|@0T{fYWhRWKQ9gkBy?EWq1v6b&s%cs*(@ug;{Sh*JmDtPO_?XV* zh_58lX66=5kj)HiT_6NYm=IjRT_6mJX})Zj5S57<9?hBi#mYv69Iu;ciaL-IrSR4e z&}C$fd2g{U zv_l}(<0WzXh)Z`8Xa^WfWanEL+YxPvo@|=SxDo17k$*R40l3~XS~a|Zof9C*p_T7Lk z+j#@B%Wfb&>GA-62Z?MnGp-E?(>64n!4!jjT=e^elucuSuiYsKzmThA)t%=VX4Dq8y%C1c1u_m z*GUN*MnH6FT~DE`>Uk(OPz4K`YVaUU9t!tKC>6pVHx2|U3kzM;qHN?Kn16D=r1x7G zRu2ycKIes6XuT;=Ro$1cf5C=;X{7q~`*eGC>Lk;9qqMzFkwva(=0OBJS zZ~cdVmA?5tZ(QD+=#o+;cJhu0J9*`5(}~~P*__4j|L$; zRk}|!dXY1y$uji>a)9Uo@x?tLR_-qksOPX343cQWQ*3bvVqduu=mt?Z=%PY{wIC}W z2Jv`y^z-8IjhMR_*1rlOV-gu7v`WIhv6^L+*Wh;)ziU|oe%Dd!0RA#5M24*ua)a~) zG1Xnzfai_nP0&c_x~?rb2-&AmVjl2hxRccSA}#r8_sA3p(8bN{ ztqZ>t^cPU}em~XkG`8@{C+ONHpdFiVzi6jH+Tjfh!>#4pxZi;!g%?d&kw|aY-5~rj z*J-++0V$VM-vl%ne`ow>?40n@iJddHxV;Gn0%#N4zeC8VB2FZ)P?d=7TvAw;Fwr?bz?&PR6?pWMqXJvOsZ!YeZxguqb^%g$? z9^-MnWw}6R>D*lVI>eqwq!B8dB0kkPHQ02{CG42KdIYvWAlbNFs~yE2AJ5}PM5uP{ z-J3txXw15M_v|T+m#T;w$F8x*y?FN%u6uBNwnL9;Z?OVb-1LKSvE*C|_-b6H`G8p% z?!bBcAUzPbz4>Wqra7vmx65yDSV|jva0SWt6f3<8EJI(EtyyNlo+KMNFnb^06}A=W z5^gII2=g6%$V9Ro23|ung+Q+$v%>w_gYAFMbbasis62TLqjq!`R2MW(VDW;2SM=5-yfOmXgb_e%w?2S$;_6FsTZrstBdpG~dx5eu@sAVJLA#8oYwatq(gm3{Vb! z1N~TS{r&%45!VOOH;IH!$@4q!+(j*0EDoTeVDmn@CAxYEFn*?d6i+M2FVUPGd!e(H zo$pYNg`%#nrlQqR^lXDl9fOKN5ona9z5lzv2GI+XhT86nxx{n51J!3th^9GGz{vhg-w?h5c_ za7osfUy{FzbxwuO_n>|%5&PQb<10vbcS8G2b&T%V$^f1LC+B=vou-Q}9QPeV%Qi>V zpORbUk`cz6?`8(OfPX_!UdX6WYC?MkVjET%;X%SYs7?gB?*g0lZu*j)%A>-XgKO&KGGTQgADE z+zmbmdbNflTVE3@(k5?JFnEEAn2VY0dL#NWS5eLPWulCSio-y_Ym*~`xelF#rLRbv z39-KWmTD-iyFcge)g&fvjA|^UFQq#}BGV^B!F8M8J_X02^ood?!|R%Tw`dquUZPI) zko(8@*tj-Z|5f(a&}6-Fga!q?_sFs3whr}&gr)^tiLc^EQJ8keNM#NeIFV2f&S7>D z*N>}tcjvSUF(@050wGQTuA^jf|kTDUcsd^MWnR zl|YHoX9Ze4dWuMwh?isa7%SM%;6+ydJfFxiQpXuQ&ENqBpT}4Dhjz})ofTWtvEG)d zhL!W{^E;_?e1wl#WGQDDkdD=pOa!BItT(F442^D#**}o3{!M(%-a`##FvU_CldL#s zHq+N&GHcSCF>b1f4}w8PR-cJyq`a9m`7Zj+x5yYwr{U2B9tbHC)&EdpBK37M%QRrQ z;adv2(b>51Vh(x1Duh-(n0TP(QBpF8GT`_JY4yLVz%?-1&XeucHF-#Op$OlA6_Oh`#0sfgCie4b zDwnb1Yq9f*HXC{<0*k_{(NNs=u`np9y@X_teiUq-vcA4{4pHNm%qrVXZN`exdMTb?!z$K~3n zH(2gr(UN((@Yk~LrE4O*hZMZRnp0V8L9(9=U}-lvW(); z>Xvp@bPM$iXiG=epqfGJew(c$n< zHcbf`R}ayl57-`EjAycpVR*uKcfGKQV6q}_Tm3h50=yyK?!RGm z;O#E`hE;_XH5w~)v7_35;T%Y2m8_I6-@*@$YtaJc86?Mqw@R?UK%t0-X8|tR;P1Kf zOogZGNE8mpGp-&F835nGrKq=kfW+0|-i+u~~VYp`U-o2jSFxA3 z>yW;|;mQl2=t*5}I*`LGGAY?5oR~5x@AtdJ?F#%%flG*Pfp3@LmY=D>*(bj=Sg<*Gim;}BWSdy#8*M&DUa-)?MO z7V>5~^5&2?*UU=ZUJjPs4K?RfqpO^Tbb-9|Rt%(>3;lNY!T^2;AqN=3?=VoW6ntgO z+f*)#mop2i5Ich3)%ab5-%{aj3BdF{4*SzS>-~xYo|vy%@#(;yO-nKR#bx&t69<-OW*yg7@+UsedhUFWAl%qo zjc`+Q4Z?hL6yfIP+KpG8RT$dWC@A~(IqAc*O=6K64z4=mE2KxeESx?B-5bg{MwgP36>rlf{+!j8MmSL6W`H!ltUb{B(ye3I|#n!Nrdx{2I26$MC*+p z=Mn&PN#wPI&<^o`-@!V#GQJ*86b`y~d<&I_!VSNgB|wC-ijWz!q}nB9R{|R*nJHhT zy6*dcb-e(x3(6PqGR%cKta@dJ);)N}-pE@LW+lyJDf+f10A-$I3pH}h>-Apo>2|Sqj$%+k zrGJD~^dSHI&NrDwAx*vn4xnBFM})fo+<+Vg_*QJ^E+o_;B(~%}nP9zxk5F{$HT5>O z;sZfbG~#-i%_BGAy9sBNA5S3g`lwA3a75iJu8#00Z6S>CC!9nKakPa3(f7FMNRC2h zmEmclmn_c2x^#SBO>&fN}rS)^621}F+> z_-*6zz=q#5wN^Q6ItDQOB0~p>FLhAW-3-M3SUi*dIS2GF5NM^KWxGIv3B4yOJpMdH50vvls`YBi<<{0b-VfenTBwkUKcPukk_ z6}bi3OS5?F^sW-cr?<0Y=!*DU0lFv!=U0!QaN1K4-9xtS4RFzg-5~{RLN+g6v(~l` zmG9lpA2GEP?Sw1)m9Tl4>gt26|78Z(5Lmqgu~Bcpp&aI-`Vix?j0<+O_7s@N>f53z z81?Ne?K>D;XTa;?)JGV6l)-m0_&9^_V({GzKEdF75Llz2RpwCn(t)tX7`Jq$i|e82 zsCbEzeOa`!tN|IP!X+^9{WLOUR#z}EA7j-WJJ&qove*&aemtZHaBU4VIJJqXA2Nno zp5V*jx)3mi;daG;qwjSvh=q3-K2d24|3D2f9xgNVJJ`?AFvVRs@vK}sV#Kq^P0dX+ z3Em7SsC1GVrC=@N8Tz9p-nxWyrWwV&7@RP`g(O+S>6>5|XN9eCD+i}9)ISA`C<(Qi z#%n0`n{Wp6DqKCok%R&F9LCqotnsgApYhLTzwuAzfbkX6znaM8)kM6GC~_5%`K5LO zzC}kaA>esV9l-Bb={g29knGpISTjHb#5rYJvj9|4y@q6+&gsutr~Twc1qg(NI3$Q$rC zNWb8>Ht4JTL5r3V0Qbr=!2KlR{w(a5w(EbEEosGxpnim9x6i*}8N?_c%TI)#Le{?| z2tS60cIN+bC9DX6yU|~QitG@Q6$?npC$s0UKk@_c~aMbR> zdJdW+oMyOrm`0J`Mi2$gJ$(8Z00u|mz~2f!4Y@4*O!jwxT^tDHpqbxqwDa>Km8{qR z_0uSYKu%R zEK?h9MuZz^Z5a|=+x!hyE|aWn2`)_**$GG=Ea7oeNTNh2LNXM5shsEuYhPa&Rbg+! ztCuh^#iU&?jid&bc$UVgh#`3cnkAs`3oPIgq8hyPxA2)z@49nKdm5QvMU#XX85l!Q z{t7bd_4e(wWg`=58njPamz(@H3iEpxvMV-A8kyyEgnS8?nv^j?hv_aIH+bP4j?^Jp z!g#=(!Dyu6%oa-0a9(D9+ z0xPand%m$5tkonKb@9b5no$?|z7_D_)|P%7Ea_Ugx5Gk{Z^`lwk@kGnhj=Gr{4Znj zU zAc2*d!nK?eC#?Q9dQjWS^@h|1TWj@FZvzLH}Q^%1E)?-KxQr5 zEEQ@%leKy5g}}20eKYU|YPO4J(%CQ_G0{%dN`p4l^Nmx}G8>HTqyMY`7nRFA^eS^4S z40GiSo(1ynW+uVENbNttQ0`{#_l@0K{-FIrtR*WnvJn9O5Z{lWdAxGLTSHYY|2_-N zOMwmprGm>H%Ia59*gkF&atd$3NKR;{i7S<|1QJ3+ic9>n$ng^*1BR?rIru}bF>e6N zuc4?%IWXMm((!h*oX`Y*XLJdXgOm6G&Rq@1ci=(8p>S+28pPrjj2a|j#P1a3UWfw+ z4{<3_?p4_ox%>ugWGu)7bDK! zJEl$Q>X5ZUAP)h{4DIl0c%bw|MmWDSu-p!E{l(aL{%G+$np@-l?n^S=6;XC^BO~K(V1?r=fIahl`L50N%z+m)v9%A5psaiZs%RxA;am60I z&BL|+rc*;UL(>f^tqSO^nj{tSQiUFhnQrL}lv2^Ul7i|gE&p(tBQ%;IS%#RzsZ_=x zG^S~_HH7&gdO8A?KgBk(yE(u04lDk2|2m@e~9oV^V zWKq=PoSs`y2y{1~F-H}tVHnoZ5-y7!1bTO1eqh~G_}u2Fdii@(BB~q2{m62brSy3` zK==xSw=(dI1>kuQ2Jddj#D#R8SPjkLh~4qRJF{%gu0ss3Ixq_+F7AP>UVMi@Zm({K zx6-ulV*t!+_<6z&uI@buGwNu2IWT0!7sexZ@_GWsYKg@xuqYBi%!%t85W?6?6_5)4 zCxCQ62P`vQ=xt)rGY;tm$>#f5syKo^I{>!>4K+u*P00Cat${wKS$iG6_CqTUPU5q3 zyeLO=fFV(7i2!xJ!kJRhpBl!J%Wz&qHfHr}D@J`N+53LN{lY)EjqC?sxv z2F^7Q{n}Oo%SVH%RT?^A)k`gt_kVn#+8QH8j^_i%V*a`;AsXHpd_8|{pyG~C&OiP* zUGP%7Mz^~qv+V39wyL!=>#GVNbKM5zXLSQpWz4X8{l25i#;Pj>H89JJlqCJ;4J_$1 zohs+`b@aPdv?y`qypGbnF8^9p^$EN|-V+@qFs|j&RrL_Q3d33bI*`|;!r zEa|S`uouMg1{3}!vwV)hhw2BwD>pHD_{ii$;lQ!UlN0YasqW&J z4GdN@*u-En1G-33>ljdpPjTZ@RJK)Q?WjD1-3(}puYRAw4hHuypiZRvJqBwT{4;~? z4E~kD*BSh824@)jHUsJ+siO=y?c_Q;b(2rS45*Q%?q;x`!6OK~YrY^X;WL7gv6 zpRB@=h;p}D#kl(zj56*Np9UBlWRPQAAA>AkckpR5gWDP0!C)7I-3;zwAS&b!;fZ_~ z97WUq*gVes`jd&_;e0|2SNnaYF6qb z4%fO;XZhh0gUbw#GoX}3QDUITu2w(9V32X-;;J8KKsKlPNd{!-DKgH4Nu`7<^z(cr z$3hVy)(0gqFvW9DaaFagG+|MnX7TSsz**?#Fd;KBV9%Se-1OPkB~Le@h5ou>W?wUM zSY(DF?&y_u9TBIQD%G<(Z^uorqx#64{ z{!3yMA?~4vb{}H+5a1YRUpIQ>i@=P2A48jvQob6Qo3ZyA$Dyk!?=o`|nb+6ej@T!` zH^cq4nVUed@C273tr=rxUNc1CB(Y4x-h%imb8Dn$up7NDeQmzRl9?tPQ|83Hb0y2pAm@ebrLr&npZH)6=o)Y}8$`Y?1njOF zL4K=SUw(~it<_if-Xrg50j$~eDIw>891ya> zsNf5ZYg4GY3Lk)A!IDGH*?&T@a?!W7I+~UjwChf~lQrwn(xjL^-R0nvb6L(&o{83o zl;K~n1u?bXL+buk5r7oh=IQ~)3`lZ{566AprJUH-R*4q^G`z%nyaP!Tg4f>l_(`g?#u$+=F7Od||(xF2FU8bETPxI7Ot#S;vKA^LWv*eE3-Ma_j33Aj-3 z!pvC=uMe_@o@653b;~5erEJ07%<3to)N^i`l&y5cocx*pM(MN)iN*$xF^le#ITzIX zxP%4nP{;8}J-660eEu5WPN{|hFQ*Wv&G literal 0 HcmV?d00001 diff --git a/vllm/engine/__pycache__/metrics.cpython-310.pyc b/vllm/engine/__pycache__/metrics.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7067853decaeb351b1f528174960537ade7f62f0 GIT binary patch literal 16454 zcmbVTd5|1ed7tj-nVp%Pz3b%Px43dUn*3JN+(rGDwP6P@<$S>(rU4~ zR?sx&b>#IzT=H?`je=266cY7hAz2?P3<=z*r3z`ZHENkc2LBVa;d-`^t&bE&>Z65G zX_Kst)yE6tk{_x~)F%s*TFf4|jFzSz!aJ$jRDHTIqs1D!WfW#?O+5-|x;9&%E6kzg zq&l!V%G3wy|4qNtTlqt2ueFq8nwny8bfIpO5@f9N)ssU zMrqQTLTSq0aXDVtV@=zpHDf1ib0M~3ShLpL6{E1%nz!$;7Vs?MnX&h+q^up*PSo6K z?eg13EoSUHP`lgOgWCP*-L&?i_q}!&y&pjN4r?FE`=opj~3Cc=D0cCm%h#bOwN-lg(DcwN=jWj8m1e>lu$#9M?-dzV23%7}`Ri*5lZLBoWaa@%)@Nc-% zsJO+Nyv5<_ej{Y(&ad0$Q+By&*-tms1zQF0mpRq$pMk;&JkGn2*s+3U#R|Hm72=j& zFziG%Sr|eJ^gx;^3|nzKYmY1dRqRo0cfv|y+s3S6D{W8{!4WYFHGpZG9cB34}34}l9lXypzxWI<2R2K|w@h%Rk3ziVru$AqC zB?UHOjdsC?1U6=kcfnGCrL75T60oTkF=8EUiGAFXjwAV;iwBFGMb z9JTI^AUma}`>Z#2^|VW1_gfEi!FF3Qkknh2+6xzJwfeC}s}59pre!;>Qv}*KKtt`r zORf5{t<2_%8P=KQmQo;s+6IY(t%#s!&b>G9#hq6DIye%M&>OLuPVW)HYt_5{W#B-*4J&TeaC=)f;#qd5>pReUtMn% z%cb&~U2Hj}RlB%ut8zP!wV97Tb+=T2*`OkC9u4@#(R=PO*@|4juL`;<59|tKy^*eV z%gL~-b!D$qo(m?o2HvbnuHEL`u?eg){pl^&n&k^78klU5Z~HxPUEB1)#oe}t#Rxml z%&yA?;}qSdTdKA1IoZOA;(#xLV@8FFOZQ}y z>n?h|U^l|f_jQgJL>^tT4Rl8why<>Ra0+WpCYf6uqR5kmg>_z%AD5`W)5zKSiv?{AL(z|Z`1nE7Ft z`Oh$8oCw-Xp9tRC9YXFrku&=C-5XkNl`q(?)4rp#b|7-G*>cxg{j1M!T|L^Ge%k@- zA5n9d@2bCwmyQRS>&I8`lixRwt9jP@rN0R4|2nMyUI_X7<7xp7)gqG}Om-q^`^RP9 zYljT02pq&cw-FuJeLWNA^^eoyzkrLG!3#a7mgx+kc5$%XNP@l0hnxE5F!NxTITdEk zhMA=kZJ%m_(Lj4$=pE2xSl2n`CQ@^ zTJP(Go$h zXsObQTP&f!b+^EddHIBTleT3rhW)8V~TblL|Wf&`1($mic=k5Mz?wLjw{ z(X7~CH?l>Ixj|u{cZGd+T5i{EZjU!#Y&Dd<3f71Tani*=QSdUw)lv&LXu0NibAHyr zy=t{;Xc@>2TPuRweQQNB0oV8Tbzbln)5V}duIe4^ZM0<>QNyXOh9OXmf#rB}kye6hl?n^0MO~Q- zHz(wC0o(Bo2H5g??8V!;wPA-Tcn8D&yZIEC8d80D6nAddu$%6^1KYLgrEPDAKdFJV z8=~8`gm^vf8IE0B5y?nRVE?Yi)M@ z%SQrLk_33f7qpH9beLb?@KVL1X!FIp(33NZ+aaYJhOYU4sT)ROn|$&O!iRz$UkiIr z4ag-mjF|FUN`AO2AB-sQptq!E+%Rq#Q3-ELtBi);x0_+GqCw;Ly%qGUYuOv^@xZo_ z4>!z9D%)+T#!{JAYWU;LO5=uT&<`Nv#$2tcSL2tpQ?UC!ctKVG0bX zR)_CRq8Be<7GNLIj5^r1f`>w1fBeZb{VxF$#?NMloH0ZX)UYi z&l@+}X0#Wb-g-FvbzMDzSC(=UL@+L+SOnLnQU@N7e70D8rd1My1WH52BJ{yxQ60le z>QN+K;tX#Hb$|d`B~+f!^dli-RQD1{5eAB-tGsxn;i@x)anfB2A%!Se4OtF_HfIt^ zDrRU#GO6)zR*!vrbVgWG8C5;MG)gD%IPXK!ZKz7?>3CYV#$b|yK?+Y27QUoyEW}`@ z8nO*o_fl5+N}@32TO121D{H6ij6J-f!)B;kuynw5m9^r~v&X&h;uEC}S@YA1im7Tp z#SBbWxZEk4mNxP&w^GZ8v@2~XxB?+ARxI0eu*>E{=gyiXM@&sy5G60Kc(J8EXB0Td&8FP zlvPDELNDc(R*M(x4F@L4gxgxL+1@Z#6bc0DNsQqoeW@Uq6to;PNpC#5Fd&(-o34YF zGbRT76&@#xL{E(4bjd-oH`@#S8ypex7WyT=5eZ+V4hq{y?5gI|<|db@ErD=~sPV=f z+l7$3Qi5%h$9~BmvYMf_=eA7Bf744`EY(_encO7&G0&Ot zP+sNoIx{;TQ_BE^^NM3$!BiAAl>T{C0Q)|kSJ`D&Yfo>P1v~lIBL#ITX}; zm^{JcJd>xG5Ltu}ee&ZX)ew`Xk$8!fTC)TjG6AQ16FD%F)nDRqOe7>qUD&g61IKnt z*Y%s3xc-`MWNxH}7Y)VMWjsB?YooZ|z?GZB#Igg418 zI_XP$A%PZdPoP2TH%6ktOTpInT0_c2oA5{SkU_^wrW9wGEKB=h^M+lI(!90&ySxoxI zwE_Df;VVddvhF>Yd_a9-5+MF>OGXY24ZMyo=t{2LZj#FBONuu;sEQNf4O3QC2%_rO^ zy4OSE`ta?ED{BA7$SZnnAc3hpSn-yNkf$nC#*46z%DS$>P0|_tB61M|c|xn&2k@GL z`!rU_Eb}Tasfvwzu!{uWi%LjGYoTnj%?-%esgE$}qX=GlKf)uI-gv^NL|l5WwRaH> zqeTaM`v$4;8IE{cYJ}7pP@~ZPmhl35ye%oJH4YOn01oL5HJ5!i+#s4$Xp!Ks7m#W1 z9JtKj{qQ1N`st`mD2qF)$Y?q@nc|RLJKv$UbyWN}p4(nwneYl@>n$`Fa(3qwYP)I9 zHVU9CI78_v*GWZ?F5wJaF+Lqz(JV-wSF(i>&>3Y-qcHkuC?AC}NR-1~21g&r>FqCx zMA!kCaB0FZYMIL$CZOjcoucCoCSk$F%YSGAhlV(Ueg7ibQ{<#WG>|)kYRI0UZeT9A z==4A%ynYbFbqTNVST3uQ+VFepD70IAk=&O!xAqPsF5(JWF_m(uKWQ&$u6{XoHBKS) zn$GwDtXf}cAL?2qtp^q5Pk{@8Q>fB#V9>0=DFhPa1?UZ*t<%+gTB)Y;5>5-Uw=c<% z=&x&Dx(Fjo38v7ZfBM22zh==o7CD@~!Mn&*qTX^3b_C z&SOa6tku4vU4rZRlD-!o#$AF)Dy6HLco#;mdp$et4irnPw=Olh>Qi+6q4Q z74zslfm0toejyv_s9tKg6~`%=vK+@ebo8)!?_u-)oQVL83T>{oUuD|Ql@T-$Pz6lZSD|!iaBZUfncn|m>WjLb3Yzbe@Lpq z_Dt-b+RKFGBf`v+Y9F&E5`=3cscAMW+&66i5P1_Ur&wdSi2dApvjNwoT#KGoAE-+_ zfERc2v}ch?rFqi2mVyW*kDk)>A1k);;*I9BUJ8UHkt5zHT#=<3Y)>!_WAl0_p9nxS z7bF-*W)p`#kHg=LZ|WBi`_e@%hWZ#9sE;#w9!Vi<*Pu4jo(?x$9~F7Cy`vU`w)b)Y z95sart4|=~jku}=7noH9yKtSJ>BUhW;V73)6VFwILtKNJxT(*_cEm1gp-L7PQO(mE z>uM5BaVjt1U)Q9CdZNq38-@L@*fG_=l{4dB% z9JyUr$9dO+A21KT%54m|A=eZBxRI1N<*a{%Oelg^b&@ekacbXH!!KP;_?oy2Ygv`i zArcL8Ij%NbMu7lNd?BBHlP3Y>{{4p~Ug6QF2G!+Xv49%F!P-a8b%!*B!3@V3WWuiQ zT?x7h3ro3KZ(@rZLwyt-db3;L#R?dbH$!-58$xfu;y|$f3V8X(GD1VvEqFJJ-F}5Y zHWI!f2Lk5jvq5q|$XLR-guJ~ZiZFY6Mgf7|EMDOZZFrwtpCs{BjDHk?}v7(G1GGoKSi# zlkn^GTX-KTQ%Xba-?e8zWW7ue#I>qMh<%i2wwMxFBfN{IYm1x;v9DhQ=S2=e3v#gc zSM@ILpZc5M5{l z(c^Fx-Ad}lO(Sl=@dUR~!uVe!nfgil=++~D9dz~#cVpf5;?3pi7Qp}m`uM<$&5gaF z`8uuUO)WKDXc{$}QE#>->c;iBKmC`{R(*|0-x@v(*cEgsmRi`0XSdFkj3SpADC5&G zs=myG6eAm==*dy+wri+J`cR)l#XGrYlvOCZPwM*H#B8KxZY>T={T2}yocsQ>l zLq!3dV>>u&7Ofp9&09M$(k^=;Xyb=(FN)H=14L|k(}Ps$_S*+RBhe|k?Yrcr;Dwfx z_b&$cuaX23JtBGrnO*i8Zj|lf&jWAW| z9UVM3q*~~sW6X!IE?`CsMo^S9J%hd+p*m>Mr`Y^?CY@PcWU+%+RF_{QUJ?DWdQu;y zQ|4BC*PxAwlxNStU0T{gVq7a(?yC&<0uu$>t%%t8vi2Db>lVQ=@J7iTusoR=oe&Vf zIp0EJRs4&;hkO?$ybDEG;=uBKU)1MA+S{qO_Y9tE)OOo>s;_g9Zy*ULLc?t^5pO84 zysJS|$;woM7B68k>JRX0d++vB3EFyTF}l&`*5fvyx-f(%y0U;12=InRJ^MlZk^xPc z*B16uc2xTsaAnh|;J@rGynUiu8)CAM6~(;-B&NFK@31T^@ZbF*hVd8mP2{|UOvp>f z1pBvqs(YP?*F&tj^JU@-=Q|!h0N&j`^ty`rRd3SDw5;>2Zf zdS`I~-YC2l;CUe(S!rc$g#=KvRqtQVsH`Vy`qgkLsrVw7vTysPMDM6iVbmURgY+FO z#E=9ooKbcr-uW89PEK0;J%t8Br?0@?oz*xt@VY7gUtOhlN^&@VM4>d`c)=( zG5H*mH!=AK z@^bta-^}_QQr{ot6|}gYEheNzR|u@*O`h~4L->)%N#BoInD6^aBn-}S)SqxNZ)Y;b zgtD!A3zC9pKI+?K`)p;?C667cnjM zB_us;iL@NvW6nEJ08R@f2X~n+{1!~gH+!JYsOo)n(LLwiEeRiiAL4k?b*COiG2G*5 zD*gL8-?yKUS?>GX*l~1`rDg9HZF>9<2j|^C&ElhediYomONO1r=UU-PX-R{k6s+5j zlH59=eDE;BU8+uE_`$AaVElQ(XC&fiK8!qQq&nZzjvwiaR2K;o^AA2g zg4`9%stB1FCCmXvAcg@M?fW6%i;<^;L1u{o>b^D76f;_2`k|#lK5NLUPxF-Z4z4Zb zcBnrkBK#Z^nmT-T^9#%gyLpkhPcRV^(Px-@p2-y^f5zlDm~=QJu~hvglXo(?%0&3w zgUnrHa-7Lmn4DnpRVH6xaym-h#T<2#eic+pm(?~Ck%5CC zk7yIrpQEafVS8Wh?LV;3qJAD`TZa)TDT6@{&kW)Nira*)!QLV#8g)2KC#>w0CYZnY{Sk5&bV_#oslFb!+i&h4-oliXz)*GQCEcKa$XXq7+9DU>VrU0!3)8k1Yz8MK^-pv z(PVu?T_%7ySCCH|^TEewA<@vZ<0uZL1e-Xw!W=0!Z$Af18e(hn3ZaYiXFF<`wZl>i zwgSto$`X=73klcs@8k~?Cf~BVZXy`5T_9oU%@I404Zw3Bs%uEsm*M&<@?)(d)> z7kF7e=xcg69c05`Xd1lx2ZL=e_npDqx7J%T*u=QUJd8cwe`^O@828x#;{mrZ-e%Us z7!9M3EW_|#F1X~rT9#b;k>Eb#vn1!tPjX)^66w#96zsx}PKyN8tSp2|^0|MSrrDV1 za|n+!u0#?`FUlDvalvN=R$9xJqJYSPS2B##gy(APK`$1^U-0j$L_i^{w4O>vbM6;2 z|HZ{5KdVxeV3C+h>1PGgf;^9=FwJX~`_W8s;ma~jls2%a6dB`Jr%7Ch++WR~VO`B= zj9<}V(*M-h+9`d>oAqCO{ruUJmw$ZnY7#zsb^P)v#*f0sPrvr2&Gw=1>ZWP8jZZim z`!APe5=Uve{H*aan5@XTKV7agcw{}#PyN$KBzmvy%4t;P@!~UoiXF-%kGX%wb?hQP zO{O9eSQC_-!>g+BSkY?8{TEBMC~`k07gDg&=sZc6tyScSs}_&Ff8neeCnKx2fB)s9 zdNZ0zC8Ai7SiweS-D8o8;xv+S&r6jQd6d>J+^UMWNFjcIQu6pQ zC;i_P;x!k~nv)Zgn#-S!!4aDL2LLyaBt~GpLTUsycX)?c4{_|#+imZ(eFr`9yKTSG z_HNtv(9=Ax?fd9AxN~5DHn@ZjTmI%H+Lp$^?NuUeMznyQ|0d zXpF?WUw0-dQu6#)j|7htf`phKA3uAdlgUT!_!UwTL=rnL=5sE_wVfAd!B$>np{Vj4 z85qi?YzSvjLMMJrkY`a@a;6Ep^Rf!#C|>YTr&dFZZy_EKu|)DFe{Lr%5)}$q7#z?d zx3lh%Npx~b6{_Mj&t;e(5rYRrpYxn!0UU%YyaSbF9F%aD2zhnd2bxuK5h8!f>MC=$ z(ZxBG(VAM(j-J7P%|l92E|qL_U$yCi_u=h!UGTkpz0N*LVImjsBmyO&a3ysjdN`N$ zuEcQ;nLf`Ua)OHBs9(Ebm`B)PSbJfJa#f{-55n-fDoR_=1?E zu+-UpNR|4qDRzggk)f{l<8mqJ&bo&te+l3jj^&!JgJ!yfQL|lhfFGbUum~HNdzPMe zBWo|NxR0^eDr=d^DQBITbzsaaW-_eJ~8_#gebN?2#t6MB2;geBU z3?VBv32YJQ5ZEK|0fCDkG#Kd<_W^#3rVVtB2bQMXa1({`h{XN`uy#{mGmAUSX3orE zC}HF#-entnqouj5_s+rn?t<#^9_ypDkaU*~*idu&Bn>BTvMtOWXeV#io9jCAWhD9e z1A@M;+Pd1T+2eXMEpRKr2gu>mC>8rK@gyd$lQ2igbg&6^%v`}G`8QE zfDR+PL$MME5FK@cog4KcxO;0#)($0hu!+k>g^g8dF&96^GCCVZmUuw8Caj~O1CI?4V2Re4j_}Nl_}5g zq~Yv|5QTW(TIG{z*Q~N(|5u>@K<;y_^E=lJ0UMjfORTnw>9?^BsntjL$`0`p473g6 zV_+k@@xw2O`YC~HY8^Vz&xt|NI^VsfmAE(dd=X%B@f-77+`N@3`h-!inK{vIWS!3@ z(J9BXi^q66>W4*}bpEH!+XSzerGA}Md78ZD>C*Gt53kB8UN{*ZY`6p?JZ4ZeeTupt zNgv+8BeXe&qw&eeY-->qx9rf*qU99#TZz{Q4{s^T6HJm+Og(#GI{!I8yyk=dH-9v| zUJbU%j2!|W5uo_!nkarom@X4k4aA55)z(W<(ecp*<40f|p-EcKMP<5US?-=`+56@l zTM&1&Bebdyf!B64jRl29>#O@0k1s9-9nQLo(}R~lQ>$pd`pOV4uwY-m*e^L}Rq*s( q))=o%CbhR&Te}aN%j|b_5+&`?G2G!cE`)t^U|IIIx&70F()ngsqk?!E^VY z|33LgLz4cB$m~-EH%$68ey|%w#E{L(}b^JwhQH*O|*IzQ1{AF`l;F`DMubQi3T=&-e zb#q;eb#KGpG&jY#;cfZb=C&*`^^ruIwDnw~Emr-y^0jR4(DtF!TlgPnFZI;4y&q2k z=Mf7Sx08^k%7GgvY3;EcMQ(7MY9B?38wR$QYEO8Wgih$Ci+mg;uFovzggKwuM;_~u zwEZD-LJC_Rgu%!?P8*Ly%DfEe$!X^hi0Rm4{G)LajguJWbl~Vm$+Jns0BanCPxqg) z&&DiH0Bs%GepHf!fYpco5u+#}C(0P5tvM8TJ_$n)$e!5T_EFAqCTE}8QNsACh_(R3?chEw!&k`W-!yErymLw*bGL=k1WwXK*N?3J7XoV`z6|k{d&Qz%? zW@_b3jcQ^>6Ek(#Rj0a`(d!NIYtlXNR)e-#lP<6p?XWgm*pTQV{C8N3wMJw^0=x_O zBK$A0F34YE%d~+jqYhoBE6=OuO3tk>xUI6)OSu(X)^aXwx<=Q5%evS_U{;|Ubo04l zZh*8+(0hw+1GWX&HefsS3Sc|+1|5_d6Ltl5?NS5y?B+JQO0NN(fx6MR>2)~6RXD>9 z_`k-ki*unYmENScVAl027UYEs3l)Tebx%4_(nMrif#caWW`AsMU0Bc9hiJwQcm%! zB-I$_A!l?OwGd*K=MI!F|0)?eWj3`0C!-WIZ?{v5AxC$nDwrNJO}Y?cQURM=kw^}y z&a;GLxioEhA-Ao3B5ftyc9?Yn!WfU!)r^B>XGx?a(<1JM+)XBq27-zQ$MbO{L!6T7 zZk}hE5oTFX#HMZJLi(!8HWe5!^s<6TvM6w-J=l$c3l1<1CLeq>V*P zHKr?bE+{C{c9f;b+!Rg?wiFMJ3;p>WKtT65*ED;(A^BRsQw__;YP^JszK0i2kY%q~^a{Hf~7&?%(EWyUg z*n9{gMBXD`LNeN!)9x(og^I=){a?v-kvxAPH7y}mns*Ml5`Tg1F?x=CNn}Ny<_SB? z;QezpTiqgAJyuNkc* zywGAroIkD1Y#}{8^gI<-Z`)%|j^ki2)%raRjM_zJKX_U@#bWd$}g{Z^`5m^7f5hjG)gJ{#L!zEZO+8aglyWigBxPbR5y1Z9Sc zhTsY+vkM??ePGAz;TH}Q9~a!f1^CvIco$bIal$!%X7MHhjBUOs7Th4=nBe&Wf*yhn zf;NIT5$qwTBj_W*RK(v#fXzF{O2U7O0Kf4#hBt2^xQF0f1osiVhu}1x#Rv;5$6gI3 z#@_(YpkpJXBa_O^s}UK0%1c=KiB!ldqL3feD*2(UlJC_T`HpDhrC!(nPr8EcyYuJ> z(CM1~vfFIe+6qyXpH#t3f7Py$DsX>=xGuQsKk7AIX)8jvVed++CKLR0%M2xsC@d%p zifNT&bvGN>nf9}^pP31*W-jH^ob}5@A&g~q#EKi>$6KG;5Y*@~b5TrV>(tLnqVSq| zP0VMRg?+)CF??M1J$nMpR9|#q(7~bU&88sCb5Z6=^lq-ne3K%)ZElGj5Mj{L6%AP_ z90SE%78GZiJrP{yRj~v)G>GB5zz21#apHP$zf99vsGz6M3xQPoV;1~>#>5z7qh?5* QN>k2$o&3KEQL!rj46p3#ivR!s literal 0 HcmV?d00001 diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py new file mode 100644 index 00000000..4825f0fd --- /dev/null +++ b/vllm/engine/arg_utils.py @@ -0,0 +1,1143 @@ +import argparse +import dataclasses +import json +from dataclasses import dataclass +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, + Tuple, Type, Union) + +import torch + +import vllm.envs as envs +from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig, + DeviceConfig, EngineConfig, LoadConfig, LoadFormat, + LoRAConfig, ModelConfig, ObservabilityConfig, + ParallelConfig, PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig, TokenizerPoolConfig) +from vllm.executor.executor_base import ExecutorBase +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS +from vllm.transformers_utils.config import ( + maybe_register_config_serialize_by_value) +from vllm.transformers_utils.utils import check_gguf_file +from vllm.utils import FlexibleArgumentParser + +if TYPE_CHECKING: + from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup + +logger = init_logger(__name__) + +ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"] + +DEVICE_OPTIONS = [ + "auto", + "cuda", + "neuron", + "cpu", + "openvino", + "tpu", + "xpu", +] + + +def nullable_str(val: str): + if not val or val == "None": + return None + return val + + +def nullable_kvs(val: str) -> Optional[Mapping[str, int]]: + """Parses a string containing comma separate key [str] to value [int] + pairs into a dictionary. + + Args: + val: String value to be parsed. + + Returns: + Dictionary with parsed values. + """ + if len(val) == 0: + return None + + out_dict: Dict[str, int] = {} + for item in val.split(","): + kv_parts = [part.lower().strip() for part in item.split("=")] + if len(kv_parts) != 2: + raise argparse.ArgumentTypeError( + "Each item should be in the form KEY=VALUE") + key, value = kv_parts + + try: + parsed_value = int(value) + except ValueError as exc: + msg = f"Failed to parse value of item {key}={value}" + raise argparse.ArgumentTypeError(msg) from exc + + if key in out_dict and out_dict[key] != parsed_value: + raise argparse.ArgumentTypeError( + f"Conflicting values specified for key: {key}") + out_dict[key] = parsed_value + + return out_dict + + +@dataclass +class EngineArgs: + """Arguments for vLLM engine.""" + model: str = 'facebook/opt-125m' + served_model_name: Optional[Union[str, List[str]]] = None + tokenizer: Optional[str] = None + skip_tokenizer_init: bool = False + tokenizer_mode: str = 'auto' + trust_remote_code: bool = False + download_dir: Optional[str] = None + load_format: str = 'auto' + config_format: str = 'auto' + dtype: str = 'auto' + kv_cache_dtype: str = 'auto' + quantization_param_path: Optional[str] = None + seed: int = 0 + max_model_len: Optional[int] = None + worker_use_ray: bool = False + # Note: Specifying a custom executor backend by passing a class + # is intended for expert use only. The API may change without + # notice. + distributed_executor_backend: Optional[Union[str, + Type[ExecutorBase]]] = None + pipeline_parallel_size: int = 1 + tensor_parallel_size: int = 1 + max_parallel_loading_workers: Optional[int] = None + block_size: int = 16 + enable_prefix_caching: bool = False + disable_sliding_window: bool = False + use_v2_block_manager: bool = True + swap_space: float = 4 # GiB + cpu_offload_gb: float = 0 # GiB + gpu_memory_utilization: float = 0.90 + max_num_batched_tokens: Optional[int] = None + max_num_seqs: int = 256 + max_logprobs: int = 20 # Default value for OpenAI Chat Completions API + disable_log_stats: bool = False + revision: Optional[str] = None + code_revision: Optional[str] = None + rope_scaling: Optional[dict] = None + rope_theta: Optional[float] = None + tokenizer_revision: Optional[str] = None + quantization: Optional[str] = None + enforce_eager: Optional[bool] = None + max_context_len_to_capture: Optional[int] = None + max_seq_len_to_capture: int = 8192 + disable_custom_all_reduce: bool = False + tokenizer_pool_size: int = 0 + # Note: Specifying a tokenizer pool by passing a class + # is intended for expert use only. The API may change without + # notice. + tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray" + tokenizer_pool_extra_config: Optional[dict] = None + limit_mm_per_prompt: Optional[Mapping[str, int]] = None + enable_lora: bool = False + max_loras: int = 1 + max_lora_rank: int = 16 + enable_prompt_adapter: bool = False + max_prompt_adapters: int = 1 + max_prompt_adapter_token: int = 0 + fully_sharded_loras: bool = False + lora_extra_vocab_size: int = 256 + long_lora_scaling_factors: Optional[Tuple[float]] = None + lora_dtype: Optional[Union[str, torch.dtype]] = 'auto' + max_cpu_loras: Optional[int] = None + device: str = 'auto' + num_scheduler_steps: int = 1 + multi_step_stream_outputs: bool = True + ray_workers_use_nsight: bool = False + num_gpu_blocks_override: Optional[int] = None + num_lookahead_slots: int = 0 + model_loader_extra_config: Optional[dict] = None + ignore_patterns: Optional[Union[str, List[str]]] = None + preemption_mode: Optional[str] = None + + scheduler_delay_factor: float = 0.0 + enable_chunked_prefill: Optional[bool] = None + + guided_decoding_backend: str = 'outlines' + # Speculative decoding configuration. + speculative_model: Optional[str] = None + speculative_model_quantization: Optional[str] = None + speculative_draft_tensor_parallel_size: Optional[int] = None + num_speculative_tokens: Optional[int] = None + speculative_disable_mqa_scorer: Optional[bool] = False + speculative_max_model_len: Optional[int] = None + speculative_disable_by_batch_size: Optional[int] = None + ngram_prompt_lookup_max: Optional[int] = None + ngram_prompt_lookup_min: Optional[int] = None + spec_decoding_acceptance_method: str = 'rejection_sampler' + typical_acceptance_sampler_posterior_threshold: Optional[float] = None + typical_acceptance_sampler_posterior_alpha: Optional[float] = None + qlora_adapter_name_or_path: Optional[str] = None + disable_logprobs_during_spec_decoding: Optional[bool] = None + + otlp_traces_endpoint: Optional[str] = None + collect_detailed_traces: Optional[str] = None + disable_async_output_proc: bool = False + override_neuron_config: Optional[Dict[str, Any]] = None + mm_processor_kwargs: Optional[Dict[str, Any]] = None + scheduling_policy: Literal["fcfs", "priority"] = "fcfs" + + def __post_init__(self): + if self.tokenizer is None: + self.tokenizer = self.model + + # Setup plugins + from vllm.plugins import load_general_plugins + load_general_plugins() + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + """Shared CLI arguments for vLLM engine.""" + + # Model arguments + parser.add_argument( + '--model', + type=str, + default=EngineArgs.model, + help='Name or path of the huggingface model to use.') + parser.add_argument( + '--tokenizer', + type=nullable_str, + default=EngineArgs.tokenizer, + help='Name or path of the huggingface tokenizer to use. ' + 'If unspecified, model name or path will be used.') + parser.add_argument( + '--skip-tokenizer-init', + action='store_true', + help='Skip initialization of tokenizer and detokenizer') + parser.add_argument( + '--revision', + type=nullable_str, + default=None, + help='The specific model version to use. It can be a branch ' + 'name, a tag name, or a commit id. If unspecified, will use ' + 'the default version.') + parser.add_argument( + '--code-revision', + type=nullable_str, + default=None, + help='The specific revision to use for the model code on ' + 'Hugging Face Hub. It can be a branch name, a tag name, or a ' + 'commit id. If unspecified, will use the default version.') + parser.add_argument( + '--tokenizer-revision', + type=nullable_str, + default=None, + help='Revision of the huggingface tokenizer to use. ' + 'It can be a branch name, a tag name, or a commit id. ' + 'If unspecified, will use the default version.') + parser.add_argument( + '--tokenizer-mode', + type=str, + default=EngineArgs.tokenizer_mode, + choices=['auto', 'slow', 'mistral'], + help='The tokenizer mode.\n\n* "auto" will use the ' + 'fast tokenizer if available.\n* "slow" will ' + 'always use the slow tokenizer. \n* ' + '"mistral" will always use the `mistral_common` tokenizer.') + parser.add_argument('--trust-remote-code', + action='store_true', + help='Trust remote code from huggingface.') + parser.add_argument('--download-dir', + type=nullable_str, + default=EngineArgs.download_dir, + help='Directory to download and load the weights, ' + 'default to the default cache dir of ' + 'huggingface.') + parser.add_argument( + '--load-format', + type=str, + default=EngineArgs.load_format, + choices=[f.value for f in LoadFormat], + help='The format of the model weights to load.\n\n' + '* "auto" will try to load the weights in the safetensors format ' + 'and fall back to the pytorch bin format if safetensors format ' + 'is not available.\n' + '* "pt" will load the weights in the pytorch bin format.\n' + '* "safetensors" will load the weights in the safetensors format.\n' + '* "npcache" will load the weights in pytorch format and store ' + 'a numpy cache to speed up the loading.\n' + '* "dummy" will initialize the weights with random values, ' + 'which is mainly for profiling.\n' + '* "tensorizer" will load the weights using tensorizer from ' + 'CoreWeave. See the Tensorize vLLM Model script in the Examples ' + 'section for more information.\n' + '* "bitsandbytes" will load the weights using bitsandbytes ' + 'quantization.\n') + parser.add_argument( + '--config-format', + default=EngineArgs.config_format, + choices=[f.value for f in ConfigFormat], + help='The format of the model config to load.\n\n' + '* "auto" will try to load the config in hf format ' + 'if available else it will try to load in mistral format ') + parser.add_argument( + '--dtype', + type=str, + default=EngineArgs.dtype, + choices=[ + 'auto', 'half', 'float16', 'bfloat16', 'float', 'float32' + ], + help='Data type for model weights and activations.\n\n' + '* "auto" will use FP16 precision for FP32 and FP16 models, and ' + 'BF16 precision for BF16 models.\n' + '* "half" for FP16. Recommended for AWQ quantization.\n' + '* "float16" is the same as "half".\n' + '* "bfloat16" for a balance between precision and range.\n' + '* "float" is shorthand for FP32 precision.\n' + '* "float32" for FP32 precision.') + parser.add_argument( + '--kv-cache-dtype', + type=str, + choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'], + default=EngineArgs.kv_cache_dtype, + help='Data type for kv cache storage. If "auto", will use model ' + 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ' + 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)') + parser.add_argument( + '--quantization-param-path', + type=nullable_str, + default=None, + help='Path to the JSON file containing the KV cache ' + 'scaling factors. This should generally be supplied, when ' + 'KV cache dtype is FP8. Otherwise, KV cache scaling factors ' + 'default to 1.0, which may cause accuracy issues. ' + 'FP8_E5M2 (without scaling) is only supported on cuda version' + 'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead ' + 'supported for common inference criteria.') + parser.add_argument('--max-model-len', + type=int, + default=EngineArgs.max_model_len, + help='Model context length. If unspecified, will ' + 'be automatically derived from the model config.') + parser.add_argument( + '--guided-decoding-backend', + type=str, + default='outlines', + choices=['outlines', 'lm-format-enforcer'], + help='Which engine will be used for guided decoding' + ' (JSON schema / regex etc) by default. Currently support ' + 'https://github.com/outlines-dev/outlines and ' + 'https://github.com/noamgat/lm-format-enforcer.' + ' Can be overridden per request via guided_decoding_backend' + ' parameter.') + # Parallel arguments + parser.add_argument( + '--distributed-executor-backend', + choices=['ray', 'mp'], + default=EngineArgs.distributed_executor_backend, + help='Backend to use for distributed serving. When more than 1 GPU ' + 'is used, will be automatically set to "ray" if installed ' + 'or "mp" (multiprocessing) otherwise.') + parser.add_argument( + '--worker-use-ray', + action='store_true', + help='Deprecated, use --distributed-executor-backend=ray.') + parser.add_argument('--pipeline-parallel-size', + '-pp', + type=int, + default=EngineArgs.pipeline_parallel_size, + help='Number of pipeline stages.') + parser.add_argument('--tensor-parallel-size', + '-tp', + type=int, + default=EngineArgs.tensor_parallel_size, + help='Number of tensor parallel replicas.') + parser.add_argument( + '--max-parallel-loading-workers', + type=int, + default=EngineArgs.max_parallel_loading_workers, + help='Load model sequentially in multiple batches, ' + 'to avoid RAM OOM when using tensor ' + 'parallel and large models.') + parser.add_argument( + '--ray-workers-use-nsight', + action='store_true', + help='If specified, use nsight to profile Ray workers.') + # KV cache arguments + parser.add_argument('--block-size', + type=int, + default=EngineArgs.block_size, + choices=[8, 16, 32], + help='Token block size for contiguous chunks of ' + 'tokens. This is ignored on neuron devices and ' + 'set to max-model-len') + + parser.add_argument('--enable-prefix-caching', + action='store_true', + help='Enables automatic prefix caching.') + parser.add_argument('--disable-sliding-window', + action='store_true', + help='Disables sliding window, ' + 'capping to sliding window size') + parser.add_argument( + '--use-v2-block-manager', + default=EngineArgs.use_v2_block_manager, + action='store_true', + help='Use BlockSpaceMangerV2. By default this is set to True. ' + 'Set to False to use BlockSpaceManagerV1') + parser.add_argument( + '--num-lookahead-slots', + type=int, + default=EngineArgs.num_lookahead_slots, + help='Experimental scheduling config necessary for ' + 'speculative decoding. This will be replaced by ' + 'speculative config in the future; it is present ' + 'to enable correctness tests until then.') + + parser.add_argument('--seed', + type=int, + default=EngineArgs.seed, + help='Random seed for operations.') + parser.add_argument('--swap-space', + type=float, + default=EngineArgs.swap_space, + help='CPU swap space size (GiB) per GPU.') + parser.add_argument( + '--cpu-offload-gb', + type=float, + default=0, + help='The space in GiB to offload to CPU, per GPU. ' + 'Default is 0, which means no offloading. Intuitively, ' + 'this argument can be seen as a virtual way to increase ' + 'the GPU memory size. For example, if you have one 24 GB ' + 'GPU and set this to 10, virtually you can think of it as ' + 'a 34 GB GPU. Then you can load a 13B model with BF16 weight,' + 'which requires at least 26GB GPU memory. Note that this ' + 'requires fast CPU-GPU interconnect, as part of the model is' + 'loaded from CPU memory to GPU memory on the fly in each ' + 'model forward pass.') + parser.add_argument( + '--gpu-memory-utilization', + type=float, + default=EngineArgs.gpu_memory_utilization, + help='The fraction of GPU memory to be used for the model ' + 'executor, which can range from 0 to 1. For example, a value of ' + '0.5 would imply 50%% GPU memory utilization. If unspecified, ' + 'will use the default value of 0.9.') + parser.add_argument( + '--num-gpu-blocks-override', + type=int, + default=None, + help='If specified, ignore GPU profiling result and use this number' + 'of GPU blocks. Used for testing preemption.') + parser.add_argument('--max-num-batched-tokens', + type=int, + default=EngineArgs.max_num_batched_tokens, + help='Maximum number of batched tokens per ' + 'iteration.') + parser.add_argument('--max-num-seqs', + type=int, + default=EngineArgs.max_num_seqs, + help='Maximum number of sequences per iteration.') + parser.add_argument( + '--max-logprobs', + type=int, + default=EngineArgs.max_logprobs, + help=('Max number of log probs to return logprobs is specified in' + ' SamplingParams.')) + parser.add_argument('--disable-log-stats', + action='store_true', + help='Disable logging statistics.') + # Quantization settings. + parser.add_argument('--quantization', + '-q', + type=nullable_str, + choices=[*QUANTIZATION_METHODS, None], + default=EngineArgs.quantization, + help='Method used to quantize the weights. If ' + 'None, we first check the `quantization_config` ' + 'attribute in the model config file. If that is ' + 'None, we assume the model weights are not ' + 'quantized and use `dtype` to determine the data ' + 'type of the weights.') + parser.add_argument('--rope-scaling', + default=None, + type=json.loads, + help='RoPE scaling configuration in JSON format. ' + 'For example, {"type":"dynamic","factor":2.0}') + parser.add_argument('--rope-theta', + default=None, + type=float, + help='RoPE theta. Use with `rope_scaling`. In ' + 'some cases, changing the RoPE theta improves the ' + 'performance of the scaled model.') + parser.add_argument('--enforce-eager', + action='store_true', + help='Always use eager-mode PyTorch. If False, ' + 'will use eager mode and CUDA graph in hybrid ' + 'for maximal performance and flexibility.') + parser.add_argument('--max-context-len-to-capture', + type=int, + default=EngineArgs.max_context_len_to_capture, + help='Maximum context length covered by CUDA ' + 'graphs. When a sequence has context length ' + 'larger than this, we fall back to eager mode. ' + '(DEPRECATED. Use --max-seq-len-to-capture instead' + ')') + parser.add_argument('--max-seq-len-to-capture', + type=int, + default=EngineArgs.max_seq_len_to_capture, + help='Maximum sequence length covered by CUDA ' + 'graphs. When a sequence has context length ' + 'larger than this, we fall back to eager mode. ' + 'Additionally for encoder-decoder models, if the ' + 'sequence length of the encoder input is larger ' + 'than this, we fall back to the eager mode.') + parser.add_argument('--disable-custom-all-reduce', + action='store_true', + default=EngineArgs.disable_custom_all_reduce, + help='See ParallelConfig.') + parser.add_argument('--tokenizer-pool-size', + type=int, + default=EngineArgs.tokenizer_pool_size, + help='Size of tokenizer pool to use for ' + 'asynchronous tokenization. If 0, will ' + 'use synchronous tokenization.') + parser.add_argument('--tokenizer-pool-type', + type=str, + default=EngineArgs.tokenizer_pool_type, + help='Type of tokenizer pool to use for ' + 'asynchronous tokenization. Ignored ' + 'if tokenizer_pool_size is 0.') + parser.add_argument('--tokenizer-pool-extra-config', + type=nullable_str, + default=EngineArgs.tokenizer_pool_extra_config, + help='Extra config for tokenizer pool. ' + 'This should be a JSON string that will be ' + 'parsed into a dictionary. Ignored if ' + 'tokenizer_pool_size is 0.') + + # Multimodal related configs + parser.add_argument( + '--limit-mm-per-prompt', + type=nullable_kvs, + default=EngineArgs.limit_mm_per_prompt, + # The default value is given in + # MultiModalRegistry.init_mm_limits_per_prompt + help=('For each multimodal plugin, limit how many ' + 'input instances to allow for each prompt. ' + 'Expects a comma-separated list of items, ' + 'e.g.: `image=16,video=2` allows a maximum of 16 ' + 'images and 2 videos per prompt. Defaults to 1 for ' + 'each modality.')) + parser.add_argument( + '--mm-processor-kwargs', + default=None, + type=json.loads, + help=('Overrides for the multimodal input mapping/processing,' + 'e.g., image processor. For example: {"num_crops": 4}.')) + + # LoRA related configs + parser.add_argument('--enable-lora', + action='store_true', + help='If True, enable handling of LoRA adapters.') + parser.add_argument('--max-loras', + type=int, + default=EngineArgs.max_loras, + help='Max number of LoRAs in a single batch.') + parser.add_argument('--max-lora-rank', + type=int, + default=EngineArgs.max_lora_rank, + help='Max LoRA rank.') + parser.add_argument( + '--lora-extra-vocab-size', + type=int, + default=EngineArgs.lora_extra_vocab_size, + help=('Maximum size of extra vocabulary that can be ' + 'present in a LoRA adapter (added to the base ' + 'model vocabulary).')) + parser.add_argument( + '--lora-dtype', + type=str, + default=EngineArgs.lora_dtype, + choices=['auto', 'float16', 'bfloat16', 'float32'], + help=('Data type for LoRA. If auto, will default to ' + 'base model dtype.')) + parser.add_argument( + '--long-lora-scaling-factors', + type=nullable_str, + default=EngineArgs.long_lora_scaling_factors, + help=('Specify multiple scaling factors (which can ' + 'be different from base model scaling factor ' + '- see eg. Long LoRA) to allow for multiple ' + 'LoRA adapters trained with those scaling ' + 'factors to be used at the same time. If not ' + 'specified, only adapters trained with the ' + 'base model scaling factor are allowed.')) + parser.add_argument( + '--max-cpu-loras', + type=int, + default=EngineArgs.max_cpu_loras, + help=('Maximum number of LoRAs to store in CPU memory. ' + 'Must be >= than max_num_seqs. ' + 'Defaults to max_num_seqs.')) + parser.add_argument( + '--fully-sharded-loras', + action='store_true', + help=('By default, only half of the LoRA computation is ' + 'sharded with tensor parallelism. ' + 'Enabling this will use the fully sharded layers. ' + 'At high sequence length, max rank or ' + 'tensor parallel size, this is likely faster.')) + parser.add_argument('--enable-prompt-adapter', + action='store_true', + help='If True, enable handling of PromptAdapters.') + parser.add_argument('--max-prompt-adapters', + type=int, + default=EngineArgs.max_prompt_adapters, + help='Max number of PromptAdapters in a batch.') + parser.add_argument('--max-prompt-adapter-token', + type=int, + default=EngineArgs.max_prompt_adapter_token, + help='Max number of PromptAdapters tokens') + parser.add_argument("--device", + type=str, + default=EngineArgs.device, + choices=DEVICE_OPTIONS, + help='Device type for vLLM execution.') + parser.add_argument('--num-scheduler-steps', + type=int, + default=1, + help=('Maximum number of forward steps per ' + 'scheduler call.')) + + parser.add_argument( + '--multi-step-stream-outputs', + action=StoreBoolean, + default=EngineArgs.multi_step_stream_outputs, + nargs="?", + const="True", + help='If False, then multi-step will stream outputs at the end ' + 'of all steps') + parser.add_argument( + '--scheduler-delay-factor', + type=float, + default=EngineArgs.scheduler_delay_factor, + help='Apply a delay (of delay factor multiplied by previous ' + 'prompt latency) before scheduling next prompt.') + parser.add_argument( + '--enable-chunked-prefill', + action=StoreBoolean, + default=EngineArgs.enable_chunked_prefill, + nargs="?", + const="True", + help='If set, the prefill requests can be chunked based on the ' + 'max_num_batched_tokens.') + + parser.add_argument( + '--speculative-model', + type=nullable_str, + default=EngineArgs.speculative_model, + help= + 'The name of the draft model to be used in speculative decoding.') + # Quantization settings for speculative model. + parser.add_argument( + '--speculative-model-quantization', + type=nullable_str, + choices=[*QUANTIZATION_METHODS, None], + default=EngineArgs.speculative_model_quantization, + help='Method used to quantize the weights of speculative model. ' + 'If None, we first check the `quantization_config` ' + 'attribute in the model config file. If that is ' + 'None, we assume the model weights are not ' + 'quantized and use `dtype` to determine the data ' + 'type of the weights.') + parser.add_argument( + '--num-speculative-tokens', + type=int, + default=EngineArgs.num_speculative_tokens, + help='The number of speculative tokens to sample from ' + 'the draft model in speculative decoding.') + parser.add_argument( + '--speculative-disable-mqa-scorer', + action='store_true', + help= + 'If set to True, the MQA scorer will be disabled in speculative ' + ' and fall back to batch expansion') + parser.add_argument( + '--speculative-draft-tensor-parallel-size', + '-spec-draft-tp', + type=int, + default=EngineArgs.speculative_draft_tensor_parallel_size, + help='Number of tensor parallel replicas for ' + 'the draft model in speculative decoding.') + + parser.add_argument( + '--speculative-max-model-len', + type=int, + default=EngineArgs.speculative_max_model_len, + help='The maximum sequence length supported by the ' + 'draft model. Sequences over this length will skip ' + 'speculation.') + + parser.add_argument( + '--speculative-disable-by-batch-size', + type=int, + default=EngineArgs.speculative_disable_by_batch_size, + help='Disable speculative decoding for new incoming requests ' + 'if the number of enqueue requests is larger than this value.') + + parser.add_argument( + '--ngram-prompt-lookup-max', + type=int, + default=EngineArgs.ngram_prompt_lookup_max, + help='Max size of window for ngram prompt lookup in speculative ' + 'decoding.') + + parser.add_argument( + '--ngram-prompt-lookup-min', + type=int, + default=EngineArgs.ngram_prompt_lookup_min, + help='Min size of window for ngram prompt lookup in speculative ' + 'decoding.') + + parser.add_argument( + '--spec-decoding-acceptance-method', + type=str, + default=EngineArgs.spec_decoding_acceptance_method, + choices=['rejection_sampler', 'typical_acceptance_sampler'], + help='Specify the acceptance method to use during draft token ' + 'verification in speculative decoding. Two types of acceptance ' + 'routines are supported: ' + '1) RejectionSampler which does not allow changing the ' + 'acceptance rate of draft tokens, ' + '2) TypicalAcceptanceSampler which is configurable, allowing for ' + 'a higher acceptance rate at the cost of lower quality, ' + 'and vice versa.') + + parser.add_argument( + '--typical-acceptance-sampler-posterior-threshold', + type=float, + default=EngineArgs.typical_acceptance_sampler_posterior_threshold, + help='Set the lower bound threshold for the posterior ' + 'probability of a token to be accepted. This threshold is ' + 'used by the TypicalAcceptanceSampler to make sampling decisions ' + 'during speculative decoding. Defaults to 0.09') + + parser.add_argument( + '--typical-acceptance-sampler-posterior-alpha', + type=float, + default=EngineArgs.typical_acceptance_sampler_posterior_alpha, + help='A scaling factor for the entropy-based threshold for token ' + 'acceptance in the TypicalAcceptanceSampler. Typically defaults ' + 'to sqrt of --typical-acceptance-sampler-posterior-threshold ' + 'i.e. 0.3') + + parser.add_argument( + '--disable-logprobs-during-spec-decoding', + action=StoreBoolean, + default=EngineArgs.disable_logprobs_during_spec_decoding, + nargs="?", + const="True", + help='If set to True, token log probabilities are not returned ' + 'during speculative decoding. If set to False, log probabilities ' + 'are returned according to the settings in SamplingParams. If ' + 'not specified, it defaults to True. Disabling log probabilities ' + 'during speculative decoding reduces latency by skipping logprob ' + 'calculation in proposal sampling, target sampling, and after ' + 'accepted tokens are determined.') + + parser.add_argument('--model-loader-extra-config', + type=nullable_str, + default=EngineArgs.model_loader_extra_config, + help='Extra config for model loader. ' + 'This will be passed to the model loader ' + 'corresponding to the chosen load_format. ' + 'This should be a JSON string that will be ' + 'parsed into a dictionary.') + parser.add_argument( + '--ignore-patterns', + action="append", + type=str, + default=[], + help="The pattern(s) to ignore when loading the model." + "Default to 'original/**/*' to avoid repeated loading of llama's " + "checkpoints.") + parser.add_argument( + '--preemption-mode', + type=str, + default=None, + help='If \'recompute\', the engine performs preemption by ' + 'recomputing; If \'swap\', the engine performs preemption by ' + 'block swapping.') + + parser.add_argument( + "--served-model-name", + nargs="+", + type=str, + default=None, + help="The model name(s) used in the API. If multiple " + "names are provided, the server will respond to any " + "of the provided names. The model name in the model " + "field of a response will be the first name in this " + "list. If not specified, the model name will be the " + "same as the `--model` argument. Noted that this name(s)" + "will also be used in `model_name` tag content of " + "prometheus metrics, if multiple names provided, metrics" + "tag will take the first one.") + parser.add_argument('--qlora-adapter-name-or-path', + type=str, + default=None, + help='Name or path of the QLoRA adapter.') + + parser.add_argument( + '--otlp-traces-endpoint', + type=str, + default=None, + help='Target URL to which OpenTelemetry traces will be sent.') + parser.add_argument( + '--collect-detailed-traces', + type=str, + default=None, + help="Valid choices are " + + ",".join(ALLOWED_DETAILED_TRACE_MODULES) + + ". It makes sense to set this only if --otlp-traces-endpoint is" + " set. If set, it will collect detailed traces for the specified " + "modules. This involves use of possibly costly and or blocking " + "operations and hence might have a performance impact.") + + parser.add_argument( + '--disable-async-output-proc', + action='store_true', + default=EngineArgs.disable_async_output_proc, + help="Disable async output processing. This may result in " + "lower performance.") + parser.add_argument( + '--override-neuron-config', + type=json.loads, + default=None, + help="Override or set neuron device configuration. " + "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'") + + parser.add_argument( + '--scheduling-policy', + choices=['fcfs', 'priority'], + default="fcfs", + help='The scheduling policy to use. "fcfs" (first come first served' + ', i.e. requests are handled in order of arrival; default) ' + 'or "priority" (requests are handled based on given ' + 'priority (lower value means earlier handling) and time of ' + 'arrival deciding any ties).') + + return parser + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # Get the list of attributes of this dataclass. + attrs = [attr.name for attr in dataclasses.fields(cls)] + # Set the attributes from the parsed arguments. + engine_args = cls(**{attr: getattr(args, attr) for attr in attrs}) + return engine_args + + def create_model_config(self) -> ModelConfig: + return ModelConfig( + model=self.model, + tokenizer=self.tokenizer, + tokenizer_mode=self.tokenizer_mode, + trust_remote_code=self.trust_remote_code, + dtype=self.dtype, + seed=self.seed, + revision=self.revision, + code_revision=self.code_revision, + rope_scaling=self.rope_scaling, + rope_theta=self.rope_theta, + tokenizer_revision=self.tokenizer_revision, + max_model_len=self.max_model_len, + quantization=self.quantization, + quantization_param_path=self.quantization_param_path, + enforce_eager=True, + max_context_len_to_capture=self.max_context_len_to_capture, + max_seq_len_to_capture=self.max_seq_len_to_capture, + max_logprobs=self.max_logprobs, + disable_sliding_window=self.disable_sliding_window, + skip_tokenizer_init=self.skip_tokenizer_init, + served_model_name=self.served_model_name, + limit_mm_per_prompt=self.limit_mm_per_prompt, + use_async_output_proc=not self.disable_async_output_proc, + override_neuron_config=self.override_neuron_config, + config_format=self.config_format, + mm_processor_kwargs=self.mm_processor_kwargs, + ) + + def create_load_config(self) -> LoadConfig: + return LoadConfig( + load_format=self.load_format, + download_dir=self.download_dir, + model_loader_extra_config=self.model_loader_extra_config, + ignore_patterns=self.ignore_patterns, + ) + + def create_engine_config(self) -> EngineConfig: + # gguf file needs a specific model loader and doesn't use hf_repo + if check_gguf_file(self.model): + self.quantization = self.load_format = "gguf" + + # bitsandbytes quantization needs a specific model loader + # so we make sure the quant method and the load format are consistent + if (self.quantization == "bitsandbytes" or + self.qlora_adapter_name_or_path is not None) and \ + self.load_format != "bitsandbytes": + raise ValueError( + "BitsAndBytes quantization and QLoRA adapter only support " + f"'bitsandbytes' load format, but got {self.load_format}") + + if (self.load_format == "bitsandbytes" or + self.qlora_adapter_name_or_path is not None) and \ + self.quantization != "bitsandbytes": + raise ValueError( + "BitsAndBytes load format and QLoRA adapter only support " + f"'bitsandbytes' quantization, but got {self.quantization}") + + assert self.cpu_offload_gb >= 0, ( + "CPU offload space must be non-negative" + f", but got {self.cpu_offload_gb}") + + device_config = DeviceConfig(device=self.device) + model_config = self.create_model_config() + + if model_config.is_multimodal_model: + if self.enable_prefix_caching: + logger.warning( + "--enable-prefix-caching is currently not " + "supported for multimodal models and has been disabled.") + self.enable_prefix_caching = False + + maybe_register_config_serialize_by_value(self.trust_remote_code) + + cache_config = CacheConfig( + block_size=self.block_size if self.device != "neuron" else + self.max_model_len, # neuron needs block_size = max_model_len + gpu_memory_utilization=self.gpu_memory_utilization, + swap_space=self.swap_space, + cache_dtype=self.kv_cache_dtype, + is_attention_free=model_config.is_attention_free, + num_gpu_blocks_override=self.num_gpu_blocks_override, + sliding_window=model_config.get_sliding_window(), + enable_prefix_caching=self.enable_prefix_caching, + cpu_offload_gb=self.cpu_offload_gb, + ) + parallel_config = ParallelConfig( + pipeline_parallel_size=self.pipeline_parallel_size, + tensor_parallel_size=self.tensor_parallel_size, + worker_use_ray=self.worker_use_ray, + max_parallel_loading_workers=self.max_parallel_loading_workers, + disable_custom_all_reduce=True, + tokenizer_pool_config=TokenizerPoolConfig.create_config( + self.tokenizer_pool_size, + self.tokenizer_pool_type, + self.tokenizer_pool_extra_config, + ), + ray_workers_use_nsight=self.ray_workers_use_nsight, + distributed_executor_backend=self.distributed_executor_backend) + + max_model_len = model_config.max_model_len + use_long_context = max_model_len > 32768 + + if self.enable_chunked_prefill is None: + # If not explicitly set, enable chunked prefill by default for + # long context (> 32K) models. This is to avoid OOM errors in the + # initial memory profiling phase. + + # Chunked prefill is currently disabled for multimodal models by + # default. + if use_long_context and not model_config.is_multimodal_model: + is_gpu = device_config.device_type == "cuda" + use_sliding_window = (model_config.get_sliding_window() + is not None) + use_spec_decode = self.speculative_model is not None + if (is_gpu and not use_sliding_window and not use_spec_decode + and not self.enable_lora + and not self.enable_prompt_adapter): + self.enable_chunked_prefill = True + logger.warning( + "Chunked prefill is enabled by default for models with " + "max_model_len > 32K. Currently, chunked prefill might " + "not work with some features or models. If you " + "encounter any issues, please disable chunked prefill " + "by setting --enable-chunked-prefill=False.") + if self.enable_chunked_prefill is None: + self.enable_chunked_prefill = False + + if not self.enable_chunked_prefill and use_long_context: + logger.warning( + "The model has a long context length (%s). This may cause OOM " + "errors during the initial memory profiling phase, or result " + "in low performance due to small KV cache space. Consider " + "setting --max-model-len to a smaller value.", max_model_len) + + if self.num_scheduler_steps > 1 and not self.use_v2_block_manager: + self.use_v2_block_manager = True + logger.warning( + "Enabled BlockSpaceManagerV2 because it is " + "required for multi-step (--num-scheduler-steps > 1)") + + speculative_config = SpeculativeConfig.maybe_create_spec_config( + target_model_config=model_config, + target_parallel_config=parallel_config, + target_dtype=self.dtype, + speculative_model=self.speculative_model, + speculative_model_quantization = \ + self.speculative_model_quantization, + speculative_draft_tensor_parallel_size = \ + self.speculative_draft_tensor_parallel_size, + num_speculative_tokens=self.num_speculative_tokens, + speculative_disable_mqa_scorer=self.speculative_disable_mqa_scorer, + speculative_disable_by_batch_size=self. + speculative_disable_by_batch_size, + speculative_max_model_len=self.speculative_max_model_len, + enable_chunked_prefill=self.enable_chunked_prefill, + use_v2_block_manager=self.use_v2_block_manager, + disable_log_stats=self.disable_log_stats, + ngram_prompt_lookup_max=self.ngram_prompt_lookup_max, + ngram_prompt_lookup_min=self.ngram_prompt_lookup_min, + draft_token_acceptance_method=\ + self.spec_decoding_acceptance_method, + typical_acceptance_sampler_posterior_threshold=self. + typical_acceptance_sampler_posterior_threshold, + typical_acceptance_sampler_posterior_alpha=self. + typical_acceptance_sampler_posterior_alpha, + disable_logprobs=self.disable_logprobs_during_spec_decoding, + ) + + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + if self.num_scheduler_steps > 1: + if speculative_config is not None: + raise ValueError("Speculative decoding is not supported with " + "multi-step (--num-scheduler-steps > 1)") + if self.enable_chunked_prefill and self.pipeline_parallel_size > 1: + raise ValueError("Multi-Step Chunked-Prefill is not supported " + "for pipeline-parallel-size > 1") + + # make sure num_lookahead_slots is set the higher value depending on + # if we are using speculative decoding or multi-step + num_lookahead_slots = max(self.num_lookahead_slots, + self.num_scheduler_steps - 1) + num_lookahead_slots = num_lookahead_slots \ + if speculative_config is None \ + else speculative_config.num_lookahead_slots + + scheduler_config = SchedulerConfig( + max_num_batched_tokens=self.max_num_batched_tokens, + max_num_seqs=self.max_num_seqs, + max_model_len=model_config.max_model_len, + use_v2_block_manager=self.use_v2_block_manager, + num_lookahead_slots=num_lookahead_slots, + delay_factor=self.scheduler_delay_factor, + enable_chunked_prefill=self.enable_chunked_prefill, + embedding_mode=model_config.embedding_mode, + is_multimodal_model=model_config.is_multimodal_model, + preemption_mode=self.preemption_mode, + num_scheduler_steps=self.num_scheduler_steps, + multi_step_stream_outputs=self.multi_step_stream_outputs, + send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER + and parallel_config.use_ray), + policy=self.scheduling_policy, + ) + lora_config = LoRAConfig( + max_lora_rank=self.max_lora_rank, + max_loras=self.max_loras, + fully_sharded_loras=self.fully_sharded_loras, + lora_extra_vocab_size=self.lora_extra_vocab_size, + long_lora_scaling_factors=self.long_lora_scaling_factors, + lora_dtype=self.lora_dtype, + max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras + and self.max_cpu_loras > 0 else None) if self.enable_lora else None + + if self.qlora_adapter_name_or_path is not None and \ + self.qlora_adapter_name_or_path != "": + if self.model_loader_extra_config is None: + self.model_loader_extra_config = {} + self.model_loader_extra_config[ + "qlora_adapter_name_or_path"] = self.qlora_adapter_name_or_path + + load_config = self.create_load_config() + + prompt_adapter_config = PromptAdapterConfig( + max_prompt_adapters=self.max_prompt_adapters, + max_prompt_adapter_token=self.max_prompt_adapter_token) \ + if self.enable_prompt_adapter else None + + decoding_config = DecodingConfig( + guided_decoding_backend=self.guided_decoding_backend) + + detailed_trace_modules = [] + if self.collect_detailed_traces is not None: + detailed_trace_modules = self.collect_detailed_traces.split(",") + for m in detailed_trace_modules: + if m not in ALLOWED_DETAILED_TRACE_MODULES: + raise ValueError( + f"Invalid module {m} in collect_detailed_traces. " + f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}") + observability_config = ObservabilityConfig( + otlp_traces_endpoint=self.otlp_traces_endpoint, + collect_model_forward_time="model" in detailed_trace_modules + or "all" in detailed_trace_modules, + collect_model_execute_time="worker" in detailed_trace_modules + or "all" in detailed_trace_modules, + ) + + if (model_config.get_sliding_window() is not None + and scheduler_config.chunked_prefill_enabled + and not scheduler_config.use_v2_block_manager): + raise ValueError( + "Chunked prefill is not supported with sliding window. " + "Set --disable-sliding-window to disable sliding window.") + + return EngineConfig( + model_config=model_config, + cache_config=cache_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + device_config=device_config, + lora_config=lora_config, + speculative_config=speculative_config, + load_config=load_config, + decoding_config=decoding_config, + observability_config=observability_config, + prompt_adapter_config=prompt_adapter_config, + ) + + +@dataclass +class AsyncEngineArgs(EngineArgs): + """Arguments for asynchronous vLLM engine.""" + disable_log_requests: bool = False + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser, + async_args_only: bool = False) -> FlexibleArgumentParser: + if not async_args_only: + parser = EngineArgs.add_cli_args(parser) + parser.add_argument('--disable-log-requests', + action='store_true', + help='Disable logging requests.') + return parser + + +class StoreBoolean(argparse.Action): + + def __call__(self, parser, namespace, values, option_string=None): + if values.lower() == "true": + setattr(namespace, self.dest, True) + elif values.lower() == "false": + setattr(namespace, self.dest, False) + else: + raise ValueError(f"Invalid boolean value: {values}. " + "Expected 'true' or 'false'.") + + +# These functions are used by sphinx to build the documentation +def _engine_args_parser(): + return EngineArgs.add_cli_args(FlexibleArgumentParser()) + + +def _async_engine_args_parser(): + return AsyncEngineArgs.add_cli_args(FlexibleArgumentParser(), + async_args_only=True) diff --git a/vllm/engine/async_llm_engine.py b/vllm/engine/async_llm_engine.py new file mode 100644 index 00000000..aeb48c39 --- /dev/null +++ b/vllm/engine/async_llm_engine.py @@ -0,0 +1,1323 @@ +import asyncio +import time +import weakref +from functools import partial +from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable, + List, Mapping, Optional, Set, Tuple, Type, Union, overload) +from weakref import ReferenceType + +import vllm.envs as envs +from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function +from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig, + ParallelConfig, SchedulerConfig) +from vllm.core.scheduler import SchedulerOutputs +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.async_timeout import asyncio_timeout +from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState +from vllm.engine.metrics_types import StatLoggerBase +from vllm.executor.executor_base import ExecutorAsyncBase +from vllm.executor.gpu_executor import GPUExecutorAsync +from vllm.executor.ray_utils import initialize_ray_cluster +from vllm.inputs import PromptType, TokensPrompt +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.guided_decoding import ( + get_guided_decoding_logits_processor) +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput, + RequestOutput) +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.sequence import ExecuteModelRequest +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.usage.usage_lib import UsageContext +from vllm.utils import (collect_from_async_generator, deprecate_kwargs, + random_uuid, weak_bind) + +logger = init_logger(__name__) +ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S + + +class AsyncEngineDeadError(RuntimeError): + pass + + +def _log_task_completion(task: asyncio.Task, + error_callback: Callable[[Exception], None]) -> None: + """This function is only intended for the `engine.run_engine_loop()` task. + + In particular, that task runs a `while True` loop that can only exit if + there is an exception. + """ + + exception = None + try: + return_value = task.result() + raise AssertionError( + f"The engine background task should never finish without an " + f"exception. {return_value}") + except asyncio.exceptions.CancelledError: + # We assume that if the task is cancelled, we are gracefully shutting + # down. This should only happen on program exit. + logger.info("Engine is gracefully shutting down.") + except Exception as e: + exception = e + logger.error("Engine background task failed", exc_info=e) + error_callback(exception) + raise AsyncEngineDeadError( + "Task finished unexpectedly. This should never happen! " + "Please open an issue on Github. See stack trace above for the " + "actual cause.") from e + + +STOP_ITERATION = Exception() # Sentinel + + +class AsyncStream: + """A stream of RequestOutputs or EmbeddingRequestOutputs for a request + that can be iterated over asynchronously via an async generator.""" + + def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None: + self.request_id = request_id + self._cancel = cancel + self._queue: asyncio.Queue = asyncio.Queue() + self._finished = False + + def put(self, item: Union[RequestOutput, EmbeddingRequestOutput, + Exception]) -> None: + if not self._finished: + self._queue.put_nowait(item) + + def finish( + self, + exception: Optional[Union[BaseException, Type[BaseException]]] = None, + ) -> None: + if not self._finished: + self._finished = True + self._queue.put_nowait( + exception if self._is_raisable(exception) else STOP_ITERATION) + + @property + def finished(self) -> bool: + return self._finished + + async def generator( + self + ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]: + try: + while True: + result = await self._queue.get() + if self._is_raisable(result): + if result == STOP_ITERATION: + return + raise result + yield result + except GeneratorExit: + self._cancel(self.request_id) + raise asyncio.CancelledError from None + + @staticmethod + def _is_raisable(value: Any): + return isinstance(value, BaseException) or \ + (isinstance(value, type) and \ + issubclass(value, BaseException)) + + +class RequestTracker: + """Synchronous abstraction for tracking requests.""" + + def __init__(self) -> None: + self._request_streams: Dict[str, AsyncStream] = {} + self._aborted_requests: asyncio.Queue[str] = asyncio.Queue() + self._new_requests: asyncio.Queue[Tuple[AsyncStream, + dict]] = asyncio.Queue() + self.new_requests_event = asyncio.Event() + + def __contains__(self, item): + return item in self._request_streams + + def __len__(self) -> int: + return len(self._request_streams) + + def propagate_exception(self, + exc: Exception, + request_id: Optional[str] = None) -> None: + """Propagate an exception to request streams + (all if request_id is None).""" + if request_id is not None: + self.abort_request(request_id, exception=exc) + else: + # NB: tuple() used here because self.abort_request pops the stream + # out of self._request_streams, so we can't iterate on it directly + for rid in tuple(self._request_streams.keys()): + self.abort_request(rid, exception=exc) + + def process_request_output(self, + request_output: Union[RequestOutput, + EmbeddingRequestOutput], + *, + verbose: bool = False) -> None: + """Process a request output from the engine.""" + request_id = request_output.request_id + finished = request_output.finished + + if finished: + stream = self._request_streams.pop(request_id, None) + else: + stream = self._request_streams.get(request_id) + # Guard against a KeyError which can occur if the request was aborted + # while the output was generated + if stream is not None: + stream.put(request_output) + if finished: + stream.finish() + + if verbose and finished: + logger.info("Finished request %s.", request_id) + + def process_exception(self, + request_id: str, + exception: BaseException, + *, + verbose: bool = False) -> None: + """Propagate an exception from the engine.""" + if verbose: + logger.info("Finished request %s.", request_id) + self.abort_request(request_id, exception=exception) + + def add_request(self, + request_id: str, + *, + verbose: bool = False, + **engine_add_request_kwargs) -> AsyncStream: + """Add a request to be sent to the engine on the next background + loop iteration.""" + if request_id in self._request_streams: + raise KeyError(f"Request {request_id} already exists.") + + abort_request = partial(self.abort_request, verbose=verbose) + stream = AsyncStream(request_id, abort_request) + self._new_requests.put_nowait((stream, { + "request_id": request_id, + **engine_add_request_kwargs + })) + + self.new_requests_event.set() + + if verbose: + logger.info("Added request %s.", request_id) + + return stream + + def abort_request(self, + request_id: str, + *, + exception: Optional[Union[BaseException, + Type[BaseException]]] = None, + verbose: bool = False) -> None: + """Abort a request during next background loop iteration.""" + if verbose: + logger.info("Aborted request %s.", request_id) + + self._aborted_requests.put_nowait(request_id) + + stream = self._request_streams.pop(request_id, None) + if stream is not None: + stream.finish(exception=exception) + + def get_new_and_aborted_requests(self) -> Tuple[List[Dict], Set[str]]: + """Get the new requests and finished requests to be + sent to the engine.""" + new_requests: List[Dict] = [] + finished_requests: Set[str] = set() + + while not self._aborted_requests.empty(): + request_id = self._aborted_requests.get_nowait() + finished_requests.add(request_id) + + while not self._new_requests.empty(): + stream, new_request = self._new_requests.get_nowait() + request_id = stream.request_id + if request_id in finished_requests: + # The request has already been aborted. + stream.finish(asyncio.CancelledError) + finished_requests.discard(request_id) + else: + self._request_streams[request_id] = stream + new_requests.append(new_request) + + return new_requests, finished_requests + + async def wait_for_new_requests(self): + if not self.has_new_requests(): + await self.new_requests_event.wait() + self.new_requests_event.clear() + + def has_new_requests(self): + return not self._new_requests.empty() + + +class _AsyncLLMEngine(LLMEngine): + """Extension of LLMEngine to add async methods.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def step_async( + self, virtual_engine: int + ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]: + """Performs one decoding iteration and returns newly generated results. + The workers are ran asynchronously if possible. + + This function performs one decoding iteration of the engine. It first + schedules the sequences to be executed in the next iteration and the + token blocks to be swapped in/out/copy. Then, it executes the model + and updates the scheduler with the model outputs. Finally, it decodes + the sequences and returns the newly generated results. + """ + # these are cached outputs from previous iterations. None if on first + # iteration + cached_outputs = self.cached_scheduler_outputs[virtual_engine] + seq_group_metadata_list = cached_outputs.seq_group_metadata_list + scheduler_outputs = cached_outputs.scheduler_outputs + allow_async_output_proc = cached_outputs.allow_async_output_proc + + ctx = self.scheduler_contexts[virtual_engine] + + # Clear outputs for each new scheduler iteration + ctx.request_outputs.clear() + + # skip the scheduler if there are any remaining steps in the seq groups. + # This ensures that the scheduler is only called again when the current + # batch has completed. + if not self._has_remaining_steps(seq_group_metadata_list): + + # Schedule iteration + (seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc + ) = self.scheduler[virtual_engine].schedule() + + ctx.seq_group_metadata_list = seq_group_metadata_list + ctx.scheduler_outputs = scheduler_outputs + + # Maybe switch from async mode to sync mode + if not allow_async_output_proc and len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + + if (self.scheduler_config.is_multi_step + and scheduler_outputs.num_lookahead_slots > 0): + # cache the scheduler outputs for the next iteration if we have + # lookahead slots + self._cache_scheduler_outputs_for_multi_step( + virtual_engine, seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc) + + assert seq_group_metadata_list is not None + assert scheduler_outputs is not None + + if not scheduler_outputs.is_empty(): + finished_requests_ids = self.scheduler[ + virtual_engine].get_and_reset_finished_requests_ids() + + # Check if we have a cached last_output from the previous iteration. + # For supporting PP this is probably the best way to pass the + # sampled_token_ids, as a separate broadcast over all the PP stages + # will cause one virtual engine's microbatch to block the pipeline. + last_sampled_token_ids = \ + self._get_last_sampled_token_ids(virtual_engine) + + execute_model_req = ExecuteModelRequest( + seq_group_metadata_list=seq_group_metadata_list, + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in, + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out, + blocks_to_copy=scheduler_outputs.blocks_to_copy, + virtual_engine=virtual_engine, + num_lookahead_slots=scheduler_outputs.num_lookahead_slots, + running_queue_size=scheduler_outputs.running_queue_size, + finished_requests_ids=finished_requests_ids, + # We use ExecuteModelRequest to pass the last sampled_token_ids + # to each of the non-last PP stages for in-place prepare_input. + last_sampled_token_ids=last_sampled_token_ids) + + if allow_async_output_proc: + execute_model_req.async_callback = self.async_callbacks[ + virtual_engine] + + # Execute the model. + outputs = await self.model_executor.execute_model_async( + execute_model_req) + + # we need to do this here so that last step's sampled_token_ids can + # be passed to the next iteration for PP. + if self.scheduler_config.is_multi_step: + self._update_cached_scheduler_output(virtual_engine, outputs) + else: + if len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + outputs = [] + + # Finish the current step for all the sequence groups. + if self.scheduler_config.is_multi_step: + for seq_group in seq_group_metadata_list: + seq_group.finish_step() + + if not self._has_remaining_steps(seq_group_metadata_list): + # Clear the cache if we have finished all the steps + if self.scheduler_config.is_multi_step: + self.cached_scheduler_outputs[ + virtual_engine] = SchedulerOutputState() + + # is_first_step_output is True only when the num_steps of all + # the sequences are 1. When the num_steps > 1, + # multi_step_model_runner does the first-step output append. + is_first_step_output: bool = False if not seq_group_metadata_list \ + else seq_group_metadata_list[0].state.num_steps == 1 + + ctx.append_output(outputs=outputs, + seq_group_metadata_list=seq_group_metadata_list, + scheduler_outputs=scheduler_outputs, + is_async=allow_async_output_proc, + is_last_step=True, + is_first_step_output=is_first_step_output) + + if outputs and allow_async_output_proc: + assert len( + outputs + ) == 1, "Async postprocessor expects only a single output set" + self._advance_to_next_step( + outputs[0], seq_group_metadata_list, + scheduler_outputs.scheduled_seq_groups) + + if not allow_async_output_proc: + self._process_model_outputs(ctx=ctx) + + # Log stats. + self.do_log_stats(scheduler_outputs, outputs) + + # Tracing + self.do_tracing(scheduler_outputs) + + else: + # Multi-step case + return ctx.request_outputs + + if not self.has_unfinished_requests(): + # Drain async postprocessor (if exists) + if len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + assert len(ctx.output_queue) == 0 + + return ctx.request_outputs + + async def stop_remote_worker_execution_loop_async(self) -> None: + """Stop the remote worker execution loop.""" + await self.model_executor.stop_remote_worker_execution_loop_async() + + @overload # DEPRECATED + async def add_request_async( + self, + request_id: str, + *, + inputs: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @overload + async def add_request_async( + self, + request_id: str, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + async def add_request_async( + self, + request_id: str, + prompt: Optional[PromptType] = None, + params: Optional[Union[SamplingParams, PoolingParams]] = None, + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> None: + """Async version of :meth:`add_request`.""" + if inputs is not None: + prompt = inputs + assert prompt is not None and params is not None + + if lora_request is not None and not self.lora_config: + raise ValueError(f"Got lora_request {lora_request} but LoRA is " + "not enabled!") + if priority != 0 and not self.scheduler_config.policy == "priority": + raise ValueError(f"Got priority {priority} but " + "Priority scheduling is not enabled.") + if arrival_time is None: + arrival_time = time.time() + + preprocessed_inputs = await self.input_preprocessor.preprocess_async( + prompt, + request_id=request_id, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + processed_inputs = self.input_processor(preprocessed_inputs) + + if isinstance(params, SamplingParams) and \ + params.guided_decoding is not None: + # Guided decoding has an async implementation for building logits + # processors in a separate threadpool. + # We want to invoke that here instead of using the blocking + # implementation in the LLMEngine + params = await build_guided_decoding_logits_processor_async( + sampling_params=params, + tokenizer=self.get_tokenizer(lora_request), + default_guided_backend=self.decoding_config. + guided_decoding_backend) + + self._add_processed_request( + request_id=request_id, + processed_inputs=processed_inputs, + params=params, + arrival_time=arrival_time, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + trace_headers=trace_headers, + priority=priority, + ) + + async def check_health_async(self) -> None: + if self.tokenizer: + self.tokenizer.check_health() + self.model_executor.check_health() + + +async def build_guided_decoding_logits_processor_async( + sampling_params: SamplingParams, tokenizer: AnyTokenizer, + default_guided_backend: str) -> SamplingParams: + """Constructs logits processors based on the guided_decoding, + logits_bias, and allowed_token_ids fields in sampling_params. Deletes + those fields and adds the constructed logits processors to the + logits_processors field. Modifies sampling params in-place and returns + the modified sampling params.""" + if (guided_decoding := sampling_params.guided_decoding) is None: + return sampling_params + + logger.debug("Building guided decoding logits processor. " + "Params: %s", guided_decoding) + + guided_decoding.backend = guided_decoding.backend or default_guided_backend + + processor = await get_guided_decoding_logits_processor( + guided_params=guided_decoding, tokenizer=tokenizer) + + if processor: + if sampling_params.logits_processors is None: + sampling_params.logits_processors = [] + sampling_params.logits_processors.append(processor) + + # Unset guided decoding params after constructing the lp from them + sampling_params.guided_decoding = None + + return sampling_params + + +class AsyncLLMEngine: + """An asynchronous wrapper for :class:`LLMEngine`. + + This class is used to wrap the :class:`LLMEngine` class to make it + asynchronous. It uses asyncio to create a background loop that keeps + processing incoming requests. The :class:`LLMEngine` is kicked by the + generate method when there are requests in the waiting queue. The generate + method yields the outputs from the :class:`LLMEngine` to the caller. + + Args: + log_requests: Whether to log the requests. + start_engine_loop: If True, the background task to run the engine + will be automatically started in the generate call. + *args: Arguments for :class:`LLMEngine`. + **kwargs: Arguments for :class:`LLMEngine`. + """ + + _engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine + + def __init__(self, + *args, + log_requests: bool = True, + start_engine_loop: bool = True, + **kwargs) -> None: + self.log_requests = log_requests + self.engine = self._engine_class(*args, **kwargs) + + # This ensures quick processing of request outputs + # so the append to asyncio queues is not delayed, + # especially for multi-step. + self.use_process_request_outputs_callback = ( + self.engine.model_config.use_async_output_proc) + + if self.use_process_request_outputs_callback: + self.engine.process_request_outputs_callback = \ + weak_bind(self.process_request_outputs) + + self.background_loop: Optional[asyncio.Future] = None + # We need to keep a reference to unshielded + # task as well to prevent it from being garbage + # collected + self._background_loop_unshielded: Optional[asyncio.Task] = None + self.start_engine_loop = start_engine_loop + self._errored_with: Optional[BaseException] = None + + # Lazy initialized fields + self._request_tracker: RequestTracker + + def __del__(self): + if rt := getattr(self, "request_tracker", None): + # Wake up engine loop so that it will exit cleanly + rt.new_requests_event.set() + + @classmethod + def _get_executor_cls( + cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]: + distributed_executor_backend = ( + engine_config.parallel_config.distributed_executor_backend) + if isinstance(distributed_executor_backend, type): + if not issubclass(distributed_executor_backend, ExecutorAsyncBase): + raise TypeError( + "distributed_executor_backend must be a subclass of " + f"ExecutorAsyncBase. Got {distributed_executor_backend}.") + executor_class = distributed_executor_backend + elif engine_config.device_config.device_type == "neuron": + from vllm.executor.neuron_executor import NeuronExecutorAsync + executor_class = NeuronExecutorAsync + elif engine_config.device_config.device_type == "tpu": + if distributed_executor_backend == "ray": + from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync + executor_class = RayTPUExecutorAsync + else: + assert distributed_executor_backend is None + from vllm.executor.tpu_executor import TPUExecutorAsync + executor_class = TPUExecutorAsync + elif engine_config.device_config.device_type == "cpu": + from vllm.executor.cpu_executor import CPUExecutorAsync + executor_class = CPUExecutorAsync + elif engine_config.device_config.device_type == "openvino": + assert distributed_executor_backend is None, ( + "Distributed execution is not supported with " + "the OpenVINO backend.") + from vllm.executor.openvino_executor import OpenVINOExecutorAsync + executor_class = OpenVINOExecutorAsync + elif engine_config.device_config.device_type == "xpu": + if distributed_executor_backend is None: + from vllm.executor.xpu_executor import XPUExecutorAsync + executor_class = XPUExecutorAsync + elif distributed_executor_backend == "ray": + from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync + executor_class = RayXPUExecutorAsync + elif distributed_executor_backend == "mp": + from vllm.executor.multiproc_xpu_executor import ( + MultiprocessingXPUExecutorAsync) + executor_class = MultiprocessingXPUExecutorAsync + else: + raise RuntimeError( + "Not supported distributed execution model on XPU device.") + elif distributed_executor_backend == "ray": + from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync + executor_class = RayGPUExecutorAsync + elif distributed_executor_backend == "mp": + from vllm.executor.multiproc_gpu_executor import ( + MultiprocessingGPUExecutorAsync) + executor_class = MultiprocessingGPUExecutorAsync + else: + from vllm.executor.gpu_executor import GPUExecutorAsync + executor_class = GPUExecutorAsync + return executor_class + + @classmethod + def from_engine_args( + cls, + engine_args: AsyncEngineArgs, + engine_config: Optional[EngineConfig] = None, + start_engine_loop: bool = True, + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, + ) -> "AsyncLLMEngine": + """Creates an async LLM engine from the engine arguments.""" + # Create the engine configs. + if engine_config is None: + engine_config = engine_args.create_engine_config() + + executor_class = cls._get_executor_cls(engine_config) + + if executor_class.uses_ray: + initialize_ray_cluster(engine_config.parallel_config) + + # Create the async LLM engine. + engine = cls( + **engine_config.to_dict(), + executor_class=executor_class, + log_requests=not engine_args.disable_log_requests, + log_stats=not engine_args.disable_log_stats, + start_engine_loop=start_engine_loop, + usage_context=usage_context, + stat_loggers=stat_loggers, + ) + return engine + + @property + def is_running(self) -> bool: + return (self.background_loop is not None + and self._background_loop_unshielded is not None + and not self._background_loop_unshielded.done()) + + @property + def is_stopped(self) -> bool: + return self.errored or (self.background_loop is not None and + self._background_loop_unshielded is not None + and self._background_loop_unshielded.done()) + + @property + def errored(self) -> bool: + return self._errored_with is not None + + @property + def dead_error(self) -> BaseException: + return AsyncEngineDeadError( + "Background loop is not running. If it was running, " + "inspect the output to find the stacktrace of the " + "error that caused the background loop to stop " + "(AsyncEngineDeadError).") + + def set_errored(self, exc: Exception) -> None: + self._errored_with = exc + + def _error_callback(self, exc: Exception) -> None: + self.set_errored(exc) + self._request_tracker.propagate_exception(exc) + + async def get_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + return await (self.engine.get_tokenizer_group(). + get_lora_tokenizer_async(lora_request)) + + def start_background_loop(self) -> None: + """Start the background loop.""" + if self.errored: + raise AsyncEngineDeadError( + "Background loop has errored already.") from self._errored_with + if self.is_running: + raise RuntimeError("Background loop is already running.") + # Initialize the RequestTracker here so it uses the right event loop. + self._request_tracker = RequestTracker() + + self._background_loop_unshielded = asyncio.get_event_loop( + ).create_task(self.run_engine_loop(weakref.ref(self))) + self._background_loop_unshielded.add_done_callback( + partial(_log_task_completion, error_callback=self._error_callback)) + self.background_loop = asyncio.shield(self._background_loop_unshielded) + + def shutdown_background_loop(self) -> None: + """ + Shut down the background loop. + + This method needs to be called during cleanup to remove + references to `self` and properly GC the resources held + by the async LLM engine (e.g., the executors as well as + their resources). + """ + if self._background_loop_unshielded is not None: + self._background_loop_unshielded.cancel() + self._background_loop_unshielded = None + self.background_loop = None + + async def engine_step(self, virtual_engine: int) -> bool: + """Kick the engine to process the waiting requests. + + Returns True if there are in-progress requests.""" + + new_requests, aborted_requests = ( + self._request_tracker.get_new_and_aborted_requests()) + + for new_request in new_requests: + # Add the request into the vLLM engine's waiting queue. + try: + await self.engine.add_request_async(**new_request) + except ValueError as e: + # TODO: use a vLLM specific error for failed validation + self._request_tracker.process_exception( + new_request["request_id"], + e, + verbose=self.log_requests, + ) + + if aborted_requests: + await self._engine_abort(aborted_requests) + + request_outputs = await self.engine.step_async(virtual_engine) + + # Put the outputs into the corresponding streams. + # If used as a callback, then already invoked inside + # LLMEngine's _process_model_outputs + if not self.use_process_request_outputs_callback: + all_finished = self.process_request_outputs(request_outputs) + else: + # For callback case, we only need to detect when all + # requests are finished + all_finished = all(request_output.finished + for request_output in request_outputs) + + return not all_finished + + def process_request_outputs(self, request_outputs) -> bool: + # Put the outputs into the corresponding streams. + all_finished = True + for request_output in request_outputs: + self._request_tracker.process_request_output( + request_output, verbose=self.log_requests) + all_finished = all_finished and request_output.finished + + return all_finished + + async def _engine_abort(self, request_ids: Iterable[str]): + self.engine.abort_request(request_ids) + + @staticmethod + async def run_engine_loop(engine_ref: ReferenceType): + """We use a weakref to the engine so that the running loop + doesn't prevent the engine being garbage collected.""" + engine: Optional["AsyncLLMEngine"] = engine_ref() + if not engine: + return + + pipeline_parallel_size = \ + engine.engine.parallel_config.pipeline_parallel_size + has_requests_in_progress = [False] * pipeline_parallel_size + while True: + if not any(has_requests_in_progress): + logger.debug("Waiting for new requests...") + # Stop the execute model loop in parallel workers until there + # are more requests to process. This avoids waiting + # indefinitely in torch.distributed ops which may otherwise + # timeout, and unblocks the RPC thread in the workers so that + # they can process any other queued control plane messages, + # such as add/remove lora adapters. + await engine.engine.stop_remote_worker_execution_loop_async() + request_tracker = engine._request_tracker + # Allow engine to be garbage collected while + # waiting for new requests + del engine + await asyncio.sleep(0) + if engine_ref() is None: + return + await request_tracker.wait_for_new_requests() + engine = engine_ref() + if not engine: + return + logger.debug("Got new requests!") + # may be we do not use this when on server mode(all vllm server in new process, but still without cuda graph) + # requests_in_progress = [ + # asyncio.create_task(engine.engine_step(ve)) + # for ve in range(pipeline_parallel_size) + # ] + # has_requests_in_progress = [True] * pipeline_parallel_size + requests_in_progress = {0: asyncio.create_task(engine.engine_step(0))} + has_requests_in_progress[0] = True + + # Abort if iteration takes too long due to unrecoverable errors + # (eg. NCCL timeouts). + try: + async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S): + done, _ = await asyncio.wait( + requests_in_progress.values(), + return_when=asyncio.FIRST_COMPLETED) + for _ in range(len(requests_in_progress)): + await asyncio.sleep(0) + add_virtual_engine = set(list(range(pipeline_parallel_size))) + for task in done: + result = task.result() + # virtual_engine = requests_in_progress.index(task) + virtual_engine = [k for k,v in requests_in_progress.items() if v == task][0] + has_unfinished_requests = ( + engine.engine. + has_unfinished_requests_for_virtual_engine( + virtual_engine)) + if result or has_unfinished_requests: + requests_in_progress[virtual_engine] = ( + asyncio.create_task( + engine.engine_step(virtual_engine))) + has_requests_in_progress[virtual_engine] = True + else: + requests_in_progress.pop(virtual_engine) + has_requests_in_progress[virtual_engine] = False + + # check other engine + for ve in list(add_virtual_engine - requests_in_progress.keys()): + has_unfinished_requests = ( + engine.engine.has_unfinished_requests_for_virtual_engine( + virtual_engine)) + if has_unfinished_requests: + requests_in_progress[ve] = asyncio.create_task(engine.engine_step(ve)) + has_requests_in_progress[ve] = True + except asyncio.TimeoutError as exc: + logger.error( + "Engine iteration timed out. This should never happen!") + engine.set_errored(exc) + raise + await asyncio.sleep(0) + + # This method does not need to be async, but kept that way + # for backwards compatibility. + @overload # DEPRECATED + def add_request( + self, + request_id: str, + *, + inputs: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> Coroutine[None, None, AsyncGenerator[Union[ + RequestOutput, EmbeddingRequestOutput], None]]: + ... + + @overload + def add_request( + self, + request_id: str, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> Coroutine[None, None, AsyncGenerator[Union[ + RequestOutput, EmbeddingRequestOutput], None]]: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + async def add_request( + self, + request_id: str, + prompt: Optional[PromptType] = None, + params: Optional[Union[SamplingParams, PoolingParams]] = None, + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]: + if inputs is not None: + prompt = inputs + assert prompt is not None and params is not None + + if not self.is_running: + if self.start_engine_loop: + self.start_background_loop() + else: + raise AsyncEngineDeadError( + "Background loop is not running. If it was running, " + "inspect the output to find the stacktrace of the " + "error that caused the background loop to stop " + "(AsyncEngineDeadError).") + + if (priority != 0 + and not self.engine.scheduler_config.policy == "priority"): + raise ValueError(f"Got priority {priority} but " + "Priority scheduling is not enabled.") + + stream = self._request_tracker.add_request( + request_id, + verbose=self.log_requests, + prompt=prompt, + params=params, + arrival_time=arrival_time or time.time(), + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=priority, + ) + + return stream.generator() + + async def generate( + self, + prompt: PromptType, + sampling_params: SamplingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + """Generate outputs for a request. + + Generate outputs for a request. This method is a coroutine. It adds the + request into the waiting queue of the LLMEngine and streams the outputs + from the LLMEngine to the caller. + + Args: + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType` + for more details about the format of each input. + sampling_params: The sampling parameters of the request. + request_id: The unique id of the request. + lora_request: LoRA request to use for generation, if any. + trace_headers: OpenTelemetry trace headers. + prompt_adapter_request: Prompt Adapter request to use + for generation, if any. + priority: The priority of the request. + Only applicable with priority scheduling. + + Yields: + The output `RequestOutput` objects from the LLMEngine + for the request. + + Details: + - If the engine is not running, start the background loop, + which iteratively invokes + :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step` + to process the waiting requests. + - Add the request to the engine's `RequestTracker`. + On the next background loop, this request will be sent to + the underlying engine. + Also, a corresponding `AsyncStream` will be created. + - Wait for the request outputs from `AsyncStream` and yield them. + + Example: + >>> # Please refer to entrypoints/api_server.py for + >>> # the complete example. + >>> + >>> # initialize the engine and the example input + >>> engine = AsyncLLMEngine.from_engine_args(engine_args) + >>> example_input = { + >>> "prompt": "What is LLM?", + >>> "stream": False, # assume the non-streaming case + >>> "temperature": 0.0, + >>> "request_id": 0, + >>> } + >>> + >>> # start the generation + >>> results_generator = engine.generate( + >>> example_input["prompt"], + >>> SamplingParams(temperature=example_input["temperature"]), + >>> example_input["request_id"]) + >>> + >>> # get the results + >>> final_output = None + >>> async for request_output in results_generator: + >>> if await request.is_disconnected(): + >>> # Abort the request if the client disconnects. + >>> await engine.abort(request_id) + >>> # Return or raise an error + >>> ... + >>> final_output = request_output + >>> + >>> # Process and return the final output + >>> ... + """ + async for output in await self.add_request( + request_id, + prompt, + sampling_params, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=priority, + ): + yield LLMEngine.validate_output(output, RequestOutput) + + async def beam_search( + self, + prompt: Union[PromptType, List[int]], + request_id: str, + params: BeamSearchParams, + ) -> AsyncGenerator[RequestOutput, None]: + + beam_width = params.beam_width + max_tokens = params.max_tokens + ignore_eos = params.ignore_eos + temperature = params.temperature + length_penalty = params.length_penalty + + tokenizer = await self.get_tokenizer() + tokenizedPrompt = prompt if isinstance( + prompt, list) else tokenizer.encode(prompt) + tokenizedLength = len(tokenizedPrompt) + + sort_beams_key = create_sort_beams_key_function( + tokenizer.eos_token_id, length_penalty) + + beam_search_params = SamplingParams(logprobs=2 * beam_width, + max_tokens=1, + temperature=temperature) + all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)] + completed = [] + + for _ in range(max_tokens): + prompts_batch = [ + TokensPrompt(prompt_token_ids=beam.tokens) + for beam in all_beams + ] + + tasks = [] + + request_id = f"beam_search-{random_uuid()}" + for i, individual_prompt in enumerate(prompts_batch): + request_id_item = f"{request_id}-{i}" + task = asyncio.create_task( + collect_from_async_generator( + self.generate(individual_prompt, beam_search_params, + request_id_item))) + tasks.append(task) + + output = await asyncio.gather(*tasks) + + output = [x[0] for x in output] + + logger.info(output) + + new_beams = [] + for i, current_beam in enumerate(all_beams): + result = output[i] + + if result.outputs[0].logprobs is not None: + logprobs = result.outputs[0].logprobs[0] + for token_id, logprob_obj in logprobs.items(): + new_beam = BeamSearchSequence( + tokens=current_beam.tokens + [token_id], + cum_logprob=current_beam.cum_logprob + + logprob_obj.logprob) + + if token_id == tokenizer.eos_token_id and \ + not ignore_eos: + completed.append(new_beam) + else: + new_beams.append(new_beam) + + sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True) + all_beams = sorted_beams[:beam_width] + + completed.extend(all_beams) + sorted_completed = sorted(completed, key=sort_beams_key, reverse=True) + best_beams = sorted_completed[:beam_width] + + for beam in best_beams: + beam.text = tokenizer.decode(beam.tokens[tokenizedLength:]) + + beam_search_output = RequestOutput( + request_id=request_id, + prompt=prompt, + outputs=[ + CompletionOutput( + text=beam.text, + cumulative_logprob=beam.cum_logprob, + token_ids=beam.tokens, + index=i, + logprobs=beam.cum_logprob, + ) for (i, beam) in enumerate(best_beams) + ], + finished=True, + prompt_token_ids=tokenizedPrompt, + prompt_logprobs=None) + + yield LLMEngine.validate_output(beam_search_output, RequestOutput) + + async def encode( + self, + prompt: PromptType, + pooling_params: PoolingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + ) -> AsyncGenerator[EmbeddingRequestOutput, None]: + """Generate outputs for a request from an embedding model. + + Generate outputs for a request. This method is a coroutine. It adds the + request into the waiting queue of the LLMEngine and streams the outputs + from the LLMEngine to the caller. + + Args: + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType` + for more details about the format of each input. + pooling_params: The pooling parameters of the request. + request_id: The unique id of the request. + lora_request: LoRA request to use for generation, if any. + trace_headers: OpenTelemetry trace headers. + priority: The priority of the request. + Only applicable with priority scheduling. + + Yields: + The output `EmbeddingRequestOutput` objects from the LLMEngine + for the request. + + Details: + - If the engine is not running, start the background loop, + which iteratively invokes + :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step` + to process the waiting requests. + - Add the request to the engine's `RequestTracker`. + On the next background loop, this request will be sent to + the underlying engine. + Also, a corresponding `AsyncStream` will be created. + - Wait for the request outputs from `AsyncStream` and yield them. + + Example: + >>> # Please refer to entrypoints/api_server.py for + >>> # the complete example. + >>> + >>> # initialize the engine and the example input + >>> engine = AsyncLLMEngine.from_engine_args(engine_args) + >>> example_input = { + >>> "input": "What is LLM?", + >>> "request_id": 0, + >>> } + >>> + >>> # start the generation + >>> results_generator = engine.encode( + >>> example_input["input"], + >>> PoolingParams(), + >>> example_input["request_id"]) + >>> + >>> # get the results + >>> final_output = None + >>> async for request_output in results_generator: + >>> if await request.is_disconnected(): + >>> # Abort the request if the client disconnects. + >>> await engine.abort(request_id) + >>> # Return or raise an error + >>> ... + >>> final_output = request_output + >>> + >>> # Process and return the final output + >>> ... + """ + async for output in await self.add_request( + request_id, + prompt, + pooling_params, + lora_request=lora_request, + trace_headers=trace_headers, + priority=priority, + ): + yield LLMEngine.validate_output(output, EmbeddingRequestOutput) + + async def abort(self, request_id: str) -> None: + """Abort a request. + + Abort a submitted request. If the request is finished or not found, + this method will be a no-op. + + Args: + request_id: The unique id of the request. + """ + if not self.is_running: + raise AsyncEngineDeadError( + "Background loop is not running. If it was running, " + "inspect the output to find the stacktrace of the " + "error that caused the background loop to stop " + "(AsyncEngineDeadError).") + + return self._abort(request_id) + + def _abort(self, request_id: str) -> None: + """Abort a request. + + Abort a submitted request. If the request is finished or not found, + this method will be a no-op. + + Args: + request_id: The unique id of the request. + """ + self._request_tracker.abort_request(request_id, + exception=asyncio.CancelledError, + verbose=self.log_requests) + + async def get_model_config(self) -> ModelConfig: + """Get the model configuration of the vLLM engine.""" + return self.engine.get_model_config() + + async def get_parallel_config(self) -> ParallelConfig: + """Get the parallel configuration of the vLLM engine.""" + return self.engine.get_parallel_config() + + async def get_decoding_config(self) -> DecodingConfig: + """Get the decoding configuration of the vLLM engine.""" + return self.engine.get_decoding_config() + + async def get_scheduler_config(self) -> SchedulerConfig: + """Get the scheduling configuration of the vLLM engine.""" + return self.engine.get_scheduler_config() + + async def get_lora_config(self) -> LoRAConfig: + """Get the lora configuration of the vLLM engine.""" + return self.engine.get_lora_config() + + async def do_log_stats( + self, + scheduler_outputs: Optional[SchedulerOutputs] = None, + model_output: Optional[List[SamplerOutput]] = None) -> None: + self.engine.do_log_stats() + + async def check_health(self) -> None: + """Raises an error if engine is unhealthy.""" + t = time.perf_counter() + logger.debug("Starting health check...") + if self.is_stopped: + raise AsyncEngineDeadError("Background loop is stopped.") + + await self.engine.check_health_async() + logger.debug("Health check took %fs", time.perf_counter() - t) + + async def is_tracing_enabled(self) -> bool: + return self.engine.is_tracing_enabled() + + def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None: + self.engine.add_logger(logger_name=logger_name, logger=logger) + + def remove_logger(self, logger_name: str) -> None: + self.engine.remove_logger(logger_name=logger_name) + + async def start_profile(self) -> None: + # using type instead of isinstance to check to avoid capturing + # inherited classes + if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721 + self.engine.model_executor.start_profile() + else: + self.engine.model_executor._run_workers("start_profile") + + async def stop_profile(self) -> None: + # using type instead of isinstance to check to avoid capturing + # inherited classes + if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721 + self.engine.model_executor.stop_profile() + else: + self.engine.model_executor._run_workers("stop_profile") \ No newline at end of file diff --git a/vllm/engine/async_timeout.py b/vllm/engine/async_timeout.py new file mode 100644 index 00000000..4b184262 --- /dev/null +++ b/vllm/engine/async_timeout.py @@ -0,0 +1,189 @@ +# Workaround for https://github.com/python/cpython/issues/86296 +# +# From https://github.com/aio-libs/async-timeout/blob/master/async_timeout/__init__.py +# Licensed under the Apache License (Apache-2.0) + +import asyncio +import enum +import sys +import warnings +from types import TracebackType +from typing import Any, Optional, Type + +if sys.version_info[:2] >= (3, 11): + from asyncio import timeout as asyncio_timeout +else: + + def asyncio_timeout(delay: Optional[float]) -> "Timeout": + """timeout context manager. + Useful in cases when you want to apply timeout logic around block + of code or in cases when asyncio.wait_for is not suitable. For example: + >>> async with timeout(0.001): + ... async with aiohttp.get('https://github.com') as r: + ... await r.text() + delay - value in seconds or None to disable timeout logic + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + delay if delay is not None else None + return Timeout(deadline, loop) + + class _State(enum.Enum): + INIT = "INIT" + ENTER = "ENTER" + TIMEOUT = "TIMEOUT" + EXIT = "EXIT" + + class Timeout: + # Internal class, please don't instantiate it directly + # Use timeout() and timeout_at() public factories instead. + # + # Implementation note: `async with timeout()` is preferred + # over `with timeout()`. + # While technically the Timeout class implementation + # doesn't need to be async at all, + # the `async with` statement explicitly points that + # the context manager should be used from async function context. + # + # This design allows to avoid many silly misusages. + # + # TimeoutError is raised immediately when scheduled + # if the deadline is passed. + # The purpose is to time out as soon as possible + # without waiting for the next await expression. + + __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler") + + def __init__(self, deadline: Optional[float], + loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._state = _State.INIT + + self._timeout_handler = None # type: Optional[asyncio.Handle] + if deadline is None: + self._deadline = None # type: Optional[float] + else: + self.update(deadline) + + def __enter__(self) -> "Timeout": + warnings.warn( + "with timeout() is deprecated, use async with timeout()", + DeprecationWarning, + stacklevel=2, + ) + self._do_enter() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + self._do_exit(exc_type) + return None + + async def __aenter__(self) -> "Timeout": + self._do_enter() + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + self._do_exit(exc_type) + return None + + @property + def expired(self) -> bool: + """Is timeout expired during execution?""" + return self._state == _State.TIMEOUT + + @property + def deadline(self) -> Optional[float]: + return self._deadline + + def reject(self) -> None: + """Reject scheduled timeout if any.""" + # cancel is maybe better name but + # task.cancel() raises CancelledError in asyncio world. + if self._state not in (_State.INIT, _State.ENTER): + raise RuntimeError(f"invalid state {self._state.value}") + self._reject() + + def _reject(self) -> None: + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + def shift(self, delay: float) -> None: + """Advance timeout on delay seconds. + The delay can be negative. + Raise RuntimeError if shift is called when deadline is not scheduled + """ + deadline = self._deadline + if deadline is None: + raise RuntimeError( + "cannot shift timeout if deadline is not scheduled") + self.update(deadline + delay) + + def update(self, deadline: float) -> None: + """Set deadline to absolute value. + deadline argument points on the time in the same clock system + as loop.time(). + If new deadline is in the past the timeout is raised immediately. + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + """ + if self._state == _State.EXIT: + raise RuntimeError( + "cannot reschedule after exit from context manager") + if self._state == _State.TIMEOUT: + raise RuntimeError("cannot reschedule expired timeout") + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._deadline = deadline + if self._state != _State.INIT: + self._reschedule() + + def _reschedule(self) -> None: + assert self._state == _State.ENTER + deadline = self._deadline + if deadline is None: + return + + now = self._loop.time() + if self._timeout_handler is not None: + self._timeout_handler.cancel() + + task = asyncio.current_task() + if deadline <= now: + self._timeout_handler = self._loop.call_soon( + self._on_timeout, task) + else: + self._timeout_handler = self._loop.call_at( + deadline, self._on_timeout, task) + + def _do_enter(self) -> None: + if self._state != _State.INIT: + raise RuntimeError(f"invalid state {self._state.value}") + self._state = _State.ENTER + self._reschedule() + + def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None: + if exc_type is asyncio.CancelledError and \ + self._state == _State.TIMEOUT: + self._timeout_handler = None + raise asyncio.TimeoutError + # timeout has not expired + self._state = _State.EXIT + self._reject() + return None + + def _on_timeout(self, task: "Optional[asyncio.Task[Any]]") -> None: + if task: + task.cancel() + self._state = _State.TIMEOUT + # drop the reference early + self._timeout_handler = None diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py new file mode 100644 index 00000000..563e52a3 --- /dev/null +++ b/vllm/engine/llm_engine.py @@ -0,0 +1,1934 @@ +import time +from collections import deque +from contextlib import contextmanager +from dataclasses import dataclass +from functools import partial +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict, + Iterable, List, Mapping, NamedTuple, Optional) +from typing import Sequence as GenericSequence +from typing import Set, Type, Union, overload + +import torch +from typing_extensions import TypeVar + +import vllm.envs as envs +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, + EngineConfig, LoadConfig, LoRAConfig, ModelConfig, + ObservabilityConfig, ParallelConfig, + PromptAdapterConfig, SchedulerConfig, + SpeculativeConfig) +from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler, + SchedulerOutputs) +from vllm.engine.arg_utils import EngineArgs +from vllm.engine.metrics_types import StatLoggerBase, Stats +from vllm.engine.output_processor.interfaces import ( + SequenceGroupOutputProcessor) +from vllm.engine.output_processor.stop_checker import StopChecker +from vllm.engine.output_processor.util import create_output_by_sequence_group +from vllm.entrypoints.openai.logits_processors import get_logits_processors +from vllm.executor.executor_base import ExecutorBase +from vllm.executor.gpu_executor import GPUExecutor +from vllm.executor.ray_utils import initialize_ray_cluster +from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs, + InputRegistry, LLMInputs, PromptType) +from vllm.inputs.preprocess import InputPreprocessor +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.model_executor.guided_decoding import ( + get_local_guided_decoding_logits_processor) +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.outputs import (EmbeddingRequestOutput, RequestOutput, + RequestOutputFactory) +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import RequestOutputKind, SamplingParams +from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest, + Sequence, SequenceGroup, SequenceGroupMetadata, + SequenceStatus) +from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context, + init_tracer) +from vllm.transformers_utils.config import try_get_generation_config +from vllm.transformers_utils.detokenizer import Detokenizer +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer_group import ( + BaseTokenizerGroup, init_tokenizer_from_configs) +from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled, + usage_message) +from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind +from vllm.version import __version__ as VLLM_VERSION + +logger = init_logger(__name__) +_LOCAL_LOGGING_INTERVAL_SEC = 5 + + +def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]: + config = try_get_generation_config( + model_config.model, + trust_remote_code=model_config.trust_remote_code, + revision=model_config.revision, + ) + + if config is None: + return {} + + return config.to_diff_dict() + + +_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup) +_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput) + + +@dataclass +class SchedulerOutputState: + """Caches the scheduler outputs for a virtual engine. Used for Multi-Step""" + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None + scheduler_outputs: Optional[SchedulerOutputs] = None + allow_async_output_proc: bool = False + last_output: Optional[SamplerOutput] = None + + +class OutputData(NamedTuple): + outputs: List[SamplerOutput] + seq_group_metadata_list: List[SequenceGroupMetadata] + scheduler_outputs: SchedulerOutputs + is_async: bool + is_last_step: bool + # Indicates if this output is from the first step of the + # multi-step. When multi-step is disabled, this is always + # set to True. + # is_first_step_output is invalid when `outputs` has + # outputs from multiple steps. + is_first_step_output: Optional[bool] + skip: List[int] + + +class SchedulerContext: + + def __init__(self, multi_step_stream_outputs: bool = False): + self.output_queue: Deque[OutputData] = deque() + self.request_outputs: List[Union[RequestOutput, + EmbeddingRequestOutput]] = [] + self.seq_group_metadata_list: Optional[ + List[SequenceGroupMetadata]] = None + self.scheduler_outputs: Optional[SchedulerOutputs] = None + + self.multi_step_stream_outputs: bool = multi_step_stream_outputs + + def append_output(self, outputs: List[SamplerOutput], + seq_group_metadata_list: List[SequenceGroupMetadata], + scheduler_outputs: SchedulerOutputs, is_async: bool, + is_last_step: bool, + is_first_step_output: Optional[bool]): + self.output_queue.append( + OutputData(outputs=outputs, + seq_group_metadata_list=seq_group_metadata_list, + scheduler_outputs=scheduler_outputs, + is_async=is_async, + is_last_step=is_last_step, + is_first_step_output=is_first_step_output, + skip=[])) + + +class LLMEngine: + """An LLM engine that receives requests and generates texts. + + This is the main class for the vLLM engine. It receives requests + from clients and generates texts from the LLM. It includes a tokenizer, a + language model (possibly distributed across multiple GPUs), and GPU memory + space allocated for intermediate states (aka KV cache). This class utilizes + iteration-level scheduling and efficient memory management to maximize the + serving throughput. + + The :class:`~vllm.LLM` class wraps this class for offline batched inference + and the :class:`AsyncLLMEngine` class wraps this class for online serving. + + The config arguments are derived from :class:`~vllm.EngineArgs`. (See + :ref:`engine_args`) + + Args: + model_config: The configuration related to the LLM model. + cache_config: The configuration related to the KV cache memory + management. + parallel_config: The configuration related to distributed execution. + scheduler_config: The configuration related to the request scheduler. + device_config: The configuration related to the device. + lora_config (Optional): The configuration related to serving multi-LoRA. + speculative_config (Optional): The configuration related to speculative + decoding. + executor_class: The model executor class for managing distributed + execution. + prompt_adapter_config (Optional): The configuration related to serving + prompt adapters. + log_stats: Whether to log statistics. + usage_context: Specified entry point, used for usage info collection. + """ + + DO_VALIDATE_OUTPUT: ClassVar[bool] = False + """A flag to toggle whether to validate the type of request output.""" + + @classmethod + @contextmanager + def enable_output_validation(cls): + cls.DO_VALIDATE_OUTPUT = True + + yield + + cls.DO_VALIDATE_OUTPUT = False + + @classmethod + def validate_output( + cls, + output: object, + output_type: Type[_O], + ) -> _O: + do_validate = cls.DO_VALIDATE_OUTPUT + + if ((TYPE_CHECKING or do_validate) + and not isinstance(output, output_type)): + raise TypeError(f"Expected output of type {output_type}, " + f"but found type {type(output)}") + + return output + + @classmethod + def validate_outputs( + cls, + outputs: GenericSequence[object], + output_type: Type[_O], + ) -> List[_O]: + do_validate = cls.DO_VALIDATE_OUTPUT + + outputs_: List[_O] + if TYPE_CHECKING or do_validate: + outputs_ = [] + for output in outputs: + if not isinstance(output, output_type): + raise TypeError(f"Expected output of type {output_type}, " + f"but found type {type(output)}") + + outputs_.append(output) + else: + outputs_ = outputs + + return outputs_ + + tokenizer: Optional[BaseTokenizerGroup] + + def __init__( + self, + model_config: ModelConfig, + cache_config: CacheConfig, + parallel_config: ParallelConfig, + scheduler_config: SchedulerConfig, + device_config: DeviceConfig, + load_config: LoadConfig, + lora_config: Optional[LoRAConfig], + speculative_config: Optional[SpeculativeConfig], + decoding_config: Optional[DecodingConfig], + observability_config: Optional[ObservabilityConfig], + prompt_adapter_config: Optional[PromptAdapterConfig], + executor_class: Type[ExecutorBase], + log_stats: bool, + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, + input_registry: InputRegistry = INPUT_REGISTRY, + use_cached_outputs: bool = False, + ) -> None: + logger.info( + "Initializing an LLM engine (v%s) with config: " + "model=%r, speculative_config=%r, tokenizer=%r, " + "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, " + "override_neuron_config=%s, " + "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, " + "trust_remote_code=%s, dtype=%s, max_seq_len=%d, " + "download_dir=%r, load_format=%s, tensor_parallel_size=%d, " + "pipeline_parallel_size=%d, " + "disable_custom_all_reduce=%s, quantization=%s, " + "enforce_eager=%s, kv_cache_dtype=%s, " + "quantization_param_path=%s, device_config=%s, " + "decoding_config=%r, observability_config=%r, " + "seed=%d, served_model_name=%s, use_v2_block_manager=%s, " + "num_scheduler_steps=%d, chunked_prefill_enabled=%s " + "multi_step_stream_outputs=%s, enable_prefix_caching=%s, " + "use_async_output_proc=%s, use_cached_outputs=%s, " + "mm_processor_kwargs=%s)", + VLLM_VERSION, + model_config.model, + speculative_config, + model_config.tokenizer, + model_config.skip_tokenizer_init, + model_config.tokenizer_mode, + model_config.revision, + model_config.override_neuron_config, + model_config.rope_scaling, + model_config.rope_theta, + model_config.tokenizer_revision, + model_config.trust_remote_code, + model_config.dtype, + model_config.max_model_len, + load_config.download_dir, + load_config.load_format, + parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size, + parallel_config.disable_custom_all_reduce, + model_config.quantization, + model_config.enforce_eager, + cache_config.cache_dtype, + model_config.quantization_param_path, + device_config.device, + decoding_config, + observability_config, + model_config.seed, + model_config.served_model_name, + scheduler_config.use_v2_block_manager, + scheduler_config.num_scheduler_steps, + scheduler_config.chunked_prefill_enabled, + scheduler_config.multi_step_stream_outputs, + cache_config.enable_prefix_caching, + model_config.use_async_output_proc, + use_cached_outputs, + model_config.mm_processor_kwargs, + ) + # TODO(woosuk): Print more configs in debug mode. + self.model_config = model_config + self.cache_config = cache_config + self.lora_config = lora_config + self.parallel_config = parallel_config + self.scheduler_config = scheduler_config + self.device_config = device_config + self.speculative_config = speculative_config + self.load_config = load_config + self.decoding_config = decoding_config or DecodingConfig() + self.prompt_adapter_config = prompt_adapter_config + self.observability_config = observability_config or ObservabilityConfig( + ) + self.log_stats = log_stats + self.use_cached_outputs = use_cached_outputs + + if not self.model_config.skip_tokenizer_init: + self.tokenizer = self._init_tokenizer() + self.detokenizer = Detokenizer(self.tokenizer) + tokenizer_group = self.get_tokenizer_group() + else: + self.tokenizer = None + self.detokenizer = None + tokenizer_group = None + + # Ensure that the function doesn't contain a reference to self, + # to avoid engine GC issues + def get_tokenizer_for_seq(sequence: Sequence) -> AnyTokenizer: + assert tokenizer_group, ("tokenizer_group cannot be None, " + "make sure skip_tokenizer_init is False") + return tokenizer_group.get_lora_tokenizer(sequence.lora_request) + + self.seq_counter = Counter() + self.generation_config_fields = _load_generation_config_dict( + model_config) + + self.input_preprocessor = InputPreprocessor(model_config, + self.tokenizer) + + self.input_registry = input_registry + self.input_processor = input_registry.create_input_processor( + model_config) + + self.model_executor = executor_class( + model_config=model_config, + cache_config=cache_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + device_config=device_config, + lora_config=lora_config, + speculative_config=speculative_config, + load_config=load_config, + prompt_adapter_config=prompt_adapter_config, + observability_config=self.observability_config, + ) + + if not self.model_config.embedding_mode: + self._initialize_kv_caches() + + # If usage stat is enabled, collect relevant info. + if is_usage_stats_enabled(): + from vllm.model_executor.model_loader import ( + get_architecture_class_name) + usage_message.report_usage( + get_architecture_class_name(model_config), + usage_context, + extra_kvs={ + # Common configuration + "dtype": + str(model_config.dtype), + "tensor_parallel_size": + parallel_config.tensor_parallel_size, + "block_size": + cache_config.block_size, + "gpu_memory_utilization": + cache_config.gpu_memory_utilization, + + # Quantization + "quantization": + model_config.quantization, + "kv_cache_dtype": + str(cache_config.cache_dtype), + + # Feature flags + "enable_lora": + bool(lora_config), + "enable_prompt_adapter": + bool(prompt_adapter_config), + "enable_prefix_caching": + cache_config.enable_prefix_caching, + "enforce_eager": + model_config.enforce_eager, + "disable_custom_all_reduce": + parallel_config.disable_custom_all_reduce, + }) + + if self.tokenizer: + # Ping the tokenizer to ensure liveness if it runs in a + # different process. + self.tokenizer.ping() + + self.cached_scheduler_outputs = [ + SchedulerOutputState() + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + + self.scheduler_contexts = [ + SchedulerContext(multi_step_stream_outputs=self.scheduler_config. + multi_step_stream_outputs) + for _ in range(self.parallel_config.pipeline_parallel_size) + ] + + if model_config.use_async_output_proc: + process_model_outputs = weak_bind(self._process_model_outputs) + + self.async_callbacks = [ + partial(process_model_outputs, + ctx=self.scheduler_contexts[v_id]) + for v_id in range(self.parallel_config.pipeline_parallel_size) + ] + else: + self.async_callbacks = [] + + # Currently used by AsyncLLMEngine to ensure quick append + # of request outputs to asyncio queues + self.process_request_outputs_callback: Optional[Callable] = None + + # Create the scheduler. + # NOTE: the cache_config here have been updated with the numbers of + # GPU and CPU blocks, which are profiled in the distributed executor. + self.scheduler = [ + Scheduler( + scheduler_config, cache_config, lora_config, + parallel_config.pipeline_parallel_size, + self.async_callbacks[v_id] + if model_config.use_async_output_proc else None) + for v_id in range(parallel_config.pipeline_parallel_size) + ] + + # Metric Logging. + if self.log_stats: + if stat_loggers is not None: + self.stat_loggers = stat_loggers + else: + # Lazy import for prometheus multiprocessing. + # We need to set PROMETHEUS_MULTIPROC_DIR environment variable + # before prometheus_client is imported. + # See https://prometheus.github.io/client_python/multiprocess/ + from vllm.engine.metrics import (LoggingStatLogger, + PrometheusStatLogger) + + self.stat_loggers = { + "logging": + LoggingStatLogger( + local_interval=_LOCAL_LOGGING_INTERVAL_SEC), + "prometheus": + PrometheusStatLogger( + local_interval=_LOCAL_LOGGING_INTERVAL_SEC, + labels=dict(model_name=model_config.served_model_name), + max_model_len=self.model_config.max_model_len), + } + self.stat_loggers["prometheus"].info("cache_config", + self.cache_config) + + self.tracer = None + if self.observability_config.otlp_traces_endpoint: + self.tracer = init_tracer( + "vllm.llm_engine", + self.observability_config.otlp_traces_endpoint) + + # Create sequence output processor, e.g. for beam search or + # speculative decoding. + self.output_processor = ( + SequenceGroupOutputProcessor.create_output_processor( + self.scheduler_config, + self.detokenizer, + self.scheduler, + self.seq_counter, + get_tokenizer_for_seq, + stop_checker=StopChecker( + self.scheduler_config.max_model_len, + get_tokenizer_for_seq, + ), + )) + + def _initialize_kv_caches(self) -> None: + """Initialize the KV cache in the worker(s). + + The workers will determine the number of blocks in both the GPU cache + and the swap CPU cache. + """ + num_gpu_blocks, num_cpu_blocks = ( + self.model_executor.determine_num_available_blocks()) + + if self.cache_config.num_gpu_blocks_override is not None: + num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override + logger.info( + "Overriding num_gpu_blocks=%d with " + "num_gpu_blocks_override=%d", num_gpu_blocks, + num_gpu_blocks_override) + num_gpu_blocks = num_gpu_blocks_override + + self.cache_config.num_gpu_blocks = num_gpu_blocks + self.cache_config.num_cpu_blocks = num_cpu_blocks + + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks) + + @classmethod + def _get_executor_cls(cls, + engine_config: EngineConfig) -> Type[ExecutorBase]: + distributed_executor_backend = ( + engine_config.parallel_config.distributed_executor_backend) + # Initialize the cluster and specify the executor class. + if isinstance(distributed_executor_backend, type): + if not issubclass(distributed_executor_backend, ExecutorBase): + raise TypeError( + "distributed_executor_backend must be a subclass of " + f"ExecutorBase. Got {distributed_executor_backend}.") + if distributed_executor_backend.uses_ray: # type: ignore + initialize_ray_cluster(engine_config.parallel_config) + executor_class = distributed_executor_backend + elif engine_config.device_config.device_type == "neuron": + from vllm.executor.neuron_executor import NeuronExecutor + executor_class = NeuronExecutor + elif engine_config.device_config.device_type == "tpu": + if distributed_executor_backend == "ray": + initialize_ray_cluster(engine_config.parallel_config) + from vllm.executor.ray_tpu_executor import RayTPUExecutor + executor_class = RayTPUExecutor + else: + assert distributed_executor_backend is None + from vllm.executor.tpu_executor import TPUExecutor + executor_class = TPUExecutor + elif engine_config.device_config.device_type == "cpu": + from vllm.executor.cpu_executor import CPUExecutor + executor_class = CPUExecutor + elif engine_config.device_config.device_type == "openvino": + from vllm.executor.openvino_executor import OpenVINOExecutor + executor_class = OpenVINOExecutor + elif engine_config.device_config.device_type == "xpu": + if distributed_executor_backend == "ray": + initialize_ray_cluster(engine_config.parallel_config) + from vllm.executor.ray_xpu_executor import RayXPUExecutor + executor_class = RayXPUExecutor + elif distributed_executor_backend == "mp": + # FIXME(kunshang): + # spawn needs calling `if __name__ == '__main__':`` + # fork is not supported for xpu start new process. + logger.error( + "Both start methods (spawn and fork) have issue " + "on XPU if you use mp backend, Please try ray instead.") + else: + from vllm.executor.xpu_executor import XPUExecutor + executor_class = XPUExecutor + elif distributed_executor_backend == "ray": + initialize_ray_cluster(engine_config.parallel_config) + from vllm.executor.ray_gpu_executor import RayGPUExecutor + executor_class = RayGPUExecutor + elif distributed_executor_backend == "mp": + from vllm.executor.multiproc_gpu_executor import ( + MultiprocessingGPUExecutor) + assert not envs.VLLM_USE_RAY_SPMD_WORKER, ( + "multiprocessing distributed executor backend does not " + "support VLLM_USE_RAY_SPMD_WORKER=1") + executor_class = MultiprocessingGPUExecutor + else: + from vllm.executor.gpu_executor import GPUExecutor + executor_class = GPUExecutor + return executor_class + + @classmethod + def from_engine_args( + cls, + engine_args: EngineArgs, + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, + ) -> "LLMEngine": + """Creates an LLM engine from the engine arguments.""" + # Create the engine configs. + engine_config = engine_args.create_engine_config() + executor_class = cls._get_executor_cls(engine_config) + # Create the LLM engine. + engine = cls( + **engine_config.to_dict(), + executor_class=executor_class, + log_stats=not engine_args.disable_log_stats, + usage_context=usage_context, + stat_loggers=stat_loggers, + ) + + return engine + + def __reduce__(self): + # This is to ensure that the LLMEngine is not referenced in + # the closure used to initialize Ray worker actors + raise RuntimeError("LLMEngine should not be pickled!") + + def __del__(self): + # Shutdown model executor when engine is garbage collected + # Use getattr since __init__ can fail before the field is set + if model_executor := getattr(self, "model_executor", None): + model_executor.shutdown() + + def get_tokenizer_group( + self, + group_type: Type[_G] = BaseTokenizerGroup, + ) -> _G: + tokenizer_group = self.tokenizer + + if tokenizer_group is None: + raise ValueError("Unable to get tokenizer because " + "skip_tokenizer_init is True") + if not isinstance(tokenizer_group, group_type): + raise TypeError("Invalid type of tokenizer group. " + f"Expected type: {group_type}, but " + f"found type: {type(tokenizer_group)}") + + return tokenizer_group + + def get_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + return self.get_tokenizer_group().get_lora_tokenizer(lora_request) + + def _init_tokenizer(self) -> BaseTokenizerGroup: + return init_tokenizer_from_configs( + model_config=self.model_config, + scheduler_config=self.scheduler_config, + parallel_config=self.parallel_config, + enable_lora=bool(self.lora_config)) + + def _verify_args(self) -> None: + self.model_config.verify_with_parallel_config(self.parallel_config) + self.cache_config.verify_with_parallel_config(self.parallel_config) + if self.lora_config: + self.lora_config.verify_with_model_config(self.model_config) + self.lora_config.verify_with_scheduler_config( + self.scheduler_config) + if self.prompt_adapter_config: + self.prompt_adapter_config.verify_with_model_config( + self.model_config) + + def _add_processed_request( + self, + request_id: str, + processed_inputs: Union[LLMInputs, EncoderDecoderLLMInputs], + params: Union[SamplingParams, PoolingParams], + arrival_time: float, + lora_request: Optional[LoRARequest], + prompt_adapter_request: Optional[PromptAdapterRequest], + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + ) -> None: + self._validate_model_inputs(processed_inputs) + # Create the sequences. + block_size = self.cache_config.block_size + seq_id = next(self.seq_counter) + eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request) + + seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id, + lora_request, prompt_adapter_request) + + encoder_seq = None + if 'encoder_prompt_token_ids' in processed_inputs: + encoder_seq = Sequence(seq_id, + processed_inputs, + block_size, + eos_token_id, + lora_request, + prompt_adapter_request, + from_decoder_prompt=False) + + # Create a SequenceGroup based on SamplingParams or PoolingParams + if isinstance(params, SamplingParams): + seq_group = self._create_sequence_group_with_sampling( + request_id, + seq, + params, + arrival_time=arrival_time, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, + priority=priority) + elif isinstance(params, PoolingParams): + seq_group = self._create_sequence_group_with_pooling( + request_id, + seq, + params, + arrival_time=arrival_time, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, + priority=priority) + else: + raise ValueError( + "Either SamplingParams or PoolingParams must be provided.") + + # Add the sequence group to the scheduler with least unfinished seqs. + costs = [ + scheduler.get_num_unfinished_seq_groups() + for scheduler in self.scheduler + ] + min_cost_scheduler = self.scheduler[costs.index(min(costs))] + min_cost_scheduler.add_seq_group(seq_group) + + def stop_remote_worker_execution_loop(self) -> None: + self.model_executor.stop_remote_worker_execution_loop() + + @overload # DEPRECATED + def add_request( + self, + request_id: str, + *, + inputs: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @overload + def add_request( + self, + request_id: str, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + def add_request( + self, + request_id: str, + prompt: Optional[PromptType] = None, + params: Optional[Union[SamplingParams, PoolingParams]] = None, + arrival_time: Optional[float] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> None: + """Add a request to the engine's request pool. + + The request is added to the request pool and will be processed by the + scheduler as `engine.step()` is called. The exact scheduling policy is + determined by the scheduler. + + Args: + request_id: The unique ID of the request. + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType` + for more details about the format of each input. + params: Parameters for sampling or pooling. + :class:`~vllm.SamplingParams` for text generation. + :class:`~vllm.PoolingParams` for pooling. + arrival_time: The arrival time of the request. If None, we use + the current monotonic time. + trace_headers: OpenTelemetry trace headers. + priority: The priority of the request. + Only applicable with priority scheduling. + + Details: + - Set arrival_time to the current time if it is None. + - Set prompt_token_ids to the encoded prompt if it is None. + - Create `n` number of :class:`~vllm.Sequence` objects. + - Create a :class:`~vllm.SequenceGroup` object + from the list of :class:`~vllm.Sequence`. + - Add the :class:`~vllm.SequenceGroup` object to the scheduler. + + Example: + >>> # initialize engine + >>> engine = LLMEngine.from_engine_args(engine_args) + >>> # set request arguments + >>> example_prompt = "Who is the president of the United States?" + >>> sampling_params = SamplingParams(temperature=0.0) + >>> request_id = 0 + >>> + >>> # add the request to the engine + >>> engine.add_request( + >>> str(request_id), + >>> example_prompt, + >>> SamplingParams(temperature=0.0)) + >>> # continue the request processing + >>> ... + """ + if inputs is not None: + prompt = inputs + assert prompt is not None and params is not None + + if lora_request is not None and not self.lora_config: + raise ValueError(f"Got lora_request {lora_request} but LoRA is " + "not enabled!") + + if priority != 0 and not self.scheduler_config.policy == "priority": + raise ValueError(f"Got priority {priority} but " + "Priority scheduling is not enabled.") + + if arrival_time is None: + arrival_time = time.time() + + preprocessed_inputs = self.input_preprocessor.preprocess( + prompt, + request_id=request_id, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + ) + processed_inputs = self.input_processor(preprocessed_inputs) + + # This is a bit of a hack - copy the mm_processor_kwargs that were + # used in the input processor to the processed output, since these + # kwargs are presumed to be immutable and the values should be aligned + # between the input processor (here) and the input mapper. + processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get( + "mm_processor_kwargs") + + self._add_processed_request( + request_id=request_id, + processed_inputs=processed_inputs, + params=params, + arrival_time=arrival_time, + lora_request=lora_request, + prompt_adapter_request=prompt_adapter_request, + trace_headers=trace_headers, + priority=priority, + ) + + def _create_sequence_group_with_sampling( + self, + request_id: str, + seq: Sequence, + sampling_params: SamplingParams, + arrival_time: float, + lora_request: Optional[LoRARequest], + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + encoder_seq: Optional[Sequence] = None, + priority: int = 0, + ) -> SequenceGroup: + """Creates a SequenceGroup with SamplingParams.""" + max_logprobs = self.get_model_config().max_logprobs + if (sampling_params.logprobs + and sampling_params.logprobs > max_logprobs) or ( + sampling_params.prompt_logprobs + and sampling_params.prompt_logprobs > max_logprobs): + raise ValueError(f"Cannot request more than " + f"{max_logprobs} logprobs.") + + sampling_params = self._build_logits_processors( + sampling_params, lora_request) + + # Defensive copy of SamplingParams, which are used by the sampler, + # this doesn't deep-copy LogitsProcessor objects + sampling_params = sampling_params.clone() + + sampling_params.update_from_generation_config( + self.generation_config_fields, seq.eos_token_id) + + # Create the sequence group. + seq_group = SequenceGroup( + request_id=request_id, + seqs=[seq], + arrival_time=arrival_time, + sampling_params=sampling_params, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, + priority=priority) + + return seq_group + + def _create_sequence_group_with_pooling( + self, + request_id: str, + seq: Sequence, + pooling_params: PoolingParams, + arrival_time: float, + lora_request: Optional[LoRARequest], + prompt_adapter_request: Optional[PromptAdapterRequest], + encoder_seq: Optional[Sequence] = None, + priority: int = 0, + ) -> SequenceGroup: + """Creates a SequenceGroup with PoolingParams.""" + # Defensive copy of PoolingParams, which are used by the pooler + pooling_params = pooling_params.clone() + # Create the sequence group. + seq_group = SequenceGroup( + request_id=request_id, + seqs=[seq], + arrival_time=arrival_time, + lora_request=lora_request, + pooling_params=pooling_params, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, + priority=priority) + return seq_group + + def abort_request(self, request_id: Union[str, Iterable[str]]) -> None: + """Aborts a request(s) with the given ID. + + Args: + request_id: The ID(s) of the request to abort. + + Details: + - Refer to the + :meth:`~vllm.core.scheduler.Scheduler.abort_seq_group` + from class :class:`~vllm.core.scheduler.Scheduler`. + + Example: + >>> # initialize engine and add a request with request_id + >>> request_id = str(0) + >>> # abort the request + >>> engine.abort_request(request_id) + """ + for scheduler in self.scheduler: + scheduler.abort_seq_group(request_id) + + def get_model_config(self) -> ModelConfig: + """Gets the model configuration.""" + return self.model_config + + def get_parallel_config(self) -> ParallelConfig: + """Gets the parallel configuration.""" + return self.parallel_config + + def get_decoding_config(self) -> DecodingConfig: + """Gets the decoding configuration.""" + return self.decoding_config + + def get_scheduler_config(self) -> SchedulerConfig: + """Gets the scheduler configuration.""" + return self.scheduler_config + + def get_lora_config(self) -> LoRAConfig: + """Gets the LoRA configuration.""" + return self.lora_config + + def get_num_unfinished_requests(self) -> int: + """Gets the number of unfinished requests.""" + return sum(scheduler.get_num_unfinished_seq_groups() + for scheduler in self.scheduler) + + def has_unfinished_requests(self) -> bool: + """Returns True if there are unfinished requests.""" + return any(scheduler.has_unfinished_seqs() + for scheduler in self.scheduler) + + def has_unfinished_requests_for_virtual_engine( + self, virtual_engine: int) -> bool: + """ + Returns True if there are unfinished requests for the virtual engine. + """ + return self.scheduler[virtual_engine].has_unfinished_seqs() + + @staticmethod + def _process_sequence_group_outputs( + seq_group: SequenceGroup, + outputs: List[EmbeddingSequenceGroupOutput], + ) -> None: + seq_group.embeddings = outputs[0].embeddings + + for seq in seq_group.get_seqs(): + seq.status = SequenceStatus.FINISHED_STOPPED + + return + + def _update_num_computed_tokens_for_multi_step_prefill( + self, seq_group: SequenceGroup, + seq_group_meta: SequenceGroupMetadata, + is_first_step_output: Optional[bool]): + """ + This function updates num_computed_tokens for prompt sequences + when Multi-Step is enabled. + + seq_group: SequenceGroup to update the num_computed_tokens for. + seq_group_meta: Metadata of the given SequenceGroup. + is_first_step_output: Optional[bool] - + When available, is_first_step_output indicates if the appended + output token is the output of the first-step in multi-step. + A value of None indicates that outputs from all steps in + in multi-step are submitted in a single burst. + """ + + assert self.scheduler_config.is_multi_step + + if not seq_group_meta.is_prompt: + # num_computed_token updates for multi-step decodes happen after + # the tokens are appended to the sequence. + return + + do_update: bool = False + if self.scheduler_config.chunked_prefill_enabled: + # In multi-step + chunked-prefill case, the prompt sequences + # that are scheduled are fully processed in the first step. + do_update = is_first_step_output is None or is_first_step_output + else: + # Normal multi-step decoding case. In this case prompt-sequences + # are actually single-stepped. Always update in this case. + assert seq_group.state.num_steps == 1 + do_update = True + + if do_update: + seq_group.update_num_computed_tokens( + seq_group_meta.token_chunk_size) + + def _process_model_outputs(self, + ctx: SchedulerContext, + request_id: Optional[str] = None) -> None: + """Apply the model output to the sequences in the scheduled seq groups + and return responses. + + ctx: The virtual engine context to work on + request_id: If provided, then only this request is going to be processed + """ + + now = time.time() + + if len(ctx.output_queue) == 0: + return None + + # Get pending async postprocessor + if request_id: + # When we process only one request, no pop is required + # (since later we will process all of the rest) + (outputs, seq_group_metadata_list, scheduler_outputs, is_async, + is_last_step, is_first_step_output, skip) = ctx.output_queue[0] + else: + (outputs, seq_group_metadata_list, scheduler_outputs, is_async, + is_last_step, is_first_step_output, + skip) = ctx.output_queue.popleft() + + # Sanity check + assert len(seq_group_metadata_list) == len( + scheduler_outputs.scheduled_seq_groups) + + has_multiple_outputs: bool = len(outputs) > 1 + if has_multiple_outputs: + assert self.scheduler_config.is_multi_step or \ + self.speculative_config + # Organize outputs by [step][sequence group] instead of + # [sequence group][step]. + outputs_by_sequence_group = create_output_by_sequence_group( + outputs, num_seq_groups=len(seq_group_metadata_list)) + # We have outputs for multiple steps submitted in a single burst, + # so invalidate is_first_step_output. + is_first_step_output = None + else: + outputs_by_sequence_group = outputs + + # Determine the requests we need to operate on + if request_id: + indices = [] + for i, seq_group_meta in enumerate(seq_group_metadata_list): + if seq_group_meta.request_id == request_id: + assert i not in skip # Cannot be called twice + indices.append(i) + break + + # If the request_id was not found, then it means that + # this is a new request that has no pending async + # postprocessor + if not indices: + return + else: + indices = range(len(seq_group_metadata_list)) # type: ignore + + finished_before: List[int] = [] + finished_now: List[int] = [] + for i in indices: + if i in skip: + continue + + seq_group_meta = seq_group_metadata_list[i] + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i] + + seq_group: SequenceGroup = scheduled_seq_group.seq_group + + if seq_group.is_finished(): + finished_before.append(i) + continue + + if has_multiple_outputs: + output = outputs_by_sequence_group[i] + else: + output = [outputs_by_sequence_group[0][i]] + + if not is_async: + if self.scheduler_config.is_multi_step: + # Updates happen only if the sequence is prefill + self._update_num_computed_tokens_for_multi_step_prefill( + seq_group, seq_group_meta, is_first_step_output) + else: + seq_group.update_num_computed_tokens( + seq_group_meta.token_chunk_size) + + if outputs: + for o in outputs: + if (isinstance(o, SamplerOutput) + and seq_group.metrics is not None): + if seq_group.metrics.model_forward_time is not None: + seq_group.metrics.model_forward_time += ( + o.model_forward_time) + else: + seq_group.metrics.model_forward_time = ( + o.model_forward_time) + if seq_group.metrics.model_execute_time is not None: + seq_group.metrics.model_execute_time += ( + o.model_execute_time) + else: + seq_group.metrics.model_execute_time = ( + o.model_execute_time) + + if self.model_config.embedding_mode: + self._process_sequence_group_outputs(seq_group, output) + else: + self.output_processor.process_prompt_logprob(seq_group, output) + if seq_group_meta.do_sample: + self.output_processor.process_outputs( + seq_group, output, is_async) + + if seq_group.is_finished(): + finished_now.append(i) + + # Generate outputs for the requests that finished this iteration + for i in finished_now: + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i] + + seq_group = scheduled_seq_group.seq_group + seq_group.maybe_set_first_token_time(now) + request_output = RequestOutputFactory.create( + seq_group, use_cache=self.use_cached_outputs) + if request_output: + ctx.request_outputs.append(request_output) + + # When we process a single request, we skip it for the next time, + # and invoke the request output callback (if there was final output) + if request_id: + assert len(indices) == 1 + skip.append(indices[0]) + + if (finished_now + and self.process_request_outputs_callback is not None): + self.process_request_outputs_callback(ctx.request_outputs) + ctx.request_outputs.clear() + return + + # Free currently finished requests + if finished_now: + for scheduler in self.scheduler: + scheduler.free_finished_seq_groups() + + # For multi-step without streaming, don't create outputs each iteration + if not is_last_step and not ctx.multi_step_stream_outputs: + # Immediately process request outputs here (if callback is given) + if (finished_now + and self.process_request_outputs_callback is not None): + self.process_request_outputs_callback(ctx.request_outputs) + ctx.request_outputs.clear() + return + + # Create the outputs + for i in indices: + if i in skip or i in finished_before or i in finished_now: + continue # Avoids double processing + + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i] + + seq_group = scheduled_seq_group.seq_group + seq_group.maybe_set_first_token_time(now) + request_output = RequestOutputFactory.create( + seq_group, use_cache=self.use_cached_outputs) + if request_output: + ctx.request_outputs.append(request_output) + + # For multi-step with streaming, create outputs each iteration + if not is_last_step and ctx.multi_step_stream_outputs: + # Immediately process request outputs here (if callback is given) + if self.process_request_outputs_callback is not None: + self.process_request_outputs_callback(ctx.request_outputs) + ctx.request_outputs.clear() + return + + for seq_group in scheduler_outputs.ignored_seq_groups: + params = seq_group.sampling_params + if params is not None and params.output_kind == ( + RequestOutputKind.DELTA) and not seq_group.is_finished(): + continue + + request_output = RequestOutputFactory.create( + seq_group, use_cache=self.use_cached_outputs) + if request_output: + ctx.request_outputs.append(request_output) + + # Immediately process request outputs here (if callback is given) + if (ctx.request_outputs + and self.process_request_outputs_callback is not None): + self.process_request_outputs_callback(ctx.request_outputs) + ctx.request_outputs.clear() + + # For async case, we need to record the stats here. + # For non-async case, the stats are done in the + # LLMEngine/AsyncLLMEngine directly + if is_async: + # Log stats. + self.do_log_stats(scheduler_outputs, outputs, finished_before, + skip) + + # Tracing + self.do_tracing(scheduler_outputs) + + return None + + def _advance_to_next_step( + self, output: List[SamplerOutput], + seq_group_metadata_list: List[SequenceGroupMetadata], + scheduled_seq_groups: List[ScheduledSequenceGroup]) -> None: + """Given model output from a single run, append the tokens to the + sequences. This is normally done inside output processor, but it is + required if the worker is to perform async forward pass to next step. + """ + for seq_group_metadata, sequence_group_outputs, scheduled_seq_group in \ + zip(seq_group_metadata_list, output, scheduled_seq_groups): + seq_group = scheduled_seq_group.seq_group + + if seq_group.is_finished(): + continue + + if self.scheduler_config.is_multi_step: + # Updates happen only if the sequence is prefill + self._update_num_computed_tokens_for_multi_step_prefill( + seq_group, seq_group_metadata, + seq_group.state.num_steps == 1) + else: + seq_group.update_num_computed_tokens( + seq_group_metadata.token_chunk_size) + + if seq_group_metadata.do_sample: + assert len(sequence_group_outputs.samples) == 1, ( + "Async output processor expects a single sample" + " (i.e sampling_params.n == 1)") + sample = sequence_group_outputs.samples[0] + + assert len(seq_group.seqs) == 1 + seq = seq_group.seqs[0] + + if self.scheduler_config.is_multi_step: + is_prefill_append = seq.data.get_num_uncomputed_tokens( + ) == 0 + seq.append_token_id(sample.output_token, sample.logprobs) + if not is_prefill_append: + seq_group.update_num_computed_tokens(1) + else: + seq.append_token_id(sample.output_token, sample.logprobs) + + def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]: + """Performs one decoding iteration and returns newly generated results. + + .. figure:: https://i.imgur.com/sv2HssD.png + :alt: Overview of the step function + :align: center + + Overview of the step function. + + Details: + - Step 1: Schedules the sequences to be executed in the next + iteration and the token blocks to be swapped in/out/copy. + + - Depending on the scheduling policy, + sequences may be `preempted/reordered`. + - A Sequence Group (SG) refer to a group of sequences + that are generated from the same prompt. + + - Step 2: Calls the distributed executor to execute the model. + - Step 3: Processes the model output. This mainly includes: + + - Decodes the relevant outputs. + - Updates the scheduled sequence groups with model outputs + based on its `sampling parameters` (`use_beam_search` or not). + - Frees the finished sequence groups. + + - Finally, it creates and returns the newly generated results. + + Example: + >>> # Please see the example/ folder for more detailed examples. + >>> + >>> # initialize engine and request arguments + >>> engine = LLMEngine.from_engine_args(engine_args) + >>> example_inputs = [(0, "What is LLM?", + >>> SamplingParams(temperature=0.0))] + >>> + >>> # Start the engine with an event loop + >>> while True: + >>> if example_inputs: + >>> req_id, prompt, sampling_params = example_inputs.pop(0) + >>> engine.add_request(str(req_id),prompt,sampling_params) + >>> + >>> # continue the request processing + >>> request_outputs = engine.step() + >>> for request_output in request_outputs: + >>> if request_output.finished: + >>> # return or show the request output + >>> + >>> if not (engine.has_unfinished_requests() or example_inputs): + >>> break + """ + if self.parallel_config.pipeline_parallel_size > 1: + raise NotImplementedError( + "Pipeline parallelism is only supported through AsyncLLMEngine " + "as performance will be severely degraded otherwise.") + + # For llm_engine, there is no pipeline parallel support, so the engine + # used is always 0. + virtual_engine = 0 + + # These are cached outputs from previous iterations. None if on first + # iteration + cached_outputs = self.cached_scheduler_outputs[virtual_engine] + seq_group_metadata_list = cached_outputs.seq_group_metadata_list + scheduler_outputs = cached_outputs.scheduler_outputs + allow_async_output_proc = cached_outputs.allow_async_output_proc + + ctx = self.scheduler_contexts[virtual_engine] + + # Clear outputs for each new scheduler iteration + ctx.request_outputs.clear() + + # Skip the scheduler if there are any remaining steps in the seq groups. + # This ensures that the scheduler is only called again when the current + # batch has completed. + if not self._has_remaining_steps(seq_group_metadata_list): + # Schedule iteration + (seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc + ) = self.scheduler[virtual_engine].schedule() + + ctx.seq_group_metadata_list = seq_group_metadata_list + ctx.scheduler_outputs = scheduler_outputs + + # Maybe switch from async mode to sync mode + if not allow_async_output_proc and len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + + if (self.scheduler_config.is_multi_step + and scheduler_outputs.num_lookahead_slots > 0): + # cache the scheduler outputs for the next iteration if we have + # lookahead slots + self._cache_scheduler_outputs_for_multi_step( + virtual_engine, seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc) + + assert seq_group_metadata_list is not None + assert scheduler_outputs is not None + + if not scheduler_outputs.is_empty(): + finished_requests_ids = self.scheduler[ + virtual_engine].get_and_reset_finished_requests_ids() + + # Check if we have a cached last_output from the previous iteration. + # For supporting PP this is probably the best way to pass the + # sampled_token_ids, as a separate broadcast over all the PP stages + # will cause one virtual engine's microbatch to block the pipeline. + last_sampled_token_ids = \ + self._get_last_sampled_token_ids(virtual_engine) + + execute_model_req = ExecuteModelRequest( + seq_group_metadata_list=seq_group_metadata_list, + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in, + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out, + blocks_to_copy=scheduler_outputs.blocks_to_copy, + num_lookahead_slots=scheduler_outputs.num_lookahead_slots, + running_queue_size=scheduler_outputs.running_queue_size, + finished_requests_ids=finished_requests_ids, + # We use ExecuteModelRequest to pass the last sampled_token_ids + # to each of the non-last PP stages for in-place prepare_input. + last_sampled_token_ids=last_sampled_token_ids) + + if allow_async_output_proc: + execute_model_req.async_callback = self.async_callbacks[ + virtual_engine] + + outputs = self.model_executor.execute_model( + execute_model_req=execute_model_req) + + # We need to do this here so that last step's sampled_token_ids can + # be passed to the next iteration for PP. + if self.scheduler_config.is_multi_step: + self._update_cached_scheduler_output(virtual_engine, outputs) + else: + # Nothing scheduled => If there is pending async postprocessor, + # then finish it here. + if len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + # No outputs in this case + outputs = [] + + # Finish the current step for all the sequence groups. + if self.scheduler_config.is_multi_step: + for seq_group in seq_group_metadata_list: + seq_group.finish_step() + + if not self._has_remaining_steps(seq_group_metadata_list): + # clear the cache if we have finished all the steps. + if self.scheduler_config.is_multi_step: + self.cached_scheduler_outputs[0] = SchedulerOutputState() + + # is_first_step_output is True only when the num_steps of all + # the sequences are 1. When the num_steps > 1, + # multi_step_model_runner does the first-step output append. + is_first_step_output: bool = False if not seq_group_metadata_list \ + else seq_group_metadata_list[0].state.num_steps == 1 + + # Add results to the output_queue + ctx.append_output(outputs=outputs, + seq_group_metadata_list=seq_group_metadata_list, + scheduler_outputs=scheduler_outputs, + is_async=allow_async_output_proc, + is_last_step=True, + is_first_step_output=is_first_step_output) + + if outputs and allow_async_output_proc: + assert len(outputs) == 1, ( + "Async postprocessor expects only a single output set") + + self._advance_to_next_step( + outputs[0], seq_group_metadata_list, + scheduler_outputs.scheduled_seq_groups) + + # Check if need to run the usual non-async path + if not allow_async_output_proc: + self._process_model_outputs(ctx=ctx) + + # Log stats. + self.do_log_stats(scheduler_outputs, outputs) + + # Tracing + self.do_tracing(scheduler_outputs) + else: + # Multi-step case + return ctx.request_outputs + + if not self.has_unfinished_requests(): + # Drain async postprocessor (if exists) + if len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + assert len(ctx.output_queue) == 0 + + # Stop the execute model loop in parallel workers until there are + # more requests to process. This avoids waiting indefinitely in + # torch.distributed ops which may otherwise timeout, and unblocks + # the RPC thread in the workers so that they can process any other + # queued control plane messages, such as add/remove lora adapters. + logger.debug("Stopping remote worker execution loop.") + self.model_executor.stop_remote_worker_execution_loop() + + return ctx.request_outputs + + def _has_remaining_steps( + self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] + ) -> bool: + if (not self.scheduler_config.is_multi_step + or not seq_group_metadata_list): + return False + + # TODO(will) this is a sanity check for nowto make sure that all the + # seqs are on the same steps. Eventually we will want to do some sort of + # dynamic scheduling when doing multi-step decoding. + ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps + if any([ + seq_group.state.remaining_steps != ref_remaining_steps + for seq_group in seq_group_metadata_list[1:] + ]): + raise AssertionError(("All running sequence groups should " + "have the same remaining steps.")) + + return ref_remaining_steps > 0 + + def _cache_scheduler_outputs_for_multi_step( + self, virtual_engine: int, + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], + scheduler_outputs: SchedulerOutputs, + allow_async_output_proc: bool) -> None: + co = self.cached_scheduler_outputs[virtual_engine] + + co.seq_group_metadata_list = seq_group_metadata_list + co.scheduler_outputs = scheduler_outputs + co.allow_async_output_proc = allow_async_output_proc + co.last_output = None + + def _update_cached_scheduler_output( + self, virtual_engine: int, + output: List[Optional[SamplerOutput]]) -> None: + if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0 + and output[0] is not None): + last_output = output[-1] + assert last_output is not None + assert last_output.sampled_token_ids_cpu is not None + assert last_output.sampled_token_ids is None + assert last_output.sampled_token_probs is None + self.cached_scheduler_outputs[ + virtual_engine].last_output = last_output + + def _get_last_sampled_token_ids( + self, virtual_engine: int) -> Optional[torch.Tensor]: + cached_last_output = self.cached_scheduler_outputs[ + virtual_engine].last_output + if (self.scheduler_config.is_multi_step + and self.parallel_config.pipeline_parallel_size > 1 + and cached_last_output is not None + and cached_last_output.sampled_token_ids_cpu is not None): + return cached_last_output.sampled_token_ids_cpu + return None + + def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None: + if not self.log_stats: + raise RuntimeError( + "Stat logging is disabled. Set `disable_log_stats=False` " + "argument to enable.") + if logger_name in self.stat_loggers: + raise KeyError(f"Logger with name {logger_name} already exists.") + self.stat_loggers[logger_name] = logger + + def remove_logger(self, logger_name: str) -> None: + if not self.log_stats: + raise RuntimeError( + "Stat logging is disabled. Set `disable_log_stats=False` " + "argument to enable.") + if logger_name not in self.stat_loggers: + raise KeyError(f"Logger with name {logger_name} does not exist.") + del self.stat_loggers[logger_name] + + def do_log_stats(self, + scheduler_outputs: Optional[SchedulerOutputs] = None, + model_output: Optional[List[SamplerOutput]] = None, + finished_before: Optional[List[int]] = None, + skip: Optional[List[int]] = None) -> None: + """Forced log when no requests active.""" + if self.log_stats: + stats = self._get_stats(scheduler_outputs, model_output, + finished_before, skip) + for logger in self.stat_loggers.values(): + logger.log(stats) + + def _get_stats(self, + scheduler_outputs: Optional[SchedulerOutputs], + model_output: Optional[List[SamplerOutput]] = None, + finished_before: Optional[List[int]] = None, + skip: Optional[List[int]] = None) -> Stats: + """Get Stats to be Logged to Prometheus. + + Args: + scheduler_outputs: Optional, used to populate metrics related to + the scheduled batch, + model_output: Optional, used to emit speculative decoding metrics + which are created by the workers. + finished_before: Optional, indices of sequences that were finished + before. These sequences will be ignored. + skip: Optional, indices of sequences that were preempted. These + sequences will be ignored. + """ + now = time.time() + + # System State + # Scheduler State + num_running_sys = sum( + len(scheduler.running) for scheduler in self.scheduler) + num_swapped_sys = sum( + len(scheduler.swapped) for scheduler in self.scheduler) + num_waiting_sys = sum( + len(scheduler.waiting) for scheduler in self.scheduler) + + # KV Cache Usage in % + num_total_gpu = self.cache_config.num_gpu_blocks + gpu_cache_usage_sys = 0. + if num_total_gpu is not None: + num_free_gpu = sum( + scheduler.block_manager.get_num_free_gpu_blocks() + for scheduler in self.scheduler) + gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu) + + num_total_cpu = self.cache_config.num_cpu_blocks + cpu_cache_usage_sys = 0. + if num_total_cpu is not None and num_total_cpu > 0: + num_free_cpu = sum( + scheduler.block_manager.get_num_free_cpu_blocks() + for scheduler in self.scheduler) + cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu) + + # Prefix Cache Hit Rate. Note that we always use + # the cache hit rate of the first virtual engine. + cpu_prefix_cache_hit_rate = self.scheduler[ + 0].get_prefix_cache_hit_rate(Device.CPU) + gpu_prefix_cache_hit_rate = self.scheduler[ + 0].get_prefix_cache_hit_rate(Device.GPU) + + # Iteration stats + num_prompt_tokens_iter = 0 + num_generation_tokens_iter = 0 + time_to_first_tokens_iter: List[float] = [] + time_per_output_tokens_iter: List[float] = [] + num_preemption_iter = (0 if scheduler_outputs is None else + scheduler_outputs.preempted) + + # Request stats + # Latency + time_e2e_requests: List[float] = [] + # Metadata + num_prompt_tokens_requests: List[int] = [] + num_generation_tokens_requests: List[int] = [] + n_requests: List[int] = [] + finished_reason_requests: List[str] = [] + + # NOTE: This loop assumes prefill seq_groups are before + # decode seq_groups in scheduled_seq_groups. + if scheduler_outputs is not None: + # For async postprocessor, already finished sequences need to be + # not counted (to avoid double counting) + actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore + + num_generation_tokens_from_prefill_groups = 0. + # NOTE: if scheduler_outputs.num_prefill_groups > 0 and + # the len of scheduler_outputs.scheduled_seq_groups is != + # scheduler_outputs.num_prefill_groups, this means that + # chunked prefills have been detected. + + for idx, scheduled_seq_group in enumerate( + scheduler_outputs.scheduled_seq_groups): + # Skip double logging when using async output proc + if finished_before and idx in finished_before: + actual_num_batched_tokens -= 1 + continue + + # Currently, skip == preempted sequences, so we need to skip + # their log stats + if skip and idx in skip: + continue + + group_was_prefill = idx < scheduler_outputs.num_prefill_groups + seq_group = scheduled_seq_group.seq_group + + # NOTE: a seq_group that completed all of its prefill tokens + # in the last iteration will have seq_group.is_prefill() = False + # with group_was_prefill = True + if group_was_prefill: + # Number of prompt tokens. + num_prompt_tokens_iter += ( + scheduled_seq_group.token_chunk_size) + + # If the seq_group just finished the prefill state + # get TTFT. + if not seq_group.is_prefill(): + latency = seq_group.get_last_latency(now) + time_to_first_tokens_iter.append(latency) + + # One generation token per finished prefill. + num_generation_tokens_from_prefill_groups += ( + seq_group.num_seqs()) + else: + # TPOTs. + latency = seq_group.get_last_latency(now) + time_per_output_tokens_iter.append(latency) + + # Because of chunked prefill, we can have a single sequence + # group that does multiple prompt_runs. To prevent logging + # the same metadata more than once per request, we standardize + # on logging request level information for finished requests, + # which can only happen once. + if seq_group.is_finished(): + # Latency timings + time_e2e_requests.append(now - + seq_group.metrics.arrival_time) + # Metadata + num_prompt_tokens_requests.append( + len(seq_group.prompt_token_ids)) + num_generation_tokens_requests.extend([ + seq.get_output_len() + for seq in seq_group.get_finished_seqs() + ]) + if seq_group.sampling_params is not None: + n_requests.append(seq_group.sampling_params.n) + finished_reason_requests.extend([ + SequenceStatus.get_finished_reason(seq.status) + for seq in seq_group.get_finished_seqs() + ]) + + # Number of generation tokens. + # num_batched_tokens equals the number of prompt_tokens plus the + # number of decode_tokens in a single iteration. So, + # num_generation_tokens = num_batched_tokens - num_prompt_tokens + # + num_generation_tokens_from_prefill_groups (since we generate + # one token on prefills on iters where the prefill finishes). + num_generation_tokens_iter = ( + actual_num_batched_tokens - num_prompt_tokens_iter + + num_generation_tokens_from_prefill_groups) + + # Spec decode, if enabled, emits specialized metrics from the worker in + # sampler output. + if model_output and (model_output[0].spec_decode_worker_metrics + is not None): + spec_decode_metrics = model_output[0].spec_decode_worker_metrics + else: + spec_decode_metrics = None + + return Stats( + now=now, + # System stats + # Scheduler State + num_running_sys=num_running_sys, + num_swapped_sys=num_swapped_sys, + num_waiting_sys=num_waiting_sys, + # KV Cache Usage in % + gpu_cache_usage_sys=gpu_cache_usage_sys, + cpu_cache_usage_sys=cpu_cache_usage_sys, + # Prefix Cache Hit Rate + cpu_prefix_cache_hit_rate=cpu_prefix_cache_hit_rate, + gpu_prefix_cache_hit_rate=gpu_prefix_cache_hit_rate, + + # Iteration stats + num_prompt_tokens_iter=num_prompt_tokens_iter, + num_generation_tokens_iter=num_generation_tokens_iter, + time_to_first_tokens_iter=time_to_first_tokens_iter, + time_per_output_tokens_iter=time_per_output_tokens_iter, + spec_decode_metrics=spec_decode_metrics, + num_preemption_iter=num_preemption_iter, + + # Request stats + # Latency + time_e2e_requests=time_e2e_requests, + # Metadata + num_prompt_tokens_requests=num_prompt_tokens_requests, + num_generation_tokens_requests=num_generation_tokens_requests, + n_requests=n_requests, + finished_reason_requests=finished_reason_requests, + ) + + def add_lora(self, lora_request: LoRARequest) -> bool: + return self.model_executor.add_lora(lora_request) + + def remove_lora(self, lora_id: int) -> bool: + return self.model_executor.remove_lora(lora_id) + + def list_loras(self) -> Set[int]: + return self.model_executor.list_loras() + + def pin_lora(self, lora_id: int) -> bool: + return self.model_executor.pin_lora(lora_id) + + def add_prompt_adapter( + self, prompt_adapter_request: PromptAdapterRequest) -> bool: + return self.model_executor.add_prompt_adapter(prompt_adapter_request) + + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: + return self.model_executor.remove_prompt_adapter(prompt_adapter_id) + + def list_prompt_adapters(self) -> List[int]: + return self.model_executor.list_prompt_adapters() + + def check_health(self) -> None: + if self.tokenizer: + self.tokenizer.check_health() + self.model_executor.check_health() + + def start_profile(self) -> None: + # using type instead of isinstance to check to avoid capturing + # inherited classes (MultiprocessingGPUExecutor) + if type(self.model_executor) == GPUExecutor: # noqa: E721 + self.model_executor.start_profile() + else: + self.model_executor._run_workers("start_profile") + + def stop_profile(self) -> None: + # using type instead of isinstance to check to avoid capturing + # inherited classes (MultiprocessingGPUExecutor) + if type(self.model_executor) == GPUExecutor: # noqa: E721 + self.model_executor.stop_profile() + else: + self.model_executor._run_workers("stop_profile") + + def is_tracing_enabled(self) -> bool: + return self.tracer is not None + + def do_tracing(self, scheduler_outputs: SchedulerOutputs) -> None: + if self.tracer is None: + return + + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups: + seq_group = scheduled_seq_group.seq_group + if seq_group.is_finished(): + self.create_trace_span(seq_group) + + def create_trace_span(self, seq_group: SequenceGroup) -> None: + if self.tracer is None or seq_group.sampling_params is None: + return + arrival_time_nano_seconds = int(seq_group.metrics.arrival_time * 1e9) + + trace_context = extract_trace_context(seq_group.trace_headers) + + with self.tracer.start_as_current_span( + "llm_request", + kind=SpanKind.SERVER, + context=trace_context, + start_time=arrival_time_nano_seconds) as seq_span: + metrics = seq_group.metrics + ttft = metrics.first_token_time - metrics.arrival_time + e2e_time = metrics.finished_time - metrics.arrival_time + # attribute names are based on + # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md + seq_span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, + self.model_config.model) + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_ID, + seq_group.request_id) + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TEMPERATURE, + seq_group.sampling_params.temperature) + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_TOP_P, + seq_group.sampling_params.top_p) + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS, + seq_group.sampling_params.max_tokens) + seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N, + seq_group.sampling_params.n) + seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES, + seq_group.num_seqs()) + seq_span.set_attribute(SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + len(seq_group.prompt_token_ids)) + seq_span.set_attribute( + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + sum([ + seq.get_output_len() + for seq in seq_group.get_finished_seqs() + ])) + seq_span.set_attribute(SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE, + metrics.time_in_queue) + seq_span.set_attribute( + SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft) + seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time) + if metrics.scheduler_time is not None: + seq_span.set_attribute( + SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER, + metrics.scheduler_time) + if metrics.model_forward_time is not None: + seq_span.set_attribute( + SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD, + metrics.model_forward_time / 1000.0) + if metrics.model_execute_time is not None: + seq_span.set_attribute( + SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE, + metrics.model_execute_time) + + def is_encoder_decoder_model(self): + return self.input_preprocessor.is_encoder_decoder_model() + + def is_embedding_model(self): + return self.model_config.is_embedding_model + + def _validate_model_inputs(self, inputs: Union[LLMInputs, + EncoderDecoderLLMInputs]): + if self.model_config.is_multimodal_model: + # For encoder-decoder multimodal models, the max_prompt_len + # restricts the decoder prompt length + prompt_ids = inputs.get("prompt_token_ids") + elif self.is_encoder_decoder_model(): + prompt_ids = inputs.get("encoder_prompt_token_ids") + else: + prompt_ids = inputs.get("prompt_token_ids") + + if prompt_ids is None or len(prompt_ids) == 0: + raise ValueError("Prompt cannot be empty") + + if self.model_config.is_multimodal_model: + max_prompt_len = self.model_config.max_model_len + + if len(prompt_ids) > max_prompt_len: + raise ValueError( + f"The prompt (total length {len(prompt_ids)}) is too long " + f"to fit into the model (context length {max_prompt_len}). " + "Make sure that `max_model_len` is no smaller than the " + "number of text tokens plus multimodal tokens. For image " + "inputs, the number of image tokens depends on the number " + "of images, and possibly their aspect ratios as well.") + + # TODO: Find out how many placeholder tokens are there so we can + # check that chunked prefill does not truncate them + # max_batch_len = self.scheduler_config.max_num_batched_tokens + + def _build_logits_processors( + self, sampling_params: SamplingParams, + lora_request: Optional[LoRARequest]) -> SamplingParams: + """Constructs logits processors based on the guided_decoding, + logits_bias, and allowed_token_ids fields in sampling_params. Deletes + those fields and adds the constructed logits processors to the + logits_processors field. Returns the modified sampling params.""" + + logits_processors = [] + if (guided_decoding := sampling_params.guided_decoding) is not None: + + logger.debug( + "Building guided decoding logits processor in " + "LLMEngine. Params: %s", guided_decoding) + + tokenizer = self.get_tokenizer(lora_request=lora_request) + guided_decoding.backend = guided_decoding.backend or \ + self.decoding_config.guided_decoding_backend + + processor = get_local_guided_decoding_logits_processor( + guided_params=guided_decoding, tokenizer=tokenizer) + if processor: + logits_processors.append(processor) + + # Unset so this doesn't get passed down to the model + sampling_params.guided_decoding = None + + if (sampling_params.logit_bias or sampling_params.allowed_token_ids): + tokenizer = self.get_tokenizer(lora_request=lora_request) + + processors = get_logits_processors( + logit_bias=sampling_params.logit_bias, + allowed_token_ids=sampling_params.allowed_token_ids, + tokenizer=tokenizer) + logits_processors.extend(processors) + + # Unset so these don't get passed down to the model + sampling_params.logit_bias = None + sampling_params.allowed_token_ids = None + + if logits_processors: + if sampling_params.logits_processors is None: + sampling_params.logits_processors = logits_processors + else: + sampling_params.logits_processors.extend(logits_processors) + + return sampling_params diff --git a/vllm/engine/metrics.py b/vllm/engine/metrics.py new file mode 100644 index 00000000..42acd3ea --- /dev/null +++ b/vllm/engine/metrics.py @@ -0,0 +1,555 @@ +from typing import TYPE_CHECKING +from typing import Counter as CollectionsCounter +from typing import Dict, List, Optional, Union + +import numpy as np +import prometheus_client + +from vllm.engine.metrics_types import (StatLoggerBase, Stats, + SupportsMetricsInfo) +from vllm.executor.ray_utils import ray +from vllm.logger import init_logger + +if ray is not None: + from ray.util import metrics as ray_metrics +else: + ray_metrics = None + +if TYPE_CHECKING: + from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics + +logger = init_logger(__name__) + +prometheus_client.disable_created_metrics() + +# The begin-* and end* here are used by the documentation generator +# to extract the metrics definitions. + + +# begin-metrics-definitions +class Metrics: + """ + vLLM uses a multiprocessing-based frontend for the OpenAI server. + This means that we need to run prometheus_client in multiprocessing mode + See https://prometheus.github.io/client_python/multiprocess/ for more + details on limitations. + """ + labelname_finish_reason = "finished_reason" + _gauge_cls = prometheus_client.Gauge + _counter_cls = prometheus_client.Counter + _histogram_cls = prometheus_client.Histogram + + def __init__(self, labelnames: List[str], max_model_len: int): + # Unregister any existing vLLM collectors (for CI/CD) + self._unregister_vllm_metrics() + + # System stats + # Scheduler State + self.gauge_scheduler_running = self._gauge_cls( + name="vllm:num_requests_running", + documentation="Number of requests currently running on GPU.", + labelnames=labelnames, + multiprocess_mode="sum") + self.gauge_scheduler_waiting = self._gauge_cls( + name="vllm:num_requests_waiting", + documentation="Number of requests waiting to be processed.", + labelnames=labelnames, + multiprocess_mode="sum") + self.gauge_scheduler_swapped = self._gauge_cls( + name="vllm:num_requests_swapped", + documentation="Number of requests swapped to CPU.", + labelnames=labelnames, + multiprocess_mode="sum") + # KV Cache Usage in % + self.gauge_gpu_cache_usage = self._gauge_cls( + name="vllm:gpu_cache_usage_perc", + documentation="GPU KV-cache usage. 1 means 100 percent usage.", + labelnames=labelnames, + multiprocess_mode="sum") + self.gauge_cpu_cache_usage = self._gauge_cls( + name="vllm:cpu_cache_usage_perc", + documentation="CPU KV-cache usage. 1 means 100 percent usage.", + labelnames=labelnames, + multiprocess_mode="sum") + # Prefix caching block hit rate + self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls( + name="vllm:cpu_prefix_cache_hit_rate", + documentation="CPU prefix cache block hit rate.", + labelnames=labelnames, + multiprocess_mode="sum") + self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls( + name="vllm:gpu_prefix_cache_hit_rate", + documentation="GPU prefix cache block hit rate.", + labelnames=labelnames, + multiprocess_mode="sum") + + # Iteration stats + self.counter_num_preemption = self._counter_cls( + name="vllm:num_preemptions_total", + documentation="Cumulative number of preemption from the engine.", + labelnames=labelnames) + self.counter_prompt_tokens = self._counter_cls( + name="vllm:prompt_tokens_total", + documentation="Number of prefill tokens processed.", + labelnames=labelnames) + self.counter_generation_tokens = self._counter_cls( + name="vllm:generation_tokens_total", + documentation="Number of generation tokens processed.", + labelnames=labelnames) + self.histogram_time_to_first_token = self._histogram_cls( + name="vllm:time_to_first_token_seconds", + documentation="Histogram of time to first token in seconds.", + labelnames=labelnames, + buckets=[ + 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, + 0.75, 1.0, 2.5, 5.0, 7.5, 10.0 + ]) + self.histogram_time_per_output_token = self._histogram_cls( + name="vllm:time_per_output_token_seconds", + documentation="Histogram of time per output token in seconds.", + labelnames=labelnames, + buckets=[ + 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, + 1.0, 2.5 + ]) + + # Request stats + # Latency + self.histogram_e2e_time_request = self._histogram_cls( + name="vllm:e2e_request_latency_seconds", + documentation="Histogram of end to end request latency in seconds.", + labelnames=labelnames, + buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0]) + # Metadata + self.histogram_num_prompt_tokens_request = self._histogram_cls( + name="vllm:request_prompt_tokens", + documentation="Number of prefill tokens processed.", + labelnames=labelnames, + buckets=build_1_2_5_buckets(max_model_len), + ) + self.histogram_num_generation_tokens_request = \ + self._histogram_cls( + name="vllm:request_generation_tokens", + documentation="Number of generation tokens processed.", + labelnames=labelnames, + buckets=build_1_2_5_buckets(max_model_len), + ) + self.histogram_n_request = self._histogram_cls( + name="vllm:request_params_n", + documentation="Histogram of the n request parameter.", + labelnames=labelnames, + buckets=[1, 2, 5, 10, 20], + ) + self.counter_request_success = self._counter_cls( + name="vllm:request_success_total", + documentation="Count of successfully processed requests.", + labelnames=labelnames + [Metrics.labelname_finish_reason]) + + # Speculatie decoding stats + self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls( + name="vllm:spec_decode_draft_acceptance_rate", + documentation="Speulative token acceptance rate.", + labelnames=labelnames, + multiprocess_mode="sum") + self.gauge_spec_decode_efficiency = self._gauge_cls( + name="vllm:spec_decode_efficiency", + documentation="Speculative decoding system efficiency.", + labelnames=labelnames, + multiprocess_mode="sum") + self.counter_spec_decode_num_accepted_tokens = (self._counter_cls( + name="vllm:spec_decode_num_accepted_tokens_total", + documentation="Number of accepted tokens.", + labelnames=labelnames)) + self.counter_spec_decode_num_draft_tokens = self._counter_cls( + name="vllm:spec_decode_num_draft_tokens_total", + documentation="Number of draft tokens.", + labelnames=labelnames) + self.counter_spec_decode_num_emitted_tokens = (self._counter_cls( + name="vllm:spec_decode_num_emitted_tokens_total", + documentation="Number of emitted tokens.", + labelnames=labelnames)) + + # Deprecated in favor of vllm:prompt_tokens_total + self.gauge_avg_prompt_throughput = self._gauge_cls( + name="vllm:avg_prompt_throughput_toks_per_s", + documentation="Average prefill throughput in tokens/s.", + labelnames=labelnames, + multiprocess_mode="sum", + ) + # Deprecated in favor of vllm:generation_tokens_total + self.gauge_avg_generation_throughput = self._gauge_cls( + name="vllm:avg_generation_throughput_toks_per_s", + documentation="Average generation throughput in tokens/s.", + labelnames=labelnames, + multiprocess_mode="sum", + ) + + +# end-metrics-definitions + + def _unregister_vllm_metrics(self) -> None: + for collector in list(prometheus_client.REGISTRY._collector_to_names): + if hasattr(collector, "_name") and "vllm" in collector._name: + prometheus_client.REGISTRY.unregister(collector) + + +class _RayGaugeWrapper: + """Wraps around ray.util.metrics.Gauge to provide same API as + prometheus_client.Gauge""" + + def __init__(self, + name: str, + documentation: str = "", + labelnames: Optional[List[str]] = None, + multiprocess_mode: str = ""): + del multiprocess_mode + labelnames_tuple = tuple(labelnames) if labelnames else None + self._gauge = ray_metrics.Gauge(name=name, + description=documentation, + tag_keys=labelnames_tuple) + + def labels(self, **labels): + self._gauge.set_default_tags(labels) + return self + + def set(self, value: Union[int, float]): + return self._gauge.set(value) + + +class _RayCounterWrapper: + """Wraps around ray.util.metrics.Counter to provide same API as + prometheus_client.Counter""" + + def __init__(self, + name: str, + documentation: str = "", + labelnames: Optional[List[str]] = None): + labelnames_tuple = tuple(labelnames) if labelnames else None + self._counter = ray_metrics.Counter(name=name, + description=documentation, + tag_keys=labelnames_tuple) + + def labels(self, **labels): + self._counter.set_default_tags(labels) + return self + + def inc(self, value: Union[int, float] = 1.0): + if value == 0: + return + return self._counter.inc(value) + + +class _RayHistogramWrapper: + """Wraps around ray.util.metrics.Histogram to provide same API as + prometheus_client.Histogram""" + + def __init__(self, + name: str, + documentation: str = "", + labelnames: Optional[List[str]] = None, + buckets: Optional[List[float]] = None): + labelnames_tuple = tuple(labelnames) if labelnames else None + self._histogram = ray_metrics.Histogram(name=name, + description=documentation, + tag_keys=labelnames_tuple, + boundaries=buckets) + + def labels(self, **labels): + self._histogram.set_default_tags(labels) + return self + + def observe(self, value: Union[int, float]): + return self._histogram.observe(value) + + +class RayMetrics(Metrics): + """ + RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics. + Provides the same metrics as Metrics but uses Ray's util.metrics library. + """ + _gauge_cls = _RayGaugeWrapper + _counter_cls = _RayCounterWrapper + _histogram_cls = _RayHistogramWrapper + + def __init__(self, labelnames: List[str], max_model_len: int): + if ray_metrics is None: + raise ImportError("RayMetrics requires Ray to be installed.") + super().__init__(labelnames, max_model_len) + + def _unregister_vllm_metrics(self) -> None: + # No-op on purpose + pass + + +def build_1_2_5_buckets(max_value: int) -> List[int]: + """ + Builds a list of buckets with increasing powers of 10 multiplied by + mantissa values (1, 2, 5) until the value exceeds the specified maximum. + + Example: + >>> build_1_2_5_buckets(100) + [1, 2, 5, 10, 20, 50, 100] + """ + mantissa_lst = [1, 2, 5] + exponent = 0 + buckets: List[int] = [] + while True: + for m in mantissa_lst: + value = m * 10**exponent + if value <= max_value: + buckets.append(value) + else: + return buckets + exponent += 1 + + +def local_interval_elapsed(now: float, last_log: float, + local_interval: float) -> bool: + elapsed_time = now - last_log + return elapsed_time > local_interval + + +def get_throughput(tracked_stats: List[int], now: float, + last_log: float) -> float: + return float(np.sum(tracked_stats) / (now - last_log)) + + +class LoggingStatLogger(StatLoggerBase): + """LoggingStatLogger is used in LLMEngine to log to Stdout.""" + + def log(self, stats: Stats) -> None: + """Called by LLMEngine. + Logs to Stdout every self.local_interval seconds.""" + + # Save tracked stats for token counters. + self.num_prompt_tokens.append(stats.num_prompt_tokens_iter) + self.num_generation_tokens.append(stats.num_generation_tokens_iter) + + # Update spec decode metrics + self.maybe_update_spec_decode_metrics(stats) + + # Log locally every local_interval seconds. + if local_interval_elapsed(stats.now, self.last_local_log, + self.local_interval): + # Compute summary metrics for tracked stats (and log them + # to promethus if applicable). + prompt_throughput = get_throughput(self.num_prompt_tokens, + now=stats.now, + last_log=self.last_local_log) + generation_throughput = get_throughput( + self.num_generation_tokens, + now=stats.now, + last_log=self.last_local_log) + + # Log to stdout. + logger.info( + "Avg prompt throughput: %.1f tokens/s, " + "Avg generation throughput: %.1f tokens/s, " + "Running: %d reqs, Swapped: %d reqs, " + "Pending: %d reqs, GPU KV cache usage: %.1f%%, " + "CPU KV cache usage: %.1f%%.", + prompt_throughput, + generation_throughput, + stats.num_running_sys, + stats.num_swapped_sys, + stats.num_waiting_sys, + stats.gpu_cache_usage_sys * 100, + stats.cpu_cache_usage_sys * 100, + ) + if (stats.cpu_prefix_cache_hit_rate >= 0 + or stats.gpu_prefix_cache_hit_rate >= 0): + logger.info( + "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%", + stats.gpu_prefix_cache_hit_rate * 100, + stats.cpu_prefix_cache_hit_rate * 100, + ) + if self.spec_decode_metrics is not None: + logger.info( + self._format_spec_decode_metrics_str( + self.spec_decode_metrics)) + + # Reset tracked stats for next interval. + self.num_prompt_tokens = [] + self.num_generation_tokens = [] + self.last_local_log = stats.now + self.spec_decode_metrics = None + + def _format_spec_decode_metrics_str( + self, metrics: "SpecDecodeWorkerMetrics") -> str: + + return ("Speculative metrics: " + f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, " + f"System efficiency: {metrics.system_efficiency:.3f}, " + f"Number of speculative tokens: {metrics.num_spec_tokens}, " + f"Number of accepted tokens: {metrics.accepted_tokens}, " + f"Number of draft tokens: {metrics.draft_tokens}, " + f"Number of emitted tokens: {metrics.emitted_tokens}.") + + def info(self, type: str, obj: SupportsMetricsInfo) -> None: + raise NotImplementedError + + +class PrometheusStatLogger(StatLoggerBase): + """PrometheusStatLogger is used LLMEngine to log to Promethus.""" + _metrics_cls = Metrics + _gauge_cls = prometheus_client.Gauge + + def __init__(self, local_interval: float, labels: Dict[str, str], + max_model_len: int) -> None: + super().__init__(local_interval) + # Prometheus metrics + self.labels = labels + self.metrics = self._metrics_cls(labelnames=list(labels.keys()), + max_model_len=max_model_len) + + def _log_gauge(self, gauge, data: Union[int, float]) -> None: + # Convenience function for logging to gauge. + gauge.labels(**self.labels).set(data) + + def _log_counter(self, counter, data: Union[int, float]) -> None: + # Convenience function for logging to counter. + counter.labels(**self.labels).inc(data) + + def _log_counter_labels(self, counter, data: CollectionsCounter, + label_key: str) -> None: + # Convenience function for collection counter of labels. + for label, count in data.items(): + counter.labels(**{**self.labels, label_key: label}).inc(count) + + def _log_histogram(self, histogram, data: Union[List[int], + List[float]]) -> None: + # Convenience function for logging list to histogram. + for datum in data: + histogram.labels(**self.labels).observe(datum) + + def _log_prometheus(self, stats: Stats) -> None: + # System state data + self._log_gauge(self.metrics.gauge_scheduler_running, + stats.num_running_sys) + self._log_gauge(self.metrics.gauge_scheduler_swapped, + stats.num_swapped_sys) + self._log_gauge(self.metrics.gauge_scheduler_waiting, + stats.num_waiting_sys) + self._log_gauge(self.metrics.gauge_gpu_cache_usage, + stats.gpu_cache_usage_sys) + self._log_gauge(self.metrics.gauge_cpu_cache_usage, + stats.cpu_cache_usage_sys) + self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate, + stats.cpu_prefix_cache_hit_rate) + self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate, + stats.gpu_prefix_cache_hit_rate) + + # Iteration level data + self._log_counter(self.metrics.counter_num_preemption, + stats.num_preemption_iter) + self._log_counter(self.metrics.counter_prompt_tokens, + stats.num_prompt_tokens_iter) + self._log_counter(self.metrics.counter_generation_tokens, + stats.num_generation_tokens_iter) + self._log_histogram(self.metrics.histogram_time_to_first_token, + stats.time_to_first_tokens_iter) + self._log_histogram(self.metrics.histogram_time_per_output_token, + stats.time_per_output_tokens_iter) + + # Request level data + # Latency + self._log_histogram(self.metrics.histogram_e2e_time_request, + stats.time_e2e_requests) + # Metadata + finished_reason_counter = CollectionsCounter( + stats.finished_reason_requests) + self._log_counter_labels(self.metrics.counter_request_success, + finished_reason_counter, + Metrics.labelname_finish_reason) + self._log_histogram(self.metrics.histogram_num_prompt_tokens_request, + stats.num_prompt_tokens_requests) + self._log_histogram( + self.metrics.histogram_num_generation_tokens_request, + stats.num_generation_tokens_requests) + self._log_histogram(self.metrics.histogram_n_request, stats.n_requests) + + def _log_prometheus_interval(self, prompt_throughput: float, + generation_throughput: float) -> None: + # Logs metrics to prometheus that are computed every logging_interval. + # Support legacy gauge metrics that make throughput calculations on + # the vLLM side. Moving forward, we should use counters like + # counter_prompt_tokens, counter_generation_tokens + # Which log raw data and calculate summaries using rate() on the + # grafana/prometheus side. See + # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666 + self.metrics.gauge_avg_prompt_throughput.labels( + **self.labels).set(prompt_throughput) + self.metrics.gauge_avg_generation_throughput.labels( + **self.labels).set(generation_throughput) + + def log(self, stats: Stats): + """Logs to prometheus and tracked stats every iteration.""" + # Log to prometheus. + self._log_prometheus(stats) + + # Save tracked stats for token counters. + self.num_prompt_tokens.append(stats.num_prompt_tokens_iter) + self.num_generation_tokens.append(stats.num_generation_tokens_iter) + + # Update spec decode metrics + self.maybe_update_spec_decode_metrics(stats) + + # Log locally every local_interval seconds. + if local_interval_elapsed(stats.now, self.last_local_log, + self.local_interval): + # Compute summary metrics for tracked stats (and log them + # to promethus if applicable). + prompt_throughput = get_throughput(self.num_prompt_tokens, + now=stats.now, + last_log=self.last_local_log) + generation_throughput = get_throughput( + self.num_generation_tokens, + now=stats.now, + last_log=self.last_local_log) + + self._log_prometheus_interval( + prompt_throughput=prompt_throughput, + generation_throughput=generation_throughput) + + if self.spec_decode_metrics is not None: + self._log_gauge( + self.metrics.gauge_spec_decode_draft_acceptance_rate, + self.spec_decode_metrics.draft_acceptance_rate) + self._log_gauge(self.metrics.gauge_spec_decode_efficiency, + self.spec_decode_metrics.system_efficiency) + self._log_counter( + self.metrics.counter_spec_decode_num_accepted_tokens, + self.spec_decode_metrics.accepted_tokens) + self._log_counter( + self.metrics.counter_spec_decode_num_draft_tokens, + self.spec_decode_metrics.draft_tokens) + self._log_counter( + self.metrics.counter_spec_decode_num_emitted_tokens, + self.spec_decode_metrics.emitted_tokens) + + # Reset tracked stats for next interval. + self.num_prompt_tokens = [] + self.num_generation_tokens = [] + self.last_local_log = stats.now + self.spec_decode_metrics = None + + def info(self, type: str, obj: SupportsMetricsInfo) -> None: + # Info type metrics are syntactic sugar for a gauge permanently set to 1 + # Since prometheus multiprocessing mode does not support Info, emulate + # info here with a gauge. + if type == "cache_config": + metrics_info = obj.metrics_info() + info_gauge = self._gauge_cls( + name="vllm:cache_config_info", + documentation="Information of the LLMEngine CacheConfig", + labelnames=metrics_info.keys(), + multiprocess_mode="mostrecent") + info_gauge.labels(**metrics_info).set(1) + + +class RayPrometheusStatLogger(PrometheusStatLogger): + """RayPrometheusStatLogger uses Ray metrics instead.""" + _metrics_cls = RayMetrics + + def info(self, type: str, obj: SupportsMetricsInfo) -> None: + return None diff --git a/vllm/engine/metrics_types.py b/vllm/engine/metrics_types.py new file mode 100644 index 00000000..bafd5fa1 --- /dev/null +++ b/vllm/engine/metrics_types.py @@ -0,0 +1,87 @@ +""" +These types are defined in this file to avoid importing vllm.engine.metrics +and therefore importing prometheus_client. + +This is required due to usage of Prometheus multiprocess mode to enable +metrics after splitting out the uvicorn process from the engine process. + +Prometheus multiprocess mode requires setting PROMETHEUS_MULTIPROC_DIR +before prometheus_client is imported. Typically, this is done by setting +the env variable before launch, but since we are a library, we need to +do this in Python code and lazily import prometheus_client. +""" + +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Dict, List, Optional, Protocol + +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics + + +@dataclass +class Stats: + """Created by LLMEngine for use by StatLogger.""" + now: float + + # System stats (should have _sys suffix) + # Scheduler State + num_running_sys: int + num_waiting_sys: int + num_swapped_sys: int + # KV Cache Usage in % + gpu_cache_usage_sys: float + cpu_cache_usage_sys: float + # Prefix caching block hit rate + cpu_prefix_cache_hit_rate: float + gpu_prefix_cache_hit_rate: float + + # Iteration stats (should have _iter suffix) + num_prompt_tokens_iter: int + num_generation_tokens_iter: int + time_to_first_tokens_iter: List[float] + time_per_output_tokens_iter: List[float] + num_preemption_iter: int + + # Request stats (should have _requests suffix) + # Latency + time_e2e_requests: List[float] + # Metadata + num_prompt_tokens_requests: List[int] + num_generation_tokens_requests: List[int] + n_requests: List[int] + finished_reason_requests: List[str] + + spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None + + +class SupportsMetricsInfo(Protocol): + + def metrics_info(self) -> Dict[str, str]: + ... + + +class StatLoggerBase(ABC): + """Base class for StatLogger.""" + + def __init__(self, local_interval: float) -> None: + # Tracked stats over current local logging interval. + self.num_prompt_tokens: List[int] = [] + self.num_generation_tokens: List[int] = [] + self.last_local_log = time.time() + self.local_interval = local_interval + self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None + + @abstractmethod + def log(self, stats: Stats) -> None: + raise NotImplementedError + + @abstractmethod + def info(self, type: str, obj: SupportsMetricsInfo) -> None: + raise NotImplementedError + + def maybe_update_spec_decode_metrics(self, stats: Stats): + """Save spec decode metrics (since they are unlikely + to be emitted at same time as log interval).""" + if stats.spec_decode_metrics is not None: + self.spec_decode_metrics = stats.spec_decode_metrics diff --git a/vllm/engine/multiprocessing/__init__.py b/vllm/engine/multiprocessing/__init__.py new file mode 100644 index 00000000..34c161e9 --- /dev/null +++ b/vllm/engine/multiprocessing/__init__.py @@ -0,0 +1,135 @@ +from dataclasses import dataclass +from enum import Enum +from typing import List, Mapping, Optional, Union, overload + +from vllm import PoolingParams +from vllm.inputs import PromptType +from vllm.lora.request import LoRARequest +from vllm.outputs import RequestOutput +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import SamplingParams +from vllm.utils import deprecate_kwargs + +VLLM_RPC_SUCCESS_STR = "SUCCESS" + +IPC_INPUT_EXT = "_input_socket" +IPC_OUTPUT_EXT = "_output_socket" +IPC_HEALTH_EXT = "_health_socket" +IPC_DATA_EXT = "_data_socket" + + +class MQEngineDeadError(RuntimeError): + pass + + +@dataclass +class RPCProcessRequest: + prompt: PromptType + params: Union[SamplingParams, PoolingParams] + request_id: str + lora_request: Optional[LoRARequest] = None + trace_headers: Optional[Mapping[str, str]] = None + prompt_adapter_request: Optional[PromptAdapterRequest] = None + priority: int = 0 + + @overload # DEPRECATED + def __init__( + self, + *, + inputs: PromptType, + params: Union[SamplingParams, PoolingParams], + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @overload + def __init__( + self, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> None: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + def __init__( + self, + prompt: Optional[PromptType] = None, + params: Optional[Union[SamplingParams, PoolingParams]] = None, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> None: + if inputs is not None: + prompt = inputs + assert (prompt is not None and params is not None + and request_id is not None) + + super().__init__() + + self.prompt = prompt + self.params = params + self.request_id = request_id + self.lora_request = lora_request + self.trace_headers = trace_headers + self.prompt_adapter_request = prompt_adapter_request + self.priority = priority + + +@dataclass +class RPCError: + request_id: Optional[str] + is_engine_errored: bool + exception: BaseException + + +@dataclass +class RPCAbortRequest: + request_id: str + + +class RPCStartupRequest(Enum): + IS_SERVER_READY = 1 + + +@dataclass +class RPCStartupResponse: + tracing_enabled: bool + + +class RPCUProfileRequest(Enum): + START_PROFILE = 1 + STOP_PROFILE = 2 + + +RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest, + RPCUProfileRequest] + +REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError] + + +def ENGINE_DEAD_ERROR( + error: Optional[BaseException] = None) -> MQEngineDeadError: + if error is None: + return MQEngineDeadError( + "Engine loop is not running. Inspect the stacktrace to " + "find the original error") + + return MQEngineDeadError( + "Engine loop is not running. Inspect the stacktrace to " + f"find the original error: {repr(error)}.") diff --git a/vllm/engine/multiprocessing/__pycache__/__init__.cpython-310.pyc b/vllm/engine/multiprocessing/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18aafdfb2009d25e00638c5b170639b66d7936b4 GIT binary patch literal 4118 zcmb_fTXP#p6`pC0Mk8r-^DS|%%7x8pi?s>&g49-wBQ^#{_Db>=^aI00-Ht_OByG=V zm!t#_I92<^FJP%!-pB)gfgdujypZP<$+Ezj@AOEHy*5>$V5a&^pVOyLcb`7D8J0=~ z1JBJr{dw=NWyAOjAxEDqklScc+cXSe2s1E3lba^>S-`>!&**+8u)-|QhBmju9M6Rg zcfvf+>oF@RghgJ|{cPZdC0^2fJ1B=0UeWzrPz`Imru$AX7tZr}-OmRL;UZtu{X(!5 zp5bTEFY>d(<;$YP&xtZWFDm@PyN0NW+7m<62>hbvxGo)eSzeJ><@H@_ z+2CIRo|Uy_L)Ld0@Hg_8#5HkQ=H!iKL)7J^T}xaMSD$40o8nT(Xk33r+HROhUih)! z3;ZYoVl@Zju#ruy&3+UoxqJR-)E_)ZoUKvZ9}fH=$?go$Nu1$_QUya_U`}a!I1IqG z?JGYdA%$%<3`cSI(MSR-ZVuaPZTZ1iMlo?uyIbRUG$z7A%DpE1Q7qLf#&X9GN5|w= zg&Zl_^JD2f{K!`iq9oVZSzm8Db!tq{sGyG^Z=-*!*J&N~+gVhg% zAY7GtvDI)K#Ql*P_GA=6)vF%P+mAhOW%NkZAeKZ&7Jy?|rly=Y6|vA0eTwtVPPJpm zjI@M#%YgCNGDoHZoB?<6&D=7C#q+Wti<+L*xC^|5afi4y-PZIn=oO9UfIGY@YmkwL ztT|!n*AazJEK!sh!CtK=GSU&*bPZu2mo0LsOp%v0;Rtm z^M8_Afy}Iyxgd(<&c9_-*|QZEvH!VGOsrFo88{Tf_fTWNGr%2uU*{afsSG;L+{ebQ zDdw?$h1UPrunfKkOIu26?d^5A2Hp77DH0nN0LQ{tyIN9Esatt{(MsH4sC;iqNlLNu zd(wm13#p=HDV^#0sh7>z6q$PBj8uQ9`thTNrOtz(E)cj#;1Z47+P9*_Rx%!|L60WU zlS6aR2wz8gP6jB9j`7rNX*+K?i51CUS6#+{S|)Igz$uHoat^fw7V@an!mj~~7=YzI zK6gSTD>JqpZDjTw-ITfls=7*mJfPtylBmuS_-}Tku4Becv~)AY$8T*1(vM_)j27?7 z`i)e@H|qLwBw|^;KZs&nz)Hj9bG{J$G=_R1uGD{k*r@e#xN}lt9*qv0p@n`9FmZ%2 zHkCbbeq~M&h|r&#u<18L-)P}S#&68n*f%~k|A4!gm=nWe69$->WB{#+1(=;=0quzm zn49DPA-R)krjbvw(Rd`4KICiYrjdVbEz;FLMV9eAont&Ny=l5+z1lnIM06WY)iSH5 zYZjQzY;*P&%!bu!wX~r&a_Ai*OKOf}%oA83K=njY@jQPp7{)&7I>KsXqzVOwp-8t$ zCW=*Re>TyI1hlzRy^-2{iF(@bGyOrV$e2>SRM1OG(}VP65X=sv1N$4qr;l|8Jun2=rNg@QogcN!^IAXh(p=`j%!W#eBrkv2 zle(^G!7IOXnfi7bSX8|UkXZLoV?z~P#+Sw}*Kz5uQJuY&t$Xc1jo!El)alTt+e zk7s*M6OW&b&LtSmkx42z$J+g&il;$XAE(vGetE-(dp&hq6gVW>a|)5PEk%?QTcQwf z8!f5=%(f&utBUez>vO z2x!{MmjkLn0(bu}IgpyrkO#vmHWUaV73gaZDd zPWbhBSl{grgeD`Wqf+&Qy5891t^cdI@6{7?rNMNqnOF*yIr7E-baL_!j7}50rYTf0txPK>oeH?C>FLiyq^SPq z=QZhg{d72qas^+|9MSZ5#Y>AbopX3W4~^q~5GhI@eBu4g&3hgk$eaGUc%5#Wmo`Ay zXl?Iwz2;B4yiDlUPWKpox4E|2y{pGv8o#sFT_Y@U+v7pp4`mu{>2E;Cavk(^n9*LX zT}oRjFVXVt=6gHMPS@-5+N_(d*zvk*m4xW4(2?~O&^PMbN`GS^j0&>s+#N$nq4)kAQN=5xby=Wvkq*N9Wb+e@NXT)fH(jE literal 0 HcmV?d00001 diff --git a/vllm/engine/multiprocessing/__pycache__/client.cpython-310.pyc b/vllm/engine/multiprocessing/__pycache__/client.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a24654bce48144d7e824b370591ff213864acb40 GIT binary patch literal 19776 zcmeHvX>=P`mR@ZrECfK1qDWCIv{({tiISK0BCBQ55^c#XQL;$c?dtYGAgV}$LIKdL zf+Qlvj@)w3nN8_*woYRAgqkGN<4HQ{oFtuXPPWc+=A8MJf2k~IGV`l{>it@*F)BQ1Ua~5Ckd#a+? zimlcZOa7{s%J-V3;a98au3;H&!b+(0T(6mK(n`81E9IuGw41Rq?tnGmX05E7vvO|U z%DaQsAh%U(1*^bg4Ov6{JIudEZOGkbZR0kH+ORufjqtTu+wP89qkNsLjk!Cl9qza_ z&iAR>PIs5J%iV45cK29&xGr7W>+ZAm@pYzFboX2P`Ffysz&&UkbPrjF+{diP+{4yk z_lR}GJ!&0wC#(tgaqDsS3F`^>n03rOZXI_gtx5NUb%Oh5YbV_&tta_9S9{8R+Irf3 z#(IYD^R>^p&sxv&^iJ>va=F>8))&Lnm8M^9)XTLnalMXu2Em8E{56Z`eL>2M0W$18@5xE zH<)vfw(^Kb5)7P$DxfW(GUOiVjKQ%X1n*QQkn5Vl7(^Hq`F7SPh?k~<i3|hHO ze{QAegxR^qO{eb38nhp%)~h~&3phl#mm1fm;?eZ5a1I@74xQDNmfvjo;pntG@7Q!; zQLCsv*K>=JM&uMzcDV_Ri-*+0!Umd1)Cu#mWlXYxN@}=^7vmTqgHH$B{T`So+Ps#EvR)f%2-^Qo189lfUSRvadoVx?C0yrL*q@w|9v$*C9p zB?q4AUw!$MVfrN3U#YVm8IRaE$^p;w_-MCYqEM7!i!^PNT&+(2GUB_Q)*j}+# zz3CLogpx(ThDWg*ga#$gFZ*8cPPJAmdOnr3FieFlpd~E_+Y>8Azfo*p)?yjs6HADe z%|lhn*s_O5SU3ZWyGmX_)l02K2cwrOH!)PX0*DYmW*Zp8!3?5t2PJmt5Q!AM7NFrT zm3?|jpo%$NvFgj7GZRLbDwXQsQKeFtDV5xY-Kx=bwp6;+D%YYX$x_K~R7#}}6!APx zn~NV`JmI^|6U}P#IB0Is@sB^}wwBx{-8(00)%g?66#%S$;&!d(o^Y5oop4(gM4|=k>BA&*e$MN+@)TWe_$~Bu%ZVXS)QnLlL>qR-ouyrS~VAzJ8c-OE@Cuy5b zYC*G;cIsUNLL%*C?6foBWXBcMX6ym3&DmKekNcdRN3TJ9&@Q~IKoAev!}uMtx7j23 z9k#bS+t6y%9z&}UjJ?CzZjU>o_D;;Z%gJKo(S?k?+up;Yj&X|}^aLx~YwzRQ@wheW ziuQhtv=i;35m0x)KFIBM*@v9nnC&t9Fxu?F*n4qz#6F6zv6CmA-Q$XV4!=)1 z2CZgX!TovMKaJmMCyzBe<9yCOLr)e8_ABLVsjG5pu zpR=dz*X%Dm(4w*98GZ@tzx2L}d7gAmU|lcc_auH#;Mc(KXW7S8)VF&-!}ev&ous)j z_VdmO=LIKE7|3HA_Kb6?Xa6btihUJl^N?q;t9hzt3_mY4qt}7_Q=3B)+|AfAQUf^^#q--?A-0{w%J(WPce~Q}!7MviWf6 z^6St|BK3-8Q+WjZXDSv{cLa$MQCtAa?P@AHF9~0W9EhU^Djeq5-9r_@5B&z@awOck z2OzQ0buXo9u`HZo6(X$;y^Tf{tvc=@z?x7feK|wKWFbl_4cBeetB~s8xW(JmauIq# z!@X6kHYK+f*8Xww%Ck0>37HO+OhZsL_^F_@?~~%7Oc;tv)dDG>d2YS`MS+mj@rJs41YP zur?H^?wgM)qk)oM}a_Q6}b0jQyl_iIiO%c}&5(9(295=BBoO*d4+GwpI z$`kuSz3tu#lM!bKljo#fA0`1|Qt-oEiPc_Mvv;asF`oxu!@-VWE6+ikK&?1K#Y^q9;VS5T6f*|EaU3+*$ZK|T8B*s>zovLUO^pZmDq_Psl}yB zVJ;$k*+lH3$qGOf;lrAbdx~*d$PRjzv&(+Dw1GWQccu>*Vv#$*FMBt`F)*Hp)JjVr zNCF|TlW-vFByYNi0M?k{Ikg4xBF%T2ijEpDUZCk=1hDB?x{5pRD2l9Vswq9CYPzA) zUrJ5=bT~Dl4(sxp*G;{kW{mdEF192^)MPwZh!Z5IM2W1)3K@*?hpshznFPrD1|)-H z?WV!V@&kH2p<1IbE*u!R4oE6O4G(thgeu-ZgZ@2iuX-i0WO#F6odFBviQhp>kEY7# z?IW9)x@Dix6pr5#^`0fu7-@VfG~g>J-r+^BslK+XFB=clRkb}FsB56xWs~VmeM3=| zHmeqiFd=XyI3yN*r)Wlx!2F=%PD3Syn-&jl745v5I z22cUiY~pf?oqC|jeo?D5wZEksil1Cgt-w!6f80MqPu}AGc$!|ht}3gVsuiyq_hpkNPq*{kqtF5riBL{(0-bme;m3{)~bRBWg zp=u~n#ppBlfhv9p5RqrwK0MPPIU$K3; zr+(oTOipCPWsDdaO?a2!IPidT5hsS%Os;F1eRigFmxT%2nQtwI`4VrZ6wNQDXq5c9 zG8~}fHj#bO9e2@$E)kvJ8mdk|`41CY>;RcL;@8m9dmM#!Pct-ihiYi1YOH5;Q!{XH zsM$|4hW0=6!v!_3?NC$qwDtC(uJef4+2(l?hpCH3)+2c)d&pnKGzK2*Tj6FuO@=T^0rkI<;ZFh<@VK7 zJ&*b{Ryt~@g7ia>6bQ!*(yL%(tEnI*?$YYKy8#HcG!U4!-crPG^z9_Mnh7#%=^)8_ zxL<#a_c<1T=vN1V0rXMDd%-~5qP{)QvH#&!QfHEDV?lNqYD%Z&ce!Pj$NEiX20NC= zSN>JR3<##oCL}Wed1u-$zUuf}>iE+vlvu&{eApZ=5m2On7hflh)SD9}oM=S7hu$n# znJCG^lQMP_=}x}#ff^c>Msub8#bHT4X~}-q1mifZq}SIK>-dgH0N6j3kafQs$P{h#7)dME{8#lfIQ?eW;E4aNh@ z&3jggh9qudr3G|xz7(Vch`Us*P|-#ak5vs5JZPBj9jTAKh}Y5QxA66vD8Ns2O&x{^ z1vk-5eV7#_bKTU2Kcu>mfo#NU3@w9aCMi>@VH&E2KjRaFm8y(^T5T9&SX<9R)yk;a zy_60Pg%MKdhgp6<|3U#z+Y?)JCAsMyEwG0*k%aEBrZ3~>9aslz%0uPr`@m+DHL#k8 z#GKR%kegr^q|NIjSwVuZIf!+!E}sGeQVqpW?(1L$h^IZNct`F)_R`;5_;QR*Z=afl z-qd6RsUp4j^A3mx4hx^$)&j=02h*L5sL0_S0o|LNVTzkbi?^}NaHK@oS3+Ni6Y|WL zaG2I0Ila*tI75bk9cJT-g<5%0q6;aoVwnm?`B_}Wf`jB5$O^QY3N$;)*y&+E(&6>swgZBJTsACH8D)uvbmtGhGvDur?`e2 z=pd{a_HI;s36K5VG3m=H&Yk_*7r4{xA zBNp-i<|m;Osl4?4K!p(~SrAbo@sp-t)-n-kuryQOsfc;heT=VMEJy_8wBFvk`7CdR{*WzB_ut3Y`(+fM5`9^LJqvnxs4mFk>*%C- z1{$%ASlHSLXtxJ^XtTcYo?<7zrQcV-N+X!^*ro7aH85T>FqTsgY6(9bB>W5n@bf|9 zt18_Ae401q4+I+bP5>2B?UUr_B7YbJM(kygS}Fy3cxyyUq~Ws{YVs9v6YH0>#JJMl zyP>q&Rl9z~$2d)>wqzs23Dnk$2tsGWJke!IMrc5%LSPu^X)U+ubqSCPYQ-%oJSu!D zT2$Od5oX>X`7uq*cfu6ELiN8&#SJQ$_!0{g4i61laNkLXJ8^8(B4@89)edR;_((OO$xTfSd3+fx-FsNn6D6tILW=`s#fJ2MIJF zolT-b%q}CAr?n^fh~Qp3ey6i?9t@F{Upd1G|KTxj&X@!!-NcN@$0yP zF~@KtX!QeVTmlGFfjO=aLyY0G18j#0M7w7uOm(-`e&SKk*rN9)-&BVdUX#Gkx_|*0 z0UJ(b+}B{)>A?M+L=C2(t_71sM3E<5H-X z?XGeM!Py>g;NG6>*2222q2gAk9BRO1Q~X&19R(Eo@pb`qu_(Af&$j|{rc`FhgC!my zs7g^_LMo2}*9h-M_4e*9pxf!zgIfej9bd++FXEyfw;&I?3}xu;glTa8vEFV%Z%=|r z>qsZp%NXj1av`SLer)T7^tnPGxg7DAX;uc6%3jnhmr-oDIf40f*(L%O@!7!BkSoMo^!3&+~x3J(hH{XOjCgx%)-Hi)*7``f*9lNp& zDaKhtJNPDXF|r0J6tE#+VMel+TWmEa+dHqbl1jFvtRxvO-A5pqU!kq_N=H(UzE0Kq zQCNc)kpeY+?TzVc$dZ~m|CTsFk4T5_LkRJwQTGMx-pk`cyGOPKSs8He{j_~}^HFT2 zS+jw(2_%G;3=$~^d}u`X)vqJxkd#+nCxwrcS4e{npNE~Gb?gN2eAo$ld+Y?x4T^*& z^qQ%cuUwmp)4+;ebrE@VUEa0`Ug8^A2`Rip;gYxiIl9_Q1uZVtAlc6Aqn89{>&y{= zq~}xeoc#KM^{td%Zv9B~^axGffkFs8(Av*jTtq&RgQzALNt^+6RNCJ*WfgTnA9+kQ z^6yT}$n)C@t+U+&eTh~qX&UOy78-AN<4p@De#mD>{8pl~t-|m~6IX=DkQ?~j4TmKV zd9!m7apd{FP#LnETF97MAk^>lmL8QOaWJgd7d?}%76;Z}>=@(_O{aU6N9&5Iwd&Hv zxw7F=oq;uT9(&o6A7rJNkex~dIKo|+a3=ZtwgO&vhfTo40MfdlQ3zn!Sa@I&?xPj~ zA4mzV!a5FliIzY|kkYnN7B(6_dTK&6nmUpQ;^C)F>-i0&AlS8dTJurUMh!PjOKUxc zua}9Yg#`x`VJRT~+9RfBP&6^;52<@bYmax~dnCZw|Kix6KA}%L1viPMpSIz4+VZ$F zwo|qxD*2dR{0&UM6`VJkNlzXaED7~6$H{Z#EVdotg`+@oq4)MR2!n| zuY@MY`@C?_i!v@bAmw`Ew+S@grs8){bat7<4dD(J$6#gIOh&K-C~+u6dDuAr0N3Jo zsbJ>CN=NUgef&O5#j)?Z@o^j3t)g&oGx+Rye-&C${OMku_*eHmTP;3h~hXETEoxz71|IDlFWZZ{n0^y`hhMuwcShS*!61$tn>-ZtTwP7)h0 z|5u2WIE^Io?;;xn(!Cs;BvFP81$TZ8aBSyQCr_trL?oRjENZ06Oi*$U-u{`BuRME8+$^_rYzoE^^^Eri0 zIMH?78KX@?8au^PoV0uD##bn_b5hd9|Rk4`Wll%}0UM20-?V?~Iua=lAr(fmD}=2YYZJlL_bS9Rg{n}M%m$gE7}D67y?`UewNk843OrPtgsJc#iO;x9gwl>lo2LH=n-CD zidY~kfpne8;-)3x4GW-FJlIA^dItZwwWN#WLN*^Igfo4^$tPr); z8-DRXJoPG5#k5T%A78@!7M!y%&pfi?sPZ6)%yz=B0c z&J0v`2^0ghZUpJI6nXGhGw>j0?ju+Rmq2=XAb^`-HS1?>EigvlqY0AmR_d!cKNnzPjBXS{wGafy0gjBS9|M{t!M;tZOtFUfUjwa?sB5hn8{wXA~Gx zw3jh^z)roZ1YXhrA}^Ye*hl&EJ|vuVRp=UT8JgX;lo0 zomTsU{a88e2fkngx~e?xT_ z4!ebs6kxI&gK~>nE}1XXtE0eRFwW0O;FsbG-VPLtP}AOqkcGOYBBr4{P`{zwB7;uF zYb%9_WAUh9wRp2h97ZScJrtNun>>kZD+F8n)tSs`$}B{lYV*v0B61?l#8fmbvu>`p zpWLib(irfB((6)_r)v#JHSY{A;xXFUt_H_{tRCOAC$gz{Pvj=zJ(;XHATK@Mj|{W0 znx=_=f@Yz5w{P9zpP~A{(}D^7BMQAQq~jf}5U~futB^NH%1Zo!hQW#p&-*xTc-|o( zsS46k;PHcEg76zsiAY6(S5SG7h*txbu|XMb)-aB$RYNN8ZAyyc6SHKx8UZ{^ASKhe zEB-!)ho(D$AMr1#_(xRmG3e1|s^XtgT^~oI9TkW-5nxQkOlm)7ZU$EZet%-F!@J5# zDsjv(8AV*f)B;k+kP>Q(e~fx-FhYbF5ho6X89tIbRU2t}|A6{tVCX|7C^-#J{6nh9 z*ANCq$|l4kQa4w`H&Mxvh0vgr4NbfihwQ;nM>{b@(`3di{l~VY0{0)zN>RM7s#zo&xTl7q2#(Z0$` z6@s48>s- zEw4e^<5@~AphTFSVw2Qv-TWkzpq5!Jt7-T0S{`~qp0e$bWQ3_`4%JK=A8Tp8{$H5_ zjORU{%IXlf*f0%$2YU(gP5B;abmSd&W8xv%pTvI7;oEQBCA;j-ke)MU_72_$P4W9^ zw;!BA{2g4hcWhp%B+m-X8-G5-8d&rCMbdD+4(o^FXJ7MM^!MV%8`t$*!T%Z4MD{fdPWRv_K+lsyP z5E-$OM*d&36g$??Tl|7buMtniyJU*R8(Y70e4_{kjI@a8ofopNI^KfL&~z$v9`tmR zfjlGrE=Vo*VSPFsZ)H3YLpepn%*NCH*mwfVh4IvJ_n?t7i0=TtY&4Ox5k?c-qDgd`9hJp%0sAqf!w5g#v1z$7sPBaYgmCsK%!)}+G-;iM-&y_R{XP);MB z4FvjWrG2k6)@v9mD_%n`WsZ`GIYlbFng*NDN?AX@Hi+Cv7+-bGCKIf%Ht18%OP7Pm z)*m4I5jl(*aU(#vpw+ctU_s>^$KmB|Fc^>v%Hb2N28zA0nrj~ka^9~7skM=ZWGc}r z^H@>d-;UkA6XfB@$}f+u=7Kz#0zpoANHNcO-^BXIsDv4%$NN>4dI8UVo1cBV|Jfad z{?HpcUmK@hGOMBc9+{M8d#Wo9CFW_@-i#Q%MOUZ8>oHIWpINvtLHNu_TC1=zT;&e)K-8%3JpH7b6M zia$lgZ7Ti^6)qLuLJ{iCMss4YOD+7=y^2-p_@7Z&samr%zv4Tdm5be3d>ba0Dj3}Fi0P;c9kUxkxmdq-R0*8j+aN$9WX60t* zrVw(xdhN<97cWiAGy57I^uaC6anP2QO8+O~dw?K^=_$Sng>QYUXn#+SM{Y{|A!eJ< z+vBqc+I-ekXJ_ZGT>bfr&Qg3QUi0r^&Ek(R*;etd?uHY_%uEy@N#d$=qsO&oSEVN2 z!jLl)FNb^<^r7B7~ny0J%R1k8=I5|b?<;ng9x>}&(C>2cb!*umms2HK*Jt|(If-C`X zfC`G7GhK?`q^mU)6!gKrXArPKe~aq=Iu%U6f0M2#f-D}Rf-TMrU6JA`j!}`If-FC_ zFU6B|MY2nfekzVr@gJ!mnI;~mg5;ne5*7cAia(}egMj(!1*$uRg6_QEg9J<|hB`c^ z9RZ~oBWgaC+LJn)%H=Z}a=$>W>mcP%ke~cBP0#*J8>hPL`fysyqHQ6s)8DY3Pp6V8 zLrY@}yvDWh)sm9l%@275L;IFkHFgRlzYlemZMhp{r|k4&uA#A?kg`kEQ-N11`_pJ+$(@t)8M z{Oc#$yj-dI(N^p1nwQrI&eYXEULi*KIFrhLqz!DbIv8i>hg+?UdVym~7(CGpfegWX zi(TZ?;HvAH4>Q;DCZFWQW$BL~i6S#CGxXLG{0R8Z8Im}3Jc5gEh5p5mM0rLoGVUbd zFiJ7%F%mMGF?RkJYH}8ZHGmf;C!_ZzMUJY8+AM=2H;jW;frB{n8y+?Gp-2bI06&xY zMS_Wm>6(bi+Ro1;kDbKJeI*3AY914~wY%pTy~ka`d{LS|3uJ3ocay&V3L-|Pzy!@4J{^sI+Y^WI*peHOkt#{Mhc~Jr=HV?8N2q3q zJ5tIc9FaO3M*`jzCe} zgT4)a6<%W`a8AaKA5o&W%bWb}s+Hl^65b~vUi%#9Qe9u0XV@$N0U^6x0Cx!LuS9|^WGIRq+ff`xs!ZEVXi6a{E83=AgE&hP6c&Kb zE<^zXI*z5Jmx+_M(@8H$T585K9d+8im)<$wH_&BRv}<>%BG|1;6JipO786-6jQZ7O#3tJ*5JHCy9f z-PZA|H4QIeCsdlFH%%{TC%u%N^3rzN%h(z2SDRDz6tBtJS^OH!oR_!r-n2dK73>1f zNi=7?qFwZ6?OCs6m%KyvA#cu}<1w>&*gIk$@#gJ$@2Gv$J7ypAj@!q*6ZQ%3qX55$7O;HrH4-NZep;nc0=@*3Q<+2*)>+M$H-VeN5tG40F zvJs{h16S69whWD>h988f8=auhZq=G$;#LclvK~%f^LwrOLTjVZa<9n^KTIzzy&W~f ze6{9vnvK@R&6=!vemN25Uw7+mfu8H_)_P+j%q+E6uEq7Kx7)&P#*I8JX*Ngg>1ut` z72T#QqY33)IJ2<)=Hl{#^ZLTI*PVrxl^ZKzcJbzQ=k;r=*PMlKu7-JPy|r*{Y4t5` z=cv87eDl`o-tdiEtHbup%EC8pEmT*XsITg*hNX7^jI)Ae)mzuEFI1~eb#*12M#r_a zwhUI>_qr~j1sw}g0!=fh-IVRR>-)n&>Z%4c8FV|tuHtx?-)XmeH>AE>Si0V5y74@a zq%bf+lAI2ih--n@Bh;l5k%f-cl-v>JiaY;O?B)G!w>zR?Xj-2ffgTRx}>tPyD& zh36EdP6QwSOQUfWkN*n9%T-9EAWkYoNfY`{RAIP=NVo}M;z^3sdP<~4=AmhuZc-%O z)Ve07ME0QpHcY!2k#na+-p$e~u-ddJ@JLR~h~h)V&Wl-5!tb; z>!dg*p23_`;#u(=el2m{J&n<_xPZ|!;&b96de4gI`TP}e3BBjU=XuU$@ddP=5ijuA zi{guDJu6-kFJr}XINvMcOXxX|+7 zix{#4kH6vKT(!WpfEBj|4dGfb&Ikc_8}++DL3@4O+TL_qI5v*K+dU<1G(A46u-1Bv zu0{)j;P`E966`yQ?>5&fF6%Bx5a5TBso(XVTE^rWhy5bIz9t0E^$1TynGq$e#yWLK z*Q(Le>Rq(zK*?}tXjLdt-s34IsQ7`~IbVMFVuW9}6}`inkZYeQXSMISl6Rq(M>H4# z%KeM$L{6thIyxQ!k$lT{gNuA(=#56*0vopL4G=(B+l^ooD6El^9^rXJ@^x?puZz$* zIwDOLwV*acL_CZ)b+ET9B6JbQ5gLf=hBKQ@&xt5-5!7weH^&e)JRA*5;OlZ6$wqIzY7=cZvTJQkcG*PA1q7_5f#on1++W^ zvk{uo4Z5Q1|AA>8|Y)w6)by@UecXI>v$4Zd@vGP;xk*Z?m zXx22(Dxjwj&zkwnSw+k$#f=)3P9Ns#Dx${C5`bX4%G&sC_j8Awx zI3bK#T|-J{uI zV&NV{B|J1X(+}Dm$E9w0oL+l$ZxfdLd$p$H!&Qay4UM4TxnaS9u@0PC6K2rip3-;+ z4&!gOo1&Zy4YDfoEP+Jih(yF784S%x7Ui?(f2_#s`1#G^rNHZ4>NGmfN2GuLWv{#G zUG}ywH5+S}I=x`C-MVxSYVuM<=u7)tinzPd>4hoBVQ0{xjs2HU7->_}RO5-JWj-}^ zQ!`aVGoNJiqFz#mPhL0ml&-0%CmAiHXSAZxFO4}emEop!lIr0f)ipf)rlY7pA`oqL zRhbDOnOn*O#1?`20NnmqT`p_#i1_9^7Uc~n#D z&yAmEZ(2ANTW&^eu1jvN1Co)STZi*C`Xfz5qQYl=2Ng)J)>J%o$AGpl9zb0U)Lre7 zvXcn(U1N}V1Wh%7#6l}2wh~(=oM3H`AT9K1s%1;LERJ?lPmaoUyaSGL?8E`E?B0CHQ7I-AEyD2*ygpp24dFXKa7p&UzG% z5ke8D*@yO2e=N~t8pCAJc0{8d*d@{M$@8Q@WT;qtduF_s4+;6u>rG!S5?plP2tqzz zE~aPnO1VU%`?#b)m>dE?bpH+}`6p3i6Q-KdGMWK?2bY7}wNEs|L|s!ezwRF&BS&Tb zCKH@ZTus>s=9Yg)rC;k`>7NXgT~(-&eIZ^^748{Y9PrJkPmgCMsOPrQnm3fd?0rd9 zc3_?QIjl2|Co=+H`ke$+owjBrRu_x2Bsh8Eayj#nJ5 zZgzu!{lND21ovaNG32xNiK&*rv`Tp@G{G?lUBiTM*SZ^F0g(yuGsVm7lglrG24RA` z!f6_f*@U%ydf|TE<)lQIc1Kl1x*Y`LUw3jF^GFn#6Jjtm2 znK4M=$ODQA-XdYw@i1>$sF0@IRf&42XhSXA^HfU;q$y;(C(CHbH>ubM_8s*79iB+<>UjX& zpB)D#M%)BABr23UsMEqXP<$r_C;%v1VAmbBUxaE+063FbRecAb-X`Ul9D+)0JKhFk z>dOq0{1&z>rx*lTL0x{1ic2VLa|Oweg%x?3dR{~kChE<$??Po>r7oU631e6wuiRk= zBl2ec2{ZjH3hjde3}6;OpVTH*#r`D|@RL!Q!o%dG%^-@(BjtySfcq3+g2EtjeoNcY zcJx5o)dvw)jDZortlWiq2WFDHDK`jXOb>*=1mVy0kdE0&LS>`|N$Q27Fm{sf zC!r|#mlOrwGBwb!cXj{_Bfmj!)CkAIFcd?1phGbjRpqkc-`oac(Z4%*sIQ#pzgUeV zEsFjrDM3LCOEE=rY`0=DfBrS=db0tKsq)5SH8kq&P7ii#I*_$G=0hk+{fp&tPQD9F zgn0^loXA&!7=&gAQGC-46J+S%39WV4*CCYim}^fjBC)ox^3KAFvw~#a9cELz*zlcw zyG!yko#8xZfLo}BsRB%?-E4-%n>Uu07MI_2;)JL3b`|OQ zHv1K%`h0m4ljS#PMPU;uDEJ&W0~}vuc}6ZFPAM0Ueh>V7?XCQR3eT^e8Ngi`K$rMCvgB`<6wTEAC;eI0sbCf*U zGs0P*8;)cYw8_I*D!a5^kK~f%3_^&^3#RNxdVs;7l@2lwH2>n)ln>QliU=fd9mA`jRwRBJtW^ibrJwQ6*rzk(<5NjL~KLd@Q zBnOa6Fwk7@`(PjhIDInnaxLG(JSnJfsaU6i(ivf<;UmaF zGNJCuBh;Zc-4+Z5UlvJnEW#-m^{$9ahdl#Nu@U%AoRn)zKRoQjDj@C~suVd#s}b=> z3Zh20HKVa(7_)QL)oUxO&drq@Z!9h?ggKl8Q?LzPz=T;&5DZ(HcWTY9yTCU*MgZv8 z1j!wG$8@A$x-B@$&0w>v7EmG%K?*Ay)WPE%*!0O~f%k%B9lSU^2xYd?crtjV@Zjd>BX}*Cm4mG0 zcc(=Hy~?A=D?y+P9z_WrE%ZdpqbG-NM+i=7z{CLVFO=Nad+DvaGm+%5FNE;@eS>y_ z>KpR0*?&3a`;$2rEg-jIgGR zEaenVI^0QqpT_7Q@*^sKK*bNK_!BD5P(d~^G#V{fWcH476C7VQBkfDP7`^y?s*y6b zA%r7!O&k#^WlG7&hbX2Y<^HRvm=s$>3#V8QYv4oVmB#*=zc$k)xJ7wAW9;ecqdN5U zfA!Cf>+7LVl1Lz}J_K`uM~PnzJj|mPPyzK3D1nbg0k@nDj9nN+?gMX89Kp#|ZMi?o z*R%U@ZZc}+pJBHM2_wwOKSkd#a$qFw!^C$n{sLiwJpMd#3Pt49`lk6o6^!dgs5D@1Et*4~Pn0jAS!B|-_*Q_BMd1?%Jdq#8ITQAr zW-tb&6gkJlJXG7ux6n>|8J9BHrWCtPlv#j}rhklsOMIN7$k(X&DvB_> z(nWybMW!K=+$>)q5;|ZF!r1$aZNr|~kmLgH_kRdAR|(x@3^UOFCB3L>i3}_Pc|k|W z_xW@(8@h+~Fd4hJ6LLiSHj<z=pTo6`VrRMA*jggrnC|%j`4h03{%V@VS-)f&!*wUJ@uU{V1xRw9)vW;!*Q;-pKYz0Wri9z9aDH5s*7l#a2 z4BJafxCiF1Eb#FLV;m4t5ZuKjKE>&{U&nvC8>`vdwNJDmIp zy?{?dL-MCo9E407y^n*)zd>b^isSJMk{-`s!gBcn-4M1AmN*W)00$_*2X&}t9p}An ztr?Fb9Y?h5=+==l#*J>f-INPhAHt(;BlZb;h??=zx2O*R8b0vw@Zm#S$Ru@JRNSDV zN(CjBB@tT^899S%n^l%c`7ZTPnoX`!LGmR}Q4vtV0!b3WjwnZT^1IZ0g$gp3vP1<- zFR59{CYr>RLo-Rd#x!LNkBK!4`0F%!8pU&X{BNR2DR5j*nZHG{0T-s)r--(G3-kNm z+RFu1Gs)paznXoLDy3$$B)P)bCz*8WP;oAm&!5bnOC?gN{A|k1Co~iH22b*4Dv{Ci zCckChjo0W?3A~p-lStr~b;66N2;}^P{?I_ySlI>3!%NgDexkhk0D6$F9T6QNkG#kD zxXxiR*J;P$ADDx$X>a3~o!0>Rh4KgF?u_57S_oadMvFcHg7zS6Ai1kLV@M->g?YcR zfiIV07hZmd{aKPzTK^dm8xObYoB*u6+HB*C1OK(khEvqT?u?(z&nS@ z>f)P7?7tl$ZNx_itM{#YOxu*HU~-OV8s;Vr@@+aIX)92|pv!PdBxQB^5A@nsA{+Pj zw989Wkn##M@g*|u-Pe|1WI04zO8&aV 1 + + @contextmanager + def get_data_socket(self) -> Iterator[Socket]: + socket = self.context.socket(zmq.constants.DEALER) + try: + socket.connect(self.data_ipc_path) + yield socket + finally: + socket.close(linger=0) + + async def run_heartbeat_loop(self, timeout: int): + """Background loop that continually listens to the RPCServer for + heartbeats. + """ + try: + while True: + if await self.heartbeat_socket.poll(timeout=timeout) == 0: + # No heartbeat was received. Set error and exit the loop + self._set_errored( + TimeoutError("No heartbeat received " + "from MQLLMEngine")) + logger.debug("Shutting down MQLLMEngineClient check " + "health loop due to timeout") + break + + else: + # Heartbeat received- check the message + await self._check_success( + error_message="Heartbeat failed.", + socket=self.heartbeat_socket) + + logger.debug("Heartbeat successful.") + + except asyncio.CancelledError: + logger.debug("Shutting down MQLLMEngineClient check health loop.") + + except Exception as e: + self._set_errored(e) + + async def run_output_handler_loop(self): + """Get RequestOutputs from Engine and stream to Request Queues""" + + try: + while True: + # Poll, checking for ENGINE_DEAD + while await self.output_socket.poll(timeout=VLLM_RPC_TIMEOUT + ) == 0: + logger.debug("Waiting for output from MQLLMEngine.") + + # If errored, alert all running requests. + if self.errored: + for queue_j in tuple(self.output_queues.values()): + queue_j.put_nowait( + ENGINE_DEAD_ERROR(self._errored_with)) + return + + message: Frame = await self.output_socket.recv(copy=False) + request_outputs = pickle.loads(message.buffer) + + is_error = isinstance(request_outputs, + (BaseException, RPCError)) + if is_error: + if isinstance(request_outputs, RPCError): + rpc_error: RPCError = request_outputs + request_id = rpc_error.request_id + exception = rpc_error.exception + is_engine_errored = rpc_error.is_engine_errored + else: + # MPLLMEngine should always return an RPCError to + # the output_socket when an issue arises. + # If we are here, we are in a bad state and + # should shut down the server. + error: BaseException = request_outputs + logger.error( + "Received Exception %s rather than RPCError from " + "MPLLMEngine. This should never happen.", error) + request_id = None + exception = error + is_engine_errored = True + + # Set to error state only on engine critical error + # (and record only the first one) + if is_engine_errored and not self._errored_with: + self._errored_with = exception + + if request_id is None: + for queue_i in tuple(self.output_queues.values()): + queue_i.put_nowait(exception) + else: + queue = self.output_queues.get(request_id) + if queue is not None: + queue.put_nowait(exception) + else: + # Put each output into the appropriate steam. + for request_output in request_outputs: + queue = self.output_queues.get( + request_output.request_id) + if queue is not None: + queue.put_nowait(request_output) + + except asyncio.CancelledError: + logger.debug("Shutting down MQLLMEngineClient output handler.") + + async def setup(self): + """Setup the client before it starts sending server requests.""" + + with self.get_data_socket() as socket: + # Wait until server is ready. + response = await self._wait_for_server_rpc(socket) + + self.tracing_flag = response.tracing_enabled + + # Start health_loop. + self.health_loop = asyncio.create_task( + self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT)) + + def close(self): + """Destroy the ZeroMQ Context.""" + # Close all sockets and terminate the context. + self.context.destroy(linger=0) + + # Cancel background tasks. + if self.health_loop is not None: + self.health_loop.cancel() + self.output_loop.cancel() + + def _set_errored(self, e: BaseException): + logger.exception(repr(e)) + if self._errored_with is None: + self._errored_with = e + + @staticmethod + async def _send_get_data_rpc_request(request: RPCStartupRequest, + expected_type: Any, + error_message: str, + socket: Socket) -> Any: + """Send an RPC request that is expecting data back.""" + + # Ping RPCServer with a request. + await socket.send_multipart((pickle.dumps(request), ), copy=False) + + # Make sure the server responds in time. + if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0: + raise TimeoutError("RPCServer didn't reply within " + f"{VLLM_RPC_TIMEOUT} ms") + + # Await the data from the Server. + frame = await socket.recv(copy=False) + data = pickle.loads(frame.buffer) + + if isinstance(data, BaseException): + raise data + elif not isinstance(data, expected_type): + raise ValueError(error_message) + + return data + + @staticmethod + async def _send_one_way_rpc_request(request: RPC_REQUEST_T, + socket: Socket): + """Send one-way RPC request to trigger an action.""" + + if socket.closed: + raise MQClientClosedError() + + await socket.send_multipart((pickle.dumps(request), )) + + async def _await_ack(self, error_message: str, socket: Socket): + """Await acknowledgement that a request succeeded.""" + + if socket.closed: + raise MQClientClosedError() + + if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0: + raise TimeoutError("MQLLMEngine didn't reply within " + f"{VLLM_RPC_TIMEOUT}ms") + + await self._check_success(error_message, socket) + + @staticmethod + async def _check_success(error_message: str, socket: Socket): + """Confirm that socket has a VLLM_RPC_SUCCESS_STR message""" + + if socket.closed: + raise MQClientClosedError() + + frame = await socket.recv(copy=False) + response = pickle.loads(frame.buffer) + + # Raise error if unsuccessful + if isinstance(response, BaseException): + raise response + elif (not isinstance(response, str) + or response != VLLM_RPC_SUCCESS_STR): + raise ValueError(error_message) + + async def get_tokenizer(self, lora_request: LoRARequest): + return await self.tokenizer.get_lora_tokenizer_async(lora_request) + + async def get_decoding_config(self) -> DecodingConfig: + return self.decoding_config + + async def get_model_config(self) -> ModelConfig: + return self.model_config + + async def is_tracing_enabled(self) -> bool: + return self.tracing_flag + + async def _wait_for_server_rpc(self, socket: Socket) -> RPCStartupResponse: + """Wait for the RPCServer to start up.""" + + return await self._send_get_data_rpc_request( + request=RPCStartupRequest.IS_SERVER_READY, + expected_type=RPCStartupResponse, + error_message="Unable to start RPC Server", + socket=socket) + + async def abort(self, request_id: str): + """Send an ABORT_REQUEST signal to the RPC Server""" + + with suppress(MQClientClosedError): + await self._send_one_way_rpc_request( + request=RPCAbortRequest(request_id), socket=self.input_socket) + + async def do_log_stats(self): + """Ignore do_log_stats (handled on MQLLMEngine polling)""" + pass + + async def check_health(self): + """ + The check health loop probes the health status of the + Engine's health every N seconds and sets _errored_with + if the engine is unhealthy. + """ + if self._errored_with is not None: + raise self._errored_with + + @property + def is_running(self) -> bool: + return not self.errored + + @property + def is_stopped(self) -> bool: + return self.errored + + @property + def errored(self) -> bool: + return self._errored_with is not None + + @property + def dead_error(self) -> BaseException: + return ENGINE_DEAD_ERROR(self._errored_with) + + @overload # DEPRECATED + def generate( + self, + *, + inputs: PromptType, + sampling_params: SamplingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + ... + + @overload + def generate( + self, + prompt: PromptType, + sampling_params: SamplingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + def generate( + self, + prompt: Optional[PromptType] = None, + sampling_params: Optional[SamplingParams] = None, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None # DEPRECATED + ) -> AsyncGenerator[RequestOutput, None]: + """Generate outputs for a request. + + Generate outputs for a request. This method is a coroutine. It adds the + request into the waiting queue of the LLMEngine and streams the outputs + from the LLMEngine to the caller. + + Args: + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType` + for more details about the format of each input. + sampling_params: The sampling parameters of the request. + request_id: The unique id of the request. + lora_request: LoRA request to use for generation, if any. + trace_headers: OpenTelemetry trace headers. + prompt_adapter_request: Prompt Adapter request to use + for generation, if any. + priority: Priority of the request (lower means earlier handling). + Any priority other than 0 will lead to an error if the + scheduling policy is not "priority". + """ + if inputs is not None: + prompt = inputs + assert (prompt is not None and sampling_params is not None + and request_id is not None) + + return self._process_request(prompt, sampling_params, request_id, + lora_request, trace_headers, + prompt_adapter_request, priority) + + async def beam_search( + self, + prompt: Union[PromptType, List[int]], + request_id: str, + params: BeamSearchParams, + ) -> AsyncGenerator[RequestOutput, None]: + + beam_width = params.beam_width + max_tokens = params.max_tokens + ignore_eos = params.ignore_eos + temperature = params.temperature + length_penalty = params.length_penalty + + tokenizer = await self.get_tokenizer(lora_request=None) + tokenizedPrompt = prompt if isinstance( + prompt, list) else tokenizer.encode(prompt) + tokenizedLength = len(tokenizedPrompt) + + sort_beams_key = create_sort_beams_key_function( + tokenizer.eos_token_id, length_penalty) + + beam_search_params = SamplingParams(logprobs=2 * beam_width, + max_tokens=1, + temperature=temperature) + all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)] + completed = [] + + for _ in range(max_tokens): + prompts_batch = [ + TokensPrompt(prompt_token_ids=beam.tokens) + for beam in all_beams + ] + + tasks = [] + + request_id = f"beam_search-{random_uuid()}" + for i, individual_prompt in enumerate(prompts_batch): + request_id_item = f"{request_id}-{i}" + task = asyncio.create_task( + collect_from_async_generator( + self.generate(individual_prompt, beam_search_params, + request_id_item))) + tasks.append(task) + + output = await asyncio.gather(*tasks) + + output = [x[0] for x in output] + + logger.info(output) + + new_beams = [] + for i, current_beam in enumerate(all_beams): + result = output[i] + + if result.outputs[0].logprobs is not None: + logprobs = result.outputs[0].logprobs[0] + for token_id, logprob_obj in logprobs.items(): + new_beam = BeamSearchSequence( + tokens=current_beam.tokens + [token_id], + cum_logprob=current_beam.cum_logprob + + logprob_obj.logprob) + + if token_id == tokenizer.eos_token_id and \ + not ignore_eos: + completed.append(new_beam) + else: + new_beams.append(new_beam) + + sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True) + all_beams = sorted_beams[:beam_width] + + completed.extend(all_beams) + sorted_completed = sorted(completed, key=sort_beams_key, reverse=True) + best_beams = sorted_completed[:beam_width] + + for beam in best_beams: + beam.text = tokenizer.decode(beam.tokens[tokenizedLength:]) + + beam_search_output = RequestOutput( + request_id=request_id, + prompt=prompt, + outputs=[ + CompletionOutput( + text=beam.text, + cumulative_logprob=beam.cum_logprob, + token_ids=beam.tokens, + index=i, + logprobs=beam.cum_logprob, + ) for (i, beam) in enumerate(best_beams) + ], + finished=True, + prompt_token_ids=tokenizedPrompt, + prompt_logprobs=None) + + logger.info(beam_search_output) + + yield beam_search_output + + @overload # DEPRECATED + def encode( + self, + *, + inputs: PromptType, + pooling_params: PoolingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + ) -> AsyncGenerator[EmbeddingRequestOutput, None]: + ... + + @overload + def encode( + self, + prompt: PromptType, + pooling_params: PoolingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + ) -> AsyncGenerator[EmbeddingRequestOutput, None]: + ... + + @deprecate_kwargs( + "inputs", + additional_message="Please use the 'prompt' parameter instead.", + ) + def encode( + self, + prompt: Optional[PromptType] = None, + pooling_params: Optional[PoolingParams] = None, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + *, + inputs: Optional[PromptType] = None # DEPRECATED + ) -> AsyncGenerator[EmbeddingRequestOutput, None]: + """Generate outputs for a request from an embedding model. + + Generate outputs for a request. This method is a coroutine. It adds the + request into the waiting queue of the LLMEngine and streams the outputs + from the LLMEngine to the caller. + + Args: + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType` + for more details about the format of each input. + pooling_params: The pooling parameters of the request. + request_id: The unique id of the request. + lora_request: LoRA request to use for generation, if any. + trace_headers: OpenTelemetry trace headers. + + Yields: + The output `EmbeddingRequestOutput` objects from the LLMEngine + for the request. + """ + if inputs is not None: + prompt = inputs + assert (prompt is not None and pooling_params is not None + and request_id is not None) + + return self._process_request(prompt, pooling_params, request_id, + lora_request, trace_headers, None, + priority) + + async def _process_request( + self, + prompt: PromptType, + params: Union[SamplingParams, PoolingParams], + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[ + EmbeddingRequestOutput, None]]: + """Send an RPCGenerateRequest to the RPCServer and stream responses.""" + + # If already dead, error out. + if self._errored_with is not None: + raise ENGINE_DEAD_ERROR(self._errored_with) + + # Constructing guided decoding logits processors is expensive, so we do + # it here to avoid contending with cpu resources and the GIL on the + # backend process. + if isinstance(params, SamplingParams) and \ + params.guided_decoding is not None: + params = await \ + build_guided_decoding_logits_processor_async( + sampling_params=params, + tokenizer=await self.get_tokenizer(lora_request), + default_guided_backend=self.decoding_config.guided_decoding_backend + ) + + # 1) Create output queue for this requests. + queue: asyncio.Queue[Union[RequestOutput, + BaseException]] = asyncio.Queue() + self.output_queues[request_id] = queue + + try: + # 2) Detach logits processors so that they can be pickled + # separately (may require cloudpickle which is slower) + if isinstance(params, SamplingParams) and params.logits_processors: + # Defensive shallow copy + params = copy.copy(params) + logits_processors = params.logits_processors + params.logits_processors = None + lp_bytes = cloudpickle.dumps(logits_processors) + else: + lp_bytes = None + + request_bytes = pickle.dumps( + RPCProcessRequest( + prompt=prompt, + params=params, + request_id=request_id, + lora_request=lora_request, + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=priority, + )) + + # 3) Send the RPCGenerateRequest to the MQLLMEngine. + parts = (request_bytes, + lp_bytes) if lp_bytes else (request_bytes, ) + await self.input_socket.send_multipart(parts, copy=False) + + # 4) Stream the RequestOutputs from the output queue. Note + # that the output_loop pushes RequestOutput objects to this + # queue after pulling them from the zmq socket. + finished = False + try: + while not finished: + request_output = await queue.get() + + if isinstance(request_output, BaseException): + raise request_output + + finished = request_output.finished + yield request_output + finally: + # Request was canceled by the client. + if not finished and not self.errored: + await self.abort(request_id) + finally: + self.output_queues.pop(request_id) + + async def start_profile(self) -> None: + """Start profiling the engine""" + + await self._send_one_way_rpc_request( + request=RPCUProfileRequest.START_PROFILE, socket=self.input_socket) + + async def stop_profile(self) -> None: + """Stop profiling the engine""" + + await self._send_one_way_rpc_request( + request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket) diff --git a/vllm/engine/multiprocessing/engine.py b/vllm/engine/multiprocessing/engine.py new file mode 100644 index 00000000..2bf0ce83 --- /dev/null +++ b/vllm/engine/multiprocessing/engine.py @@ -0,0 +1,395 @@ +import pickle +import signal +import threading +import time +from contextlib import contextmanager +from typing import Iterator, List, Optional, Union + +import cloudpickle +import zmq + +from vllm import AsyncEngineArgs, LLMEngine, SamplingParams +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig, + ParallelConfig, SchedulerConfig) +# yapf conflicts with isort for this block +# yapf: disable +from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT, + IPC_HEALTH_EXT, IPC_INPUT_EXT, + IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T, + VLLM_RPC_SUCCESS_STR, RPCAbortRequest, + RPCError, RPCProcessRequest, + RPCStartupRequest, RPCStartupResponse, + RPCUProfileRequest) +# yapf: enable +from vllm.envs import VLLM_RPC_TIMEOUT +from vllm.executor.gpu_executor import GPUExecutor +from vllm.logger import init_logger +from vllm.outputs import RequestOutput +from vllm.usage.usage_lib import UsageContext + +CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig, + SchedulerConfig, LoRAConfig] + +logger = init_logger(__name__) + +POLLING_TIMEOUT_MS = 10000 +HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), ) + + +class MQLLMEngine: + """A multiprocessing wrapper for :class:`LLMEngine`. + + This class is used to wrap the :class:`LLMEngine` class to enable use + in concurrnet manner. It runs a background loop and uses zeromq to + receive new requests and stream outputs incrementally via ipc. + + The :class:`LLMEngine` generate or encode process is kicked off when a new + RPCProcessRequest is received by the input_socket. + + The self.engine_loop checks the input_socket for new requests, + adds them to the LLMEngine if there are any, calls the internal + :class:`LLMEngine.step()`, and sends the RequestOutputs back over + the output_socket. + + If use_async_sockets is set, the logic associated with reading new + requests from the socket and sending data to the socket is passed + as a callback to the llm_engine, which calls the logic asynchronously + such that the IPC can be overlapped with the GPU. + + Args: + ipc_path: Base path for zeromq interprocess messaging + use_async_sockets: Whether to make send/recv async with GPU + log_requests: Whether to log the requests. + *args: Arguments for :class:`LLMEngine`. + **kwargs: Arguments for :class:`LLMEngine`. + """ + + def __init__(self, + ipc_path: str, + use_async_sockets: bool, + *args, + log_requests: bool = True, + **kwargs) -> None: + # For MQLLMEngine, we can use cached outputs, since each new request + # output is immediately pickled and send over the socket, which frees + # the python object to be reused again. + use_cached_outputs = True + + self.engine = LLMEngine(*args, + **kwargs, + use_cached_outputs=use_cached_outputs) + self.log_requests = log_requests + + self.use_async_sockets = use_async_sockets + if self.use_async_sockets: + self.engine.process_request_outputs_callback = \ + self._async_socket_engine_callback + + self.ctx = zmq.Context() # type: ignore[attr-defined] + + # Receive input from the client. + self.input_socket = self.ctx.socket(zmq.constants.PULL) + self.input_socket.bind(f"{ipc_path}{IPC_INPUT_EXT}") + + # Send output stream back to client. + self.output_socket = self.ctx.socket(zmq.constants.PUSH) + self.output_socket.bind(f"{ipc_path}{IPC_OUTPUT_EXT}") + + # Send heartbeats back to client. + self.heartbeat_socket = self.ctx.socket(zmq.constants.PUSH) + self.heartbeat_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}") + + # IPC path for the data socket. + self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}" + + # Error state. + self._errored_with: Optional[BaseException] = None + + # Heartbeat thread + self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop, + daemon=True) + self._heartbeat_stop_event = threading.Event() + # The heartbeat needs to be faster than what the client will wait for + # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds + self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0 + + self._last_alive_time = time.time() + # The heartbeats can tolerate a long period of the engine chugging + # away at a generation request. + # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds + self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0 + + @property + def dead_error(self) -> BaseException: + if self._errored_with is not None: + return ENGINE_DEAD_ERROR(self._errored_with) + else: + return ENGINE_DEAD_ERROR() + + @classmethod + def from_engine_args(cls, engine_args: AsyncEngineArgs, + usage_context: UsageContext, ipc_path: str): + """Creates an MQLLMEngine from the engine arguments.""" + # Setup plugins for each process + from vllm.plugins import load_general_plugins + load_general_plugins() + + engine_config = engine_args.create_engine_config() + + executor_class = LLMEngine._get_executor_cls(engine_config) + + return cls( + ipc_path=ipc_path, + use_async_sockets=engine_config.model_config.use_async_output_proc, + **engine_config.to_dict(), + executor_class=executor_class, + log_requests=not engine_args.disable_log_requests, + log_stats=not engine_args.disable_log_stats, + usage_context=usage_context) + + def start(self): + try: + try: + logger.debug("Starting Startup Loop.") + self.run_startup_loop() + logger.debug("Starting heartbeat thread") + self.heartbeat_thread.start() + logger.debug("Starting Engine Loop.") + self.run_engine_loop() + except Exception as e: + logger.exception(repr(e)) + except KeyboardInterrupt: + logger.debug("Shutting down MQLLMEngine.") + finally: + logger.debug("MQLLMEngine is shut down.") + self.cleanup() + + def cleanup(self): + """Cleanup zeromq state on shutdown.""" + # Closes all sockets and destroys context. + self._heartbeat_stop_event.set() + self.ctx.destroy(linger=0) + del self.engine + + @contextmanager + def make_data_socket( + self) -> Iterator[zmq.Socket]: # type: ignore[name-defined] + socket = self.ctx.socket(zmq.constants.ROUTER) + try: + socket.bind(self.data_ipc_path) + yield socket + finally: + socket.close(linger=0) + + def run_startup_loop(self) -> None: + """Startup loop for sending data from Engine -> Client.""" + + with self.make_data_socket() as socket: + response: Union[RPCStartupResponse, BaseException] + try: + identity, message = socket.recv_multipart(copy=False) + request: RPCStartupRequest = pickle.loads(message.buffer) + + # Handle the query from the Client. + if request == RPCStartupRequest.IS_SERVER_READY: + tracing_enabled = self.engine.is_tracing_enabled() + response = RPCStartupResponse( + tracing_enabled=tracing_enabled) + + except Exception as e: + response = e + + socket.send_multipart((identity, pickle.dumps(response)), + copy=False) + + def run_engine_loop(self): + """Core busy loop of the LLMEngine.""" + + while True: + self._alive() + if not self.engine.has_unfinished_requests(): + # Poll until there is work to do. + while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0: + self._alive() + self.engine.do_log_stats() + logger.debug("Waiting for new requests in engine loop.") + + # Handle any input from the client. + self.handle_new_input() + + # Engine step. + request_outputs = self.engine_step() + + # Send request outputs (if async, done in engine_step callback). + if not self.use_async_sockets: + self._send_outputs(request_outputs) + + def engine_step(self) -> List[RequestOutput]: + """Engine step wrapper with error handling.""" + try: + return self.engine.step() + except SystemExit: + raise + except BaseException as e: + self._set_errored(e) + rpc_err = RPCError(request_id=None, + is_engine_errored=True, + exception=e) + self._send_outputs(rpc_err) + raise e + + def handle_new_input(self): + """Handle new input from the socket""" + try: + while self.input_socket.poll(timeout=0) != 0: + frames = self.input_socket.recv_multipart(copy=False) + request = pickle.loads(frames[0].buffer) + + if isinstance(request, RPCProcessRequest): + if len(frames) > 1: + # Use cloudpickle for logits processors + assert isinstance(request.params, SamplingParams) + lprocs = cloudpickle.loads(frames[1].buffer) + request.params.logits_processors = lprocs + self._handle_process_request(request) + elif isinstance(request, RPCAbortRequest): + self._handle_abort_request(request) + elif isinstance(request, RPCUProfileRequest): + if request == RPCUProfileRequest.START_PROFILE: + self.start_profile() + else: + self.stop_profile() + else: + raise ValueError("Unknown RPCRequest Type: " + f"{type(request)}") + + except Exception as e: + self._set_errored(e) + self._send_unhealthy(e) + raise e + + def _handle_process_request(self, request: RPCProcessRequest): + """Handle RPCProcessRequest by adding it to the LLMEngine.""" + request_id = request.request_id + + if self._errored_with is not None: + rpc_err = RPCError(request_id=request_id, + is_engine_errored=True, + exception=ENGINE_DEAD_ERROR(self._errored_with)) + self._send_outputs(rpc_err) + + try: + self.engine.add_request( + request_id=request_id, + prompt=request.prompt, + params=request.params, + lora_request=request.lora_request, + trace_headers=request.trace_headers, + prompt_adapter_request=request.prompt_adapter_request, + priority=request.priority) + + if self.log_requests: + logger.info("Added request %s.", request.request_id) + + except Exception as e: + # We do not set self._errored = True here, since the error + # is due to an issue adding this request to the engine, + # rather than an issue with the engine itself. + is_errored = self._errored_with is not None + rpc_err = RPCError(request_id=request_id, + is_engine_errored=is_errored, + exception=e) + self._send_outputs(rpc_err) + + # Remove request from the engine. + self.engine.abort_request(request_id) + + def _handle_abort_request(self, request: RPCAbortRequest): + self.engine.abort_request(request.request_id) + if self.log_requests: + logger.info("Aborted request %s.", request.request_id) + + def _heartbeat_loop(self): + while not self._heartbeat_stop_event.wait( + timeout=self.heartbeat_interval_seconds): + # Loops until the stop event is set + self._heartbeat() + + logger.debug("Exiting MQLLMEngine heartbeat thread") + + def _heartbeat(self): + # Send unhealthy if engine has already errored + if self._errored_with is not None: + self._send_unhealthy(self._errored_with) + + # Check for life of the main loop + elif time.time() - self._last_alive_time > self.last_alive_threshold: + self._send_unhealthy(RuntimeError("Engine loop has died")) + + else: + # Otherwise- check health of the engine + # self.engine.check_health() raises on unhealthy + try: + self.engine.check_health() + self._send_healthy() + except Exception as e: + self._set_errored(e) + self._send_unhealthy(e) + + def _send_outputs(self, outputs: REQUEST_OUTPUTS_T): + """Send List of RequestOutput to RPCClient.""" + if outputs: + output_bytes = pickle.dumps(outputs) + self.output_socket.send_multipart((output_bytes, ), copy=False) + + def _send_healthy(self): + """Send HEALTHY message to RPCClient.""" + if not self.heartbeat_socket.closed: + self.heartbeat_socket.send_multipart(HEALTHY_RESPONSE, copy=False) + + def _send_unhealthy(self, error: BaseException): + """Send UNHEALTHY message to RPCClient.""" + if not self.heartbeat_socket.closed: + error_bytes = pickle.dumps(error) + self.heartbeat_socket.send_multipart((error_bytes, ), copy=False) + + def _async_socket_engine_callback(self, + request_outputs: REQUEST_OUTPUTS_T): + """Callback used by engine to make socket handling async with GPU.""" + self._send_outputs(request_outputs) + self.handle_new_input() + + def _set_errored(self, e: BaseException): + """Log and set errored status if this is the first issue.""" + if self._errored_with is None: + self._errored_with = e + + def _alive(self): + self._last_alive_time = time.time() + + def start_profile(self) -> None: + if type(self.engine.model_executor) is GPUExecutor: + self.engine.model_executor.start_profile() + else: + self.engine.model_executor._run_workers("start_profile") + + def stop_profile(self) -> None: + if type(self.engine.model_executor) is GPUExecutor: + self.engine.model_executor.stop_profile() + else: + self.engine.model_executor._run_workers("stop_profile") + + +def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext, + ipc_path: str): + + def signal_handler(*_) -> None: + # Interrupt server on sigterm + raise KeyboardInterrupt("MQLLMEngine terminated") + + signal.signal(signal.SIGTERM, signal_handler) + + engine = MQLLMEngine.from_engine_args(engine_args=engine_args, + usage_context=usage_context, + ipc_path=ipc_path) + engine.start() diff --git a/vllm/engine/output_processor/__init__.py b/vllm/engine/output_processor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/engine/output_processor/__pycache__/__init__.cpython-310.pyc b/vllm/engine/output_processor/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d16adb25c09c70ac4491fa1b1c091d6ae355248b GIT binary patch literal 169 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HIeo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_erjHNW?rg(erZWTX-RxRQGRl2 madCc;etdjpUS>&ryk0@&Ee@O9{FKt1R6CF@#Y{kgg#iF;QYt+F literal 0 HcmV?d00001 diff --git a/vllm/engine/output_processor/__pycache__/interfaces.cpython-310.pyc b/vllm/engine/output_processor/__pycache__/interfaces.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13ebd7df29a4aa63b613b467a652f233ece5caf4 GIT binary patch literal 3062 zcmbVOOOM<{5Vk#@x971tyLk~rhaAXAn;AtQAw>~NvOxhU2}lzzzE~c2&y3mjBi(kE zT}@8OjSGJRq5K7Yh_9SDb4wths{Pny*hC1AT+`iEU0u(w9rb!04W1h>e_j39)wJJm za{4i0avNUx6A-FVooj(^>w#e#I?fGV3ryP#>ULetYrGLO?WP)=ycM+Vwi?%YC+ON; zHEwV#=-IuXZ})?NJy2^+J`A?(Ej4cO?Op& zFJY-&Ey1KaOQLvx#k>P1#MSrk0sFo3;ko* zu4mSr@aXZWd2>HX!-NSB%|89~U~(H?`2q;jY@KShL3O*vOlq(?t+56*S(CLcYP8PU zw81*8yVRMrFkq!gTMLV}Y3GIcT(^6y4?!AG3*?8u?XfM|r-Ma{4(XPX*anI1QxeE+ z&=K8H+z}l>Xm_*AXSn#Sh&(1`B-~4erhSdREL@L~MI;E1mcB=lm79<_7h{ry?2xF4 z5T8ooheQ?uB}-0Ki{0ylJ*ez+vG44<3$QMgAWWP?D)yaV9w5jUh{AsBPk`*Je%=4t9)*i2XIA4;Lzg+Pn;=$pZQMCtbX5_(xhhK(Z15<~{Rg`i{ z=AehMkhl_73Hu>op#-~>!e~MKWI`Up-rRFgE?MjfHvzp#M5G_Y++Q4RsQ0)lQ5_{v zo6(`tRqi9Wg#|23l>bu2Rf3nCU6%=qAwFJ2(d%o6E8klYnZ|J>5($>NL7~bO$SA{H zglVi!3{sx>XKSJnW-Cfcl}Z_E1y!hVkId7AD5>I5B4g&I94vjxD#OMTwJ{r3@pnAc zXj!+)EfZP0VrN}26=YOjWV=h2I8_HZV7vqD%)&ZwR2?uOTF@h214#Mc0?zm0m3M)x zweJC{4h%7(IzSY_)w71Kts84?t*?!BZC!s}pMeCeA$&apxLP+0j+yhu=JCsY!628I z`AGG5S=Ny#Y6?qN)sf~=o4bg=AYmFfJc5Bwy83YHtrNU_zjSAai%PoLAB(lRJANHFFeEoIEz=eM`xI3Uwg`VFkz}O zCi&q`(cBP)+*t=JQ=Eh2?N8n-uKdWQ&!e#|F@!&{g=8Da2+0nTT_mVdpybb*xB!2z z9!--Vp2mKBJ#ht8%Jt8KbQRnP4yW9oPvav%{cwsVOmmB;n-)*~60zhYKFZE}D#OmE z*hM%@UIxA3PJ^s+{t`bQpKzU6~sOflB@A2^{pptN!POOf9Vq(wkhFj(S%I&DI}0^s4$={aP{ z6rjRADZi>k{}22ExWkE6DOVkSQlpPl8pmd4O2!xB1Mos%n8k-kFvU5+_-BKn&Wh~F zo3JS1($UP;@!oqwzp)X(1`tO1@6}*``QW&WsQ3sb5Y3z4#T0uUR0MH?QG(H2LNj^{ zU=DAeC@A1SDv&Q#PYUO1f`BUVF%sq5hFj-K>+&N24Ac?#KtA`*JpS}QdH2u#I8Dc)k%!9?uNW`?QW6NB!a`C<(nQinf`um9Nbn0q zTtb3z6Zq9A;Db@i%y|@XaRujBk(~IeMwm-60D=nSEg&ro8mg<>=q%+H7)1&eEtme)=fz_bQ$r{N~47iuui$2(01x5MY zqx`c+#fo5)sy3CY_V~T9^{>0f?kV}Qk@NUe0kN+tE_{fF61p&8LON;Ub2+KH3+^xs xwteMYlGP2U!ftnzoKljf1><>EptIJkJn>&(VM~P4p*GYl!_qB%2VV7G`x}(Jha><1 literal 0 HcmV?d00001 diff --git a/vllm/engine/output_processor/__pycache__/multi_step.cpython-310.pyc b/vllm/engine/output_processor/__pycache__/multi_step.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8dc16c92fd6dc503119de1c3fadf1ddf90ee50f0 GIT binary patch literal 6569 zcmb7IO>^7E83sTQ1V2Q{vMfuElOXoAF;kn0Go7S$-87EBs>G4q$jJfI3<42L5fVP= zT~M|()S;C-Z4a4F+dm*3-I@M@9y-1BPvF{9E}hQgmey9EcL7k8Y^4n(EEoIn?z`{j zvl~xL)D%3I|NPh98pB4K@}pLNi0*iW1wUPWyMRO-t~+`ryqopnjH?nl1t zUX`ECWjE{xpmWdVZYWwtdg{|VcRq3!mp;9AXYqz}|L%u3mz>2LsaY_SP82oCR(kT# zXU1jPEE`2^jCV*@;}2QNdw8WSFvV6q z#nwF4E_oW$y%H-kV_xxeR`JTrWYtxb)w&v53|8;fyozT&*6oH@WfNWvW1HxydyTFk z^`zUAY>J)Oo!(QtNpI@0vZF$aC#xsCm%Zt+87J9{oMn15-l@lB`;<4!s@`cfo70-} z=CR6Y*?I=6%wd(H*YsZU&Z2kTYrqy>N#{Nq1c|>46&_Kr`*%%I3NgOO0<4EUztgjN zZsY|_SkSoNu{aCdgn3pH*DO3a;#GL&nppC_-FX2L-2-IeL0(l_ieklR|<6Svu%W8)UBzrcnIq7-OM;>TnU| zIz@wXD2D1#ag?EkQXQ61YC|1mX;?<7dkXwbU22v2DU9W_RGg+_j*58{2!6U?LHF1J zPZa(V#(llCkc9n(zTdx;xEyR>dM6z8!pq^qg}`4~=x-*yI9k{Uf^dOFtA4~5GT9X| zI_3+~LuBA?_cv41ab#3>&V#z32xpjvN`EzV>$T$oM7x*;kwT8ElqAwhwv&XNI- zmO3SnD;`RuWxP^kXHXexH^6rx2Cqo>mp*UY1tK36NIEV<@c% zn6`)!E-2JUCuJ_|J@cBb@lPRW(@?Q>+@4uZJ-+i#sZL$Rvx-MLYNs73V9KilDSAo<%n3EK{R5?$tOE8Qoy^I zCgr77_*~#`no30p{-19m<7DDT^^R#%D+_-NXQUJFsN05DlV1kS+pY{NVp+~q zcB`0Gk@E;AV_vK7{t2P;noLOtnMFo$N(P0m0S6SK8TAznmJB#~P9*=989k1W>T>NN z3+8go#tKR$0@2;NoFR_P?gV-nA_?VAHyXKE9I*q=B!noa9bFTdHSh@wNCZTPK!ThD zfMdz@L79#K&?p(@0f=~~2N=i=L_7jilo5e%ij+uU2XCHTS?DL^F4Ad5(F&BA}%d;``#xMs^B9Y zdI3mifAH#@5bgL7g`D{j;^}j?;l~3g3egQZ$#D@V0A4VFj2=5s2(b{wg#aRk-NQas zAQovx_(baOFw0Mf1vh|Fy-mx59uM)r9zoTS1bP=%7`TT_v6Tqzqi}nn1w0Z2teXcR#*=^Tx>Z#Fxlo%GTxg z5AflUpa;|`t6D-yRdVNXDTfVu>09N&AQrCR$<{u=1uEN5hmV>zRCHNcsWQsuinT98&h zt8868Hs%#MB6r^Q)c}Ay5bvT=5PEJq6yKJKI>8~ioyxXK5sWsIZG^vpu9nKbhaY~C z3K_jeA)LR7wgj)_aQ1nTjn<%CDc7iGYb5G=iEBeOUxz3{T#bzX3<~l(8lPzV5`G{f zIsmX2t^Wx98E{fGfF^CC9*-X)u?4%glhmcR^7C1O7TO0Cf34q$QM6n(4#_&RZZX zwjgvHC5j?H3Pp>$ccJsT5lSLuA$e|O--jf zdEx)H8U90B{T&p?!$xAzQ@2Y)ZB5~gAxwT3Ivi?y8rqb#(wa^{a;SOQ7nE%XdX$z5 zUufba0Wk0K4ED>VT4lR7ipflZZ=9Vaz98jgiP?|(f!kpcr{hM(-CUu251b5&5d{_- z?SNmUy&SRCx6$?sykZ804yhR=$$gR+{negsT|QP;BSjyv34R5%4k3jG8b&btpZM@p zPaEZF+)|8vm4~G@gpXZyPf2iI*-=j`bIP_cG=LKnejfNh85+OB4F(WKX{{t~0tGbrhLv@V-;gs+uY6j`W~)l1K6nehUD-AFa6_`LZC8^j zW|Vj65ODtI+IDTbj{e%PHmrNb7r+?9DtEogb~Q5Oyt-E=I0+6Kq~kprq3%xXDLlzE z{orR(-{#VDfrda61)8s>QOF=r&Vs~puW7quE2-(oVd0n1K(*|nU4A4o-+>KUoQ|P&gYtIt_oX-K3v%?hga@uKW z%kSHhSxz3!w$BXun8@UuBQrQTMOp_$>EaYBaR1n9WZ_Ht5?QzOQ+sMWGJ_>yPZoQ~ zkA?OWXQ2x!(Q3{Pe~*Z(zC8!}fE!&u2=aiJb37taNX%{?rh3#AegKNU!z;+an}~q5 z68so3u%Q}8L!Bz&g;#6Onz~-n)W*K9k#8H%^s=Eg^oAy*<-Y!}txMm^h+68c))_jh zM{dX*C#^Y77}H%X>J7(vfXl1Ar{XwX+;JR!0vvL(D^5v}6L8>Uz?>okCs*Kfs^BkE zF+l|#-1s>ZbnyXz+6ftM6TGzAg{G4@4n$fF_`vDlyv_J`XrR7=X8t3ZNLL3^oqUpN zB&jj!NtdG3!YTUEEr|FOg{eST6Pl(yGfLW1P1pX@C~N=LjM~>m6KQ21VR2uZfap}M zray%!_YI8IXl$wW)VM+|G$tQSeOmLGT%%?253xdOB%5^QnC+e9VV|Vykv8t4SbKEF zmO`>$mp$3VNjtZM5ioB1bSd0LCJ>w=tbO4dM~_{Q^9l`bee>|Kt0t)!yDrCP2U%Vv z3Ny0h2nvvbwSQt$t@_xn8Mu(i_ z%(;B$%%rVWL&NjtZ~w4;@)b?{3w_K#Ha_0LD}IL}&;mWuV%^na!!_cHThXb_h|IX^ zR%N{sS#iy+$+{WYaow%UdNpdq3+{rfTTwG^xh+|*MeVrbcH*wvjTha;c*$Lgm)&LA zXGbgX3HOAo*Q1m1Dfg7Fu@-yozV5CD4OR~pST|_0#h?|mhj!2jx(`hE`RiJ+7%V-| zf+beJUvWVw77nt){4f+*QN1-DMXaY6osGdZ3&s)SSJPw|ZlS3@ zYr@Bw4YoUG$$(ww>3DQ=oQ=lWM?4)cAySV1g^es7UEOAbJ4}9tNtk(2y0wMRYGIeN zi&j~abc*F8pNeeJE*p=3Z)AQp7PQ(mmZf)C65f-m)vl)FBxAg2i!Cpeym=!qSJVaD z^|pxP!cy&p#aTAy$@-u|;+Q@reyez|;uR5!+}P13mAn$@W1YX1>pPlQ3iK^Kube{r zmX_cE@Q#%@o=EFjN_IimC+DRIfwda)49TdLa` zr6SD2H1X2mP_V3L7agT=au%uZq6TI7S;mWw@Z(Vg;@-&Tek_W{9I+FyQuoDx4bp&- zvBPF&B)l*gMgD*lm53#U#Xiee67)>Ih&gr*cRA`9 zpTv%EP*@$iX6X&x(%Xiuo4Rpm8HR4?wr>5STQ&d2U&NpR-Xjme_B(jR0~Abi^+0ot zKzA#F!OWn-s)5NYR$JAADzgKN)rXCs7T5sDMo?!9fKQ7x(YwG}K{IFpKHEVDz3reo z*GoN?gj3T6yvm;D+??fL1#`MV3lMvvSly7eumP7-HZrqU|M!NbJ;h&V1Dw0t;b0q{ zm;@0M&N(Gua!CSNkUd9k6b{Zy%PI$tMl6Ae4}pHt>|>ZilXygJc4l@WMbK+EFz)5AQYC&fNNruT@I+*T8NqWSY+A;8r z;^P=%#b%WN#2AVLf)Wqo2pxOOk)A1~%AI%|W#I+#j9GKvxwQ?hS>kV!wl|p{OV$MQ zIf#{>e|~Be9MTIp210LlKy^)tP}VqOv`#5t0q(JCNs76!yODK9) zVG0%vpE*E+oGM#pk;qWIckv|%Nvw(m{2>&&ckY>n)-R_a)0o=xG`*-PcNK*l z3eOk&$$+nb>iU3Ei}E>=X1syc40}dupx-e>=L4Bg+}D4Bq~e!oMM|L^7?6w+R8}>l z3>8Sh^lw3MSIFxF$B!Z>VRx0?lD{0a0SeV6myS$=`p$bWANZ#-4N4TmnD2#qWItqr z&H(5T20#q)J!c5RqOs06c`_)!}I`=uppaKRz5-Gf$-+{YK zMW~#J$iatZ*q{=WeJ5rtgHfg<^1MI>PjEn^>6ukZu{#&QR1m^@!4{k!QXpi62O#uj zQ#1v_aO{daWk!IgQJjq;0O%~!y8h88<70N7#wgIE<>^5dHl>j;1!bGif>by@fm8$Lgc%X=BMu*hKSL)+ zc0uu;C)mio^O{0>$%@4F(pO*WLDx~#n@#)@DX4bubx?!ckR-w~Mx&GiBMH0+7^W63 z0g`sZX-2QYpT|#OX8R*noSjAbQn$)Irb)R#4)Se`<~V0*{8cJmqk=M!Ip?H|h+gHh+xq*3julH;(JvLvaJpYpKlQUi=%I8tcIb!1j%IeAeq~lF0l;=|> zf1l2JCY-S9pZyQ=b!bN_pTM*XH8K`ln906WN-#~HWoUVdjnC)IWGfJ|lJEbX@)9|v z;QvF~OT>5&V!ijWQj!vY*iOe0f=U9zqyxvB(-r&DlGb~MG$onrm4+gdRcRs17|;Tr^dg7^P#QEgL-b|<|AWLOE7;@Lr-I8 z0c|w|KYgc(6QhY))eLStUxAIcLb_?Xl&vzsmM zECuEl`lJz51M7j7H?H9XI$6l;ps)x&7Y>%e)ygB?)F#b9!x?v!HxEwaRY}#`(k3mM z^XU&K?R+6`@0`S*I@zhbgJ5OcG5F81!gG1sTg~-@=c%9O9pkVqY5jC}oOf_TQu_(y z0*a?6-Rv|*o&i6LdHdjneDRS!S<06nt1e{Koyw{OZkF=)(2!Jq1sX`?C(D>&=F1qj zJU6aA8#m5dSmW%EjEOZ_k@EgC*0Q-Q|2=u{{!!9!B#kz-(H`p$b-sk1zNnk~6y!Fxg`JodVDA%(KKYey;f zJ>-G7{;+WH#YM<~0JwZ$m>NOa23|p_M+k*{YE9^3Lfg`QX6zap3S=b#AV(PVjKb)@ z;WiNVkQ#b`ljBg_xP(xe(UJc0Ln2IquPgKcWBS zA)cPYpTiP`o)k5i12f?wv)dJJN5w!-21oGjHWgf{1`>}tKJ!(~mpP()YI-`J#iHr4 zSf-5ba_sMKGMrbDq?RMxW+~+{!|m7tl8Q1Pkr}6JQ&L33T^omFstDf3f^O~OPuACO ztY0rGXm?vrBnnmwOYzRvh&2kAZuh+#>o+#ufA^Yq<@+~3zV+@k-bXWCPQ~dSD@=;j zGQBNK%J_?l9|Tw&Ra&!mLWR*M=qSUel$@o+R#b?|Hqt|6;67JLxyqW~A_f+oa46ix z>BnF@i~^4|?lx3C!X;r@r_C0Y8tS%Gf2pJ9{HNPh%Mvk6A_-Jwzq^QQs|*au9=hCM zh3*}*IsQ8MOYw?dqp&Isy4@|UStpn|fc!rV;Lh*NsQFHk{(ofEbUrgR$OXHmS3S6o72YmjOS(mjMM z4-Lb3Y#5ElR{hA=iu}?Whiar6)qX4~_0AL)vO%0FIDwMll%UC7pbMn_^z2AmahInp zob}5jh|68MugV^E+0>u9`ZV09ydTnaFYY0k;1rwP-q((vKZr_6O*sK(;C`Cdi9$l{iW3Bn%MG#<>_s0>O%;$Oe?Ppb-+FgMjF5a(8;S zA0x77Uud1A3 zY`SCTQdcCSBpkj`R+x>>jW?bT^7xEf zHXY8vw=W@(l!adA;NB>VttFV0v1-j|m8~_0r3%x7@D!fLCo+_0iIhSLYYQ$3^HRtx z%w#dOV$IOpY>6Di=7U37wT75*^Mg`xEng-kiPyud8NP@3^cYi3Yr;v*FeCgk@-^K< zPL22t&B|oT#A#eDvjb%yV<}$~wT1P!&vvVPvAakYpH#7e4WE3LFX#EE`RQ(!?(Z(n zt9e=Mo@80RD{UNi5vs+q;tN$KQtMLfniz2#AE~B`^Ty{qEz*i}W1@C3c}_qtZeL4{ z>6)eSvr@%e!BgYedYCnmr}gwhENZI!njVpMo{^dy(HT(#98FlXot4EQx0;OWcuqC6 zCu)Qrh2qdUYdu!RtY(Ls_rCE^<{s@~3aCdtcJb322LBKHUgID@&8U#4@IF>NDVIgn zZ0QxoxykKZH=dH!QWZ%Dn%AR?h1uIk{tJtmUXr(@c8=&VQ9S|Qux9!m=nm-g$USE2 zUd_~9&^!7rD4w7->yy6#;5Fr5)jh%s@mAJKz3S8*a0e$uJ%QwXD|!F_OM;v7zP+1V z4v<@{cHTO#yc#zxsru9UhT7Zk(aRyMBbTGveM^sBbp)QRYgV*AjbZ!g^%HZ)gk4;k z2e$Y)K*H!bz=9XGKzbO>OGT#T#%L9~xSm`hp7Hg$BTuYj|u~vA` z))*viSjn?01iV@`@-#aS4`qRJh4RoAjE(%>WIGy}z^`&1i4_HTTC5626r_hmsU%0K ziM92jjX&aSx~E^zTGPs>#}HWwLsk5Vaj9FwWa|Sw~XcwH=3V5-h1-m z3;ytd(GSNqm2lE+fpZ)D>H{+wY*-vc!y7&}{@R-sTc)J60K1jP=lg4CS5f^Kr~0>; zMq4iRoDp>$m$IuJ=2MsYZh+bD9Ulk;#bMOF@^NZ*JD|FBWM}vLi=WMY=rq&!+;yXJ z8dRIl;_VEJnrhc(Iy)kd$e(}t%Bg7Wyr!>QR2FvW)J{$Nq;@|dZ<)c7V94Q*Uv_JD zx?p?TdeFZQeGcCit zvXE_4GKksx<_crdLBoJpCJE>tJ8*Qy~0k+tl7FYtLsv zp?(a3KjPCNCf9dqV8YCh0th(6%4Z|$v$1P}elytbN%6K3Y@60o#JS|W3ApL4%gn3? zoWESg+3KXrxhNBmowTTw=`_luhVsp{Z6?!amFZg+Fv=8a^%G3Zz(6ofa{S3C1WAJGaF~=K!b7 zP=zF<2}!9(skRx~%rk1+K-=`p+BTC$YI#;|Hb~Xtk6%4l-#HSz$d`wzCa(|6s-G30Mb_0#qTgfa%ODvGCi#`4Xo* zKo!!ET@gh}dPxt-C0($R9Fp<(9x&J@SJyQ|By!}?x8vGlcQeAsR>!YMaVMWiHSb3r5WiOsOauS9fk8R=xhCl#I4 zUafu|yDH7yJkB30UqqlDzetN|x|^Q6Nqp+&b3~N6ACe?>A&cS++$GtQT<`#-{i_t99#7#LLp zDU6_M0rnLH2B<7Q&mjv3me5aGv`~55|B70fn%s)U-qzi8t2+L&Ao>`ndib9zb+(#b z%d!xaZEVs_%5F?UY~jx9LASEhJdd+T=!OX05pUxNG7xbvTHzI@UVWqq9i}{lWC9-{ zC=?gN#GiwZL%I9`gnp!6uRgf0UT8{16IEs43#rrI0-;;HL}Kq@>>`1E{YvCEsxIkL S91IIK`dBn$?7GeR*S`T1=O!@# literal 0 HcmV?d00001 diff --git a/vllm/engine/output_processor/interfaces.py b/vllm/engine/output_processor/interfaces.py new file mode 100644 index 00000000..50adaf4e --- /dev/null +++ b/vllm/engine/output_processor/interfaces.py @@ -0,0 +1,72 @@ +from abc import ABC, abstractmethod +from typing import Callable, List + +from vllm.config import SchedulerConfig +from vllm.core.scheduler import Scheduler +from vllm.engine.output_processor.stop_checker import StopChecker +from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput +from vllm.transformers_utils.detokenizer import Detokenizer +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import Counter + + +class SequenceGroupOutputProcessor(ABC): + """Interface for logic that processes new token ids in sequence groups, + managing detokenization, stop checking, and freeing/forking sequences with + the scheduler. + + This is highly coupled with the LLMEngine and should be seen as an extension + of it. The logic is separated to simplify the LLMEngine class and allow + separate implementations for single-step decoding (which supports beam + search sequence forking) and multi-step decoding (which does not support + beam search, but does support speculative decoding). + """ + + @staticmethod + def create_output_processor( + scheduler_config: SchedulerConfig, + detokenizer: Detokenizer, + scheduler: List[Scheduler], + seq_counter: Counter, + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer], + stop_checker: "StopChecker", + ): + """Create an output processor. + + This returns a single-step output processor if num_lookahead_slots is + zero, else returns a multi-step output processor. + """ + if scheduler_config.num_lookahead_slots == 0: + # Importing here to avoid cycle. + from vllm.engine.output_processor.single_step import ( + SingleStepOutputProcessor) + return SingleStepOutputProcessor(scheduler_config, detokenizer, + scheduler, seq_counter, + stop_checker) + else: + # Importing here to avoid cycle. + from vllm.engine.output_processor.multi_step import ( + MultiStepOutputProcessor) + return MultiStepOutputProcessor( + detokenizer, + scheduler, + seq_counter, + get_tokenizer_for_seq, + stop_checker, + ) + + @abstractmethod + def process_outputs(self, sequence_group: SequenceGroup, + outputs: List[SequenceGroupOutput], + is_async: bool) -> None: + """Process new token ids for the sequence group. Handles logic such as + detokenization, stop checking, and freeing/forking sequences in the + scheduler. + """ + pass + + @abstractmethod + def process_prompt_logprob(self, seq_group: SequenceGroup, + outputs: List[SequenceGroupOutput]) -> None: + """Update prompt logprobs received from outputs to seq_group.""" + pass diff --git a/vllm/engine/output_processor/multi_step.py b/vllm/engine/output_processor/multi_step.py new file mode 100644 index 00000000..74ddb250 --- /dev/null +++ b/vllm/engine/output_processor/multi_step.py @@ -0,0 +1,188 @@ +import functools +from typing import Callable, List + +from vllm.core.scheduler import Scheduler +from vllm.engine.output_processor.interfaces import ( + SequenceGroupOutputProcessor) +from vllm.engine.output_processor.single_step import ( + single_step_process_prompt_logprob) +from vllm.engine.output_processor.stop_checker import StopChecker +from vllm.logger import init_logger +from vllm.sampling_params import SamplingParams +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup, + SequenceGroupOutput, SequenceOutput, SequenceStatus) +from vllm.transformers_utils.detokenizer import Detokenizer +from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.utils import Counter + +logger = init_logger(__name__) + + +class MultiStepOutputProcessor(SequenceGroupOutputProcessor): + """SequenceGroupOutputProcessor which handles logic related to + detokenization and stopping conditions. It specializes to "multi-step + decoding", where vLLM's worker may generate multiple tokens per invocation. + This is currently mutually exclusive with advanced sampling techniques like + beam search, which motivates the separation of this logic from the single + step output processor. + + This class is responsible for things such as correctly appending all new + token ids to their sequence, detokenizing new token ids, truncating new + output tokens after an eos token, and correctly handling the case where the + number of new output tokens per sequence differs in a single batch. + """ + + def __init__( + self, + detokenizer: Detokenizer, + scheduler: List[Scheduler], + seq_counter: Counter, + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer], + stop_checker: StopChecker, + ): + self.detokenizer = detokenizer + self.scheduler = scheduler + self.seq_counter = seq_counter + self.get_tokenizer_for_seq = get_tokenizer_for_seq + self.stop_checker = stop_checker + + def process_prompt_logprob(self, seq_group: SequenceGroup, + outputs: List[SequenceGroupOutput]) -> None: + """Process prompt logprobs associated with each step of a multi-step- + scheduled computation. + + Args: + seq_group: the outputs are associated with this :class:`SequenceGroup` + outputs: the :class:`SequenceGroupOutput`s for all scheduler steps + """ + for output in outputs: + # Concatenate single-step prompt logprob processing results. + single_step_process_prompt_logprob(self, seq_group, output) + + @staticmethod + @functools.lru_cache() + def _log_prompt_logprob_unsupported_warning_once(): + # Reminder: Please update docs/source/serving/compatibility_matrix.rst + # If the feature combo become valid + logger.warning( + "Prompt logprob is not supported by multi step workers. " + "(e.g., speculative decode uses multi step workers).") + + def process_outputs(self, + sequence_group: SequenceGroup, + outputs: List[SequenceGroupOutput], + is_async: bool = False) -> None: + """Append new tokens in the outputs to sequences in the sequence group. + + This only supports sequence groups of size 1. It supports greater than + one new token per sequence. + + This applies logic like stop condition checking and detokenization. + It also handles cases where there are tokens emitted after + the EOS token. + + is_async - Indicates whether this postprocessor runs in + parallel with the GPU forward pass and is processing + tokens from the previous step. If this is true, then + no tokens need to be appended since it is already done + externally (before the next schedule() call) + """ + # Sequences can be in RUNNING or FINISHED_ABORTED state + # once scheduled, as a sequence is moved to FINSIHED_ABORTED + # if a client disconnects from the api server. + seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING) + if seqs is None: + seqs = sequence_group.get_seqs( + status=SequenceStatus.FINISHED_ABORTED) + + assert seqs, "Expected RUNNING or FINISHED_ABORTED sequences" + assert len(seqs) == 1, ( + "Beam search not supported in multi-step decoding.") + seq = seqs[0] + seq_id = seq.seq_id + assert all( + [seq_id == output.samples[0].parent_seq_id for output in outputs]) + + if is_async: + # Async case: We process tokens one by one. Here, we know the token + # was already appended, so we only need to do the rest of the + # postprocessor: Detokenization + stopping logic + self._process_decode_and_stop(seq, sequence_group.sampling_params) + else: + # Standard multi-step case + + # Since there's only one sequence per sequence group, + # we can take the first sample. + samples = [output.samples[0] for output in outputs] + + # entries in sample tokens may be invalid (eg. due to spec decode + # rejecting tokens). + valid_samples = [ + sample for sample in samples + if sample.output_token != VLLM_INVALID_TOKEN_ID + ] + assert valid_samples + + self._process_seq_outputs(seq, valid_samples, + sequence_group.sampling_params) + + def _process_decode_and_stop(self, seq: Sequence, + sampling_params: SamplingParams) -> None: + new_char_count = 0 + if sampling_params.detokenize: + new_char_count = self.detokenizer.decode_sequence_inplace( + seq, sampling_params) + + # TODO(sang): Support lora. + self.stop_checker.maybe_stop_sequence( + seq, + new_char_count=new_char_count, + sampling_params=sampling_params, + ) + + def _process_seq_outputs(self, seq: Sequence, + valid_samples: List[SequenceOutput], + sampling_params: SamplingParams) -> None: + output_token_ids = [sample.output_token for sample in valid_samples] + output_logprobs = [sample.logprobs for sample in valid_samples] + + # Truncate to max_tokens if necessary. + remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() + + len(output_token_ids)) + if remaining_tokens < 0: + output_token_ids = output_token_ids[:remaining_tokens] + + # Truncate any tokens after EOS. This is required as spec decode + # generates a fixed number of tokens without evaluating stopping + # conditions within the block. This can cause an eos token to be + # unintentionally ignored. + if not sampling_params.ignore_eos: + eos_token_id = self.get_tokenizer_for_seq(seq).eos_token_id + # Avoiding .index calls as exception throwing in the happy path + # is expensive. + for i in range(len(output_token_ids)): + if output_token_ids[i] == eos_token_id: + output_token_ids = output_token_ids[:i + 1] + break + + is_prefill_sampled_token = seq.data.get_num_uncomputed_tokens() == 0 + # Incrementally append tokens to the sequence, as if we had only one new + # token. + for output_token_id, output_logprob in zip(output_token_ids, + output_logprobs): + seq.append_token_id( + token_id=output_token_id, + logprobs=output_logprob, + ) + + if is_prefill_sampled_token: + is_prefill_sampled_token = False + else: + # Update num_computed_tokens iff the sampled token is not from + # a prefill step. + seq.data.update_num_computed_tokens(1) + + self._process_decode_and_stop(seq, sampling_params) + + if seq.is_finished(): + break diff --git a/vllm/engine/output_processor/single_step.py b/vllm/engine/output_processor/single_step.py new file mode 100644 index 00000000..cfa84077 --- /dev/null +++ b/vllm/engine/output_processor/single_step.py @@ -0,0 +1,215 @@ +from typing import Dict, List, Tuple + +from vllm.config import SchedulerConfig +from vllm.core.scheduler import Scheduler +from vllm.engine.output_processor.interfaces import ( + SequenceGroupOutputProcessor) +from vllm.engine.output_processor.stop_checker import StopChecker +from vllm.logger import init_logger +from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput, + SequenceOutput, SequenceStatus) +from vllm.transformers_utils.detokenizer import Detokenizer +from vllm.utils import Counter + +logger = init_logger(__name__) + + +def single_step_process_prompt_logprob( + sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup, + output: SequenceGroupOutput) -> None: + """Process prompt logprobs associated with the :class:`SequenceGroupOutput` + for a given step. + + Do nothing if the output has no prompt logprobs. + + Account for the fact that transformers do not compute first-token logprobs. + + Args: + sg_output_proc: :class:`SequenceGroupOutputProcessor` instance + seq_group: the output is associated with this :class:`SequenceGroup` + output: the :class:`SequenceGroupOutput` for a single scheduler step + """ + prompt_logprobs = output.prompt_logprobs + + # If this is the first (or only) "chunk" of the prefill, we need + # to prepend None to the list of prompt logprobs. The reason for this + # is that for N prompt tokens, the Sampler will generate N-1 total + # prompt logprobs during prefill since the token at idx 0 will not + # have a logprob associated with it. + if prompt_logprobs is not None: + if not seq_group.prompt_logprobs: + prompt_logprobs = [None] + prompt_logprobs + seq_group.prompt_logprobs = [] + + assert hasattr(sg_output_proc, 'detokenizer') + if (seq_group.sampling_params.detokenize + and sg_output_proc.detokenizer): + sg_output_proc.detokenizer.decode_prompt_logprobs_inplace( + seq_group, + prompt_logprobs, + position_offset=len(seq_group.prompt_logprobs)) + + seq_group.prompt_logprobs.extend(prompt_logprobs) + + +class SingleStepOutputProcessor(SequenceGroupOutputProcessor): + """SequenceGroupOutputProcessor which handles "output processing" logic, + which happens after the model returns generated token ids and before + scheduling of the next batch. Output processing logic includes + detokenization, and determining if a sequence is finished (e.g. via max len + or eos token). + + The SingleStepOutputProcessor is specialized to the case where the model + emits at most a single token per invocation, which precludes configurations + such as speculative decoding or multi-step decoding. This enables beam + search sampling, which requires forking/finishing/freeing sequences in a way + that is currently difficult to schedule multiple steps ahead of time. + """ + + def __init__(self, scheduler_config: SchedulerConfig, + detokenizer: Detokenizer, scheduler: List[Scheduler], + seq_counter: Counter, stop_checker: StopChecker): + self.scheduler_config = scheduler_config + self.detokenizer = detokenizer + self.scheduler = scheduler + self.seq_counter = seq_counter + self.stop_checker = stop_checker + + def process_outputs(self, sequence_group: SequenceGroup, + outputs: List[SequenceGroupOutput], + is_async: bool) -> None: + """Append all new tokens to sequences in the sequence group. Fork any + surviving beam candidates; free any unsurviving ones. + + Invokes detokenizer to detokenize new tokens, and also marks sequences + as finished if they meet stop conditions. + + is_async - Indicates whether this postprocessor runs in + parallel with the GPU forward pass and is processing + tokens from the previous step. If this is true, then + no tokens need to be appended since it is already done + externally (before the next schedule() call) + """ + assert (len(outputs) == 1 + ), f"{type(self)} does not support multiple outputs per step" + return self._process_sequence_group_outputs(sequence_group, outputs[0], + is_async) + + def process_prompt_logprob(self, seq_group: SequenceGroup, + outputs: List[SequenceGroupOutput]) -> None: + """Process prompt logprobs associated with one step of a single-step- + scheduled computation. + + Args: + seq_group: the output is associated with this :class:`SequenceGroup` + output: the :class:`SequenceGroupOutput` for a single scheduler step + """ + assert len(outputs) == 1, ("Single step should only has 1 output.") + output = outputs[0] + single_step_process_prompt_logprob(self, seq_group, output) + + def _process_sequence_group_outputs(self, seq_group: SequenceGroup, + outputs: SequenceGroupOutput, + is_async: bool) -> None: + sampling_params = seq_group.sampling_params + if sampling_params.n == 1: + # only have one output sample + sample = outputs.samples[0] + # only have one sequence + seq = seq_group.seqs[0] + if not is_async: + seq.append_token_id(sample.output_token, sample.logprobs) + if sampling_params.detokenize and self.detokenizer: + new_char_count = self.detokenizer.decode_sequence_inplace( + seq, sampling_params) + else: + new_char_count = 0 + self.stop_checker.maybe_stop_sequence( + seq, + new_char_count, + sampling_params, + lora_req=seq_group.lora_request, + ) + if seq.is_finished(): + for scheduler in self.scheduler: + scheduler.free_seq(seq) + return + + # TODO: Add support for async for beam search + assert not is_async + + # Process samples + samples = outputs.samples + parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING) + parent_child_dict: Dict[int, List[SequenceOutput]] = { + parent_seq.seq_id: [] + for parent_seq in parent_seqs + } + for sample in samples: + # Guard against a KeyError which can occur if the request was + # aborted while the output was generated + if (child_list := + parent_child_dict.get(sample.parent_seq_id)) is not None: + child_list.append(sample) + # List of (child, parent) + child_seqs: List[Tuple[Sequence, Sequence]] = [] + + # Process the child samples for each parent sequence + for parent in parent_seqs: + child_samples: List[SequenceOutput] = parent_child_dict[ + parent.seq_id] + if len(child_samples) == 0: + # This parent sequence has no children samples. Remove + # the parent sequence from the sequence group since it will + # not be used in the future iterations. + parent.status = SequenceStatus.FINISHED_ABORTED + seq_group.remove(parent.seq_id) + for scheduler in self.scheduler: + scheduler.free_seq(parent) + continue + # Fork the parent sequence if there are multiple child samples. + for child_sample in child_samples[:-1]: + new_child_seq_id: int = next(self.seq_counter) + child = parent.fork(new_child_seq_id) + child.append_token_id(child_sample.output_token, + child_sample.logprobs) + child_seqs.append((child, parent)) + # Continue the parent sequence for the last child sample. + # We reuse the parent sequence here to reduce redundant memory + # copies, especially when using non-beam search sampling methods. + last_child_sample = child_samples[-1] + parent.append_token_id(last_child_sample.output_token, + last_child_sample.logprobs) + child_seqs.append((parent, parent)) + + for seq, _ in child_seqs: + if sampling_params.detokenize and self.detokenizer: + new_char_count = self.detokenizer.decode_sequence_inplace( + seq, sampling_params) + else: + new_char_count = 0 + self.stop_checker.maybe_stop_sequence( + seq, + new_char_count, + sampling_params, + lora_req=seq_group.lora_request, + ) + + # For newly created child sequences, add them to the sequence group + # and fork them in block manager if they are not finished. + for seq, parent in child_seqs: + if seq is not parent: + seq_group.add(seq) + if not seq.is_finished(): + for scheduler in self.scheduler: + scheduler.fork_seq(parent, seq) + + # Free the finished and selected parent sequences' memory in block + # manager. Keep them in the sequence group as candidate output. + # NOTE: we need to fork the new sequences before freeing the + # old sequences. + for seq, parent in child_seqs: + if seq is parent and seq.is_finished(): + for scheduler in self.scheduler: + scheduler.free_seq(seq) + return diff --git a/vllm/engine/output_processor/stop_checker.py b/vllm/engine/output_processor/stop_checker.py new file mode 100644 index 00000000..0c5f8fb7 --- /dev/null +++ b/vllm/engine/output_processor/stop_checker.py @@ -0,0 +1,117 @@ +from typing import Callable, Optional + +from vllm.lora.request import LoRARequest +from vllm.sampling_params import SamplingParams +from vllm.sequence import Sequence, SequenceStatus +from vllm.transformers_utils.tokenizer import AnyTokenizer + + +class StopChecker: + """LLMEngine helper class which separates out the logic involving stop + checking. This checks things such as: whether the eos token was emitted, + whether the max_tokens has been consumed, whether a stop string has been + emitted, or if we have exceeded the max model len. + """ + + def __init__(self, max_model_len: int, + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer]): + # Do not use it directly, but use `self._get_max_model_len`. + self._max_model_len = max_model_len + self.get_tokenizer_for_seq = get_tokenizer_for_seq + + def _get_max_model_len(self, lora_req: Optional[LoRARequest]): + if lora_req and lora_req.long_lora_max_len: + return lora_req.long_lora_max_len + else: + return self._max_model_len + + def maybe_stop_sequence( + self, + seq: Sequence, + new_char_count: int, + sampling_params: SamplingParams, + lora_req: Optional[LoRARequest] = None, + ) -> None: + """Stop the finished sequences. + + new_char_count is the number of chars added to the + sequence's output text for the newly generated token + """ + + # Check if the minimum number of tokens has been generated yet; + # skip the stop string/token checks if not + if seq.get_output_len() < sampling_params.min_tokens: + return + + # Check if the sequence has generated the EOS token. + if ((not sampling_params.ignore_eos) + and seq.get_last_token_id() == seq.eos_token_id): + # Remove the last EOS token unless explicitly specified + # This prevents unintended exposure of the EOS token + if new_char_count and ( + not sampling_params.include_stop_str_in_output): + seq.output_text = seq.output_text[:-new_char_count] + seq.status = SequenceStatus.FINISHED_STOPPED + return + + # Check if a stop token was encountered. + # This assumes a single token produced per step. + last_token_id = seq.get_last_token_id() + if last_token_id in sampling_params.stop_token_ids: + if new_char_count and ( + not sampling_params.include_stop_str_in_output): + # Remove last token + seq.output_text = seq.output_text[:-new_char_count] + seq.status = SequenceStatus.FINISHED_STOPPED + seq.stop_reason = last_token_id + return + + # Check if any stop strings are matched. + stop_str = self._check_stop_strings(seq, new_char_count, + sampling_params) + if stop_str is not None: + seq.status = SequenceStatus.FINISHED_STOPPED + seq.stop_reason = stop_str + return + + # Check if the sequence has reached max_model_len. + if seq.get_len() > self._get_max_model_len(lora_req): + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED + return + + # Check if the sequence has reached max_tokens. + if seq.get_output_len() == sampling_params.max_tokens: + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED + return + + @staticmethod + def _check_stop_strings(seq: Sequence, new_char_count: int, + sampling_params: SamplingParams) -> Optional[str]: + """Check if any stop strings are matched and truncate sequence + output text accordingly. + + Returns the stop string if matched or else None. + """ + if not new_char_count: + return None + + for stop_str in sampling_params.stop: + stop_string_len = len(stop_str) + # Avoid searching already-searched text. + stop_index = seq.output_text.find( + stop_str, -new_char_count - stop_string_len) + if stop_index == -1: + continue + + if sampling_params.include_stop_str_in_output: + # Truncate to end of stop string. + stop_index += stop_string_len + if stop_index >= len(seq.output_text): + # No truncation required. + return stop_str + + # Truncate the output text to either the beginning + # or end of the stop string. + seq.output_text = seq.output_text[:stop_index] + return stop_str + return None diff --git a/vllm/engine/output_processor/util.py b/vllm/engine/output_processor/util.py new file mode 100644 index 00000000..76782888 --- /dev/null +++ b/vllm/engine/output_processor/util.py @@ -0,0 +1,22 @@ +from typing import List +from typing import Sequence as GenericSequence +from typing import Union + +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.sequence import PoolerOutput, SequenceGroupOutput + + +def create_output_by_sequence_group( + outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]], + num_seq_groups: int) -> List[List[SequenceGroupOutput]]: + """Helper method which transforms a 2d list organized by + [step][sequence group] into [sequence group][step]. + """ + output_by_sequence_group: List[List[SequenceGroupOutput]] = [ + [] for _ in range(num_seq_groups) + ] + for step in outputs: + for i, sequence_group_output in enumerate(step): + output_by_sequence_group[i].append(sequence_group_output) + + return output_by_sequence_group diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py new file mode 100644 index 00000000..d7ff743e --- /dev/null +++ b/vllm/engine/protocol.py @@ -0,0 +1,103 @@ +from typing import (AsyncGenerator, List, Mapping, Optional, Protocol, + runtime_checkable) + +from vllm.config import DecodingConfig, ModelConfig +from vllm.core.scheduler import SchedulerOutputs +from vllm.inputs.data import PromptType +from vllm.lora.request import LoRARequest +from vllm.model_executor.layers.sampler import SamplerOutput +from vllm.outputs import EmbeddingRequestOutput, RequestOutput +from vllm.pooling_params import PoolingParams +from vllm.prompt_adapter.request import PromptAdapterRequest +from vllm.sampling_params import SamplingParams +from vllm.transformers_utils.tokenizer import AnyTokenizer + + +@runtime_checkable +class EngineClient(Protocol): + """Protocol class for Clients to Engine""" + + @property + def is_running(self) -> bool: + ... + + @property + def is_stopped(self) -> bool: + ... + + @property + def errored(self) -> bool: + ... + + @property + def dead_error(self) -> BaseException: + ... + + def generate( + self, + prompt: PromptType, + sampling_params: SamplingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + """Generate outputs for a request.""" + ... + + def encode( + self, + prompt: PromptType, + pooling_params: PoolingParams, + request_id: str, + lora_request: Optional[LoRARequest] = None, + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, + ) -> AsyncGenerator[EmbeddingRequestOutput, None]: + """Generate outputs for a request from an embedding model.""" + ... + + async def abort(self, request_id: str) -> None: + """Abort a request. + + Args: + request_id: The unique id of the request. + """ + + async def get_model_config(self) -> ModelConfig: + """Get the model configuration of the vLLM engine.""" + ... + + async def get_decoding_config(self) -> DecodingConfig: + ... + """Get the decoding configuration of the vLLM engine.""" + + async def get_tokenizer( + self, + lora_request: Optional[LoRARequest] = None, + ) -> AnyTokenizer: + """Get the appropriate tokenizer for the request""" + ... + + async def is_tracing_enabled(self) -> bool: + ... + + async def do_log_stats( + self, + scheduler_outputs: Optional[SchedulerOutputs] = None, + model_output: Optional[List[SamplerOutput]] = None, + ) -> None: + ... + + async def check_health(self) -> None: + """Raise if unhealthy""" + ... + + async def start_profile(self) -> None: + """Start profiling the engine""" + ... + + async def stop_profile(self) -> None: + """Start profiling the engine""" + ... diff --git a/vllm/entrypoints/__init__.py b/vllm/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm/entrypoints/__pycache__/__init__.cpython-310.pyc b/vllm/entrypoints/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ebcd41a9a84d82f59c0b18c574f6a1f2343ea4a GIT binary patch literal 157 zcmd1j<>g`k0>icYGC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6eo1bDenDn| zZb@QMdTNQTd2VS&u3>JueokhRenDkPMt+`tSx!!_erjGxQDs4XW?o6LetdjpUS>&r ayk0@&Ee@O9{FKt1R6CH##Y{kgg#iFqZ6#{} literal 0 HcmV?d00001 diff --git a/vllm/entrypoints/__pycache__/api_server.cpython-310.pyc b/vllm/entrypoints/__pycache__/api_server.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48c4537ad35b84cce26bcbc6e3202c976913b95e GIT binary patch literal 4991 zcmZ`+TW=f372erhF3A-|U7|R?#o1gOn@LF7ahjyA>qK$nMs6Y-mXn|hw3`)YD6YKA zrDunBL{LCd8K_;rMf=_d3Fu26`U?v5B`*c~5B9YWMVExtq;CJ@Vf9?Kd&M^K#orAwBI`5)nr%l7)hHr4j&6xR2e@)-iU(2`fwPHId z`6Whk?AS@lemSZ5m89xdHLVo8$%H?V)cl(6JMm;v_v^`&Kb1`T)0$R}XOdZeR<|qh zQ^}k^r`y%|>Ew`qNVnbindGp4ShpwQBgs+!sBYKdW65#V_^g=fnEWX|ch|{Wh5lt7#4*!(CzcGvb_tD5|EO+7BEV(y5cM;-nwKXJF@H=h`N zPL%m9H#-&X@#it4CFc1HH_hKOIV(ryi@&9hrhM{#unqv1N4@jIe&n129&eovu3w{B+l$G;%fWN9m9W1 zu_g4X=|* zkBcPjWlDxB>aAfcToYc}@fNa;UVE{(7WIT1_PCctNk0}|Uq~!U!d_c=tD@KLCZW8U zwY()oJNHt2dP?*-PN2>EGUbD|iqamP+Hx;HL3_=+BRnbEX_8ZL@Ir|aXQ@7MEu}Ro^}=>r^p$t*dgdi5k2)KgP6Fsev3SkH1;R|Y z?QYn^B|p0z$H~0tDY?V6j2zkr>hFo*G!Sd>In24+& zwnf9rt%cr3Uf1IKndk|~F_n4cYM(A0#tkzsUk)<`4dmtP;?_W93X5u2R<15z#~u*9 z$SZrTnH42Pn1JtH5{{cXpo;~$mVs29_u+>h6&eD?)f@+1rTXYiM7>A_ak{o9Bu!rl zp*`$+EtFwG8`M4~MPEvxtj~#|LziRmIY}7@44?^xnbuP!aex2 z(D@oAtD*XmDUE#(*ox7xhLa!YwO+d`+BaL@{0%d{0tcB2)gTMnDHjbU2|T|toHr#OUM@U~m*+R(x!MM}N*fw^ICEV--W0Q^8Hqd>C-8Xd0 z;`WZYX)2S~BT)Xte#bT~Wo;Rowz4@Ju1LE=QKB;8k^-& zdDFRd$6YGS>%!I?pSHOZ^oeZU;!7siTl#~3-9pPR(b$XT!7H|}8x9#-`rKiJKKZIjTf#v#O>?RpOvA6`q7h#ttO3*T+KR2EmBx5=Lm}X}PY4 z-;I5Wmvz%Y%mGJU=#gJ}>FT;@E00W8FYTmpoZcZY=tLss*?GN1lh7&YtMhtJ(IylD zP-*O#{8|LNFT7K5c?dSvM%nvWXq8c( zLLQO*3XLyKt?tuMxA6f~ylKE}46Qf=iksb298405GuD|_9MX&tgW_P8+tvui%I#;2 zZJU>N*e5WITh^P{59)N5AF+L$YLK@)h4L*iw}#_>S>9x@T()jJWBkFqh3KW-6pm~Y zD1Bw-X6sDuqyyC-sIQnjfpvv9A44m5FT_!%C_G$TKtFR(Ifl!ehr>5^J+XBmPTPnR z7h8p-0x7h9_UZeZNb>3Lm@zC3RHylNqofba%fN#-%FOVdng2 zxNDJO{|b*TkG_Q;$-`Lxe>EUIj24o7o?0(brTNiVNleL8s6Ix?NOgyd;qfO$`k3hS zhK(8d3Kry6KkdsYv?QsV#6^ijUfJD6&VuYu!DxAuwbNb?{tCM#8_7$n8%ku7@G>ul zl!ip9U%#|#+>25|d<5==MKq{Iz}j%-v-tS6U5)Ult>kk=n%ecZK>J(fySsBDF3;*k zoe0r-tT)*NNw%n(r;5bVFpJ~P5J8LR4YW?9WN)J~zo=JTD638?GwaO6*Lir##bOWGy35s$J*^aUjvB>GDzIs%e=Di)xuAl5A-uc*p2Rk#ph zs1^1R_2pZbT5XC&e1fRAd30Ewe$)BK+yNJ?3p_?|Hw)>-5RK7;h*Q5A5 zBIGs|+crnW4r0-WaX^Kf9kH-Hg7+$G-5y!0w2o--z$|DD6Zw)qsk2JNg% zHuonxz~G=AM*)dIQmpr zFNccGh#-a#Gc&><^NWh@+7;7&XxsMp4znMcXi4(Fygs-cwNu#(Zr;(VIVc=TUx@d= zqhv!=Bpifprpz4^!I9b|B5GSB0Nj>|5D%0^__j%8*)bisOCzg8!P-G7qaakHc2Oo! zYABQ3`eS8eQs^J=V2|vgg>;{A)?OYSy^S>RV1_@=!n~8&utLnAN2^iE?QWW>-0r7R zEVB7j}NO(FV07)Q4S5~(ake}l5-yhfe2km^7$)PyXb+#Lh3-pU&F z+$mDd+>UykG_QRE)&l5jW$Eg&7OYmBoP^NiJ5;?()dH$~_Qujji&sCs5&V2{@mjF( z;nF9I@;w?Q50LLuwTKF^tgT!t{UTAd<|ni6K=JfWFHm82^Kl<=<)Amt=xol7rNcf+vHu}i*xafq4 z`7}VXjot;~3@4jSh^9G~;jmtGccr-351T~FhKDJgxwPO>FX}{XB;3WDrD4er>5?B2 z6Lwd`{o&O9UQLqv@ZC?xsR+CX8TY|W=uJjQ7fHOwPG`t3GR2YXdSH&8NP}3M${NFo zW|Od_Nmw#`XZ&38_TYtbot_8Rq3u>hH^MtDMCO9*Y8CG(j+q1fctl$A`3CIVp#XAmHdEFO8akRw8j zAAY%ob-9HHabCT&c>PAOe06!z*Hd<3I(PNL0-MPlk>?D02fXe8>|Q9$>>>rU?6;_F zqf(~GM-gz^oI|v%p&7L)@_7CKuj%og2TJHDeOy!}2J{nqR-Br)M}E@g+)>WVt0vrpA`2hh-pQ8k)|Z@blWg9*zEHshh!zNt19F!HNiM+NB^O*CNPr-Z&f_i#kl_9c zpkRS`*Jgg-*E7SJA*uBuL(IqOs;{c6zWVBW)i>4t{;Yz}@xS%Ew|{g^QT_)#bbk8r za1P&akD(|x6`=^#SDfU(>ZtszIU4?Jz8)Bk5u}`yO6|ID25BeFWy80EjFSnnPL`ij zeqWGta$Gk3{vhw8dCz<2JlmU^H^vlauXn^dFZN6- zz;@TTWA;O$5I^TI~E-EP0&jfj0{yXfr{`!TOezYeelxP zzOQ2BF|Xjg#B{mZoCnbv)Dtt_x5TWMXIkaN+v3_I(jTNS=0>**{TQRN9A~EPyXDBe<$JMl0lnh%n_k6}ix?q2x4h`R^^^DQ-x6)k`+(l-26)}BZmiB3<5fLlG+XDYRe##`oA2qFFg13KqQ+S)V!pgU z;VuAAA;suuHx*C!AdsMpFCjHrh8C%6-#SOe-G0ql?K~>azD(8yDdo|ZjTBMC(V;z%- zw-vGGN;PtcS;J!Si6UP?BZJS*AB}>=ql@LmBathy+#|0CwcEk*;Qmp+eCz1qGEQRU z=#uXTN5Q>vd9hlqMB&jA4nVOMmHlvHaakV5Xov6(O@OSDQw^2tE@3-T5nk+Xz518X zW%Jd85~ff|po1D3sG)-z#xu?plPNh28Q$6=$Q2tAxM;SJmJ@iBJWAjg0h<8LT%I71 zC(tHKQaVX>uK`f6&;p>v4QN;%)o#7018D7Blu(8CiRu_$$}_Q?w9v5}1ItOp2c~Ob zR1Lb1Wv>))QaFCq`$SHhW$DUOc)@JJ>Rr=wsCI^n#l>X-aa1l9OMDWG#bmX|(VVPy zvxUBaR$5kbp;@pAl^IX39z#JLC(4O+(ci)9gK+fU(ap}&XTak^L()zU`t4inMH z4ra>90V?GP>?W`o@z0=P8sBgyKvpr-oT{lKnxOqp4>~H(;TygKu)YtS zf&|nyn_?x|k=9eP6RoFYH%#o8)jW3C5n+P<>3A4Cx#Y=^g-Da`auRQpUm@^S0%r-F z1DGvjdPvpFfM0?n=P?kmRyYbk2M>fGVl=;e3<(DkGL?wyR3gw*OJ(6zW34#1fmU>e z=@b3{phYX*oRrs_sDpt!rtu@ zIWH&rxuxIBg2zpf7u#S-<#EmjVAl@7V#;^}u;2#85cP5fxzDgSg!)z=vgHkD7}oTd z*dcZ@-u7=RhT@EZ{=4F_^KR&^t2P)j0kmrAt|#k%Cu`%wPoW)<_B#Td;QBTUoPbY& zYcG_SyoxQMTD>r`p#p8M64oRRNJJ-tj!2~z-0~!kRh|EG}>*E~e=;5HTgo-m!%Qh$xKes+q*6^kO;U$1bd8n+a zj}=vEs12p4H8eoIq24j%5WzdCidt4s2h}08M=0GfFDMTQH@!ghmQX-XeYT`xwAP1S zLE`KNzTpr64e@?zMQuPUe^v`v! zRCNaYasb;1LlhI*$N3^_5>FJFIba)L1HxL+t}v4Z94aI9{CW8nE&4LR6E)5hizVL; zVM281$62)1@nyPYRZ}xnL(8sdsKwmvO>m_|iNp@@dE3T}7|kK&0+Lz%)Wfk*|7=Pkbhl-`XROxy^Okgo_U>6d9V z8ak`jUw+1v+5PC~7h6p6au#&ZVq5??BJ)D^v<-_ceK zu$g*SlkYYRq2d2_p*OVe!9w^HCJ*XTsM}3)80GYmBKHvwqimu6WszarY=XP0i@wLI zJO!EK7Xu^iYN)Ld3%h3}5yLSTZ4%Z) z$;uMM6NY!p*>|qa6fe}%&8fs@fKGv1&on39I9t#hi{5jhxa2$8r1UDv^_M3bOXU!T zH<%2*U*l|Ror|uZhi0Q!!IP5;s(w(7mHNod%H2x!e#Pd~Zj%Xn+LjlHU=~ifbeF0h z)b~s_AizDjgu@#ydL_8SO@2PsI4ds#d9$~(a)bl4;qCRIj+cUr(eh~b_Sg=<*7lz}7+WuD9%@@!u!^wR=P4RUrKk zU-l48#ng7LnQPC~Zcv4{;Q4Q=-AqXmF*f*3^Hj{@pj0T5Qu1OGjt&k*Y$7;=|6m(J zfnZ3c7CL$0F2L_ag@B(g%hXIWr_Jh_ITUDfCch1=a0CGCqYr>(24GohKQYtLTVFg= zS2YKS4H|!ga3{}~dEw_#AmP4BLQ{Q=y~mTrPugQ_#SR7H-e~A|4f%YeL(uCa=t(e& z)Tb#Hwj{W*iIX&rv6^aV#3vBwCOE|)VO}un%WOoE-8UtK?QSsF?5LVpV~I*?nqF-9 zB`TE+Kd(JV2glpkcTkoFsD<8PyF8Cx%ZbrO?~@ByA=^TAYbt_(f<5_}YS&*l6c+5j zUAAv|^fnSRg1oP4#cSD}6Y#>nj^443K!dzarR+PdU-MWpNMdFg5byxH1Vt4UPJhdE zVf~s2k8Tc)bytW#04^IN`P2~3J6tEogshQ+w5UI~^{GeCcz1U*T%Kmk4pO%3_|xH` zFKzi#V&h2pNRFq4L7tBCO-I2gbf@B}aJiJFMLp858dq~}In5|5Jqfkq=6#L(*F z-BY;?rfb^!6J33tX!1SukP$#(N3Yj4MNKadAR!^iR*Q2uVTcVRK~_mZEY_j#Bu58G zigSRC6~K*1CeABuNaD~Qq*8?dnd34fpc7aIfO(o&h$Rp1bJI%XCEp|zlIrqJ0O}Rm z02VymQB&)))Lp8nYpS+ps<0hj-l^^y)CNaX#x4P|>qGB5hwoDgI<0$giN}lRzb16} zQt;PAN|-3YgTtZ8#M?LZpO}6dc9-5UjW2tjbf9GhM-bxujzX3uSzx^1urhA|Eg{JV zRO0nAU@g{1k%Xp@`Xd^UHh5I6@7r`T6CK#RryC|l-DzS^$(MzohjY`KiW=`&AGC{k z@6@)v;oWJ5HK{dwk%kI>FTcMzgY4f!Ej{|mkxDzua# zEw6K333Xi`qW?L3Li@}PaI{4i2BIyxNF{MYoM4<-KDTNTJLEep4d*$7T0~$3B8KGO zBc_7*3&%?OP}|?G